From 4b9336ec073d1b7e45131b15254025570fbd49c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Tue, 24 Sep 2024 10:37:08 +0200 Subject: [PATCH] API tweaks for OpenAPI, errors and endpoints (#886) * All sort of tweaks * docs: availability's minPrice doc * Revert changes to the two node test example * Change default EC params in REST API Change default EC params in REST API to 3 nodes and 1 tolerance. Adjust integration tests to honour these settings. --------- Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> --- codex/contracts/requests.nim | 8 +-- codex/rest/api.nim | 34 +++++++------ codex/rest/json.nim | 2 + codex/sales.nim | 14 ++++-- codex/sales/reservations.nim | 4 +- codex/sales/states/preparing.nim | 4 +- openapi.yaml | 75 ++++++++++++++++++++++------ tests/integration/codexclient.nim | 8 +-- tests/integration/testpurchasing.nim | 8 +-- tests/integration/testrestapi.nim | 29 +++++++++-- 10 files changed, 133 insertions(+), 53 deletions(-) diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 1363fb9d..d94baa17 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -163,12 +163,12 @@ func id*(request: StorageRequest): RequestId = let encoding = AbiEncoder.encode((request, )) RequestId(keccak256.digest(encoding).data) -func slotId*(requestId: RequestId, slot: UInt256): SlotId = - let encoding = AbiEncoder.encode((requestId, slot)) +func slotId*(requestId: RequestId, slotIndex: UInt256): SlotId = + let encoding = AbiEncoder.encode((requestId, slotIndex)) SlotId(keccak256.digest(encoding).data) -func slotId*(request: StorageRequest, slot: UInt256): SlotId = - slotId(request.id, slot) +func slotId*(request: StorageRequest, slotIndex: UInt256): SlotId = + slotId(request.id, slotIndex) func id*(slot: Slot): SlotId = slotId(slot.request, slot.slotIndex) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 8abb91ce..1323654a 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -232,7 +232,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = ## Returns active slots for the host try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") let json = %(await contracts.sales.mySlots()) return RestApiResponse.response($json, contentType="application/json") @@ -247,7 +247,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = ## slot is not active for the host. without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without slotId =? slotId.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -258,7 +258,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = let restAgent = RestSalesAgent( state: agent.state() |? "none", slotIndex: agent.data.slotIndex, - requestId: agent.data.requestId + requestId: agent.data.requestId, + request: agent.data.request, + reservation: agent.data.reservation, ) return RestApiResponse.response(restAgent.toJson, contentType="application/json") @@ -270,7 +272,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without avails =? (await contracts.sales.context.reservations.all(Availability)), err: return RestApiResponse.error(Http500, err.msg) @@ -289,7 +291,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = ## ## totalSize - size of available storage in bytes ## duration - maximum time the storage should be sold for (in seconds) - ## minPrice - minimum price to be paid (in amount of tokens) + ## minPrice - minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration ## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens) var headers = newSeq[(string,string)]() @@ -301,7 +303,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable", headers = headers) + return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers) let body = await request.getBody() @@ -359,7 +361,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without id =? id.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -415,7 +417,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Sales unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without id =? id.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -423,6 +425,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = return RestApiResponse.error(Http400, error.msg) let reservations = contracts.sales.context.reservations + let market = contracts.sales.context.market if error =? (await reservations.get(keyId, Availability)).errorOption: if error of NotExistsError: @@ -462,10 +465,10 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = headers.add(("Access-Control-Allow-Origin", corsOrigin)) headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS")) headers.add(("Access-Control-Max-Age", "86400")) - + try: without contracts =? node.contracts.client: - return RestApiResponse.error(Http503, "Purchasing unavailable", headers = headers) + return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers) without cid =? cid.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg, headers = headers) @@ -475,8 +478,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = without params =? StorageRequestParams.fromJson(body), error: return RestApiResponse.error(Http400, error.msg, headers = headers) - let nodes = params.nodes |? 1 - let tolerance = params.tolerance |? 0 + let nodes = params.nodes |? 3 + let tolerance = params.tolerance |? 1 + + if tolerance == 0: + return RestApiResponse.error(Http400, "Tolerance needs to be bigger then zero", headers = headers) # prevent underflow if tolerance > nodes: @@ -524,7 +530,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = try: without contracts =? node.contracts.client: - return RestApiResponse.error(Http503, "Purchasing unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") without id =? id.tryGet.catch, error: return RestApiResponse.error(Http400, error.msg) @@ -549,7 +555,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = "/api/codex/v1/storage/purchases") do () -> RestApiResponse: try: without contracts =? node.contracts.client: - return RestApiResponse.error(Http503, "Purchasing unavailable") + return RestApiResponse.error(Http503, "Persistence is not enabled") let purchaseIds = contracts.purchasing.getPurchaseIds() return RestApiResponse.response($ %purchaseIds, contentType="application/json") diff --git a/codex/rest/json.nim b/codex/rest/json.nim index fba708be..24936b34 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -38,6 +38,8 @@ type state* {.serialize.}: string requestId* {.serialize.}: RequestId slotIndex* {.serialize.}: UInt256 + request* {.serialize.}: ?StorageRequest + reservation* {.serialize.}: ?Reservation RestContent* = object cid* {.serialize.}: Cid diff --git a/codex/sales.nim b/codex/sales.nim index c4fcb217..974d4f29 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -180,7 +180,7 @@ proc filled( processing.complete() proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = - debug "processing slot from queue", requestId = item.requestId, + debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex let agent = newSalesAgent( @@ -202,13 +202,17 @@ proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} = let reservations = sales.context.reservations without reservs =? await reservations.all(Reservation): - info "no unused reservations found for deletion" + return let unused = reservs.filter(r => ( let slotId = slotId(r.requestId, r.slotIndex) not activeSlots.any(slot => slot.id == slotId) )) - info "found unused reservations for deletion", unused = unused.len + + if unused.len == 0: + return + + info "Found unused reservations for deletion", unused = unused.len for reservation in unused: @@ -219,9 +223,9 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} if err =? (await reservations.deleteReservation( reservation.id, reservation.availabilityId )).errorOption: - error "failed to delete unused reservation", error = err.msg + error "Failed to delete unused reservation", error = err.msg else: - trace "deleted unused reservation" + trace "Deleted unused reservation" proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} = let market = sales.context.market diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 0b5eaaf5..027bda95 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -16,7 +16,7 @@ ## |----------------------------------------| |--------------------------------------| ## | UInt256 | totalSize | | | UInt256 | size | | ## |----------------------------------------| |--------------------------------------| -## | UInt256 | freeSize | | | SlotId | slotId | | +## | UInt256 | freeSize | | | UInt256 | slotIndex | | ## |----------------------------------------| +--------------------------------------+ ## | UInt256 | duration | | ## |----------------------------------------| @@ -65,7 +65,7 @@ type totalSize* {.serialize.}: UInt256 freeSize* {.serialize.}: UInt256 duration* {.serialize.}: UInt256 - minPrice* {.serialize.}: UInt256 + minPrice* {.serialize.}: UInt256 # minimal price paid for the whole hosted slot for the request's duration maxCollateral* {.serialize.}: UInt256 Reservation* = ref object id* {.serialize.}: ReservationId diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index e5a441d3..d7f38f5b 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -69,11 +69,11 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} = request.ask.duration, request.ask.pricePerSlot, request.ask.collateral): - debug "no availability found for request, ignoring" + debug "No availability found for request, ignoring" return some State(SaleIgnored()) - info "availability found for request, creating reservation" + info "Availability found for request, creating reservation" without reservation =? await reservations.createReservation( availability.id, diff --git a/openapi.yaml b/openapi.yaml index 94450bf3..9a2056ca 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -23,6 +23,8 @@ components: Id: type: string description: 32bits identifier encoded in hex-decimal string. + minLength: 66 + maxLength: 66 example: 0x... BigInt: @@ -136,7 +138,7 @@ components: $ref: "#/components/schemas/Duration" minPrice: type: string - description: Minimum price to be paid (in amount of tokens) as decimal string + description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string maxCollateral: type: string description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string @@ -168,7 +170,39 @@ components: $ref: "#/components/schemas/StorageRequest" slotIndex: type: string - description: Slot Index as hexadecimal string + description: Slot Index as decimal string + + SlotAgent: + type: object + properties: + id: + $ref: "#/components/schemas/SlotId" + slotIndex: + type: string + description: Slot Index as decimal string + requestId: + $ref: "#/components/schemas/Id" + request: + $ref: "#/components/schemas/StorageRequest" + reservation: + $ref: "#/components/schemas/Reservation" + state: + type: string + description: Description of the slot's + enum: + - SaleCancelled + - SaleDownloading + - SaleErrored + - SaleFailed + - SaleFilled + - SaleFilling + - SaleFinished + - SaleIgnored + - SaleInitialProving + - SalePayout + - SalePreparing + - SaleProving + - SaleUnknown Reservation: type: object @@ -183,7 +217,7 @@ components: $ref: "#/components/schemas/Id" slotIndex: type: string - description: Slot Index as hexadecimal string + description: Slot Index as decimal string StorageRequestCreation: type: object @@ -259,6 +293,15 @@ components: state: type: string description: Description of the Request's state + enum: + - cancelled + - error + - failed + - finished + - pending + - started + - submitted + - unknown error: type: string description: If Request failed, then here is presented the error message @@ -491,7 +534,7 @@ paths: $ref: "#/components/schemas/Slot" "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/slots/{slotId}": get: @@ -511,7 +554,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Slot" + $ref: "#/components/schemas/SlotAgent" "400": description: Invalid or missing SlotId @@ -520,13 +563,13 @@ paths: description: Host is not in an active sale for the slot "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/availability": get: summary: "Returns storage that is for sale" tags: [ Marketplace ] - operationId: getOfferedStorage + operationId: getAvailabilities responses: "200": description: Retrieved storage availabilities of the node @@ -535,11 +578,11 @@ paths: schema: type: array items: - $ref: "#/components/schemas/SalesAvailability" + $ref: "#/components/schemas/SalesAvailabilityREAD" "500": description: Error getting unused availabilities "503": - description: Sales are unavailable + description: Persistence is not enabled post: summary: "Offers storage for sale" @@ -564,7 +607,7 @@ paths: "500": description: Error reserving availability "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/availability/{id}": patch: summary: "Updates availability" @@ -597,10 +640,10 @@ paths: "500": description: Error reserving availability "503": - description: Sales are unavailable + description: Persistence is not enabled "/sales/availability/{id}/reservations": - patch: + get: summary: "Get availability's reservations" description: Return's list of Reservations for ongoing Storage Requests that the node hosts. operationId: getReservations @@ -628,7 +671,7 @@ paths: "500": description: Error getting reservations "503": - description: Sales are unavailable + description: Persistence is not enabled "/storage/request/{cid}": post: @@ -659,7 +702,7 @@ paths: "404": description: Request ID not found "503": - description: Purchasing is unavailable + description: Persistence is not enabled "/storage/purchases": get: @@ -676,7 +719,7 @@ paths: items: type: string "503": - description: Purchasing is unavailable + description: Persistence is not enabled "/storage/purchases/{id}": get: @@ -702,7 +745,7 @@ paths: "404": description: Purchase not found "503": - description: Purchasing is unavailable + description: Persistence is not enabled "/node/spr": get: diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index d2d78b46..2b1e324c 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -96,8 +96,8 @@ proc requestStorageRaw*( proofProbability: UInt256, collateral: UInt256, expiry: uint = 0, - nodes: uint = 2, - tolerance: uint = 0 + nodes: uint = 3, + tolerance: uint = 1 ): Response = ## Call request storage REST endpoint @@ -125,8 +125,8 @@ proc requestStorage*( proofProbability: UInt256, expiry: uint, collateral: UInt256, - nodes: uint = 2, - tolerance: uint = 0 + nodes: uint = 3, + tolerance: uint = 1 ): ?!PurchaseId = ## Call request storage REST endpoint ## diff --git a/tests/integration/testpurchasing.nim b/tests/integration/testpurchasing.nim index bc87f51b..5da4df08 100644 --- a/tests/integration/testpurchasing.nim +++ b/tests/integration/testpurchasing.nim @@ -60,7 +60,9 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false: reward=2.u256, proofProbability=3.u256, expiry=30, - collateral=200.u256).get + collateral=200.u256, + nodes=3.uint, + tolerance=1.uint).get check eventually client1.purchaseStateIs(id, "submitted") node1.restart() @@ -73,8 +75,8 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false: check request.ask.proofProbability == 3.u256 check request.expiry == 30 check request.ask.collateral == 200.u256 - check request.ask.slots == 2'u64 - check request.ask.maxSlotLoss == 0'u64 + check request.ask.slots == 3'u64 + check request.ask.maxSlotLoss == 1'u64 test "node requires expiry and its value to be in future": let data = await RandomChunker.example(blocks=2) diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 1834dcf2..1848ba0e 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -41,7 +41,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false: test "request storage fails for datasets that are too small": let cid = client1.upload("some file contents").get - let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, nodes=2, collateral=200.u256, expiry=9) + let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=9) check: response.status == "400 Bad Request" @@ -55,6 +55,29 @@ twonodessuite "REST API", debug1 = false, debug2 = false: check: response.status == "200 OK" + test "request storage fails if tolerance is zero": + let data = await RandomChunker.example(blocks=2) + let cid = client1.upload(data).get + let duration = 100.u256 + let reward = 2.u256 + let proofProbability = 3.u256 + let expiry = 30.uint + let collateral = 200.u256 + let nodes = 3 + let tolerance = 0 + + var responseBefore = client1.requestStorageRaw(cid, + duration, + reward, + proofProbability, + collateral, + expiry, + nodes.uint, + tolerance.uint) + + check responseBefore.status == "400 Bad Request" + check responseBefore.body == "Tolerance needs to be bigger then zero" + test "request storage fails if nodes and tolerance aren't correct": let data = await RandomChunker.example(blocks=2) let cid = client1.upload(data).get @@ -63,7 +86,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false: let proofProbability = 3.u256 let expiry = 30.uint let collateral = 200.u256 - let ecParams = @[(1, 0), (1, 1), (2, 1), (3, 2), (3, 3)] + let ecParams = @[(1, 1), (2, 1), (3, 2), (3, 3)] for ecParam in ecParams: let (nodes, tolerance) = ecParam @@ -113,7 +136,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false: let proofProbability = 3.u256 let expiry = 30.uint let collateral = 200.u256 - let ecParams = @[(2, 0), (3, 1), (5, 2)] + let ecParams = @[(3, 1), (5, 2)] for ecParam in ecParams: let (nodes, tolerance) = ecParam