API tweaks for OpenAPI, errors and endpoints (#886)

* All sort of tweaks

* docs: availability's minPrice doc

* Revert changes to the two node test example

* Change default EC params in REST API

Change default EC params in REST API to 3 nodes and 1 tolerance.

Adjust integration tests to honour these settings.

---------

Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com>
This commit is contained in:
Adam Uhlíř 2024-09-24 10:37:08 +02:00 committed by GitHub
parent 098c4bb6e9
commit 4b9336ec07
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 133 additions and 53 deletions

View File

@ -163,12 +163,12 @@ func id*(request: StorageRequest): RequestId =
let encoding = AbiEncoder.encode((request, )) let encoding = AbiEncoder.encode((request, ))
RequestId(keccak256.digest(encoding).data) RequestId(keccak256.digest(encoding).data)
func slotId*(requestId: RequestId, slot: UInt256): SlotId = func slotId*(requestId: RequestId, slotIndex: UInt256): SlotId =
let encoding = AbiEncoder.encode((requestId, slot)) let encoding = AbiEncoder.encode((requestId, slotIndex))
SlotId(keccak256.digest(encoding).data) SlotId(keccak256.digest(encoding).data)
func slotId*(request: StorageRequest, slot: UInt256): SlotId = func slotId*(request: StorageRequest, slotIndex: UInt256): SlotId =
slotId(request.id, slot) slotId(request.id, slotIndex)
func id*(slot: Slot): SlotId = func id*(slot: Slot): SlotId =
slotId(slot.request, slot.slotIndex) slotId(slot.request, slot.slotIndex)

View File

@ -232,7 +232,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
## Returns active slots for the host ## Returns active slots for the host
try: try:
without contracts =? node.contracts.host: without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
let json = %(await contracts.sales.mySlots()) let json = %(await contracts.sales.mySlots())
return RestApiResponse.response($json, contentType="application/json") return RestApiResponse.response($json, contentType="application/json")
@ -247,7 +247,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
## slot is not active for the host. ## slot is not active for the host.
without contracts =? node.contracts.host: without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
without slotId =? slotId.tryGet.catch, error: without slotId =? slotId.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg) return RestApiResponse.error(Http400, error.msg)
@ -258,7 +258,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
let restAgent = RestSalesAgent( let restAgent = RestSalesAgent(
state: agent.state() |? "none", state: agent.state() |? "none",
slotIndex: agent.data.slotIndex, slotIndex: agent.data.slotIndex,
requestId: agent.data.requestId requestId: agent.data.requestId,
request: agent.data.request,
reservation: agent.data.reservation,
) )
return RestApiResponse.response(restAgent.toJson, contentType="application/json") return RestApiResponse.response(restAgent.toJson, contentType="application/json")
@ -270,7 +272,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try: try:
without contracts =? node.contracts.host: without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
without avails =? (await contracts.sales.context.reservations.all(Availability)), err: without avails =? (await contracts.sales.context.reservations.all(Availability)), err:
return RestApiResponse.error(Http500, err.msg) return RestApiResponse.error(Http500, err.msg)
@ -289,7 +291,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
## ##
## totalSize - size of available storage in bytes ## totalSize - size of available storage in bytes
## duration - maximum time the storage should be sold for (in seconds) ## duration - maximum time the storage should be sold for (in seconds)
## minPrice - minimum price to be paid (in amount of tokens) ## minPrice - minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens) ## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
var headers = newSeq[(string,string)]() var headers = newSeq[(string,string)]()
@ -301,7 +303,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try: try:
without contracts =? node.contracts.host: without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable", headers = headers) return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
let body = await request.getBody() let body = await request.getBody()
@ -359,7 +361,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try: try:
without contracts =? node.contracts.host: without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
without id =? id.tryGet.catch, error: without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg) return RestApiResponse.error(Http400, error.msg)
@ -415,7 +417,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try: try:
without contracts =? node.contracts.host: without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
without id =? id.tryGet.catch, error: without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg) return RestApiResponse.error(Http400, error.msg)
@ -423,6 +425,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
return RestApiResponse.error(Http400, error.msg) return RestApiResponse.error(Http400, error.msg)
let reservations = contracts.sales.context.reservations let reservations = contracts.sales.context.reservations
let market = contracts.sales.context.market
if error =? (await reservations.get(keyId, Availability)).errorOption: if error =? (await reservations.get(keyId, Availability)).errorOption:
if error of NotExistsError: if error of NotExistsError:
@ -462,10 +465,10 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
headers.add(("Access-Control-Allow-Origin", corsOrigin)) headers.add(("Access-Control-Allow-Origin", corsOrigin))
headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS")) headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS"))
headers.add(("Access-Control-Max-Age", "86400")) headers.add(("Access-Control-Max-Age", "86400"))
try: try:
without contracts =? node.contracts.client: without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Purchasing unavailable", headers = headers) return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
without cid =? cid.tryGet.catch, error: without cid =? cid.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg, headers = headers) return RestApiResponse.error(Http400, error.msg, headers = headers)
@ -475,8 +478,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
without params =? StorageRequestParams.fromJson(body), error: without params =? StorageRequestParams.fromJson(body), error:
return RestApiResponse.error(Http400, error.msg, headers = headers) return RestApiResponse.error(Http400, error.msg, headers = headers)
let nodes = params.nodes |? 1 let nodes = params.nodes |? 3
let tolerance = params.tolerance |? 0 let tolerance = params.tolerance |? 1
if tolerance == 0:
return RestApiResponse.error(Http400, "Tolerance needs to be bigger then zero", headers = headers)
# prevent underflow # prevent underflow
if tolerance > nodes: if tolerance > nodes:
@ -524,7 +530,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
try: try:
without contracts =? node.contracts.client: without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Purchasing unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
without id =? id.tryGet.catch, error: without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg) return RestApiResponse.error(Http400, error.msg)
@ -549,7 +555,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
"/api/codex/v1/storage/purchases") do () -> RestApiResponse: "/api/codex/v1/storage/purchases") do () -> RestApiResponse:
try: try:
without contracts =? node.contracts.client: without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Purchasing unavailable") return RestApiResponse.error(Http503, "Persistence is not enabled")
let purchaseIds = contracts.purchasing.getPurchaseIds() let purchaseIds = contracts.purchasing.getPurchaseIds()
return RestApiResponse.response($ %purchaseIds, contentType="application/json") return RestApiResponse.response($ %purchaseIds, contentType="application/json")

View File

@ -38,6 +38,8 @@ type
state* {.serialize.}: string state* {.serialize.}: string
requestId* {.serialize.}: RequestId requestId* {.serialize.}: RequestId
slotIndex* {.serialize.}: UInt256 slotIndex* {.serialize.}: UInt256
request* {.serialize.}: ?StorageRequest
reservation* {.serialize.}: ?Reservation
RestContent* = object RestContent* = object
cid* {.serialize.}: Cid cid* {.serialize.}: Cid

View File

@ -180,7 +180,7 @@ proc filled(
processing.complete() processing.complete()
proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
debug "processing slot from queue", requestId = item.requestId, debug "Processing slot from queue", requestId = item.requestId,
slot = item.slotIndex slot = item.slotIndex
let agent = newSalesAgent( let agent = newSalesAgent(
@ -202,13 +202,17 @@ proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} = proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} =
let reservations = sales.context.reservations let reservations = sales.context.reservations
without reservs =? await reservations.all(Reservation): without reservs =? await reservations.all(Reservation):
info "no unused reservations found for deletion" return
let unused = reservs.filter(r => ( let unused = reservs.filter(r => (
let slotId = slotId(r.requestId, r.slotIndex) let slotId = slotId(r.requestId, r.slotIndex)
not activeSlots.any(slot => slot.id == slotId) not activeSlots.any(slot => slot.id == slotId)
)) ))
info "found unused reservations for deletion", unused = unused.len
if unused.len == 0:
return
info "Found unused reservations for deletion", unused = unused.len
for reservation in unused: for reservation in unused:
@ -219,9 +223,9 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.}
if err =? (await reservations.deleteReservation( if err =? (await reservations.deleteReservation(
reservation.id, reservation.availabilityId reservation.id, reservation.availabilityId
)).errorOption: )).errorOption:
error "failed to delete unused reservation", error = err.msg error "Failed to delete unused reservation", error = err.msg
else: else:
trace "deleted unused reservation" trace "Deleted unused reservation"
proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} = proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} =
let market = sales.context.market let market = sales.context.market

View File

@ -16,7 +16,7 @@
## |----------------------------------------| |--------------------------------------| ## |----------------------------------------| |--------------------------------------|
## | UInt256 | totalSize | | | UInt256 | size | | ## | UInt256 | totalSize | | | UInt256 | size | |
## |----------------------------------------| |--------------------------------------| ## |----------------------------------------| |--------------------------------------|
## | UInt256 | freeSize | | | SlotId | slotId | | ## | UInt256 | freeSize | | | UInt256 | slotIndex | |
## |----------------------------------------| +--------------------------------------+ ## |----------------------------------------| +--------------------------------------+
## | UInt256 | duration | | ## | UInt256 | duration | |
## |----------------------------------------| ## |----------------------------------------|
@ -65,7 +65,7 @@ type
totalSize* {.serialize.}: UInt256 totalSize* {.serialize.}: UInt256
freeSize* {.serialize.}: UInt256 freeSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256 duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256 minPrice* {.serialize.}: UInt256 # minimal price paid for the whole hosted slot for the request's duration
maxCollateral* {.serialize.}: UInt256 maxCollateral* {.serialize.}: UInt256
Reservation* = ref object Reservation* = ref object
id* {.serialize.}: ReservationId id* {.serialize.}: ReservationId

View File

@ -69,11 +69,11 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
request.ask.duration, request.ask.duration,
request.ask.pricePerSlot, request.ask.pricePerSlot,
request.ask.collateral): request.ask.collateral):
debug "no availability found for request, ignoring" debug "No availability found for request, ignoring"
return some State(SaleIgnored()) return some State(SaleIgnored())
info "availability found for request, creating reservation" info "Availability found for request, creating reservation"
without reservation =? await reservations.createReservation( without reservation =? await reservations.createReservation(
availability.id, availability.id,

View File

@ -23,6 +23,8 @@ components:
Id: Id:
type: string type: string
description: 32bits identifier encoded in hex-decimal string. description: 32bits identifier encoded in hex-decimal string.
minLength: 66
maxLength: 66
example: 0x... example: 0x...
BigInt: BigInt:
@ -136,7 +138,7 @@ components:
$ref: "#/components/schemas/Duration" $ref: "#/components/schemas/Duration"
minPrice: minPrice:
type: string type: string
description: Minimum price to be paid (in amount of tokens) as decimal string description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
maxCollateral: maxCollateral:
type: string type: string
description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
@ -168,7 +170,39 @@ components:
$ref: "#/components/schemas/StorageRequest" $ref: "#/components/schemas/StorageRequest"
slotIndex: slotIndex:
type: string type: string
description: Slot Index as hexadecimal string description: Slot Index as decimal string
SlotAgent:
type: object
properties:
id:
$ref: "#/components/schemas/SlotId"
slotIndex:
type: string
description: Slot Index as decimal string
requestId:
$ref: "#/components/schemas/Id"
request:
$ref: "#/components/schemas/StorageRequest"
reservation:
$ref: "#/components/schemas/Reservation"
state:
type: string
description: Description of the slot's
enum:
- SaleCancelled
- SaleDownloading
- SaleErrored
- SaleFailed
- SaleFilled
- SaleFilling
- SaleFinished
- SaleIgnored
- SaleInitialProving
- SalePayout
- SalePreparing
- SaleProving
- SaleUnknown
Reservation: Reservation:
type: object type: object
@ -183,7 +217,7 @@ components:
$ref: "#/components/schemas/Id" $ref: "#/components/schemas/Id"
slotIndex: slotIndex:
type: string type: string
description: Slot Index as hexadecimal string description: Slot Index as decimal string
StorageRequestCreation: StorageRequestCreation:
type: object type: object
@ -259,6 +293,15 @@ components:
state: state:
type: string type: string
description: Description of the Request's state description: Description of the Request's state
enum:
- cancelled
- error
- failed
- finished
- pending
- started
- submitted
- unknown
error: error:
type: string type: string
description: If Request failed, then here is presented the error message description: If Request failed, then here is presented the error message
@ -491,7 +534,7 @@ paths:
$ref: "#/components/schemas/Slot" $ref: "#/components/schemas/Slot"
"503": "503":
description: Sales are unavailable description: Persistence is not enabled
"/sales/slots/{slotId}": "/sales/slots/{slotId}":
get: get:
@ -511,7 +554,7 @@ paths:
content: content:
application/json: application/json:
schema: schema:
$ref: "#/components/schemas/Slot" $ref: "#/components/schemas/SlotAgent"
"400": "400":
description: Invalid or missing SlotId description: Invalid or missing SlotId
@ -520,13 +563,13 @@ paths:
description: Host is not in an active sale for the slot description: Host is not in an active sale for the slot
"503": "503":
description: Sales are unavailable description: Persistence is not enabled
"/sales/availability": "/sales/availability":
get: get:
summary: "Returns storage that is for sale" summary: "Returns storage that is for sale"
tags: [ Marketplace ] tags: [ Marketplace ]
operationId: getOfferedStorage operationId: getAvailabilities
responses: responses:
"200": "200":
description: Retrieved storage availabilities of the node description: Retrieved storage availabilities of the node
@ -535,11 +578,11 @@ paths:
schema: schema:
type: array type: array
items: items:
$ref: "#/components/schemas/SalesAvailability" $ref: "#/components/schemas/SalesAvailabilityREAD"
"500": "500":
description: Error getting unused availabilities description: Error getting unused availabilities
"503": "503":
description: Sales are unavailable description: Persistence is not enabled
post: post:
summary: "Offers storage for sale" summary: "Offers storage for sale"
@ -564,7 +607,7 @@ paths:
"500": "500":
description: Error reserving availability description: Error reserving availability
"503": "503":
description: Sales are unavailable description: Persistence is not enabled
"/sales/availability/{id}": "/sales/availability/{id}":
patch: patch:
summary: "Updates availability" summary: "Updates availability"
@ -597,10 +640,10 @@ paths:
"500": "500":
description: Error reserving availability description: Error reserving availability
"503": "503":
description: Sales are unavailable description: Persistence is not enabled
"/sales/availability/{id}/reservations": "/sales/availability/{id}/reservations":
patch: get:
summary: "Get availability's reservations" summary: "Get availability's reservations"
description: Return's list of Reservations for ongoing Storage Requests that the node hosts. description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
operationId: getReservations operationId: getReservations
@ -628,7 +671,7 @@ paths:
"500": "500":
description: Error getting reservations description: Error getting reservations
"503": "503":
description: Sales are unavailable description: Persistence is not enabled
"/storage/request/{cid}": "/storage/request/{cid}":
post: post:
@ -659,7 +702,7 @@ paths:
"404": "404":
description: Request ID not found description: Request ID not found
"503": "503":
description: Purchasing is unavailable description: Persistence is not enabled
"/storage/purchases": "/storage/purchases":
get: get:
@ -676,7 +719,7 @@ paths:
items: items:
type: string type: string
"503": "503":
description: Purchasing is unavailable description: Persistence is not enabled
"/storage/purchases/{id}": "/storage/purchases/{id}":
get: get:
@ -702,7 +745,7 @@ paths:
"404": "404":
description: Purchase not found description: Purchase not found
"503": "503":
description: Purchasing is unavailable description: Persistence is not enabled
"/node/spr": "/node/spr":
get: get:

View File

@ -96,8 +96,8 @@ proc requestStorageRaw*(
proofProbability: UInt256, proofProbability: UInt256,
collateral: UInt256, collateral: UInt256,
expiry: uint = 0, expiry: uint = 0,
nodes: uint = 2, nodes: uint = 3,
tolerance: uint = 0 tolerance: uint = 1
): Response = ): Response =
## Call request storage REST endpoint ## Call request storage REST endpoint
@ -125,8 +125,8 @@ proc requestStorage*(
proofProbability: UInt256, proofProbability: UInt256,
expiry: uint, expiry: uint,
collateral: UInt256, collateral: UInt256,
nodes: uint = 2, nodes: uint = 3,
tolerance: uint = 0 tolerance: uint = 1
): ?!PurchaseId = ): ?!PurchaseId =
## Call request storage REST endpoint ## Call request storage REST endpoint
## ##

View File

@ -60,7 +60,9 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
reward=2.u256, reward=2.u256,
proofProbability=3.u256, proofProbability=3.u256,
expiry=30, expiry=30,
collateral=200.u256).get collateral=200.u256,
nodes=3.uint,
tolerance=1.uint).get
check eventually client1.purchaseStateIs(id, "submitted") check eventually client1.purchaseStateIs(id, "submitted")
node1.restart() node1.restart()
@ -73,8 +75,8 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
check request.ask.proofProbability == 3.u256 check request.ask.proofProbability == 3.u256
check request.expiry == 30 check request.expiry == 30
check request.ask.collateral == 200.u256 check request.ask.collateral == 200.u256
check request.ask.slots == 2'u64 check request.ask.slots == 3'u64
check request.ask.maxSlotLoss == 0'u64 check request.ask.maxSlotLoss == 1'u64
test "node requires expiry and its value to be in future": test "node requires expiry and its value to be in future":
let data = await RandomChunker.example(blocks=2) let data = await RandomChunker.example(blocks=2)

View File

@ -41,7 +41,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
test "request storage fails for datasets that are too small": test "request storage fails for datasets that are too small":
let cid = client1.upload("some file contents").get let cid = client1.upload("some file contents").get
let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, nodes=2, collateral=200.u256, expiry=9) let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=9)
check: check:
response.status == "400 Bad Request" response.status == "400 Bad Request"
@ -55,6 +55,29 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
check: check:
response.status == "200 OK" response.status == "200 OK"
test "request storage fails if tolerance is zero":
let data = await RandomChunker.example(blocks=2)
let cid = client1.upload(data).get
let duration = 100.u256
let reward = 2.u256
let proofProbability = 3.u256
let expiry = 30.uint
let collateral = 200.u256
let nodes = 3
let tolerance = 0
var responseBefore = client1.requestStorageRaw(cid,
duration,
reward,
proofProbability,
collateral,
expiry,
nodes.uint,
tolerance.uint)
check responseBefore.status == "400 Bad Request"
check responseBefore.body == "Tolerance needs to be bigger then zero"
test "request storage fails if nodes and tolerance aren't correct": test "request storage fails if nodes and tolerance aren't correct":
let data = await RandomChunker.example(blocks=2) let data = await RandomChunker.example(blocks=2)
let cid = client1.upload(data).get let cid = client1.upload(data).get
@ -63,7 +86,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
let proofProbability = 3.u256 let proofProbability = 3.u256
let expiry = 30.uint let expiry = 30.uint
let collateral = 200.u256 let collateral = 200.u256
let ecParams = @[(1, 0), (1, 1), (2, 1), (3, 2), (3, 3)] let ecParams = @[(1, 1), (2, 1), (3, 2), (3, 3)]
for ecParam in ecParams: for ecParam in ecParams:
let (nodes, tolerance) = ecParam let (nodes, tolerance) = ecParam
@ -113,7 +136,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
let proofProbability = 3.u256 let proofProbability = 3.u256
let expiry = 30.uint let expiry = 30.uint
let collateral = 200.u256 let collateral = 200.u256
let ecParams = @[(2, 0), (3, 1), (5, 2)] let ecParams = @[(3, 1), (5, 2)]
for ecParam in ecParams: for ecParam in ecParams:
let (nodes, tolerance) = ecParam let (nodes, tolerance) = ecParam