codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

API tweaks for OpenAPI, errors and endpoints (#886) 

* All sort of tweaks

* docs: availability's minPrice doc

* Revert changes to the two node test example

* Change default EC params in REST API

Change default EC params in REST API to 3 nodes and 1 tolerance.

Adjust integration tests to honour these settings.

---------

Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com>
This commit is contained in:
Adam Uhlíř 2024-09-24 10:37:08 +02:00 committed by GitHub
parent 098c4bb6e9
commit 4b9336ec07
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 133 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
codex/contracts/requests.nim
View File

@ -163,12 +163,12 @@ func id*(request: StorageRequest): RequestId =
let encoding = AbiEncoder.encode((request, ))
RequestId(keccak256.digest(encoding).data)
func slotId*(requestId: RequestId, slot: UInt256): SlotId =
let encoding = AbiEncoder.encode((requestId, slot))
func slotId*(requestId: RequestId, slotIndex: UInt256): SlotId =
let encoding = AbiEncoder.encode((requestId, slotIndex))
SlotId(keccak256.digest(encoding).data)
func slotId*(request: StorageRequest, slot: UInt256): SlotId =
slotId(request.id, slot)
func slotId*(request: StorageRequest, slotIndex: UInt256): SlotId =
slotId(request.id, slotIndex)
func id*(slot: Slot): SlotId =
slotId(slot.request, slot.slotIndex)

34
codex/rest/api.nim
View File

@ -232,7 +232,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
## Returns active slots for the host
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
let json = %(await contracts.sales.mySlots())
return RestApiResponse.response($json, contentType="application/json")
@ -247,7 +247,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
## slot is not active for the host.
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
without slotId =? slotId.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
@ -258,7 +258,9 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
let restAgent = RestSalesAgent(
state: agent.state() |? "none",
slotIndex: agent.data.slotIndex,
requestId: agent.data.requestId
requestId: agent.data.requestId,
request: agent.data.request,
reservation: agent.data.reservation,
)
return RestApiResponse.response(restAgent.toJson, contentType="application/json")
@ -270,7 +272,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
without avails =? (await contracts.sales.context.reservations.all(Availability)), err:
return RestApiResponse.error(Http500, err.msg)
@ -289,7 +291,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
##
## totalSize - size of available storage in bytes
## duration - maximum time the storage should be sold for (in seconds)
## minPrice - minimum price to be paid (in amount of tokens)
## minPrice - minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
var headers = newSeq[(string,string)]()
@ -301,7 +303,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable", headers = headers)
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
let body = await request.getBody()
@ -359,7 +361,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
@ -415,7 +417,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
@ -423,6 +425,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
return RestApiResponse.error(Http400, error.msg)
let reservations = contracts.sales.context.reservations
let market = contracts.sales.context.market
if error =? (await reservations.get(keyId, Availability)).errorOption:
if error of NotExistsError:
@ -462,10 +465,10 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
headers.add(("Access-Control-Allow-Origin", corsOrigin))
headers.add(("Access-Control-Allow-Methods", "POST, OPTIONS"))
headers.add(("Access-Control-Max-Age", "86400"))
try:
without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Purchasing unavailable", headers = headers)
return RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
without cid =? cid.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
@ -475,8 +478,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
without params =? StorageRequestParams.fromJson(body), error:
return RestApiResponse.error(Http400, error.msg, headers = headers)
let nodes = params.nodes |? 1
let tolerance = params.tolerance |? 0
let nodes = params.nodes |? 3
let tolerance = params.tolerance |? 1
if tolerance == 0:
return RestApiResponse.error(Http400, "Tolerance needs to be bigger then zero", headers = headers)
# prevent underflow
if tolerance > nodes:
@ -524,7 +530,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
try:
without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Purchasing unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
@ -549,7 +555,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
"/api/codex/v1/storage/purchases") do () -> RestApiResponse:
try:
without contracts =? node.contracts.client:
return RestApiResponse.error(Http503, "Purchasing unavailable")
return RestApiResponse.error(Http503, "Persistence is not enabled")
let purchaseIds = contracts.purchasing.getPurchaseIds()
return RestApiResponse.response($ %purchaseIds, contentType="application/json")

2
codex/rest/json.nim
View File

@ -38,6 +38,8 @@ type
state* {.serialize.}: string
requestId* {.serialize.}: RequestId
slotIndex* {.serialize.}: UInt256
request* {.serialize.}: ?StorageRequest
reservation* {.serialize.}: ?Reservation
RestContent* = object
cid* {.serialize.}: Cid

14
codex/sales.nim
View File

@ -180,7 +180,7 @@ proc filled(
processing.complete()
proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
debug "processing slot from queue", requestId = item.requestId,
debug "Processing slot from queue", requestId = item.requestId,
slot = item.slotIndex
let agent = newSalesAgent(
@ -202,13 +202,17 @@ proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} =
let reservations = sales.context.reservations
without reservs =? await reservations.all(Reservation):
info "no unused reservations found for deletion"
return
let unused = reservs.filter(r => (
let slotId = slotId(r.requestId, r.slotIndex)
not activeSlots.any(slot => slot.id == slotId)
))
info "found unused reservations for deletion", unused = unused.len
if unused.len == 0:
return
info "Found unused reservations for deletion", unused = unused.len
for reservation in unused:
@ -219,9 +223,9 @@ proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.}
if err =? (await reservations.deleteReservation(
reservation.id, reservation.availabilityId
)).errorOption:
error "failed to delete unused reservation", error = err.msg
error "Failed to delete unused reservation", error = err.msg
else:
trace "deleted unused reservation"
trace "Deleted unused reservation"
proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} =
let market = sales.context.market

4
codex/sales/reservations.nim
View File

@ -16,7 +16,7 @@
## |----------------------------------------| |--------------------------------------|
## | UInt256 | totalSize | | | UInt256 | size | |
## |----------------------------------------| |--------------------------------------|
## | UInt256 | freeSize | | | SlotId | slotId | |
## | UInt256 | freeSize | | | UInt256 | slotIndex | |
## |----------------------------------------| +--------------------------------------+
## | UInt256 | duration | |
## |----------------------------------------|
@ -65,7 +65,7 @@ type
totalSize* {.serialize.}: UInt256
freeSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256 # minimal price paid for the whole hosted slot for the request's duration
maxCollateral* {.serialize.}: UInt256
Reservation* = ref object
id* {.serialize.}: ReservationId

4
codex/sales/states/preparing.nim
View File

@ -69,11 +69,11 @@ method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
request.ask.duration,
request.ask.pricePerSlot,
request.ask.collateral):
debug "no availability found for request, ignoring"
debug "No availability found for request, ignoring"
return some State(SaleIgnored())
info "availability found for request, creating reservation"
info "Availability found for request, creating reservation"
without reservation =? await reservations.createReservation(
availability.id,

75
openapi.yaml
View File

@ -23,6 +23,8 @@ components:
Id:
type: string
description: 32bits identifier encoded in hex-decimal string.
minLength: 66
maxLength: 66
example: 0x...
BigInt:
@ -136,7 +138,7 @@ components:
$ref: "#/components/schemas/Duration"
minPrice:
type: string
description: Minimum price to be paid (in amount of tokens) as decimal string
description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
maxCollateral:
type: string
description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
@ -168,7 +170,39 @@ components:
$ref: "#/components/schemas/StorageRequest"
slotIndex:
type: string
description: Slot Index as hexadecimal string
description: Slot Index as decimal string
SlotAgent:
type: object
properties:
id:
$ref: "#/components/schemas/SlotId"
slotIndex:
type: string
description: Slot Index as decimal string
requestId:
$ref: "#/components/schemas/Id"
request:
$ref: "#/components/schemas/StorageRequest"
reservation:
$ref: "#/components/schemas/Reservation"
state:
type: string
description: Description of the slot's
enum:
- SaleCancelled
- SaleDownloading
- SaleErrored
- SaleFailed
- SaleFilled
- SaleFilling
- SaleFinished
- SaleIgnored
- SaleInitialProving
- SalePayout
- SalePreparing
- SaleProving
- SaleUnknown
Reservation:
type: object
@ -183,7 +217,7 @@ components:
$ref: "#/components/schemas/Id"
slotIndex:
type: string
description: Slot Index as hexadecimal string
description: Slot Index as decimal string
StorageRequestCreation:
type: object
@ -259,6 +293,15 @@ components:
state:
type: string
description: Description of the Request's state
enum:
- cancelled
- error
- failed
- finished
- pending
- started
- submitted
- unknown
error:
type: string
description: If Request failed, then here is presented the error message
@ -491,7 +534,7 @@ paths:
$ref: "#/components/schemas/Slot"
"503":
description: Sales are unavailable
description: Persistence is not enabled
"/sales/slots/{slotId}":
get:
@ -511,7 +554,7 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/Slot"
$ref: "#/components/schemas/SlotAgent"
"400":
description: Invalid or missing SlotId
@ -520,13 +563,13 @@ paths:
description: Host is not in an active sale for the slot
"503":
description: Sales are unavailable
description: Persistence is not enabled
"/sales/availability":
get:
summary: "Returns storage that is for sale"
tags: [ Marketplace ]
operationId: getOfferedStorage
operationId: getAvailabilities
responses:
"200":
description: Retrieved storage availabilities of the node
@ -535,11 +578,11 @@ paths:
schema:
type: array
items:
$ref: "#/components/schemas/SalesAvailability"
$ref: "#/components/schemas/SalesAvailabilityREAD"
"500":
description: Error getting unused availabilities
"503":
description: Sales are unavailable
description: Persistence is not enabled
post:
summary: "Offers storage for sale"
@ -564,7 +607,7 @@ paths:
"500":
description: Error reserving availability
"503":
description: Sales are unavailable
description: Persistence is not enabled
"/sales/availability/{id}":
patch:
summary: "Updates availability"
@ -597,10 +640,10 @@ paths:
"500":
description: Error reserving availability
"503":
description: Sales are unavailable
description: Persistence is not enabled
"/sales/availability/{id}/reservations":
patch:
get:
summary: "Get availability's reservations"
description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
operationId: getReservations
@ -628,7 +671,7 @@ paths:
"500":
description: Error getting reservations
"503":
description: Sales are unavailable
description: Persistence is not enabled
"/storage/request/{cid}":
post:
@ -659,7 +702,7 @@ paths:
"404":
description: Request ID not found
"503":
description: Purchasing is unavailable
description: Persistence is not enabled
"/storage/purchases":
get:
@ -676,7 +719,7 @@ paths:
items:
type: string
"503":
description: Purchasing is unavailable
description: Persistence is not enabled
"/storage/purchases/{id}":
get:
@ -702,7 +745,7 @@ paths:
"404":
description: Purchase not found
"503":
description: Purchasing is unavailable
description: Persistence is not enabled
"/node/spr":
get:

8
tests/integration/codexclient.nim
View File

@ -96,8 +96,8 @@ proc requestStorageRaw*(
proofProbability: UInt256,
collateral: UInt256,
expiry: uint = 0,
nodes: uint = 2,
tolerance: uint = 0
nodes: uint = 3,
tolerance: uint = 1
): Response =
## Call request storage REST endpoint
@ -125,8 +125,8 @@ proc requestStorage*(
proofProbability: UInt256,
expiry: uint,
collateral: UInt256,
nodes: uint = 2,
tolerance: uint = 0
nodes: uint = 3,
tolerance: uint = 1
): ?!PurchaseId =
## Call request storage REST endpoint
##

8
tests/integration/testpurchasing.nim
View File

@ -60,7 +60,9 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
reward=2.u256,
proofProbability=3.u256,
expiry=30,
collateral=200.u256).get
collateral=200.u256,
nodes=3.uint,
tolerance=1.uint).get
check eventually client1.purchaseStateIs(id, "submitted")
node1.restart()
@ -73,8 +75,8 @@ twonodessuite "Purchasing", debug1 = false, debug2 = false:
check request.ask.proofProbability == 3.u256
check request.expiry == 30
check request.ask.collateral == 200.u256
check request.ask.slots == 2'u64
check request.ask.maxSlotLoss == 0'u64
check request.ask.slots == 3'u64
check request.ask.maxSlotLoss == 1'u64
test "node requires expiry and its value to be in future":
let data = await RandomChunker.example(blocks=2)

29
tests/integration/testrestapi.nim
View File

@ -41,7 +41,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
test "request storage fails for datasets that are too small":
let cid = client1.upload("some file contents").get
let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, nodes=2, collateral=200.u256, expiry=9)
let response = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=9)
check:
response.status == "400 Bad Request"
@ -55,6 +55,29 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
check:
response.status == "200 OK"
test "request storage fails if tolerance is zero":
let data = await RandomChunker.example(blocks=2)
let cid = client1.upload(data).get
let duration = 100.u256
let reward = 2.u256
let proofProbability = 3.u256
let expiry = 30.uint
let collateral = 200.u256
let nodes = 3
let tolerance = 0
var responseBefore = client1.requestStorageRaw(cid,
duration,
reward,
proofProbability,
collateral,
expiry,
nodes.uint,
tolerance.uint)
check responseBefore.status == "400 Bad Request"
check responseBefore.body == "Tolerance needs to be bigger then zero"
test "request storage fails if nodes and tolerance aren't correct":
let data = await RandomChunker.example(blocks=2)
let cid = client1.upload(data).get
@ -63,7 +86,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
let proofProbability = 3.u256
let expiry = 30.uint
let collateral = 200.u256
let ecParams = @[(1, 0), (1, 1), (2, 1), (3, 2), (3, 3)]
let ecParams = @[(1, 1), (2, 1), (3, 2), (3, 3)]
for ecParam in ecParams:
let (nodes, tolerance) = ecParam
@ -113,7 +136,7 @@ twonodessuite "REST API", debug1 = false, debug2 = false:
let proofProbability = 3.u256
let expiry = 30.uint
let collateral = 200.u256
let ecParams = @[(2, 0), (3, 1), (5, 2)]
let ecParams = @[(3, 1), (5, 2)]
for ecParam in ecParams:
let (nodes, tolerance) = ecParam