From de1714ed0601b85c7247766b4ebc2bd04f8b0482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Thu, 21 Mar 2024 11:53:45 +0100 Subject: [PATCH] feat(api): availabilities patch endpoint (#692) * feat(api): availabilities patch and reservations endpoints * test: fixing tests and writing * test: test reservations endpoint * chore: feedback implementation * chore: feedback implementation * test: fix integration tests --- codex/rest/api.nim | 115 ++++++++++++++- codex/rest/coders.nim | 2 +- codex/rest/json.nim | 3 +- codex/sales/reservations.nim | 137 ++++++++++++------ codex/utils/optionalcast.nim | 16 -- codex/utils/options.nim | 61 ++++++++ openapi.yaml | 128 ++++++++++++++-- tests/codex/examples.nim | 3 +- tests/codex/sales/testreservations.nim | 65 ++++++--- tests/codex/sales/testsales.nim | 29 ++-- tests/codex/testutils.nim | 2 +- .../{testoptionalcast.nim => testoptions.nim} | 21 ++- tests/integration/codexclient.nim | 49 ++++++- tests/integration/marketplacesuite.nim | 2 +- tests/integration/testIntegration.nim | 92 ++++++++++-- tests/integration/testmarketplace.nim | 2 +- tests/integration/testproofs.nim | 6 +- vendor/nim-presto | 2 +- 18 files changed, 604 insertions(+), 131 deletions(-) delete mode 100644 codex/utils/optionalcast.nim create mode 100644 codex/utils/options.nim rename tests/codex/utils/{testoptionalcast.nim => testoptions.nim} (62%) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index ccb213ab..18f65006 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -35,6 +35,7 @@ import ../contracts import ../manifest import ../streams/asyncstreamwrapper import ../stores +import ../utils/options import ./coders import ./json @@ -253,9 +254,10 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = router.rawApi( MethodPost, "/api/codex/v1/sales/availability") do () -> RestApiResponse: - ## Add available storage to sell + ## Add available storage to sell. + ## Every time Availability's offer finishes, its capacity is returned to the availability. ## - ## size - size of available storage in bytes + ## totalSize - size of available storage in bytes ## duration - maximum time the storage should be sold for (in seconds) ## minPrice - minimum price to be paid (in amount of tokens) ## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens) @@ -271,12 +273,15 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = let reservations = contracts.sales.context.reservations - if not reservations.hasAvailable(restAv.size.truncate(uint)): + if restAv.totalSize == 0: + return RestApiResponse.error(Http400, "Total size must be larger then zero") + + if not reservations.hasAvailable(restAv.totalSize.truncate(uint)): return RestApiResponse.error(Http422, "Not enough storage quota") without availability =? ( await reservations.createAvailability( - restAv.size, + restAv.totalSize, restAv.duration, restAv.minPrice, restAv.maxCollateral) @@ -284,11 +289,106 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = return RestApiResponse.error(Http500, error.msg) return RestApiResponse.response(availability.toJson, + Http201, contentType="application/json") except CatchableError as exc: trace "Excepting processing request", exc = exc.msg return RestApiResponse.error(Http500) + router.rawApi( + MethodPatch, + "/api/codex/v1/sales/availability/{id}") do (id: AvailabilityId) -> RestApiResponse: + ## Updates Availability. + ## The new parameters will be only considered for new requests. + ## Existing Requests linked to this Availability will continue as is. + ## + ## totalSize - size of available storage in bytes. When decreasing the size, then lower limit is the currently `totalSize - freeSize`. + ## duration - maximum time the storage should be sold for (in seconds) + ## minPrice - minimum price to be paid (in amount of tokens) + ## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens) + + try: + without contracts =? node.contracts.host: + return RestApiResponse.error(Http503, "Sales unavailable") + + without id =? id.tryGet.catch, error: + return RestApiResponse.error(Http400, error.msg) + without keyId =? id.key.tryGet.catch, error: + return RestApiResponse.error(Http400, error.msg) + + let + body = await request.getBody() + reservations = contracts.sales.context.reservations + + type OptRestAvailability = Optionalize(RestAvailability) + without restAv =? OptRestAvailability.fromJson(body), error: + return RestApiResponse.error(Http400, error.msg) + + without availability =? (await reservations.get(keyId, Availability)), error: + if error of NotExistsError: + return RestApiResponse.error(Http404, "Availability not found") + + return RestApiResponse.error(Http500, error.msg) + + if isSome restAv.freeSize: + return RestApiResponse.error(Http400, "Updating freeSize is not allowed") + + if size =? restAv.totalSize: + # we don't allow lowering the totalSize bellow currently utilized size + if size < (availability.totalSize - availability.freeSize): + return RestApiResponse.error(Http400, "New totalSize must be larger then current totalSize - freeSize, which is currently: " & $(availability.totalSize - availability.freeSize)) + + availability.freeSize += size - availability.totalSize + availability.totalSize = size + + if duration =? restAv.duration: + availability.duration = duration + + if minPrice =? restAv.minPrice: + availability.minPrice = minPrice + + if maxCollateral =? restAv.maxCollateral: + availability.maxCollateral = maxCollateral + + if err =? (await reservations.update(availability)).errorOption: + return RestApiResponse.error(Http500, err.msg) + + return RestApiResponse.response(Http200) + except CatchableError as exc: + trace "Excepting processing request", exc = exc.msg + return RestApiResponse.error(Http500) + + router.rawApi( + MethodGet, + "/api/codex/v1/sales/availability/{id}/reservations") do (id: AvailabilityId) -> RestApiResponse: + ## Gets Availability's reservations. + + try: + without contracts =? node.contracts.host: + return RestApiResponse.error(Http503, "Sales unavailable") + + without id =? id.tryGet.catch, error: + return RestApiResponse.error(Http400, error.msg) + without keyId =? id.key.tryGet.catch, error: + return RestApiResponse.error(Http400, error.msg) + + let reservations = contracts.sales.context.reservations + + if error =? (await reservations.get(keyId, Availability)).errorOption: + if error of NotExistsError: + return RestApiResponse.error(Http404, "Availability not found") + else: + return RestApiResponse.error(Http500, error.msg) + + without availabilitysReservations =? (await reservations.all(Reservation, id)), err: + return RestApiResponse.error(Http500, err.msg) + + # TODO: Expand this structure with information about the linked StorageRequest not only RequestID + return RestApiResponse.response(availabilitysReservations.toJson, contentType="application/json") + except CatchableError as exc: + trace "Excepting processing request", exc = exc.msg + return RestApiResponse.error(Http500) + proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = router.rawApi( MethodPost, @@ -329,10 +429,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = return RestApiResponse.error(Http500) if expiry <= node.clock.now.u256: - return RestApiResponse.error(Http400, "Expiry needs to be in future") + return RestApiResponse.error(Http400, "Expiry needs to be in future. Now: " & $node.clock.now) - if expiry > node.clock.now.u256 + params.duration: - return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration)") + let expiryLimit = node.clock.now.u256 + params.duration + if expiry > expiryLimit: + return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration). Limit: " & $expiryLimit) without purchaseId =? await node.requestStorage( cid, diff --git a/codex/rest/coders.nim b/codex/rest/coders.nim index 2fa4da49..0be1a638 100644 --- a/codex/rest/coders.nim +++ b/codex/rest/coders.nim @@ -84,7 +84,7 @@ proc decodeString*(_: type array[32, byte], except ValueError as e: err e.msg.cstring -proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId](_: type T, +proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId | AvailabilityId](_: type T, value: string): Result[T, cstring] = array[32, byte].decodeString(value).map(id => T(id)) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 748a8b5b..7fe13c32 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -27,10 +27,11 @@ type error* {.serialize.}: ?string RestAvailability* = object - size* {.serialize.}: UInt256 + totalSize* {.serialize.}: UInt256 duration* {.serialize.}: UInt256 minPrice* {.serialize.}: UInt256 maxCollateral* {.serialize.}: UInt256 + freeSize* {.serialize.}: ?UInt256 RestSalesAgent* = object state* {.serialize.}: string diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 997a085f..40793e68 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -9,24 +9,27 @@ ## ## +--------------------------------------+ ## | RESERVATION | -## +--------------------------------------+ |--------------------------------------| -## | AVAILABILITY | | ReservationId | id | PK | -## |--------------------------------------| |--------------------------------------| -## | AvailabilityId | id | PK |<-||-------o<-| AvailabilityId | availabilityId | FK | -## |--------------------------------------| |--------------------------------------| -## | UInt256 | size | | | UInt256 | size | | -## |--------------------------------------| |--------------------------------------| -## | UInt256 | duration | | | SlotId | slotId | | -## |--------------------------------------| +--------------------------------------+ -## | UInt256 | minPrice | | -## |--------------------------------------| -## | UInt256 | maxCollateral | | -## +--------------------------------------+ +## +----------------------------------------+ |--------------------------------------| +## | AVAILABILITY | | ReservationId | id | PK | +## |----------------------------------------| |--------------------------------------| +## | AvailabilityId | id | PK |<-||-------o<-| AvailabilityId | availabilityId | FK | +## |----------------------------------------| |--------------------------------------| +## | UInt256 | totalSize | | | UInt256 | size | | +## |----------------------------------------| |--------------------------------------| +## | UInt256 | freeSize | | | SlotId | slotId | | +## |----------------------------------------| +--------------------------------------+ +## | UInt256 | duration | | +## |----------------------------------------| +## | UInt256 | minPrice | | +## |----------------------------------------| +## | UInt256 | maxCollateral | | +## +----------------------------------------+ import pkg/upraises push: {.upraises: [].} import std/typetraits +import std/sequtils import pkg/chronos import pkg/datastore import pkg/nimcrypto @@ -35,7 +38,9 @@ import pkg/questionable/results import pkg/stint import pkg/stew/byteutils import ../logutils +import ../clock import ../stores +import ../market import ../contracts/requests import ../utils/json @@ -52,7 +57,8 @@ type SomeStorableId = AvailabilityId | ReservationId Availability* = ref object id* {.serialize.}: AvailabilityId - size* {.serialize.}: UInt256 + totalSize* {.serialize.}: UInt256 + freeSize* {.serialize.}: UInt256 duration* {.serialize.}: UInt256 minPrice* {.serialize.}: UInt256 maxCollateral* {.serialize.}: UInt256 @@ -91,14 +97,15 @@ proc new*(T: type Reservations, proc init*( _: type Availability, - size: UInt256, + totalSize: UInt256, + freeSize: UInt256, duration: UInt256, minPrice: UInt256, maxCollateral: UInt256): Availability = var id: array[32, byte] doAssert randomBytes(id) == 32 - Availability(id: AvailabilityId(id), size: size, duration: duration, minPrice: minPrice, maxCollateral: maxCollateral) + Availability(id: AvailabilityId(id), totalSize:totalSize, freeSize: freeSize, duration: duration, minPrice: minPrice, maxCollateral: maxCollateral) proc init*( _: type Reservation, @@ -118,17 +125,9 @@ func toArray(id: SomeStorableId): array[32, byte] = proc `==`*(x, y: AvailabilityId): bool {.borrow.} proc `==`*(x, y: ReservationId): bool {.borrow.} proc `==`*(x, y: Reservation): bool = - x.id == y.id and - x.availabilityId == y.availabilityId and - x.size == y.size and - x.requestId == y.requestId and - x.slotIndex == y.slotIndex + x.id == y.id proc `==`*(x, y: Availability): bool = - x.id == y.id and - x.size == y.size and - x.duration == y.duration and - x.maxCollateral == y.maxCollateral and - x.minPrice == y.minPrice + x.id == y.id proc `$`*(id: SomeStorableId): string = id.toArray.toHex @@ -198,11 +197,11 @@ proc get*( return success obj -proc update( +proc updateImpl( self: Reservations, obj: SomeStorableObject): Future[?!void] {.async.} = - trace "updating " & $(obj.type), id = obj.id, size = obj.size + trace "updating " & $(obj.type), id = obj.id without key =? obj.key, error: return failure(error) @@ -215,6 +214,39 @@ proc update( return success() +proc update*( + self: Reservations, + obj: Reservation): Future[?!void] {.async.} = + return await self.updateImpl(obj) + +proc update*( + self: Reservations, + obj: Availability): Future[?!void] {.async.} = + + without key =? obj.key, error: + return failure(error) + + let getResult = await self.get(key, Availability) + + if getResult.isOk: + let oldAvailability = !getResult + + # Sizing of the availability changed, we need to adjust the repo reservation accordingly + if oldAvailability.totalSize != obj.totalSize: + if oldAvailability.totalSize < obj.totalSize: # storage added + if reserveErr =? (await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).truncate(uint))).errorOption: + return failure(reserveErr.toErr(ReserveFailedError)) + + elif oldAvailability.totalSize > obj.totalSize: # storage removed + if reserveErr =? (await self.repo.release((oldAvailability.totalSize - obj.totalSize).truncate(uint))).errorOption: + return failure(reserveErr.toErr(ReleaseFailedError)) + else: + let err = getResult.error() + if not (err of NotExistsError): + return failure(err) + + return await self.updateImpl(obj) + proc delete( self: Reservations, key: Key): Future[?!void] {.async.} = @@ -258,7 +290,7 @@ proc deleteReservation*( without var availability =? await self.get(availabilityKey, Availability), error: return failure(error) - availability.size += reservation.size + availability.freeSize += reservation.size if updateErr =? (await self.update(availability)).errorOption: return failure(updateErr) @@ -278,9 +310,9 @@ proc createAvailability*( trace "creating availability", size, duration, minPrice, maxCollateral let availability = Availability.init( - size, duration, minPrice, maxCollateral + size, size, duration, minPrice, maxCollateral ) - let bytes = availability.size.truncate(uint) + let bytes = availability.freeSize.truncate(uint) if reserveErr =? (await self.repo.reserve(bytes)).errorOption: return failure(reserveErr.toErr(ReserveFailedError)) @@ -324,7 +356,7 @@ proc createReservation*( without var availability =? await self.get(availabilityKey, Availability), error: return failure(error) - if availability.size < slotSize: + if availability.freeSize < slotSize: let error = newException( BytesOutOfBoundsError, "trying to reserve an amount of bytes that is greater than the total size of the Availability") @@ -333,9 +365,9 @@ proc createReservation*( if createResErr =? (await self.update(reservation)).errorOption: return failure(createResErr) - # reduce availability size by the slot size, which is now accounted for in + # reduce availability freeSize by the slot size, which is now accounted for in # the newly created Reservation - availability.size -= slotSize + availability.freeSize -= slotSize # update availability with reduced size if updateErr =? (await self.update(availability)).errorOption: @@ -393,7 +425,7 @@ proc returnBytesToAvailability*( without var availability =? await self.get(availabilityKey, Availability), error: return failure(error) - availability.size += bytesToBeReturned + availability.freeSize += bytesToBeReturned # Update availability with returned size if updateErr =? (await self.update(availability)).errorOption: @@ -456,11 +488,12 @@ iterator items(self: StorableIter): Future[?seq[byte]] = proc storables( self: Reservations, - T: type SomeStorableObject + T: type SomeStorableObject, + queryKey: Key = ReservationsKey ): Future[?!StorableIter] {.async.} = var iter = StorableIter() - let query = Query.init(ReservationsKey) + let query = Query.init(queryKey) when T is Availability: # should indicate key length of 4, but let the .key logic determine it without defaultKey =? AvailabilityId.default.key, error: @@ -475,6 +508,7 @@ proc storables( without results =? await self.repo.metaDs.query(query), error: return failure(error) + # /sales/reservations proc next(): Future[?seq[byte]] {.async.} = await idleAsync() iter.finished = results.finished @@ -491,14 +525,15 @@ proc storables( iter.next = next return success iter -proc all*( +proc allImpl( self: Reservations, - T: type SomeStorableObject + T: type SomeStorableObject, + queryKey: Key = ReservationsKey ): Future[?!seq[T]] {.async.} = var ret: seq[T] = @[] - without storables =? (await self.storables(T)), error: + without storables =? (await self.storables(T, queryKey)), error: return failure(error) for storable in storables.items: @@ -515,6 +550,22 @@ proc all*( return success(ret) +proc all*( + self: Reservations, + T: type SomeStorableObject +): Future[?!seq[T]] {.async.} = + return await self.allImpl(T) + +proc all*( + self: Reservations, + T: type SomeStorableObject, + availabilityId: AvailabilityId +): Future[?!seq[T]] {.async.} = + without key =? (ReservationsKey / $availabilityId): + return failure("no key") + + return await self.allImpl(T, key) + proc findAvailability*( self: Reservations, size, duration, minPrice, collateral: UInt256 @@ -528,13 +579,13 @@ proc findAvailability*( if bytes =? (await item) and availability =? Availability.fromJson(bytes): - if size <= availability.size and + if size <= availability.freeSize and duration <= availability.duration and collateral <= availability.maxCollateral and minPrice >= availability.minPrice: trace "availability matched", - size, availsize = availability.size, + size, availFreeSize = availability.freeSize, duration, availDuration = availability.duration, minPrice, availMinPrice = availability.minPrice, collateral, availMaxCollateral = availability.maxCollateral @@ -542,7 +593,7 @@ proc findAvailability*( return some availability trace "availability did not match", - size, availsize = availability.size, + size, availFreeSize = availability.freeSize, duration, availDuration = availability.duration, minPrice, availMinPrice = availability.minPrice, collateral, availMaxCollateral = availability.maxCollateral diff --git a/codex/utils/optionalcast.nim b/codex/utils/optionalcast.nim deleted file mode 100644 index 15830769..00000000 --- a/codex/utils/optionalcast.nim +++ /dev/null @@ -1,16 +0,0 @@ -import pkg/questionable -import pkg/questionable/operators - -export questionable - -proc `as`*[T](value: T, U: type): ?U = - ## Casts a value to another type, returns an Option. - ## When the cast succeeds, the option will contain the casted value. - ## When the cast fails, the option will have no value. - when value is U: - return some value - elif value is ref object: - if value of U: - return some U(value) - -Option.liftBinary `as` diff --git a/codex/utils/options.nim b/codex/utils/options.nim new file mode 100644 index 00000000..115af782 --- /dev/null +++ b/codex/utils/options.nim @@ -0,0 +1,61 @@ +import macros +import strutils +import pkg/questionable +import pkg/questionable/operators + +export questionable + +proc `as`*[T](value: T, U: type): ?U = + ## Casts a value to another type, returns an Option. + ## When the cast succeeds, the option will contain the casted value. + ## When the cast fails, the option will have no value. + when value is U: + return some value + elif value is ref object: + if value of U: + return some U(value) + +Option.liftBinary `as` + +# Template that wraps type with `Option[]` only if it is already not `Option` type +template WrapOption*(input: untyped): type = + when input is Option: + input + else: + Option[input] + + +macro createType(t: typedesc): untyped = + var objectType = getType(t) + + # Work around for https://github.com/nim-lang/Nim/issues/23112 + while objectType.kind == nnkBracketExpr and objectType[0].eqIdent"typeDesc": + objectType = getType(objectType[1]) + + expectKind(objectType, NimNodeKind.nnkObjectTy) + var fields = nnkRecList.newTree() + + # Generates the list of fields that are wrapped in `Option[T]`. + # Technically wrapped with `WrapOption` which is template used to prevent + # re-wrapping already filed which is `Option[T]`. + for field in objectType[2]: + let fieldType = getTypeInst(field) + let newFieldNode = + nnkIdentDefs.newTree(ident($field), nnkCall.newTree(ident("WrapOption"), fieldType), newEmptyNode()) + + fields.add(newFieldNode) + + # Creates new object type T with the fields lists from steps above. + let tSym = genSym(nskType, "T") + nnkStmtList.newTree( + nnkTypeSection.newTree( + nnkTypeDef.newTree(tSym, newEmptyNode(), nnkObjectTy.newTree(newEmptyNode(), newEmptyNode(), fields)) + ), + tSym + ) + +template Optionalize*(t: typed): untyped = + ## Takes object type and wraps all the first level fields into + ## Option type unless it is already Option type. + createType(t) + diff --git a/openapi.yaml b/openapi.yaml index b3ac2df5..adfc887a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -20,6 +20,15 @@ components: description: Peer Identity reference as specified at https://docs.libp2p.io/concepts/fundamentals/peers/ example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N + Id: + type: string + description: 32bits identifier encoded in hex-decimal string. + example: 0x... + + BigInt: + type: string + description: Integer represented as decimal string + Cid: type: string description: Content Identifier as specified at https://github.com/multiformats/cid @@ -102,16 +111,12 @@ components: SalesAvailability: type: object - required: - - size - - minPrice properties: id: + $ref: "#/components/schemas/Id" + totalSize: type: string - description: Hexadecimal identifier of the availability - size: - type: string - description: Size of available storage in bytes as decimal string + description: Total size of availability's storage in bytes as decimal string duration: $ref: "#/components/schemas/Duration" minPrice: @@ -121,6 +126,24 @@ components: type: string description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string + SalesAvailabilityREAD: + allOf: + - $ref: "#/components/schemas/SalesAvailability" + - type: object + properties: + freeSize: + type: string + description: Unused size of availability's storage in bytes as decimal string + + SalesAvailabilityCREATE: + allOf: + - $ref: "#/components/schemas/SalesAvailability" + - required: + - totalSize + - minPrice + - maxCollateral + - duration + Slot: type: object properties: @@ -132,6 +155,21 @@ components: type: string description: Slot Index as hexadecimal string + Reservation: + type: object + properties: + id: + $ref: "#/components/schemas/Id" + availabilityId: + $ref: "#/components/schemas/Id" + size: + $ref: "#/components/schemas/BigInt" + requestId: + $ref: "#/components/schemas/Id" + slotIndex: + type: string + description: Slot Index as hexadecimal string + StorageRequestCreation: type: object required: @@ -493,16 +531,84 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SalesAvailability" + $ref: "#/components/schemas/SalesAvailabilityCREATE" responses: - "200": + "201": description: Created storage availability content: application/json: schema: - $ref: "#/components/schemas/SalesAvailability" + $ref: "#/components/schemas/SalesAvailabilityREAD" + "400": + description: Invalid data input + "422": + description: Not enough node's storage quota available "500": - description: Error reserving availablility + description: Error reserving availability + "503": + description: Sales are unavailable + "/sales/availability/{id}": + patch: + summary: "Updates availability" + description: | + The new parameters will be only considered for new requests. + Existing Requests linked to this Availability will continue as is. + operationId: updateOfferedStorage + tags: [ Marketplace ] + parameters: + - in: path + name: id + required: true + schema: + type: string + description: ID of Availability + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SalesAvailability" + responses: + "204": + description: Availability successfully updated + "400": + description: Invalid data input + "404": + description: Availability not found + "422": + description: Not enough node's storage quota available + "500": + description: Error reserving availability + "503": + description: Sales are unavailable + + "/sales/availability/{id}/reservations": + patch: + summary: "Get availability's reservations" + description: Return's list of Reservations for ongoing Storage Requests that the node hosts. + operationId: getReservations + tags: [ Marketplace ] + parameters: + - in: path + name: id + required: true + schema: + type: string + description: ID of Availability + responses: + "200": + description: Retrieved storage availabilities of the node + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Reservation" + "400": + description: Invalid Availability ID + "404": + description: Availability not found + "500": + description: Error getting reservations "503": description: Sales are unavailable diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 895839c3..2e68d236 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -60,7 +60,8 @@ proc example*(_: type MultiHash, mcodec = Sha256HashCodec): MultiHash = proc example*(_: type Availability): Availability = Availability.init( - size = uint16.example.u256, + totalSize = uint16.example.u256, + freeSize = uint16.example.u256, duration = uint16.example.u256, minPrice = uint64.example.u256, maxCollateral = uint16.example.u256 diff --git a/tests/codex/sales/testreservations.nim b/tests/codex/sales/testreservations.nim index 08c0bf95..4b82fb89 100644 --- a/tests/codex/sales/testreservations.nim +++ b/tests/codex/sales/testreservations.nim @@ -29,9 +29,9 @@ asyncchecksuite "Reservations module": proc createAvailability(): Availability = let example = Availability.example - let size = rand(100000..200000) + let totalSize = rand(100000..200000) let availability = waitFor reservations.createAvailability( - size.u256, + totalSize.u256, example.duration, example.minPrice, example.maxCollateral @@ -39,7 +39,7 @@ asyncchecksuite "Reservations module": return availability.get proc createReservation(availability: Availability): Reservation = - let size = rand(1.. orig - check (updated.size - orig) == 200.u256 + check updated.freeSize > orig + check (updated.freeSize - orig) == 200.u256 check (repo.quotaReservedBytes - origQuota) == 200 + test "update releases quota when lowering size": + let + availability = createAvailability() + origQuota = repo.quotaReservedBytes + availability.totalSize = availability.totalSize - 100 + + check isOk await reservations.update(availability) + check (origQuota - repo.quotaReservedBytes) == 100 + + test "update reserves quota when growing size": + let + availability = createAvailability() + origQuota = repo.quotaReservedBytes + availability.totalSize = availability.totalSize + 100 + + check isOk await reservations.update(availability) + check (repo.quotaReservedBytes - origQuota) == 100 + test "reservation can be partially released": let availability = createAvailability() let reservation = createReservation(availability) @@ -240,7 +271,7 @@ asyncchecksuite "Reservations module": let availability = createAvailability() let found = await reservations.findAvailability( - availability.size, + availability.freeSize, availability.duration, availability.minPrice, availability.maxCollateral) @@ -252,7 +283,7 @@ asyncchecksuite "Reservations module": let availability = createAvailability() let found = await reservations.findAvailability( - availability.size + 1, + availability.freeSize + 1, availability.duration, availability.minPrice, availability.maxCollateral) @@ -262,7 +293,7 @@ asyncchecksuite "Reservations module": test "non-existant availability cannot be found": let availability = Availability.example let found = (await reservations.findAvailability( - availability.size, + availability.freeSize, availability.duration, availability.minPrice, availability.maxCollateral diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index b5fe4821..c25a0572 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -127,7 +127,8 @@ asyncchecksuite "Sales": setup: availability = Availability( - size: 100.u256, + totalSize: 100.u256, + freeSize: 100.u256, duration: 60.u256, minPrice: 600.u256, maxCollateral: 400.u256 @@ -189,7 +190,7 @@ asyncchecksuite "Sales": proc createAvailability() = let a = waitFor reservations.createAvailability( - availability.size, + availability.totalSize, availability.duration, availability.minPrice, availability.maxCollateral @@ -224,7 +225,7 @@ asyncchecksuite "Sales": proc wasIgnored(): bool = let run = proc(): Future[bool] {.async.} = always ( - getAvailability().size == availability.size and + getAvailability().freeSize == availability.freeSize and (waitFor reservations.all(Reservation)).get.len == 0 ) waitFor run() @@ -300,7 +301,7 @@ asyncchecksuite "Sales": createAvailability() await market.requestStorage(request) - check eventually getAvailability().size == availability.size - request.ask.slotSize + check eventually getAvailability().freeSize == availability.freeSize - request.ask.slotSize test "non-downloaded bytes are returned to availability once finished": var slotIndex = 0.u256 @@ -316,7 +317,7 @@ asyncchecksuite "Sales": sold.complete() createAvailability() - let origSize = availability.size + let origSize = availability.freeSize await market.requestStorage(request) await allowRequestToStart() await sold @@ -325,7 +326,7 @@ asyncchecksuite "Sales": market.slotState[request.slotId(slotIndex)] = SlotState.Finished clock.advance(request.ask.duration.truncate(int64)) - check eventually getAvailability().size == origSize - 1 + check eventually getAvailability().freeSize == origSize - 1 test "ignores download when duration not long enough": availability.duration = request.ask.duration - 1 @@ -334,7 +335,7 @@ asyncchecksuite "Sales": check wasIgnored() test "ignores request when slot size is too small": - availability.size = request.ask.slotSize - 1 + availability.totalSize = request.ask.slotSize - 1 createAvailability() await market.requestStorage(request) check wasIgnored() @@ -399,7 +400,7 @@ asyncchecksuite "Sales": return failure(error) createAvailability() await market.requestStorage(request) - check getAvailability().size == availability.size + check getAvailability().freeSize == availability.freeSize test "generates proof of storage": var provingRequest: StorageRequest @@ -472,7 +473,7 @@ asyncchecksuite "Sales": check eventually (await reservations.all(Availability)).get == @[availability] test "makes storage available again when request expires": - let origSize = availability.size + let origSize = availability.freeSize sales.onStore = proc(request: StorageRequest, slot: UInt256, onBatch: BatchProc): Future[?!void] {.async.} = @@ -487,10 +488,10 @@ asyncchecksuite "Sales": market.requestState[request.id]=RequestState.Cancelled clock.set(request.expiry.truncate(int64)+1) check eventually (await reservations.all(Availability)).get == @[availability] - check getAvailability().size == origSize + check getAvailability().freeSize == origSize test "verifies that request is indeed expired from onchain before firing onCancelled": - let origSize = availability.size + let origSize = availability.freeSize sales.onStore = proc(request: StorageRequest, slot: UInt256, onBatch: BatchProc): Future[?!void] {.async.} = @@ -504,10 +505,10 @@ asyncchecksuite "Sales": # would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation. await sleepAsync(chronos.milliseconds(100)) clock.set(request.expiry.truncate(int64)+1) - check getAvailability().size == 0 + check getAvailability().freeSize == 0 market.requestState[request.id]=RequestState.Cancelled # Now "on-chain" is also expired - check eventually getAvailability().size == origSize + check eventually getAvailability().freeSize == origSize test "loads active slots from market": let me = await market.getSigner() @@ -556,4 +557,4 @@ asyncchecksuite "Sales": check (await reservations.all(Reservation)).get.len == 1 await sales.load() check (await reservations.all(Reservation)).get.len == 0 - check getAvailability().size == availability.size # was restored + check getAvailability().freeSize == availability.freeSize # was restored diff --git a/tests/codex/testutils.nim b/tests/codex/testutils.nim index 6b4b2366..82b5ecad 100644 --- a/tests/codex/testutils.nim +++ b/tests/codex/testutils.nim @@ -1,4 +1,4 @@ -import ./utils/testoptionalcast +import ./utils/testoptions import ./utils/testkeyutils import ./utils/testasyncstatemachine import ./utils/testtimer diff --git a/tests/codex/utils/testoptionalcast.nim b/tests/codex/utils/testoptions.nim similarity index 62% rename from tests/codex/utils/testoptionalcast.nim rename to tests/codex/utils/testoptions.nim index d1d3d8e5..eb566ad7 100644 --- a/tests/codex/utils/testoptionalcast.nim +++ b/tests/codex/utils/testoptions.nim @@ -1,5 +1,5 @@ import std/unittest -import codex/utils/optionalcast +import codex/utils/options import ../helpers checksuite "optional casts": @@ -28,3 +28,22 @@ checksuite "optional casts": check 42.some as int == some 42 check 42.some as string == string.none check int.none as int == int.none + +checksuite "Optionalize": + test "does not except non-object types": + static: + doAssert not compiles(Optionalize(int)) + + test "converts object fields to option": + type BaseType = object + a: int + b: bool + c: string + d: Option[string] + + type OptionalizedType = Optionalize(BaseType) + + check OptionalizedType.a is Option[int] + check OptionalizedType.b is Option[bool] + check OptionalizedType.c is Option[string] + check OptionalizedType.d is Option[string] diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 972b4708..47a1dbad 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -133,27 +133,70 @@ proc getSlots*(client: CodexClient): ?!seq[Slot] = proc postAvailability*( client: CodexClient, - size, duration, minPrice, maxCollateral: UInt256 + totalSize, duration, minPrice, maxCollateral: UInt256 ): ?!Availability = ## Post sales availability endpoint ## let url = client.baseurl & "/sales/availability" let json = %*{ - "size": size, + "totalSize": totalSize, "duration": duration, "minPrice": minPrice, "maxCollateral": maxCollateral, } let response = client.http.post(url, $json) - doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body + doAssert response.status == "201 Created", "expected 201 Created, got " & response.status & ", body: " & response.body Availability.fromJson(response.body) +proc patchAvailabilityRaw*( + client: CodexClient, + availabilityId: AvailabilityId, + totalSize, freeSize, duration, minPrice, maxCollateral: ?UInt256 = UInt256.none +): Response = + ## Updates availability + ## + let url = client.baseurl & "/sales/availability/" & $availabilityId + + # TODO: Optionalize macro does not keep `serialize` pragmas so we can't use `Optionalize(RestAvailability)` here. + var json = %*{} + + if totalSize =? totalSize: + json["totalSize"] = %totalSize + + if freeSize =? freeSize: + json["freeSize"] = %freeSize + + if duration =? duration: + json["duration"] = %duration + + if minPrice =? minPrice: + json["minPrice"] = %minPrice + + if maxCollateral =? maxCollateral: + json["maxCollateral"] = %maxCollateral + + client.http.patch(url, $json) + +proc patchAvailability*( + client: CodexClient, + availabilityId: AvailabilityId, + totalSize, duration, minPrice, maxCollateral: ?UInt256 = UInt256.none +): void = + let response = client.patchAvailabilityRaw(availabilityId, totalSize=totalSize, duration=duration, minPrice=minPrice, maxCollateral=maxCollateral) + doAssert response.status == "200 OK", "expected 200 OK, got " & response.status + proc getAvailabilities*(client: CodexClient): ?!seq[Availability] = ## Call sales availability REST endpoint let url = client.baseurl & "/sales/availability" let body = client.http.getContent(url) seq[Availability].fromJson(body) +proc getAvailabilityReservations*(client: CodexClient, availabilityId: AvailabilityId): ?!seq[Reservation] = + ## Retrieves Availability's Reservations + let url = client.baseurl & "/sales/availability/" & $availabilityId & "/reservations" + let body = client.http.getContent(url) + seq[Reservation].fromJson(body) + proc close*(client: CodexClient) = client.http.close() diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 0c0b67db..82a82619 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -53,7 +53,7 @@ template marketplacesuite*(name: string, body: untyped) = let provider = providers()[i].client discard provider.postAvailability( - size=datasetSize.u256, # should match 1 slot only + totalSize=datasetSize.u256, # should match 1 slot only duration=duration.u256, minPrice=300.u256, maxCollateral=200.u256 diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim index 4d2bc15a..57ce0955 100644 --- a/tests/integration/testIntegration.nim +++ b/tests/integration/testIntegration.nim @@ -1,5 +1,6 @@ import std/options import std/sequtils +import std/strutils import std/httpclient from pkg/libp2p import `==` import pkg/chronos @@ -12,8 +13,16 @@ import ../contracts/time import ../contracts/deployment import ../codex/helpers import ../examples +import ../codex/examples import ./twonodes +proc findItem[T](items: seq[T], item: T): ?!T = + for tmp in items: + if tmp == item: + return success tmp + + return failure("Not found") + # For debugging you can enable logging output with debugX = true # You can also pass a string in same format like for the `--log-level` parameter # to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace" @@ -38,7 +47,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: test "node shows used and available space": discard client1.upload("some file contents").get - discard client1.postAvailability(size=12.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get + discard client1.postAvailability(totalSize=12.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get let space = client1.space().tryGet() check: space.totalBlocks == 2.uint @@ -94,12 +103,12 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: [cid1, cid2].allIt(it in list.mapIt(it.cid)) test "node handles new storage availability": - let availability1 = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get - let availability2 = client1.postAvailability(size=4.u256, duration=5.u256, minPrice=6.u256, maxCollateral=7.u256).get + let availability1 = client1.postAvailability(totalSize=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get + let availability2 = client1.postAvailability(totalSize=4.u256, duration=5.u256, minPrice=6.u256, maxCollateral=7.u256).get check availability1 != availability2 test "node lists storage that is for sale": - let availability = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get + let availability = client1.postAvailability(totalSize=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get check availability in client1.getAvailabilities().get test "node handles storage request": @@ -177,7 +186,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: let size = 0xFFFFFF.u256 let data = await RandomChunker.example(blocks=8) # client 2 makes storage available - discard client2.postAvailability(size=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256) + let availability = client2.postAvailability(totalSize=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get # client 1 requests storage let expiry = (await ethProvider.currentTime()) + 5*60 @@ -197,9 +206,13 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: check purchase.error == none string let availabilities = client2.getAvailabilities().get check availabilities.len == 1 - let newSize = availabilities[0].size + let newSize = availabilities[0].freeSize check newSize > 0 and newSize < size + let reservations = client2.getAvailabilityReservations(availability.id).get + check reservations.len == 5 + check reservations[0].requestId == purchase.requestId + test "node slots gets paid out": let size = 0xFFFFFF.u256 let data = await RandomChunker.example(blocks = 8) @@ -212,7 +225,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: # client 2 makes storage available let startBalance = await token.balanceOf(account2) - discard client2.postAvailability(size=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get + discard client2.postAvailability(totalSize=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get # client 1 requests storage let expiry = (await ethProvider.currentTime()) + 5*60 @@ -263,8 +276,69 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: let responsePast = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=currentTime-10) check responsePast.status == "400 Bad Request" - check responsePast.body == "Expiry needs to be in future" + check "Expiry needs to be in future" in responsePast.body let responseBefore = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=currentTime+10) check responseBefore.status == "400 Bad Request" - check responseBefore.body == "Expiry has to be before the request's end (now + duration)" + check "Expiry has to be before the request's end (now + duration)" in responseBefore.body + + test "updating non-existing availability": + let nonExistingResponse = client1.patchAvailabilityRaw(AvailabilityId.example, duration=100.u256.some, minPrice=200.u256.some, maxCollateral=200.u256.some) + check nonExistingResponse.status == "404 Not Found" + + test "updating availability": + let availability = client1.postAvailability(totalSize=140000.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get + + client1.patchAvailability(availability.id, duration=100.u256.some, minPrice=200.u256.some, maxCollateral=200.u256.some) + + let updatedAvailability = (client1.getAvailabilities().get).findItem(availability).get + check updatedAvailability.duration == 100 + check updatedAvailability.minPrice == 200 + check updatedAvailability.maxCollateral == 200 + check updatedAvailability.totalSize == 140000 + check updatedAvailability.freeSize == 140000 + + test "updating availability - freeSize is not allowed to be changed": + let availability = client1.postAvailability(totalSize=140000.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get + let freeSizeResponse = client1.patchAvailabilityRaw(availability.id, freeSize=110000.u256.some) + check freeSizeResponse.status == "400 Bad Request" + check "not allowed" in freeSizeResponse.body + + test "updating availability - updating totalSize": + let availability = client1.postAvailability(totalSize=140000.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get + client1.patchAvailability(availability.id, totalSize=100000.u256.some) + let updatedAvailability = (client1.getAvailabilities().get).findItem(availability).get + check updatedAvailability.totalSize == 100000 + check updatedAvailability.freeSize == 100000 + + test "updating availability - updating totalSize does not allow bellow utilized": + let originalSize = 0xFFFFFF.u256 + let data = await RandomChunker.example(blocks=8) + let availability = client1.postAvailability(totalSize=originalSize, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get + + # Lets create storage request that will utilize some of the availability's space + let expiry = (await ethProvider.currentTime()) + 5*60 + let cid = client2.upload(data).get + let id = client2.requestStorage( + cid, + duration=10*60.u256, + reward=400.u256, + proofProbability=3.u256, + expiry=expiry, + collateral=200.u256, + nodes = 5, + tolerance = 2).get + + check eventually(client2.purchaseStateIs(id, "started"), timeout=5*60*1000) + let updatedAvailability = (client1.getAvailabilities().get).findItem(availability).get + check updatedAvailability.totalSize != updatedAvailability.freeSize + + let utilizedSize = updatedAvailability.totalSize - updatedAvailability.freeSize + let totalSizeResponse = client1.patchAvailabilityRaw(availability.id, totalSize=(utilizedSize-1.u256).some) + check totalSizeResponse.status == "400 Bad Request" + check "totalSize must be larger then current totalSize" in totalSizeResponse.body + + client1.patchAvailability(availability.id, totalSize=(originalSize + 20000).some) + let newUpdatedAvailability = (client1.getAvailabilities().get).findItem(availability).get + check newUpdatedAvailability.totalSize == originalSize + 20000 + check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000 diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index 7acde13f..79acae68 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -41,7 +41,7 @@ marketplacesuite "Marketplace payouts": discard providerApi.postAvailability( # make availability size small enough that we can't fill all the slots, # thus causing a cancellation - size=(data.len div 2).u256, + totalSize=(data.len div 2).u256, duration=duration.u256, minPrice=reward, maxCollateral=collateral) diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index 85fb2cf0..057d7f46 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -230,7 +230,7 @@ marketplacesuite "Simulate invalid proofs": # let slotSize = (DefaultBlockSize * 4.NBytes).Natural.u256 # discard provider0.client.postAvailability( - # size=slotSize, # should match 1 slot only + # totalSize=slotSize, # should match 1 slot only # duration=totalPeriods.periods.u256, # minPrice=300.u256, # maxCollateral=200.u256 @@ -263,7 +263,7 @@ marketplacesuite "Simulate invalid proofs": # # now add availability for providers 1 and 2, which should allow them to to # # put the remaining slots in their queues # discard provider1.client.postAvailability( - # size=slotSize, # should match 1 slot only + # totalSize=slotSize, # should match 1 slot only # duration=totalPeriods.periods.u256, # minPrice=300.u256, # maxCollateral=200.u256 @@ -272,7 +272,7 @@ marketplacesuite "Simulate invalid proofs": # check eventually filledSlotIds.len > 1 # discard provider2.client.postAvailability( - # size=slotSize, # should match 1 slot only + # totalSize=slotSize, # should match 1 slot only # duration=totalPeriods.periods.u256, # minPrice=300.u256, # maxCollateral=200.u256 diff --git a/vendor/nim-presto b/vendor/nim-presto index 3984431d..c17bfdda 160000 --- a/vendor/nim-presto +++ b/vendor/nim-presto @@ -1 +1 @@ -Subproject commit 3984431dc0fc829eb668e12e57e90542b041d298 +Subproject commit c17bfdda2c60cf5fadb043feb22e328b7659c719