codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/ceremony-files

This commit is contained in:
Ben 2024-03-22 11:04:35 +01:00
parent e4ddb942a1 de1714ed06
commit 2a34c480c3
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
21 changed files with 769 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

270
codex/contracts/market.nim
View File

@ -2,9 +2,9 @@ import std/sequtils
import std/strutils
import std/sugar
import pkg/ethers
import pkg/ethers/testing
import pkg/upraises
import pkg/questionable
import ../utils/exceptions
import ../logutils
import ../market
import ./marketplace
@ -32,147 +32,177 @@ func new*(_: type OnChainMarket, contract: Marketplace): OnChainMarket =
signer: signer,
)
proc raiseMarketError(message: string) {.raises: [MarketError].} =
raise newException(MarketError, message)
template convertEthersError(body) =
try:
body
except EthersError as error:
raiseMarketError(error.msgDetail)
proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
debug "Approving tokens", amount
let tokenAddress = await market.contract.token()
let token = Erc20Token.new(tokenAddress, market.signer)
discard await token.increaseAllowance(market.contract.address(), amount).confirm(1)
convertEthersError:
let tokenAddress = await market.contract.token()
let token = Erc20Token.new(tokenAddress, market.signer)
discard await token.increaseAllowance(market.contract.address(), amount).confirm(1)
method getZkeyHash*(market: OnChainMarket): Future[?string] {.async.} =
let config = await market.contract.config()
return some config.proofs.zkeyHash
method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
return await market.signer.getAddress()
convertEthersError:
return await market.signer.getAddress()
method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} =
let config = await market.contract.config()
let period = config.proofs.period
return Periodicity(seconds: period)
convertEthersError:
let config = await market.contract.config()
let period = config.proofs.period
return Periodicity(seconds: period)
method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} =
let config = await market.contract.config()
return config.proofs.timeout
convertEthersError:
let config = await market.contract.config()
return config.proofs.timeout
method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} =
let config = await market.contract.config()
return config.proofs.downtime
convertEthersError:
let config = await market.contract.config()
return config.proofs.downtime
method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} =
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.getPointer(slotId, overrides)
convertEthersError:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.getPointer(slotId, overrides)
method myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} =
return await market.contract.myRequests
convertEthersError:
return await market.contract.myRequests
method mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} =
let slots = await market.contract.mySlots()
debug "Fetched my slots", numSlots=len(slots)
convertEthersError:
let slots = await market.contract.mySlots()
debug "Fetched my slots", numSlots=len(slots)
return slots
return slots
method requestStorage(market: OnChainMarket, request: StorageRequest){.async.} =
debug "Requesting storage"
await market.approveFunds(request.price())
await market.contract.requestStorage(request)
convertEthersError:
debug "Requesting storage"
await market.approveFunds(request.price())
await market.contract.requestStorage(request)
method getRequest(market: OnChainMarket,
id: RequestId): Future[?StorageRequest] {.async.} =
try:
return some await market.contract.getRequest(id)
except ProviderError as e:
if e.revertReason.contains("Unknown request"):
return none StorageRequest
raise e
convertEthersError:
try:
return some await market.contract.getRequest(id)
except ProviderError as e:
if e.msgDetail.contains("Unknown request"):
return none StorageRequest
raise e
method requestState*(market: OnChainMarket,
requestId: RequestId): Future[?RequestState] {.async.} =
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return some await market.contract.requestState(requestId, overrides)
except ProviderError as e:
if e.revertReason.contains("Unknown request"):
return none RequestState
raise e
convertEthersError:
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return some await market.contract.requestState(requestId, overrides)
except ProviderError as e:
if e.msgDetail.contains("Unknown request"):
return none RequestState
raise e
method slotState*(market: OnChainMarket,
slotId: SlotId): Future[SlotState] {.async.} =
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.slotState(slotId, overrides)
convertEthersError:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.slotState(slotId, overrides)
method getRequestEnd*(market: OnChainMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
return await market.contract.requestEnd(id)
convertEthersError:
return await market.contract.requestEnd(id)
method getHost(market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.async.} =
let slotId = slotId(requestId, slotIndex)
let address = await market.contract.getHost(slotId)
if address != Address.default:
return some address
else:
return none Address
convertEthersError:
let slotId = slotId(requestId, slotIndex)
let address = await market.contract.getHost(slotId)
if address != Address.default:
return some address
else:
return none Address
method getActiveSlot*(market: OnChainMarket,
slotId: SlotId): Future[?Slot] {.async.} =
try:
return some await market.contract.getActiveSlot(slotId)
except ProviderError as e:
if e.revertReason.contains("Slot is free"):
return none Slot
raise e
convertEthersError:
try:
return some await market.contract.getActiveSlot(slotId)
except ProviderError as e:
if e.msgDetail.contains("Slot is free"):
return none Slot
raise e
method fillSlot(market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256,
proof: Groth16Proof,
collateral: UInt256) {.async.} =
await market.approveFunds(collateral)
await market.contract.fillSlot(requestId, slotIndex, proof)
convertEthersError:
await market.approveFunds(collateral)
await market.contract.fillSlot(requestId, slotIndex, proof)
method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
await market.contract.freeSlot(slotId)
convertEthersError:
await market.contract.freeSlot(slotId)
method withdrawFunds(market: OnChainMarket,
requestId: RequestId) {.async.} =
await market.contract.withdrawFunds(requestId)
convertEthersError:
await market.contract.withdrawFunds(requestId)
method isProofRequired*(market: OnChainMarket,
id: SlotId): Future[bool] {.async.} =
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.isProofRequired(id, overrides)
except ProviderError as e:
if e.revertReason.contains("Slot is free"):
return false
raise e
convertEthersError:
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.isProofRequired(id, overrides)
except ProviderError as e:
if e.msgDetail.contains("Slot is free"):
return false
raise e
method willProofBeRequired*(market: OnChainMarket,
id: SlotId): Future[bool] {.async.} =
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.willProofBeRequired(id, overrides)
except ProviderError as e:
if e.revertReason.contains("Slot is free"):
return false
raise e
convertEthersError:
try:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.willProofBeRequired(id, overrides)
except ProviderError as e:
if e.msgDetail.contains("Slot is free"):
return false
raise e
method getChallenge*(market: OnChainMarket, id: SlotId): Future[ProofChallenge] {.async.} =
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.getChallenge(id, overrides)
convertEthersError:
let overrides = CallOverrides(blockTag: some BlockTag.pending)
return await market.contract.getChallenge(id, overrides)
method submitProof*(market: OnChainMarket,
id: SlotId,
proof: Groth16Proof) {.async.} =
await market.contract.submitProof(id, proof)
convertEthersError:
await market.contract.submitProof(id, proof)
method markProofAsMissing*(market: OnChainMarket,
id: SlotId,
period: Period) {.async.} =
await market.contract.markProofAsMissing(id, period)
convertEthersError:
await market.contract.markProofAsMissing(id, period)
method canProofBeMarkedAsMissing*(
market: OnChainMarket,
@ -186,7 +216,7 @@ method canProofBeMarkedAsMissing*(
await contractWithoutSigner.markProofAsMissing(id, period, overrides)
return true
except EthersError as e:
trace "Proof can not be marked as missing", msg = e.msg
trace "Proof cannot be marked as missing", msg = e.msg
return false
method subscribeRequests*(market: OnChainMarket,
@ -196,16 +226,20 @@ method subscribeRequests*(market: OnChainMarket,
callback(event.requestId,
event.ask,
event.expiry)
let subscription = await market.contract.subscribe(StorageRequested, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(StorageRequested, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeSlotFilled*(market: OnChainMarket,
callback: OnSlotFilled):
Future[MarketSubscription] {.async.} =
proc onEvent(event: SlotFilled) {.upraises:[].} =
callback(event.requestId, event.slotIndex)
let subscription = await market.contract.subscribe(SlotFilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(SlotFilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeSlotFilled*(market: OnChainMarket,
requestId: RequestId,
@ -215,23 +249,29 @@ method subscribeSlotFilled*(market: OnChainMarket,
proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: UInt256) =
if eventRequestId == requestId and eventSlotIndex == slotIndex:
callback(requestId, slotIndex)
return await market.subscribeSlotFilled(onSlotFilled)
convertEthersError:
return await market.subscribeSlotFilled(onSlotFilled)
method subscribeSlotFreed*(market: OnChainMarket,
callback: OnSlotFreed):
Future[MarketSubscription] {.async.} =
proc onEvent(event: SlotFreed) {.upraises:[].} =
callback(event.requestId, event.slotIndex)
let subscription = await market.contract.subscribe(SlotFreed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(SlotFreed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeFulfillment(market: OnChainMarket,
callback: OnFulfillment):
Future[MarketSubscription] {.async.} =
proc onEvent(event: RequestFulfilled) {.upraises:[].} =
callback(event.requestId)
let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeFulfillment(market: OnChainMarket,
requestId: RequestId,
@ -240,16 +280,20 @@ method subscribeFulfillment(market: OnChainMarket,
proc onEvent(event: RequestFulfilled) {.upraises:[].} =
if event.requestId == requestId:
callback(event.requestId)
let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestCancelled*(market: OnChainMarket,
callback: OnRequestCancelled):
Future[MarketSubscription] {.async.} =
proc onEvent(event: RequestCancelled) {.upraises:[].} =
callback(event.requestId)
let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestCancelled*(market: OnChainMarket,
requestId: RequestId,
@ -258,16 +302,20 @@ method subscribeRequestCancelled*(market: OnChainMarket,
proc onEvent(event: RequestCancelled) {.upraises:[].} =
if event.requestId == requestId:
callback(event.requestId)
let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestFailed*(market: OnChainMarket,
callback: OnRequestFailed):
Future[MarketSubscription] {.async.} =
proc onEvent(event: RequestFailed) {.upraises:[]} =
callback(event.requestId)
let subscription = await market.contract.subscribe(RequestFailed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(RequestFailed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeRequestFailed*(market: OnChainMarket,
requestId: RequestId,
@ -276,16 +324,20 @@ method subscribeRequestFailed*(market: OnChainMarket,
proc onEvent(event: RequestFailed) {.upraises:[]} =
if event.requestId == requestId:
callback(event.requestId)
let subscription = await market.contract.subscribe(RequestFailed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(RequestFailed, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method subscribeProofSubmission*(market: OnChainMarket,
callback: OnProofSubmitted):
Future[MarketSubscription] {.async.} =
proc onEvent(event: ProofSubmitted) {.upraises: [].} =
callback(event.id)
let subscription = await market.contract.subscribe(ProofSubmitted, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
convertEthersError:
let subscription = await market.contract.subscribe(ProofSubmitted, onEvent)
return OnChainMarketSubscription(eventSubscription: subscription)
method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} =
await subscription.eventSubscription.unsubscribe()
@ -293,20 +345,20 @@ method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} =
method queryPastStorageRequests*(market: OnChainMarket,
blocksAgo: int):
Future[seq[PastStorageRequest]] {.async.} =
convertEthersError:
let contract = market.contract
let provider = contract.provider
let contract = market.contract
let provider = contract.provider
let head = await provider.getBlockNumber()
let fromBlock = BlockTag.init(head - blocksAgo.abs.u256)
let head = await provider.getBlockNumber()
let fromBlock = BlockTag.init(head - blocksAgo.abs.u256)
let events = await contract.queryFilter(StorageRequested,
fromBlock,
BlockTag.latest)
return events.map(event =>
PastStorageRequest(
requestId: event.requestId,
ask: event.ask,
expiry: event.expiry
let events = await contract.queryFilter(StorageRequested,
fromBlock,
BlockTag.latest)
return events.map(event =>
PastStorageRequest(
requestId: event.requestId,
ask: event.ask,
expiry: event.expiry
)
)
)

2
codex/market.nim
View File

@ -5,6 +5,7 @@ import pkg/ethers/erc20
import ./contracts/requests
import ./contracts/proofs
import ./clock
import ./errors
import ./periods
export chronos
@ -16,6 +17,7 @@ export periods
type
Market* = ref object of RootObj
MarketError* = object of CodexError
Subscription* = ref object of RootObj
OnRequest* = proc(id: RequestId,
ask: StorageAsk,

115
codex/rest/api.nim
View File

@ -35,6 +35,7 @@ import ../contracts
import ../manifest
import ../streams/asyncstreamwrapper
import ../stores
import ../utils/options
import ./coders
import ./json
@ -253,9 +254,10 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
router.rawApi(
MethodPost,
"/api/codex/v1/sales/availability") do () -> RestApiResponse:
## Add available storage to sell
## Add available storage to sell.
## Every time Availability's offer finishes, its capacity is returned to the availability.
##
## size - size of available storage in bytes
## totalSize - size of available storage in bytes
## duration - maximum time the storage should be sold for (in seconds)
## minPrice - minimum price to be paid (in amount of tokens)
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
@ -271,12 +273,15 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
let reservations = contracts.sales.context.reservations
if not reservations.hasAvailable(restAv.size.truncate(uint)):
if restAv.totalSize == 0:
return RestApiResponse.error(Http400, "Total size must be larger then zero")
if not reservations.hasAvailable(restAv.totalSize.truncate(uint)):
return RestApiResponse.error(Http422, "Not enough storage quota")
without availability =? (
await reservations.createAvailability(
restAv.size,
restAv.totalSize,
restAv.duration,
restAv.minPrice,
restAv.maxCollateral)
@ -284,11 +289,106 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
return RestApiResponse.error(Http500, error.msg)
return RestApiResponse.response(availability.toJson,
Http201,
contentType="application/json")
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500)
router.rawApi(
MethodPatch,
"/api/codex/v1/sales/availability/{id}") do (id: AvailabilityId) -> RestApiResponse:
## Updates Availability.
## The new parameters will be only considered for new requests.
## Existing Requests linked to this Availability will continue as is.
##
## totalSize - size of available storage in bytes. When decreasing the size, then lower limit is the currently `totalSize - freeSize`.
## duration - maximum time the storage should be sold for (in seconds)
## minPrice - minimum price to be paid (in amount of tokens)
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
without keyId =? id.key.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
let
body = await request.getBody()
reservations = contracts.sales.context.reservations
type OptRestAvailability = Optionalize(RestAvailability)
without restAv =? OptRestAvailability.fromJson(body), error:
return RestApiResponse.error(Http400, error.msg)
without availability =? (await reservations.get(keyId, Availability)), error:
if error of NotExistsError:
return RestApiResponse.error(Http404, "Availability not found")
return RestApiResponse.error(Http500, error.msg)
if isSome restAv.freeSize:
return RestApiResponse.error(Http400, "Updating freeSize is not allowed")
if size =? restAv.totalSize:
# we don't allow lowering the totalSize bellow currently utilized size
if size < (availability.totalSize - availability.freeSize):
return RestApiResponse.error(Http400, "New totalSize must be larger then current totalSize - freeSize, which is currently: " & $(availability.totalSize - availability.freeSize))
availability.freeSize += size - availability.totalSize
availability.totalSize = size
if duration =? restAv.duration:
availability.duration = duration
if minPrice =? restAv.minPrice:
availability.minPrice = minPrice
if maxCollateral =? restAv.maxCollateral:
availability.maxCollateral = maxCollateral
if err =? (await reservations.update(availability)).errorOption:
return RestApiResponse.error(Http500, err.msg)
return RestApiResponse.response(Http200)
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500)
router.rawApi(
MethodGet,
"/api/codex/v1/sales/availability/{id}/reservations") do (id: AvailabilityId) -> RestApiResponse:
## Gets Availability's reservations.
try:
without contracts =? node.contracts.host:
return RestApiResponse.error(Http503, "Sales unavailable")
without id =? id.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
without keyId =? id.key.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
let reservations = contracts.sales.context.reservations
if error =? (await reservations.get(keyId, Availability)).errorOption:
if error of NotExistsError:
return RestApiResponse.error(Http404, "Availability not found")
else:
return RestApiResponse.error(Http500, error.msg)
without availabilitysReservations =? (await reservations.all(Reservation, id)), err:
return RestApiResponse.error(Http500, err.msg)
# TODO: Expand this structure with information about the linked StorageRequest not only RequestID
return RestApiResponse.response(availabilitysReservations.toJson, contentType="application/json")
except CatchableError as exc:
trace "Excepting processing request", exc = exc.msg
return RestApiResponse.error(Http500)
proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
router.rawApi(
MethodPost,
@ -329,10 +429,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
return RestApiResponse.error(Http500)
if expiry <= node.clock.now.u256:
return RestApiResponse.error(Http400, "Expiry needs to be in future")
return RestApiResponse.error(Http400, "Expiry needs to be in future. Now: " & $node.clock.now)
if expiry > node.clock.now.u256 + params.duration:
return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration)")
let expiryLimit = node.clock.now.u256 + params.duration
if expiry > expiryLimit:
return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration). Limit: " & $expiryLimit)
without purchaseId =? await node.requestStorage(
cid,

2
codex/rest/coders.nim
View File

@ -84,7 +84,7 @@ proc decodeString*(_: type array[32, byte],
except ValueError as e:
err e.msg.cstring
proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId](_: type T,
proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId | AvailabilityId](_: type T,
value: string): Result[T, cstring] =
array[32, byte].decodeString(value).map(id => T(id))

3
codex/rest/json.nim
View File

@ -27,10 +27,11 @@ type
error* {.serialize.}: ?string
RestAvailability* = object
size* {.serialize.}: UInt256
totalSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256
maxCollateral* {.serialize.}: UInt256
freeSize* {.serialize.}: ?UInt256
RestSalesAgent* = object
state* {.serialize.}: string

137
codex/sales/reservations.nim
View File

@ -9,24 +9,27 @@
##
## +--------------------------------------+
## | RESERVATION |
## +--------------------------------------+ |--------------------------------------|
## | AVAILABILITY | | ReservationId | id | PK |
## |--------------------------------------| |--------------------------------------|
## | AvailabilityId | id | PK |<-||-------o<-| AvailabilityId | availabilityId | FK |
## |--------------------------------------| |--------------------------------------|
## | UInt256 | size | | | UInt256 | size | |
## |--------------------------------------| |--------------------------------------|
## | UInt256 | duration | | | SlotId | slotId | |
## |--------------------------------------| +--------------------------------------+
## | UInt256 | minPrice | |
## |--------------------------------------|
## | UInt256 | maxCollateral | |
## +--------------------------------------+
## +----------------------------------------+ |--------------------------------------|
## | AVAILABILITY | | ReservationId | id | PK |
## |----------------------------------------| |--------------------------------------|
## | AvailabilityId | id | PK |<-||-------o<-| AvailabilityId | availabilityId | FK |
## |----------------------------------------| |--------------------------------------|
## | UInt256 | totalSize | | | UInt256 | size | |
## |----------------------------------------| |--------------------------------------|
## | UInt256 | freeSize | | | SlotId | slotId | |
## |----------------------------------------| +--------------------------------------+
## | UInt256 | duration | |
## |----------------------------------------|
## | UInt256 | minPrice | |
## |----------------------------------------|
## | UInt256 | maxCollateral | |
## +----------------------------------------+
import pkg/upraises
push: {.upraises: [].}
import std/typetraits
import std/sequtils
import pkg/chronos
import pkg/datastore
import pkg/nimcrypto
@ -35,7 +38,9 @@ import pkg/questionable/results
import pkg/stint
import pkg/stew/byteutils
import ../logutils
import ../clock
import ../stores
import ../market
import ../contracts/requests
import ../utils/json
@ -52,7 +57,8 @@ type
SomeStorableId = AvailabilityId | ReservationId
Availability* = ref object
id* {.serialize.}: AvailabilityId
size* {.serialize.}: UInt256
totalSize* {.serialize.}: UInt256
freeSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256
maxCollateral* {.serialize.}: UInt256
@ -91,14 +97,15 @@ proc new*(T: type Reservations,
proc init*(
_: type Availability,
size: UInt256,
totalSize: UInt256,
freeSize: UInt256,
duration: UInt256,
minPrice: UInt256,
maxCollateral: UInt256): Availability =
var id: array[32, byte]
doAssert randomBytes(id) == 32
Availability(id: AvailabilityId(id), size: size, duration: duration, minPrice: minPrice, maxCollateral: maxCollateral)
Availability(id: AvailabilityId(id), totalSize:totalSize, freeSize: freeSize, duration: duration, minPrice: minPrice, maxCollateral: maxCollateral)
proc init*(
_: type Reservation,
@ -118,17 +125,9 @@ func toArray(id: SomeStorableId): array[32, byte] =
proc `==`*(x, y: AvailabilityId): bool {.borrow.}
proc `==`*(x, y: ReservationId): bool {.borrow.}
proc `==`*(x, y: Reservation): bool =
x.id == y.id and
x.availabilityId == y.availabilityId and
x.size == y.size and
x.requestId == y.requestId and
x.slotIndex == y.slotIndex
x.id == y.id
proc `==`*(x, y: Availability): bool =
x.id == y.id and
x.size == y.size and
x.duration == y.duration and
x.maxCollateral == y.maxCollateral and
x.minPrice == y.minPrice
x.id == y.id
proc `$`*(id: SomeStorableId): string = id.toArray.toHex
@ -198,11 +197,11 @@ proc get*(
return success obj
proc update(
proc updateImpl(
self: Reservations,
obj: SomeStorableObject): Future[?!void] {.async.} =
trace "updating " & $(obj.type), id = obj.id, size = obj.size
trace "updating " & $(obj.type), id = obj.id
without key =? obj.key, error:
return failure(error)
@ -215,6 +214,39 @@ proc update(
return success()
proc update*(
self: Reservations,
obj: Reservation): Future[?!void] {.async.} =
return await self.updateImpl(obj)
proc update*(
self: Reservations,
obj: Availability): Future[?!void] {.async.} =
without key =? obj.key, error:
return failure(error)
let getResult = await self.get(key, Availability)
if getResult.isOk:
let oldAvailability = !getResult
# Sizing of the availability changed, we need to adjust the repo reservation accordingly
if oldAvailability.totalSize != obj.totalSize:
if oldAvailability.totalSize < obj.totalSize: # storage added
if reserveErr =? (await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).truncate(uint))).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
elif oldAvailability.totalSize > obj.totalSize: # storage removed
if reserveErr =? (await self.repo.release((oldAvailability.totalSize - obj.totalSize).truncate(uint))).errorOption:
return failure(reserveErr.toErr(ReleaseFailedError))
else:
let err = getResult.error()
if not (err of NotExistsError):
return failure(err)
return await self.updateImpl(obj)
proc delete(
self: Reservations,
key: Key): Future[?!void] {.async.} =
@ -258,7 +290,7 @@ proc deleteReservation*(
without var availability =? await self.get(availabilityKey, Availability), error:
return failure(error)
availability.size += reservation.size
availability.freeSize += reservation.size
if updateErr =? (await self.update(availability)).errorOption:
return failure(updateErr)
@ -278,9 +310,9 @@ proc createAvailability*(
trace "creating availability", size, duration, minPrice, maxCollateral
let availability = Availability.init(
size, duration, minPrice, maxCollateral
size, size, duration, minPrice, maxCollateral
)
let bytes = availability.size.truncate(uint)
let bytes = availability.freeSize.truncate(uint)
if reserveErr =? (await self.repo.reserve(bytes)).errorOption:
return failure(reserveErr.toErr(ReserveFailedError))
@ -324,7 +356,7 @@ proc createReservation*(
without var availability =? await self.get(availabilityKey, Availability), error:
return failure(error)
if availability.size < slotSize:
if availability.freeSize < slotSize:
let error = newException(
BytesOutOfBoundsError,
"trying to reserve an amount of bytes that is greater than the total size of the Availability")
@ -333,9 +365,9 @@ proc createReservation*(
if createResErr =? (await self.update(reservation)).errorOption:
return failure(createResErr)
# reduce availability size by the slot size, which is now accounted for in
# reduce availability freeSize by the slot size, which is now accounted for in
# the newly created Reservation
availability.size -= slotSize
availability.freeSize -= slotSize
# update availability with reduced size
if updateErr =? (await self.update(availability)).errorOption:
@ -393,7 +425,7 @@ proc returnBytesToAvailability*(
without var availability =? await self.get(availabilityKey, Availability), error:
return failure(error)
availability.size += bytesToBeReturned
availability.freeSize += bytesToBeReturned
# Update availability with returned size
if updateErr =? (await self.update(availability)).errorOption:
@ -456,11 +488,12 @@ iterator items(self: StorableIter): Future[?seq[byte]] =
proc storables(
self: Reservations,
T: type SomeStorableObject
T: type SomeStorableObject,
queryKey: Key = ReservationsKey
): Future[?!StorableIter] {.async.} =
var iter = StorableIter()
let query = Query.init(ReservationsKey)
let query = Query.init(queryKey)
when T is Availability:
# should indicate key length of 4, but let the .key logic determine it
without defaultKey =? AvailabilityId.default.key, error:
@ -475,6 +508,7 @@ proc storables(
without results =? await self.repo.metaDs.query(query), error:
return failure(error)
# /sales/reservations
proc next(): Future[?seq[byte]] {.async.} =
await idleAsync()
iter.finished = results.finished
@ -491,14 +525,15 @@ proc storables(
iter.next = next
return success iter
proc all*(
proc allImpl(
self: Reservations,
T: type SomeStorableObject
T: type SomeStorableObject,
queryKey: Key = ReservationsKey
): Future[?!seq[T]] {.async.} =
var ret: seq[T] = @[]
without storables =? (await self.storables(T)), error:
without storables =? (await self.storables(T, queryKey)), error:
return failure(error)
for storable in storables.items:
@ -515,6 +550,22 @@ proc all*(
return success(ret)
proc all*(
self: Reservations,
T: type SomeStorableObject
): Future[?!seq[T]] {.async.} =
return await self.allImpl(T)
proc all*(
self: Reservations,
T: type SomeStorableObject,
availabilityId: AvailabilityId
): Future[?!seq[T]] {.async.} =
without key =? (ReservationsKey / $availabilityId):
return failure("no key")
return await self.allImpl(T, key)
proc findAvailability*(
self: Reservations,
size, duration, minPrice, collateral: UInt256
@ -528,13 +579,13 @@ proc findAvailability*(
if bytes =? (await item) and
availability =? Availability.fromJson(bytes):
if size <= availability.size and
if size <= availability.freeSize and
duration <= availability.duration and
collateral <= availability.maxCollateral and
minPrice >= availability.minPrice:
trace "availability matched",
size, availsize = availability.size,
size, availFreeSize = availability.freeSize,
duration, availDuration = availability.duration,
minPrice, availMinPrice = availability.minPrice,
collateral, availMaxCollateral = availability.maxCollateral
@ -542,7 +593,7 @@ proc findAvailability*(
return some availability
trace "availability did not match",
size, availsize = availability.size,
size, availFreeSize = availability.freeSize,
duration, availDuration = availability.duration,
minPrice, availMinPrice = availability.minPrice,
collateral, availMaxCollateral = availability.maxCollateral

4
codex/sales/states/provingsimulated.nim
View File

@ -33,8 +33,8 @@ when codex_enable_proof_failures:
try:
warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id
await market.submitProof(slot.id, Groth16Proof.default)
except SignerError as e:
if not e.msgDetail.contains("Invalid proof"):
except MarketError as e:
if not e.msg.contains("Invalid proof"):
onSubmitProofError(e, currentPeriod, slot.id)
except CatchableError as e:
onSubmitProofError(e, currentPeriod, slot.id)

16
codex/utils/optionalcast.nim
View File

@ -1,16 +0,0 @@
import pkg/questionable
import pkg/questionable/operators
export questionable
proc `as`*[T](value: T, U: type): ?U =
## Casts a value to another type, returns an Option.
## When the cast succeeds, the option will contain the casted value.
## When the cast fails, the option will have no value.
when value is U:
return some value
elif value is ref object:
if value of U:
return some U(value)
Option.liftBinary `as`

61
codex/utils/options.nim Normal file
View File

@ -0,0 +1,61 @@
import macros
import strutils
import pkg/questionable
import pkg/questionable/operators
export questionable
proc `as`*[T](value: T, U: type): ?U =
## Casts a value to another type, returns an Option.
## When the cast succeeds, the option will contain the casted value.
## When the cast fails, the option will have no value.
when value is U:
return some value
elif value is ref object:
if value of U:
return some U(value)
Option.liftBinary `as`
# Template that wraps type with `Option[]` only if it is already not `Option` type
template WrapOption*(input: untyped): type =
when input is Option:
input
else:
Option[input]
macro createType(t: typedesc): untyped =
var objectType = getType(t)
# Work around for https://github.com/nim-lang/Nim/issues/23112
while objectType.kind == nnkBracketExpr and objectType[0].eqIdent"typeDesc":
objectType = getType(objectType[1])
expectKind(objectType, NimNodeKind.nnkObjectTy)
var fields = nnkRecList.newTree()
# Generates the list of fields that are wrapped in `Option[T]`.
# Technically wrapped with `WrapOption` which is template used to prevent
# re-wrapping already filed which is `Option[T]`.
for field in objectType[2]:
let fieldType = getTypeInst(field)
let newFieldNode =
nnkIdentDefs.newTree(ident($field), nnkCall.newTree(ident("WrapOption"), fieldType), newEmptyNode())
fields.add(newFieldNode)
# Creates new object type T with the fields lists from steps above.
let tSym = genSym(nskType, "T")
nnkStmtList.newTree(
nnkTypeSection.newTree(
nnkTypeDef.newTree(tSym, newEmptyNode(), nnkObjectTy.newTree(newEmptyNode(), newEmptyNode(), fields))
),
tSym
)
template Optionalize*(t: typed): untyped =
## Takes object type and wraps all the first level fields into
## Option type unless it is already Option type.
createType(t)

128
openapi.yaml
View File

@ -20,6 +20,15 @@ components:
description: Peer Identity reference as specified at https://docs.libp2p.io/concepts/fundamentals/peers/
example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
Id:
type: string
description: 32bits identifier encoded in hex-decimal string.
example: 0x...
BigInt:
type: string
description: Integer represented as decimal string
Cid:
type: string
description: Content Identifier as specified at https://github.com/multiformats/cid
@ -102,16 +111,12 @@ components:
SalesAvailability:
type: object
required:
- size
- minPrice
properties:
id:
$ref: "#/components/schemas/Id"
totalSize:
type: string
description: Hexadecimal identifier of the availability
size:
type: string
description: Size of available storage in bytes as decimal string
description: Total size of availability's storage in bytes as decimal string
duration:
$ref: "#/components/schemas/Duration"
minPrice:
@ -121,6 +126,24 @@ components:
type: string
description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
SalesAvailabilityREAD:
allOf:
- $ref: "#/components/schemas/SalesAvailability"
- type: object
properties:
freeSize:
type: string
description: Unused size of availability's storage in bytes as decimal string
SalesAvailabilityCREATE:
allOf:
- $ref: "#/components/schemas/SalesAvailability"
- required:
- totalSize
- minPrice
- maxCollateral
- duration
Slot:
type: object
properties:
@ -132,6 +155,21 @@ components:
type: string
description: Slot Index as hexadecimal string
Reservation:
type: object
properties:
id:
$ref: "#/components/schemas/Id"
availabilityId:
$ref: "#/components/schemas/Id"
size:
$ref: "#/components/schemas/BigInt"
requestId:
$ref: "#/components/schemas/Id"
slotIndex:
type: string
description: Slot Index as hexadecimal string
StorageRequestCreation:
type: object
required:
@ -493,16 +531,84 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/SalesAvailability"
$ref: "#/components/schemas/SalesAvailabilityCREATE"
responses:
"200":
"201":
description: Created storage availability
content:
application/json:
schema:
$ref: "#/components/schemas/SalesAvailability"
$ref: "#/components/schemas/SalesAvailabilityREAD"
"400":
description: Invalid data input
"422":
description: Not enough node's storage quota available
"500":
description: Error reserving availablility
description: Error reserving availability
"503":
description: Sales are unavailable
"/sales/availability/{id}":
patch:
summary: "Updates availability"
description: |
The new parameters will be only considered for new requests.
Existing Requests linked to this Availability will continue as is.
operationId: updateOfferedStorage
tags: [ Marketplace ]
parameters:
- in: path
name: id
required: true
schema:
type: string
description: ID of Availability
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/SalesAvailability"
responses:
"204":
description: Availability successfully updated
"400":
description: Invalid data input
"404":
description: Availability not found
"422":
description: Not enough node's storage quota available
"500":
description: Error reserving availability
"503":
description: Sales are unavailable
"/sales/availability/{id}/reservations":
patch:
summary: "Get availability's reservations"
description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
operationId: getReservations
tags: [ Marketplace ]
parameters:
- in: path
name: id
required: true
schema:
type: string
description: ID of Availability
responses:
"200":
description: Retrieved storage availabilities of the node
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Reservation"
"400":
description: Invalid Availability ID
"404":
description: Availability not found
"500":
description: Error getting reservations
"503":
description: Sales are unavailable

3
tests/codex/examples.nim
View File

@ -60,7 +60,8 @@ proc example*(_: type MultiHash, mcodec = Sha256HashCodec): MultiHash =
proc example*(_: type Availability): Availability =
Availability.init(
size = uint16.example.u256,
totalSize = uint16.example.u256,
freeSize = uint16.example.u256,
duration = uint16.example.u256,
minPrice = uint64.example.u256,
maxCollateral = uint16.example.u256

65
tests/codex/sales/testreservations.nim
View File

@ -29,9 +29,9 @@ asyncchecksuite "Reservations module":
proc createAvailability(): Availability =
let example = Availability.example
let size = rand(100000..200000)
let totalSize = rand(100000..200000)
let availability = waitFor reservations.createAvailability(
size.u256,
totalSize.u256,
example.duration,
example.minPrice,
example.maxCollateral
@ -39,7 +39,7 @@ asyncchecksuite "Reservations module":
return availability.get
proc createReservation(availability: Availability): Reservation =
let size = rand(1..<availability.size.truncate(int))
let size = rand(1..<availability.freeSize.truncate(int))
let reservation = waitFor reservations.createReservation(
availability.id,
size.u256,
@ -57,8 +57,8 @@ asyncchecksuite "Reservations module":
check (await reservations.all(Availability)).get.len == 0
test "generates unique ids for storage availability":
let availability1 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256)
let availability2 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256)
let availability1 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256)
let availability2 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256)
check availability1.id != availability2.id
test "can reserve available storage":
@ -68,7 +68,7 @@ asyncchecksuite "Reservations module":
test "creating availability reserves bytes in repo":
let orig = repo.available
let availability = createAvailability()
check repo.available == (orig.u256 - availability.size).truncate(uint)
check repo.available == (orig.u256 - availability.freeSize).truncate(uint)
test "can get all availabilities":
let availability1 = createAvailability()
@ -106,6 +106,19 @@ asyncchecksuite "Reservations module":
reservations.contains(reservation1)
reservations.contains(reservation2)
test "can get reservations of specific availability":
let availability1 = createAvailability()
let availability2 = createAvailability()
let reservation1 = createReservation(availability1)
let reservation2 = createReservation(availability2)
let reservations = !(await reservations.all(Reservation, availability1.id))
check:
# perform unordered checks
reservations.len == 1
reservations.contains(reservation1)
not reservations.contains(reservation2)
test "cannot create reservation with non-existant availability":
let availability = Availability.example
let created = await reservations.createReservation(
@ -121,7 +134,7 @@ asyncchecksuite "Reservations module":
let availability = createAvailability()
let created = await reservations.createReservation(
availability.id,
availability.size + 1,
availability.totalSize + 1,
RequestId.example,
UInt256.example
)
@ -130,11 +143,11 @@ asyncchecksuite "Reservations module":
test "creating reservation reduces availability size":
let availability = createAvailability()
let orig = availability.size
let orig = availability.freeSize
let reservation = createReservation(availability)
let key = availability.id.key.get
let updated = (await reservations.get(key, Availability)).get
check updated.size == orig - reservation.size
check updated.freeSize == orig - reservation.size
test "can check if reservation exists":
let availability = createAvailability()
@ -166,19 +179,19 @@ asyncchecksuite "Reservations module":
test "deleting reservation returns bytes back to availability":
let availability = createAvailability()
let orig = availability.size
let orig = availability.freeSize
let reservation = createReservation(availability)
discard await reservations.deleteReservation(
reservation.id, reservation.availabilityId
)
let key = availability.key.get
let updated = !(await reservations.get(key, Availability))
check updated.size == orig
check updated.freeSize == orig
test "calling returnBytesToAvailability returns bytes back to availability":
let availability = createAvailability()
let reservation = createReservation(availability)
let orig = availability.size - reservation.size
let orig = availability.freeSize - reservation.size
let origQuota = repo.quotaReservedBytes
let returnedBytes = reservation.size + 200.u256
@ -189,10 +202,28 @@ asyncchecksuite "Reservations module":
let key = availability.key.get
let updated = !(await reservations.get(key, Availability))
check updated.size > orig
check (updated.size - orig) == 200.u256
check updated.freeSize > orig
check (updated.freeSize - orig) == 200.u256
check (repo.quotaReservedBytes - origQuota) == 200
test "update releases quota when lowering size":
let
availability = createAvailability()
origQuota = repo.quotaReservedBytes
availability.totalSize = availability.totalSize - 100
check isOk await reservations.update(availability)
check (origQuota - repo.quotaReservedBytes) == 100
test "update reserves quota when growing size":
let
availability = createAvailability()
origQuota = repo.quotaReservedBytes
availability.totalSize = availability.totalSize + 100
check isOk await reservations.update(availability)
check (repo.quotaReservedBytes - origQuota) == 100
test "reservation can be partially released":
let availability = createAvailability()
let reservation = createReservation(availability)
@ -240,7 +271,7 @@ asyncchecksuite "Reservations module":
let availability = createAvailability()
let found = await reservations.findAvailability(
availability.size,
availability.freeSize,
availability.duration,
availability.minPrice,
availability.maxCollateral)
@ -252,7 +283,7 @@ asyncchecksuite "Reservations module":
let availability = createAvailability()
let found = await reservations.findAvailability(
availability.size + 1,
availability.freeSize + 1,
availability.duration,
availability.minPrice,
availability.maxCollateral)
@ -262,7 +293,7 @@ asyncchecksuite "Reservations module":
test "non-existant availability cannot be found":
let availability = Availability.example
let found = (await reservations.findAvailability(
availability.size,
availability.freeSize,
availability.duration,
availability.minPrice,
availability.maxCollateral

29
tests/codex/sales/testsales.nim
View File

@ -127,7 +127,8 @@ asyncchecksuite "Sales":
setup:
availability = Availability(
size: 100.u256,
totalSize: 100.u256,
freeSize: 100.u256,
duration: 60.u256,
minPrice: 600.u256,
maxCollateral: 400.u256
@ -189,7 +190,7 @@ asyncchecksuite "Sales":
proc createAvailability() =
let a = waitFor reservations.createAvailability(
availability.size,
availability.totalSize,
availability.duration,
availability.minPrice,
availability.maxCollateral
@ -224,7 +225,7 @@ asyncchecksuite "Sales":
proc wasIgnored(): bool =
let run = proc(): Future[bool] {.async.} =
always (
getAvailability().size == availability.size and
getAvailability().freeSize == availability.freeSize and
(waitFor reservations.all(Reservation)).get.len == 0
)
waitFor run()
@ -300,7 +301,7 @@ asyncchecksuite "Sales":
createAvailability()
await market.requestStorage(request)
check eventually getAvailability().size == availability.size - request.ask.slotSize
check eventually getAvailability().freeSize == availability.freeSize - request.ask.slotSize
test "non-downloaded bytes are returned to availability once finished":
var slotIndex = 0.u256
@ -316,7 +317,7 @@ asyncchecksuite "Sales":
sold.complete()
createAvailability()
let origSize = availability.size
let origSize = availability.freeSize
await market.requestStorage(request)
await allowRequestToStart()
await sold
@ -325,7 +326,7 @@ asyncchecksuite "Sales":
market.slotState[request.slotId(slotIndex)] = SlotState.Finished
clock.advance(request.ask.duration.truncate(int64))
check eventually getAvailability().size == origSize - 1
check eventually getAvailability().freeSize == origSize - 1
test "ignores download when duration not long enough":
availability.duration = request.ask.duration - 1
@ -334,7 +335,7 @@ asyncchecksuite "Sales":
check wasIgnored()
test "ignores request when slot size is too small":
availability.size = request.ask.slotSize - 1
availability.totalSize = request.ask.slotSize - 1
createAvailability()
await market.requestStorage(request)
check wasIgnored()
@ -399,7 +400,7 @@ asyncchecksuite "Sales":
return failure(error)
createAvailability()
await market.requestStorage(request)
check getAvailability().size == availability.size
check getAvailability().freeSize == availability.freeSize
test "generates proof of storage":
var provingRequest: StorageRequest
@ -472,7 +473,7 @@ asyncchecksuite "Sales":
check eventually (await reservations.all(Availability)).get == @[availability]
test "makes storage available again when request expires":
let origSize = availability.size
let origSize = availability.freeSize
sales.onStore = proc(request: StorageRequest,
slot: UInt256,
onBatch: BatchProc): Future[?!void] {.async.} =
@ -487,10 +488,10 @@ asyncchecksuite "Sales":
market.requestState[request.id]=RequestState.Cancelled
clock.set(request.expiry.truncate(int64)+1)
check eventually (await reservations.all(Availability)).get == @[availability]
check getAvailability().size == origSize
check getAvailability().freeSize == origSize
test "verifies that request is indeed expired from onchain before firing onCancelled":
let origSize = availability.size
let origSize = availability.freeSize
sales.onStore = proc(request: StorageRequest,
slot: UInt256,
onBatch: BatchProc): Future[?!void] {.async.} =
@ -504,10 +505,10 @@ asyncchecksuite "Sales":
# would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation.
await sleepAsync(chronos.milliseconds(100))
clock.set(request.expiry.truncate(int64)+1)
check getAvailability().size == 0
check getAvailability().freeSize == 0
market.requestState[request.id]=RequestState.Cancelled # Now "on-chain" is also expired
check eventually getAvailability().size == origSize
check eventually getAvailability().freeSize == origSize
test "loads active slots from market":
let me = await market.getSigner()
@ -556,4 +557,4 @@ asyncchecksuite "Sales":
check (await reservations.all(Reservation)).get.len == 1
await sales.load()
check (await reservations.all(Reservation)).get.len == 0
check getAvailability().size == availability.size # was restored
check getAvailability().freeSize == availability.freeSize # was restored

2
tests/codex/testutils.nim
View File

@ -1,4 +1,4 @@
import ./utils/testoptionalcast
import ./utils/testoptions
import ./utils/testkeyutils
import ./utils/testasyncstatemachine
import ./utils/testtimer

21
tests/codex/utils/testoptionalcast.nim → tests/codex/utils/testoptions.nim
View File

@ -1,5 +1,5 @@
import std/unittest
import codex/utils/optionalcast
import codex/utils/options
import ../helpers
checksuite "optional casts":
@ -28,3 +28,22 @@ checksuite "optional casts":
check 42.some as int == some 42
check 42.some as string == string.none
check int.none as int == int.none
checksuite "Optionalize":
test "does not except non-object types":
static:
doAssert not compiles(Optionalize(int))
test "converts object fields to option":
type BaseType = object
a: int
b: bool
c: string
d: Option[string]
type OptionalizedType = Optionalize(BaseType)
check OptionalizedType.a is Option[int]
check OptionalizedType.b is Option[bool]
check OptionalizedType.c is Option[string]
check OptionalizedType.d is Option[string]

49
tests/integration/codexclient.nim
View File

@ -133,27 +133,70 @@ proc getSlots*(client: CodexClient): ?!seq[Slot] =
proc postAvailability*(
client: CodexClient,
size, duration, minPrice, maxCollateral: UInt256
totalSize, duration, minPrice, maxCollateral: UInt256
): ?!Availability =
## Post sales availability endpoint
##
let url = client.baseurl & "/sales/availability"
let json = %*{
"size": size,
"totalSize": totalSize,
"duration": duration,
"minPrice": minPrice,
"maxCollateral": maxCollateral,
}
let response = client.http.post(url, $json)
doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body
doAssert response.status == "201 Created", "expected 201 Created, got " & response.status & ", body: " & response.body
Availability.fromJson(response.body)
proc patchAvailabilityRaw*(
client: CodexClient,
availabilityId: AvailabilityId,
totalSize, freeSize, duration, minPrice, maxCollateral: ?UInt256 = UInt256.none
): Response =
## Updates availability
##
let url = client.baseurl & "/sales/availability/" & $availabilityId
# TODO: Optionalize macro does not keep `serialize` pragmas so we can't use `Optionalize(RestAvailability)` here.
var json = %*{}
if totalSize =? totalSize:
json["totalSize"] = %totalSize
if freeSize =? freeSize:
json["freeSize"] = %freeSize
if duration =? duration:
json["duration"] = %duration
if minPrice =? minPrice:
json["minPrice"] = %minPrice
if maxCollateral =? maxCollateral:
json["maxCollateral"] = %maxCollateral
client.http.patch(url, $json)
proc patchAvailability*(
client: CodexClient,
availabilityId: AvailabilityId,
totalSize, duration, minPrice, maxCollateral: ?UInt256 = UInt256.none
): void =
let response = client.patchAvailabilityRaw(availabilityId, totalSize=totalSize, duration=duration, minPrice=minPrice, maxCollateral=maxCollateral)
doAssert response.status == "200 OK", "expected 200 OK, got " & response.status
proc getAvailabilities*(client: CodexClient): ?!seq[Availability] =
## Call sales availability REST endpoint
let url = client.baseurl & "/sales/availability"
let body = client.http.getContent(url)
seq[Availability].fromJson(body)
proc getAvailabilityReservations*(client: CodexClient, availabilityId: AvailabilityId): ?!seq[Reservation] =
## Retrieves Availability's Reservations
let url = client.baseurl & "/sales/availability/" & $availabilityId & "/reservations"
let body = client.http.getContent(url)
seq[Reservation].fromJson(body)
proc close*(client: CodexClient) =
client.http.close()

2
tests/integration/marketplacesuite.nim
View File

@ -53,7 +53,7 @@ template marketplacesuite*(name: string, body: untyped) =
let provider = providers()[i].client
discard provider.postAvailability(
size=datasetSize.u256, # should match 1 slot only
totalSize=datasetSize.u256, # should match 1 slot only
duration=duration.u256,
minPrice=300.u256,
maxCollateral=200.u256

92
tests/integration/testIntegration.nim
View File

@ -1,5 +1,6 @@
import std/options
import std/sequtils
import std/strutils
import std/httpclient
from pkg/libp2p import `==`
import pkg/chronos
@ -12,8 +13,16 @@ import ../contracts/time
import ../contracts/deployment
import ../codex/helpers
import ../examples
import ../codex/examples
import ./twonodes
proc findItem[T](items: seq[T], item: T): ?!T =
for tmp in items:
if tmp == item:
return success tmp
return failure("Not found")
# For debugging you can enable logging output with debugX = true
# You can also pass a string in same format like for the `--log-level` parameter
# to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace"
@ -38,7 +47,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
test "node shows used and available space":
discard client1.upload("some file contents").get
discard client1.postAvailability(size=12.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
discard client1.postAvailability(totalSize=12.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
let space = client1.space().tryGet()
check:
space.totalBlocks == 2.uint
@ -94,12 +103,12 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
[cid1, cid2].allIt(it in list.mapIt(it.cid))
test "node handles new storage availability":
let availability1 = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
let availability2 = client1.postAvailability(size=4.u256, duration=5.u256, minPrice=6.u256, maxCollateral=7.u256).get
let availability1 = client1.postAvailability(totalSize=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
let availability2 = client1.postAvailability(totalSize=4.u256, duration=5.u256, minPrice=6.u256, maxCollateral=7.u256).get
check availability1 != availability2
test "node lists storage that is for sale":
let availability = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
let availability = client1.postAvailability(totalSize=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
check availability in client1.getAvailabilities().get
test "node handles storage request":
@ -177,7 +186,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
let size = 0xFFFFFF.u256
let data = await RandomChunker.example(blocks=8)
# client 2 makes storage available
discard client2.postAvailability(size=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256)
let availability = client2.postAvailability(totalSize=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
# client 1 requests storage
let expiry = (await ethProvider.currentTime()) + 5*60
@ -197,9 +206,13 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
check purchase.error == none string
let availabilities = client2.getAvailabilities().get
check availabilities.len == 1
let newSize = availabilities[0].size
let newSize = availabilities[0].freeSize
check newSize > 0 and newSize < size
let reservations = client2.getAvailabilityReservations(availability.id).get
check reservations.len == 5
check reservations[0].requestId == purchase.requestId
test "node slots gets paid out":
let size = 0xFFFFFF.u256
let data = await RandomChunker.example(blocks = 8)
@ -212,7 +225,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
# client 2 makes storage available
let startBalance = await token.balanceOf(account2)
discard client2.postAvailability(size=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
discard client2.postAvailability(totalSize=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
# client 1 requests storage
let expiry = (await ethProvider.currentTime()) + 5*60
@ -263,8 +276,69 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
let responsePast = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=currentTime-10)
check responsePast.status == "400 Bad Request"
check responsePast.body == "Expiry needs to be in future"
check "Expiry needs to be in future" in responsePast.body
let responseBefore = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=currentTime+10)
check responseBefore.status == "400 Bad Request"
check responseBefore.body == "Expiry has to be before the request's end (now + duration)"
check "Expiry has to be before the request's end (now + duration)" in responseBefore.body
test "updating non-existing availability":
let nonExistingResponse = client1.patchAvailabilityRaw(AvailabilityId.example, duration=100.u256.some, minPrice=200.u256.some, maxCollateral=200.u256.some)
check nonExistingResponse.status == "404 Not Found"
test "updating availability":
let availability = client1.postAvailability(totalSize=140000.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get
client1.patchAvailability(availability.id, duration=100.u256.some, minPrice=200.u256.some, maxCollateral=200.u256.some)
let updatedAvailability = (client1.getAvailabilities().get).findItem(availability).get
check updatedAvailability.duration == 100
check updatedAvailability.minPrice == 200
check updatedAvailability.maxCollateral == 200
check updatedAvailability.totalSize == 140000
check updatedAvailability.freeSize == 140000
test "updating availability - freeSize is not allowed to be changed":
let availability = client1.postAvailability(totalSize=140000.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get
let freeSizeResponse = client1.patchAvailabilityRaw(availability.id, freeSize=110000.u256.some)
check freeSizeResponse.status == "400 Bad Request"
check "not allowed" in freeSizeResponse.body
test "updating availability - updating totalSize":
let availability = client1.postAvailability(totalSize=140000.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get
client1.patchAvailability(availability.id, totalSize=100000.u256.some)
let updatedAvailability = (client1.getAvailabilities().get).findItem(availability).get
check updatedAvailability.totalSize == 100000
check updatedAvailability.freeSize == 100000
test "updating availability - updating totalSize does not allow bellow utilized":
let originalSize = 0xFFFFFF.u256
let data = await RandomChunker.example(blocks=8)
let availability = client1.postAvailability(totalSize=originalSize, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
# Lets create storage request that will utilize some of the availability's space
let expiry = (await ethProvider.currentTime()) + 5*60
let cid = client2.upload(data).get
let id = client2.requestStorage(
cid,
duration=10*60.u256,
reward=400.u256,
proofProbability=3.u256,
expiry=expiry,
collateral=200.u256,
nodes = 5,
tolerance = 2).get
check eventually(client2.purchaseStateIs(id, "started"), timeout=5*60*1000)
let updatedAvailability = (client1.getAvailabilities().get).findItem(availability).get
check updatedAvailability.totalSize != updatedAvailability.freeSize
let utilizedSize = updatedAvailability.totalSize - updatedAvailability.freeSize
let totalSizeResponse = client1.patchAvailabilityRaw(availability.id, totalSize=(utilizedSize-1.u256).some)
check totalSizeResponse.status == "400 Bad Request"
check "totalSize must be larger then current totalSize" in totalSizeResponse.body
client1.patchAvailability(availability.id, totalSize=(originalSize + 20000).some)
let newUpdatedAvailability = (client1.getAvailabilities().get).findItem(availability).get
check newUpdatedAvailability.totalSize == originalSize + 20000
check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000

2
tests/integration/testmarketplace.nim
View File

@ -41,7 +41,7 @@ marketplacesuite "Marketplace payouts":
discard providerApi.postAvailability(
# make availability size small enough that we can't fill all the slots,
# thus causing a cancellation
size=(data.len div 2).u256,
totalSize=(data.len div 2).u256,
duration=duration.u256,
minPrice=reward,
maxCollateral=collateral)

6
tests/integration/testproofs.nim
View File

@ -230,7 +230,7 @@ marketplacesuite "Simulate invalid proofs":
# let slotSize = (DefaultBlockSize * 4.NBytes).Natural.u256
# discard provider0.client.postAvailability(
# size=slotSize, # should match 1 slot only
# totalSize=slotSize, # should match 1 slot only
# duration=totalPeriods.periods.u256,
# minPrice=300.u256,
# maxCollateral=200.u256
@ -263,7 +263,7 @@ marketplacesuite "Simulate invalid proofs":
# # now add availability for providers 1 and 2, which should allow them to to
# # put the remaining slots in their queues
# discard provider1.client.postAvailability(
# size=slotSize, # should match 1 slot only
# totalSize=slotSize, # should match 1 slot only
# duration=totalPeriods.periods.u256,
# minPrice=300.u256,
# maxCollateral=200.u256
@ -272,7 +272,7 @@ marketplacesuite "Simulate invalid proofs":
# check eventually filledSlotIds.len > 1
# discard provider2.client.postAvailability(
# size=slotSize, # should match 1 slot only
# totalSize=slotSize, # should match 1 slot only
# duration=totalPeriods.periods.u256,
# minPrice=300.u256,
# maxCollateral=200.u256

2
vendor/nim-presto vendored

@ -1 +1 @@
Subproject commit 3984431dc0fc829eb668e12e57e90542b041d298
Subproject commit c17bfdda2c60cf5fadb043feb22e328b7659c719