2022-05-19 19:56:03 +00:00
|
|
|
## Nim-Codex
|
2022-01-10 15:32:56 +00:00
|
|
|
## Copyright (c) 2021 Status Research & Development GmbH
|
|
|
|
|
## Licensed under either of
|
|
|
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
|
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
|
|
|
## at your option.
|
|
|
|
|
## This file may not be copied, modified, or distributed except according to
|
|
|
|
|
## those terms.
|
|
|
|
|
|
2022-03-18 22:17:51 +00:00
|
|
|
import pkg/upraises
|
|
|
|
|
|
|
|
|
|
push: {.upraises: [].}
|
|
|
|
|
|
2022-01-10 15:32:56 +00:00
|
|
|
|
|
|
|
|
import std/sequtils
|
|
|
|
|
|
|
|
|
|
import pkg/questionable
|
|
|
|
|
import pkg/questionable/results
|
|
|
|
|
import pkg/chronicles
|
|
|
|
|
import pkg/chronos
|
|
|
|
|
import pkg/presto
|
|
|
|
|
import pkg/libp2p
|
2023-07-20 07:56:28 +00:00
|
|
|
import pkg/metrics
|
2022-04-06 00:34:29 +00:00
|
|
|
import pkg/stew/base10
|
2022-05-11 08:51:59 +00:00
|
|
|
import pkg/stew/byteutils
|
2022-05-12 21:52:03 +00:00
|
|
|
import pkg/confutils
|
2022-01-10 15:32:56 +00:00
|
|
|
|
2023-06-19 06:21:03 +00:00
|
|
|
import pkg/libp2p
|
2022-01-10 15:32:56 +00:00
|
|
|
import pkg/libp2p/routing_record
|
2023-08-01 23:47:57 +00:00
|
|
|
import pkg/codexdht/discv5/spr as spr
|
2022-01-10 15:32:56 +00:00
|
|
|
|
|
|
|
|
import ../node
|
2022-04-06 00:34:29 +00:00
|
|
|
import ../blocktype
|
2022-05-12 21:52:03 +00:00
|
|
|
import ../conf
|
2023-09-01 05:44:41 +00:00
|
|
|
import ../contracts except `%*`, `%` # imported from contracts/marketplace (exporting ethers)
|
2022-06-14 15:19:35 +00:00
|
|
|
import ../streams
|
2022-01-10 15:32:56 +00:00
|
|
|
|
2022-11-14 23:42:57 +00:00
|
|
|
import ./coders
|
2022-05-09 13:15:23 +00:00
|
|
|
import ./json
|
|
|
|
|
|
2022-11-14 23:42:57 +00:00
|
|
|
logScope:
|
|
|
|
|
topics = "codex restapi"
|
|
|
|
|
|
2023-11-03 15:21:54 +00:00
|
|
|
declareCounter(codex_api_uploads, "codex API uploads")
|
|
|
|
|
declareCounter(codex_api_downloads, "codex API downloads")
|
2023-07-20 07:56:28 +00:00
|
|
|
|
2022-01-10 15:32:56 +00:00
|
|
|
proc validate(
|
|
|
|
|
pattern: string,
|
|
|
|
|
value: string): int
|
|
|
|
|
{.gcsafe, raises: [Defect].} =
|
|
|
|
|
0
|
|
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
proc formatManifestBlocks(node: CodexNodeRef): Future[JsonNode] {.async.} =
|
|
|
|
|
var content: seq[RestContent] = @[]
|
2023-06-19 06:21:03 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
proc formatManifest(cid: Cid, manifest: Manifest) =
|
|
|
|
|
let restContent = RestContent.init(cid, manifest)
|
|
|
|
|
content.add(restContent)
|
2022-04-06 00:34:29 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
await node.iterateManifests(formatManifest)
|
|
|
|
|
return %content
|
2022-04-06 00:34:29 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
proc initDataApi(node: CodexNodeRef, router: var RestRouter) =
|
2022-01-10 15:32:56 +00:00
|
|
|
router.rawApi(
|
|
|
|
|
MethodPost,
|
2023-11-09 08:47:09 +00:00
|
|
|
"/api/codex/v1/data") do (
|
2022-01-10 15:32:56 +00:00
|
|
|
) -> RestApiResponse:
|
2023-03-15 13:10:53 +00:00
|
|
|
## Upload a file in a streaming manner
|
2022-01-10 20:35:52 +00:00
|
|
|
##
|
|
|
|
|
|
2022-01-10 15:32:56 +00:00
|
|
|
trace "Handling file upload"
|
|
|
|
|
var bodyReader = request.getBodyReader()
|
|
|
|
|
if bodyReader.isErr():
|
|
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
|
|
|
|
|
|
# Attempt to handle `Expect` header
|
2022-01-10 20:35:52 +00:00
|
|
|
# some clients (curl), wait 1000ms
|
2022-01-10 15:32:56 +00:00
|
|
|
# before giving up
|
|
|
|
|
#
|
|
|
|
|
await request.handleExpect()
|
|
|
|
|
|
|
|
|
|
let
|
|
|
|
|
reader = bodyReader.get()
|
|
|
|
|
|
|
|
|
|
try:
|
2022-06-14 15:19:35 +00:00
|
|
|
without cid =? (
|
|
|
|
|
await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)))), error:
|
|
|
|
|
trace "Error uploading file", exc = error.msg
|
2022-04-06 00:34:29 +00:00
|
|
|
return RestApiResponse.error(Http500, error.msg)
|
2022-01-10 15:32:56 +00:00
|
|
|
|
2023-11-03 15:21:54 +00:00
|
|
|
codex_api_uploads.inc()
|
2022-10-27 13:41:34 +00:00
|
|
|
trace "Uploaded file", cid
|
2022-01-10 15:32:56 +00:00
|
|
|
return RestApiResponse.response($cid)
|
2023-03-09 11:23:45 +00:00
|
|
|
except CancelledError:
|
2023-08-22 06:35:16 +00:00
|
|
|
trace "Upload cancelled error"
|
2022-01-10 15:32:56 +00:00
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
|
except AsyncStreamError:
|
2023-08-22 06:35:16 +00:00
|
|
|
trace "Async stream error"
|
2022-01-10 15:32:56 +00:00
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
|
finally:
|
|
|
|
|
await reader.closeWait()
|
|
|
|
|
|
2023-08-22 06:35:16 +00:00
|
|
|
trace "Something went wrong error"
|
2022-01-10 15:32:56 +00:00
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
|
|
2022-11-14 23:42:57 +00:00
|
|
|
router.api(
|
2023-11-09 08:47:09 +00:00
|
|
|
MethodGet,
|
|
|
|
|
"/api/codex/v1/local") do () -> RestApiResponse:
|
|
|
|
|
let json = await formatManifestBlocks(node)
|
|
|
|
|
return RestApiResponse.response($json, contentType="application/json")
|
2022-11-14 23:42:57 +00:00
|
|
|
|
2022-01-10 15:32:56 +00:00
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
2023-11-09 08:47:09 +00:00
|
|
|
"/api/codex/v1/data/{cid}") do (
|
|
|
|
|
cid: Cid, resp: HttpResponseRef) -> RestApiResponse:
|
|
|
|
|
## Download a file from the node in a streaming
|
|
|
|
|
## manner
|
2022-01-10 20:35:52 +00:00
|
|
|
##
|
|
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
if cid.isErr:
|
|
|
|
|
return RestApiResponse.error(
|
|
|
|
|
Http400,
|
|
|
|
|
$cid.error())
|
2022-01-10 15:32:56 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
var
|
|
|
|
|
stream: LPStream
|
2022-01-10 15:32:56 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
var bytes = 0
|
|
|
|
|
try:
|
|
|
|
|
without stream =? (await node.retrieve(cid.get())), error:
|
|
|
|
|
return RestApiResponse.error(Http404, error.msg)
|
2023-06-19 06:21:03 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
resp.addHeader("Content-Type", "application/octet-stream")
|
|
|
|
|
await resp.prepareChunked()
|
2023-06-19 06:21:03 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
while not stream.atEof:
|
|
|
|
|
var
|
|
|
|
|
buff = newSeqUninitialized[byte](DefaultBlockSize.int)
|
|
|
|
|
len = await stream.readOnce(addr buff[0], buff.len)
|
|
|
|
|
|
|
|
|
|
buff.setLen(len)
|
|
|
|
|
if buff.len <= 0:
|
|
|
|
|
break
|
2023-06-19 06:21:03 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
bytes += buff.len
|
|
|
|
|
trace "Sending chunk", size = buff.len
|
|
|
|
|
await resp.sendChunk(addr buff[0], buff.len)
|
|
|
|
|
await resp.finish()
|
|
|
|
|
codex_api_downloads.inc()
|
|
|
|
|
except CatchableError as exc:
|
|
|
|
|
trace "Excepting streaming blocks", exc = exc.msg
|
|
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
|
finally:
|
|
|
|
|
trace "Sent bytes", cid = cid.get(), bytes
|
|
|
|
|
if not stream.isNil:
|
|
|
|
|
await stream.close()
|
|
|
|
|
|
|
|
|
|
proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
2023-06-20 12:52:15 +00:00
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
|
|
|
|
"/api/codex/v1/sales/slots") do () -> RestApiResponse:
|
|
|
|
|
## Returns active slots for the host
|
|
|
|
|
|
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
|
return RestApiResponse.error(Http503, "Sales unavailable")
|
|
|
|
|
|
|
|
|
|
let json = %(await contracts.sales.mySlots())
|
|
|
|
|
return RestApiResponse.response($json, contentType="application/json")
|
|
|
|
|
|
2022-05-09 14:51:08 +00:00
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
2022-05-19 19:56:03 +00:00
|
|
|
"/api/codex/v1/sales/availability") do () -> RestApiResponse:
|
2022-05-09 14:51:08 +00:00
|
|
|
## Returns storage that is for sale
|
|
|
|
|
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 07:05:16 +00:00
|
|
|
without contracts =? node.contracts.host:
|
2022-05-09 14:51:08 +00:00
|
|
|
return RestApiResponse.error(Http503, "Sales unavailable")
|
|
|
|
|
|
[marketplace] Availability improvements (#535)
## Problem
When Availabilities are created, the amount of bytes in the Availability are reserved in the repo, so those bytes on disk cannot be written to otherwise. When a request for storage is received by a node, if a previously created Availability is matched, an attempt will be made to fill a slot in the request (more accurately, the request's slots are added to the SlotQueue, and eventually those slots will be processed). During download, bytes that were reserved for the Availability were released (as they were written to disk). To prevent more bytes from being released than were reserved in the Availability, the Availability was marked as used during the download, so that no other requests would match the Availability, and therefore no new downloads (and byte releases) would begin. The unfortunate downside to this, is that the number of Availabilities a node has determines the download concurrency capacity. If, for example, a node creates a single Availability that covers all available disk space the operator is willing to use, that single Availability would mean that only one download could occur at a time, meaning the node could potentially miss out on storage opportunities.
## Solution
To alleviate the concurrency issue, each time a slot is processed, a Reservation is created, which takes size (aka reserved bytes) away from the Availability and stores them in the Reservation object. This can be done as many times as needed as long as there are enough bytes remaining in the Availability. Therefore, concurrent downloads are no longer limited by the number of Availabilities. Instead, they would more likely be limited to the SlotQueue's `maxWorkers`.
From a database design perspective, an Availability has zero or more Reservations.
Reservations are persisted in the RepoStore's metadata, along with Availabilities. The metadata store key path for Reservations is ` meta / sales / reservations / <availabilityId> / <reservationId>`, while Availabilities are stored one level up, eg `meta / sales / reservations / <availabilityId> `, allowing all Reservations for an Availability to be queried (this is not currently needed, but may be useful when work to restore Availability size is implemented, more on this later).
### Lifecycle
When a reservation is created, its size is deducted from the Availability, and when a reservation is deleted, any remaining size (bytes not written to disk) is returned to the Availability. If the request finishes, is cancelled (expired), or an error occurs, the Reservation is deleted (and any undownloaded bytes returned to the Availability). In addition, when the Sales module starts, any Reservations that are not actively being used in a filled slot, are deleted.
Having a Reservation persisted until after a storage request is completed, will allow for the originally set Availability size to be reclaimed once a request contract has been completed. This is a feature that is yet to be implemented, however the work in this PR is a step in the direction towards enabling this.
### Unknowns
Reservation size is determined by the `StorageAsk.slotSize`. If during download, more bytes than `slotSize` are attempted to be downloaded than this, then the Reservation update will fail, and the state machine will move to a `SaleErrored` state, deleting the Reservation. This will likely prevent the slot from being filled.
### Notes
Based on #514
2023-09-29 04:33:08 +00:00
|
|
|
without avails =? (await contracts.sales.context.reservations.all(Availability)), err:
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 07:05:16 +00:00
|
|
|
return RestApiResponse.error(Http500, err.msg)
|
|
|
|
|
|
[marketplace] Availability improvements (#535)
## Problem
When Availabilities are created, the amount of bytes in the Availability are reserved in the repo, so those bytes on disk cannot be written to otherwise. When a request for storage is received by a node, if a previously created Availability is matched, an attempt will be made to fill a slot in the request (more accurately, the request's slots are added to the SlotQueue, and eventually those slots will be processed). During download, bytes that were reserved for the Availability were released (as they were written to disk). To prevent more bytes from being released than were reserved in the Availability, the Availability was marked as used during the download, so that no other requests would match the Availability, and therefore no new downloads (and byte releases) would begin. The unfortunate downside to this, is that the number of Availabilities a node has determines the download concurrency capacity. If, for example, a node creates a single Availability that covers all available disk space the operator is willing to use, that single Availability would mean that only one download could occur at a time, meaning the node could potentially miss out on storage opportunities.
## Solution
To alleviate the concurrency issue, each time a slot is processed, a Reservation is created, which takes size (aka reserved bytes) away from the Availability and stores them in the Reservation object. This can be done as many times as needed as long as there are enough bytes remaining in the Availability. Therefore, concurrent downloads are no longer limited by the number of Availabilities. Instead, they would more likely be limited to the SlotQueue's `maxWorkers`.
From a database design perspective, an Availability has zero or more Reservations.
Reservations are persisted in the RepoStore's metadata, along with Availabilities. The metadata store key path for Reservations is ` meta / sales / reservations / <availabilityId> / <reservationId>`, while Availabilities are stored one level up, eg `meta / sales / reservations / <availabilityId> `, allowing all Reservations for an Availability to be queried (this is not currently needed, but may be useful when work to restore Availability size is implemented, more on this later).
### Lifecycle
When a reservation is created, its size is deducted from the Availability, and when a reservation is deleted, any remaining size (bytes not written to disk) is returned to the Availability. If the request finishes, is cancelled (expired), or an error occurs, the Reservation is deleted (and any undownloaded bytes returned to the Availability). In addition, when the Sales module starts, any Reservations that are not actively being used in a filled slot, are deleted.
Having a Reservation persisted until after a storage request is completed, will allow for the originally set Availability size to be reclaimed once a request contract has been completed. This is a feature that is yet to be implemented, however the work in this PR is a step in the direction towards enabling this.
### Unknowns
Reservation size is determined by the `StorageAsk.slotSize`. If during download, more bytes than `slotSize` are attempted to be downloaded than this, then the Reservation update will fail, and the state machine will move to a `SaleErrored` state, deleting the Reservation. This will likely prevent the slot from being filled.
### Notes
Based on #514
2023-09-29 04:33:08 +00:00
|
|
|
let json = %avails
|
2023-06-13 13:33:01 +00:00
|
|
|
return RestApiResponse.response($json, contentType="application/json")
|
2022-05-09 14:51:08 +00:00
|
|
|
|
2022-05-09 13:15:23 +00:00
|
|
|
router.rawApi(
|
|
|
|
|
MethodPost,
|
2022-05-19 19:56:03 +00:00
|
|
|
"/api/codex/v1/sales/availability") do () -> RestApiResponse:
|
2022-04-21 08:12:16 +00:00
|
|
|
## Add available storage to sell
|
|
|
|
|
##
|
2023-04-14 09:04:17 +00:00
|
|
|
## size - size of available storage in bytes
|
|
|
|
|
## duration - maximum time the storage should be sold for (in seconds)
|
|
|
|
|
## minPrice - minimum price to be paid (in amount of tokens)
|
|
|
|
|
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
|
2022-04-21 08:12:16 +00:00
|
|
|
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 07:05:16 +00:00
|
|
|
without contracts =? node.contracts.host:
|
2022-05-09 14:51:08 +00:00
|
|
|
return RestApiResponse.error(Http503, "Sales unavailable")
|
|
|
|
|
|
2022-05-09 13:15:23 +00:00
|
|
|
let body = await request.getBody()
|
2022-04-21 08:12:16 +00:00
|
|
|
|
2023-09-01 05:44:41 +00:00
|
|
|
without restAv =? RestAvailability.fromJson(body), error:
|
2022-05-09 13:15:23 +00:00
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
2022-04-21 08:12:16 +00:00
|
|
|
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 07:05:16 +00:00
|
|
|
let reservations = contracts.sales.context.reservations
|
|
|
|
|
|
[marketplace] Availability improvements (#535)
## Problem
When Availabilities are created, the amount of bytes in the Availability are reserved in the repo, so those bytes on disk cannot be written to otherwise. When a request for storage is received by a node, if a previously created Availability is matched, an attempt will be made to fill a slot in the request (more accurately, the request's slots are added to the SlotQueue, and eventually those slots will be processed). During download, bytes that were reserved for the Availability were released (as they were written to disk). To prevent more bytes from being released than were reserved in the Availability, the Availability was marked as used during the download, so that no other requests would match the Availability, and therefore no new downloads (and byte releases) would begin. The unfortunate downside to this, is that the number of Availabilities a node has determines the download concurrency capacity. If, for example, a node creates a single Availability that covers all available disk space the operator is willing to use, that single Availability would mean that only one download could occur at a time, meaning the node could potentially miss out on storage opportunities.
## Solution
To alleviate the concurrency issue, each time a slot is processed, a Reservation is created, which takes size (aka reserved bytes) away from the Availability and stores them in the Reservation object. This can be done as many times as needed as long as there are enough bytes remaining in the Availability. Therefore, concurrent downloads are no longer limited by the number of Availabilities. Instead, they would more likely be limited to the SlotQueue's `maxWorkers`.
From a database design perspective, an Availability has zero or more Reservations.
Reservations are persisted in the RepoStore's metadata, along with Availabilities. The metadata store key path for Reservations is ` meta / sales / reservations / <availabilityId> / <reservationId>`, while Availabilities are stored one level up, eg `meta / sales / reservations / <availabilityId> `, allowing all Reservations for an Availability to be queried (this is not currently needed, but may be useful when work to restore Availability size is implemented, more on this later).
### Lifecycle
When a reservation is created, its size is deducted from the Availability, and when a reservation is deleted, any remaining size (bytes not written to disk) is returned to the Availability. If the request finishes, is cancelled (expired), or an error occurs, the Reservation is deleted (and any undownloaded bytes returned to the Availability). In addition, when the Sales module starts, any Reservations that are not actively being used in a filled slot, are deleted.
Having a Reservation persisted until after a storage request is completed, will allow for the originally set Availability size to be reclaimed once a request contract has been completed. This is a feature that is yet to be implemented, however the work in this PR is a step in the direction towards enabling this.
### Unknowns
Reservation size is determined by the `StorageAsk.slotSize`. If during download, more bytes than `slotSize` are attempted to be downloaded than this, then the Reservation update will fail, and the state machine will move to a `SaleErrored` state, deleting the Reservation. This will likely prevent the slot from being filled.
### Notes
Based on #514
2023-09-29 04:33:08 +00:00
|
|
|
if not reservations.hasAvailable(restAv.size.truncate(uint)):
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 07:05:16 +00:00
|
|
|
return RestApiResponse.error(Http422, "Not enough storage quota")
|
|
|
|
|
|
[marketplace] Availability improvements (#535)
## Problem
When Availabilities are created, the amount of bytes in the Availability are reserved in the repo, so those bytes on disk cannot be written to otherwise. When a request for storage is received by a node, if a previously created Availability is matched, an attempt will be made to fill a slot in the request (more accurately, the request's slots are added to the SlotQueue, and eventually those slots will be processed). During download, bytes that were reserved for the Availability were released (as they were written to disk). To prevent more bytes from being released than were reserved in the Availability, the Availability was marked as used during the download, so that no other requests would match the Availability, and therefore no new downloads (and byte releases) would begin. The unfortunate downside to this, is that the number of Availabilities a node has determines the download concurrency capacity. If, for example, a node creates a single Availability that covers all available disk space the operator is willing to use, that single Availability would mean that only one download could occur at a time, meaning the node could potentially miss out on storage opportunities.
## Solution
To alleviate the concurrency issue, each time a slot is processed, a Reservation is created, which takes size (aka reserved bytes) away from the Availability and stores them in the Reservation object. This can be done as many times as needed as long as there are enough bytes remaining in the Availability. Therefore, concurrent downloads are no longer limited by the number of Availabilities. Instead, they would more likely be limited to the SlotQueue's `maxWorkers`.
From a database design perspective, an Availability has zero or more Reservations.
Reservations are persisted in the RepoStore's metadata, along with Availabilities. The metadata store key path for Reservations is ` meta / sales / reservations / <availabilityId> / <reservationId>`, while Availabilities are stored one level up, eg `meta / sales / reservations / <availabilityId> `, allowing all Reservations for an Availability to be queried (this is not currently needed, but may be useful when work to restore Availability size is implemented, more on this later).
### Lifecycle
When a reservation is created, its size is deducted from the Availability, and when a reservation is deleted, any remaining size (bytes not written to disk) is returned to the Availability. If the request finishes, is cancelled (expired), or an error occurs, the Reservation is deleted (and any undownloaded bytes returned to the Availability). In addition, when the Sales module starts, any Reservations that are not actively being used in a filled slot, are deleted.
Having a Reservation persisted until after a storage request is completed, will allow for the originally set Availability size to be reclaimed once a request contract has been completed. This is a feature that is yet to be implemented, however the work in this PR is a step in the direction towards enabling this.
### Unknowns
Reservation size is determined by the `StorageAsk.slotSize`. If during download, more bytes than `slotSize` are attempted to be downloaded than this, then the Reservation update will fail, and the state machine will move to a `SaleErrored` state, deleting the Reservation. This will likely prevent the slot from being filled.
### Notes
Based on #514
2023-09-29 04:33:08 +00:00
|
|
|
without availability =? (
|
|
|
|
|
await reservations.createAvailability(
|
|
|
|
|
restAv.size,
|
|
|
|
|
restAv.duration,
|
|
|
|
|
restAv.minPrice,
|
|
|
|
|
restAv.maxCollateral)
|
|
|
|
|
), error:
|
|
|
|
|
return RestApiResponse.error(Http500, error.msg)
|
2022-05-09 14:51:08 +00:00
|
|
|
|
[marketplace] Availability improvements (#535)
## Problem
When Availabilities are created, the amount of bytes in the Availability are reserved in the repo, so those bytes on disk cannot be written to otherwise. When a request for storage is received by a node, if a previously created Availability is matched, an attempt will be made to fill a slot in the request (more accurately, the request's slots are added to the SlotQueue, and eventually those slots will be processed). During download, bytes that were reserved for the Availability were released (as they were written to disk). To prevent more bytes from being released than were reserved in the Availability, the Availability was marked as used during the download, so that no other requests would match the Availability, and therefore no new downloads (and byte releases) would begin. The unfortunate downside to this, is that the number of Availabilities a node has determines the download concurrency capacity. If, for example, a node creates a single Availability that covers all available disk space the operator is willing to use, that single Availability would mean that only one download could occur at a time, meaning the node could potentially miss out on storage opportunities.
## Solution
To alleviate the concurrency issue, each time a slot is processed, a Reservation is created, which takes size (aka reserved bytes) away from the Availability and stores them in the Reservation object. This can be done as many times as needed as long as there are enough bytes remaining in the Availability. Therefore, concurrent downloads are no longer limited by the number of Availabilities. Instead, they would more likely be limited to the SlotQueue's `maxWorkers`.
From a database design perspective, an Availability has zero or more Reservations.
Reservations are persisted in the RepoStore's metadata, along with Availabilities. The metadata store key path for Reservations is ` meta / sales / reservations / <availabilityId> / <reservationId>`, while Availabilities are stored one level up, eg `meta / sales / reservations / <availabilityId> `, allowing all Reservations for an Availability to be queried (this is not currently needed, but may be useful when work to restore Availability size is implemented, more on this later).
### Lifecycle
When a reservation is created, its size is deducted from the Availability, and when a reservation is deleted, any remaining size (bytes not written to disk) is returned to the Availability. If the request finishes, is cancelled (expired), or an error occurs, the Reservation is deleted (and any undownloaded bytes returned to the Availability). In addition, when the Sales module starts, any Reservations that are not actively being used in a filled slot, are deleted.
Having a Reservation persisted until after a storage request is completed, will allow for the originally set Availability size to be reclaimed once a request contract has been completed. This is a feature that is yet to be implemented, however the work in this PR is a step in the direction towards enabling this.
### Unknowns
Reservation size is determined by the `StorageAsk.slotSize`. If during download, more bytes than `slotSize` are attempted to be downloaded than this, then the Reservation update will fail, and the state machine will move to a `SaleErrored` state, deleting the Reservation. This will likely prevent the slot from being filled.
### Notes
Based on #514
2023-09-29 04:33:08 +00:00
|
|
|
return RestApiResponse.response(availability.toJson,
|
|
|
|
|
contentType="application/json")
|
2022-04-21 08:12:16 +00:00
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
|
|
|
|
router.rawApi(
|
|
|
|
|
MethodPost,
|
|
|
|
|
"/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse:
|
|
|
|
|
## Create a request for storage
|
|
|
|
|
##
|
|
|
|
|
## cid - the cid of a previously uploaded dataset
|
|
|
|
|
## duration - the duration of the request in seconds
|
|
|
|
|
## proofProbability - how often storage proofs are required
|
|
|
|
|
## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
|
|
|
|
|
## expiry - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
|
|
|
|
|
## nodes - minimal number of nodes the content should be stored on
|
|
|
|
|
## tolerance - allowed number of nodes that can be lost before pronouncing the content lost
|
|
|
|
|
## colateral - requested collateral from hosts when they fill slot
|
|
|
|
|
|
|
|
|
|
without cid =? cid.tryGet.catch, error:
|
|
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
|
|
|
|
|
|
|
|
|
let body = await request.getBody()
|
|
|
|
|
|
|
|
|
|
without params =? StorageRequestParams.fromJson(body), error:
|
|
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
|
|
|
|
|
|
|
|
|
let nodes = params.nodes |? 1
|
|
|
|
|
let tolerance = params.tolerance |? 0
|
|
|
|
|
|
|
|
|
|
without purchaseId =? await node.requestStorage(
|
|
|
|
|
cid,
|
|
|
|
|
params.duration,
|
|
|
|
|
params.proofProbability,
|
|
|
|
|
nodes,
|
|
|
|
|
tolerance,
|
|
|
|
|
params.reward,
|
|
|
|
|
params.collateral,
|
|
|
|
|
params.expiry), error:
|
|
|
|
|
|
|
|
|
|
return RestApiResponse.error(Http500, error.msg)
|
|
|
|
|
|
|
|
|
|
return RestApiResponse.response(purchaseId.toHex)
|
|
|
|
|
|
2022-05-11 08:51:59 +00:00
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
2022-05-19 19:56:03 +00:00
|
|
|
"/api/codex/v1/storage/purchases/{id}") do (
|
2022-08-17 02:29:44 +00:00
|
|
|
id: PurchaseId) -> RestApiResponse:
|
2022-05-11 08:51:59 +00:00
|
|
|
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 07:05:16 +00:00
|
|
|
without contracts =? node.contracts.client:
|
2022-05-11 08:51:59 +00:00
|
|
|
return RestApiResponse.error(Http503, "Purchasing unavailable")
|
|
|
|
|
|
|
|
|
|
without id =? id.tryGet.catch, error:
|
|
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
|
|
|
|
|
|
|
|
|
without purchase =? contracts.purchasing.getPurchase(id):
|
|
|
|
|
return RestApiResponse.error(Http404)
|
|
|
|
|
|
2023-09-01 05:44:41 +00:00
|
|
|
let json = % RestPurchase(
|
|
|
|
|
state: purchase.state |? "none",
|
|
|
|
|
error: purchase.error.?msg,
|
|
|
|
|
request: purchase.request,
|
|
|
|
|
requestId: purchase.requestId
|
|
|
|
|
)
|
2022-05-11 08:51:59 +00:00
|
|
|
|
2023-06-13 13:33:01 +00:00
|
|
|
return RestApiResponse.response($json, contentType="application/json")
|
2022-05-11 08:51:59 +00:00
|
|
|
|
2023-11-13 11:30:27 +00:00
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
|
|
|
|
"/api/codex/v1/storage/purchases") do () -> RestApiResponse:
|
|
|
|
|
without contracts =? node.contracts.client:
|
|
|
|
|
return RestApiResponse.error(Http503, "Purchasing unavailable")
|
|
|
|
|
|
|
|
|
|
let purchaseIds = contracts.purchasing.getPurchaseIds()
|
|
|
|
|
return RestApiResponse.response($ %purchaseIds, contentType="application/json")
|
|
|
|
|
|
2023-11-09 08:47:09 +00:00
|
|
|
proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
|
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
|
|
|
|
"/api/codex/v1/debug/info") do () -> RestApiResponse:
|
|
|
|
|
## Print rudimentary node information
|
|
|
|
|
##
|
|
|
|
|
|
|
|
|
|
let table = RestRoutingTable.init(node.discovery.protocol.routingTable)
|
|
|
|
|
|
|
|
|
|
let
|
|
|
|
|
json = %*{
|
|
|
|
|
"id": $node.switch.peerInfo.peerId,
|
|
|
|
|
"addrs": node.switch.peerInfo.addrs.mapIt( $it ),
|
|
|
|
|
"repo": $conf.dataDir,
|
|
|
|
|
"spr":
|
|
|
|
|
if node.discovery.dhtRecord.isSome:
|
|
|
|
|
node.discovery.dhtRecord.get.toURI
|
|
|
|
|
else:
|
|
|
|
|
"",
|
2023-11-12 08:58:40 +00:00
|
|
|
"announceAddresses": node.discovery.announceAddrs,
|
2023-11-09 08:47:09 +00:00
|
|
|
"table": table,
|
|
|
|
|
"codex": {
|
|
|
|
|
"version": $codexVersion,
|
|
|
|
|
"revision": $codexRevision
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return RestApiResponse.response($json, contentType="application/json")
|
|
|
|
|
|
|
|
|
|
router.api(
|
|
|
|
|
MethodPost,
|
|
|
|
|
"/api/codex/v1/debug/chronicles/loglevel") do (
|
|
|
|
|
level: Option[string]) -> RestApiResponse:
|
|
|
|
|
## Set log level at run time
|
|
|
|
|
##
|
|
|
|
|
## e.g. `chronicles/loglevel?level=DEBUG`
|
|
|
|
|
##
|
|
|
|
|
## `level` - chronicles log level
|
|
|
|
|
##
|
|
|
|
|
|
|
|
|
|
without res =? level and level =? res:
|
|
|
|
|
return RestApiResponse.error(Http400, "Missing log level")
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
{.gcsafe.}:
|
|
|
|
|
updateLogLevel(level)
|
|
|
|
|
except CatchableError as exc:
|
|
|
|
|
return RestApiResponse.error(Http500, exc.msg)
|
|
|
|
|
|
|
|
|
|
return RestApiResponse.response("")
|
|
|
|
|
|
|
|
|
|
when codex_enable_api_debug_peers:
|
|
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
|
|
|
|
"/api/codex/v1/debug/peer/{peerId}") do (peerId: PeerId) -> RestApiResponse:
|
|
|
|
|
|
|
|
|
|
trace "debug/peer start"
|
|
|
|
|
without peerRecord =? (await node.findPeer(peerId.get())):
|
|
|
|
|
trace "debug/peer peer not found!"
|
|
|
|
|
return RestApiResponse.error(
|
|
|
|
|
Http400,
|
|
|
|
|
"Unable to find Peer!")
|
|
|
|
|
|
|
|
|
|
let json = %RestPeerRecord.init(peerRecord)
|
|
|
|
|
trace "debug/peer returning peer record"
|
|
|
|
|
return RestApiResponse.response($json)
|
|
|
|
|
|
|
|
|
|
proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
|
|
|
|
|
var router = RestRouter.init(validate)
|
|
|
|
|
|
|
|
|
|
initDataApi(node, router)
|
|
|
|
|
initSalesApi(node, router)
|
|
|
|
|
initPurchasingApi(node, router)
|
|
|
|
|
initDebugApi(node, conf, router)
|
|
|
|
|
|
|
|
|
|
router.api(
|
|
|
|
|
MethodGet,
|
|
|
|
|
"/api/codex/v1/connect/{peerId}") do (
|
|
|
|
|
peerId: PeerId,
|
|
|
|
|
addrs: seq[MultiAddress]) -> RestApiResponse:
|
|
|
|
|
## Connect to a peer
|
|
|
|
|
##
|
|
|
|
|
## If `addrs` param is supplied, it will be used to
|
|
|
|
|
## dial the peer, otherwise the `peerId` is used
|
|
|
|
|
## to invoke peer discovery, if it succeeds
|
|
|
|
|
## the returned addresses will be used to dial
|
|
|
|
|
##
|
|
|
|
|
## `addrs` the listening addresses of the peers to dial, eg the one specified with `--listen-addrs`
|
|
|
|
|
##
|
|
|
|
|
|
|
|
|
|
if peerId.isErr:
|
|
|
|
|
return RestApiResponse.error(
|
|
|
|
|
Http400,
|
|
|
|
|
$peerId.error())
|
|
|
|
|
|
|
|
|
|
let addresses = if addrs.isOk and addrs.get().len > 0:
|
|
|
|
|
addrs.get()
|
|
|
|
|
else:
|
|
|
|
|
without peerRecord =? (await node.findPeer(peerId.get())):
|
|
|
|
|
return RestApiResponse.error(
|
|
|
|
|
Http400,
|
|
|
|
|
"Unable to find Peer!")
|
|
|
|
|
peerRecord.addresses.mapIt(it.address)
|
|
|
|
|
try:
|
|
|
|
|
await node.connect(peerId.get(), addresses)
|
|
|
|
|
return RestApiResponse.response("Successfully connected to peer")
|
|
|
|
|
except DialFailedError:
|
|
|
|
|
return RestApiResponse.error(Http400, "Unable to dial peer")
|
|
|
|
|
except CatchableError:
|
|
|
|
|
return RestApiResponse.error(Http400, "Unknown error dialling peer")
|
|
|
|
|
|
2022-01-10 15:32:56 +00:00
|
|
|
return router
|
