2022-05-19 14:56:03 -05:00
|
|
|
## Nim-Codex
|
2022-01-10 09:32:56 -06:00
|
|
|
## Copyright (c) 2021 Status Research & Development GmbH
|
|
|
|
## Licensed under either of
|
|
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
|
|
## at your option.
|
|
|
|
## This file may not be copied, modified, or distributed except according to
|
|
|
|
## those terms.
|
|
|
|
|
2022-03-18 16:17:51 -06:00
|
|
|
import pkg/upraises
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
push:
|
|
|
|
{.upraises: [].}
|
2022-01-10 09:32:56 -06:00
|
|
|
|
|
|
|
import std/sequtils
|
2024-10-25 14:43:19 +01:00
|
|
|
import mimetypes
|
|
|
|
import os
|
2022-01-10 09:32:56 -06:00
|
|
|
|
|
|
|
import pkg/questionable
|
|
|
|
import pkg/questionable/results
|
|
|
|
import pkg/chronos
|
2023-12-07 01:16:36 +00:00
|
|
|
import pkg/presto except toJson
|
|
|
|
import pkg/metrics except toJson
|
2022-04-05 18:34:29 -06:00
|
|
|
import pkg/stew/base10
|
2022-05-11 10:51:59 +02:00
|
|
|
import pkg/stew/byteutils
|
2022-05-12 15:52:03 -06:00
|
|
|
import pkg/confutils
|
2022-01-10 09:32:56 -06:00
|
|
|
|
2023-06-19 08:21:03 +02:00
|
|
|
import pkg/libp2p
|
2022-01-10 09:32:56 -06:00
|
|
|
import pkg/libp2p/routing_record
|
2023-08-01 16:47:57 -07:00
|
|
|
import pkg/codexdht/discv5/spr as spr
|
2022-01-10 09:32:56 -06:00
|
|
|
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 18:35:03 +11:00
|
|
|
import ../logutils
|
2022-01-10 09:32:56 -06:00
|
|
|
import ../node
|
2022-04-05 18:34:29 -06:00
|
|
|
import ../blocktype
|
2022-05-12 15:52:03 -06:00
|
|
|
import ../conf
|
2023-12-07 01:16:36 +00:00
|
|
|
import ../contracts
|
2024-06-28 07:26:19 +10:00
|
|
|
import ../erasure/erasure
|
2023-12-07 01:16:36 +00:00
|
|
|
import ../manifest
|
|
|
|
import ../streams/asyncstreamwrapper
|
2023-12-14 11:57:16 +01:00
|
|
|
import ../stores
|
2024-03-21 11:53:45 +01:00
|
|
|
import ../utils/options
|
2022-01-10 09:32:56 -06:00
|
|
|
|
2022-11-14 17:42:57 -06:00
|
|
|
import ./coders
|
2022-05-09 15:15:23 +02:00
|
|
|
import ./json
|
|
|
|
|
2022-11-14 17:42:57 -06:00
|
|
|
logScope:
|
|
|
|
topics = "codex restapi"
|
|
|
|
|
2023-11-03 17:21:54 +02:00
|
|
|
declareCounter(codex_api_uploads, "codex API uploads")
|
|
|
|
declareCounter(codex_api_downloads, "codex API downloads")
|
2023-07-20 09:56:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} =
|
2022-01-10 09:32:56 -06:00
|
|
|
0
|
|
|
|
|
2024-10-17 18:54:28 +02:00
|
|
|
proc formatManifest(cid: Cid, manifest: Manifest): RestContent =
|
|
|
|
return RestContent.init(cid, manifest)
|
|
|
|
|
2023-11-09 09:47:09 +01:00
|
|
|
proc formatManifestBlocks(node: CodexNodeRef): Future[JsonNode] {.async.} =
|
2023-11-20 18:14:06 -06:00
|
|
|
var content: seq[RestContent]
|
2023-06-19 08:21:03 +02:00
|
|
|
|
2024-10-17 18:54:28 +02:00
|
|
|
proc addManifest(cid: Cid, manifest: Manifest) =
|
|
|
|
content.add(formatManifest(cid, manifest))
|
2025-01-21 21:54:46 +01:00
|
|
|
|
2024-10-17 18:54:28 +02:00
|
|
|
await node.iterateManifests(addManifest)
|
2022-04-05 18:34:29 -06:00
|
|
|
|
2024-03-26 16:12:15 +01:00
|
|
|
return %RestContentList.init(content)
|
2022-04-05 18:34:29 -06:00
|
|
|
|
2023-11-20 18:14:06 -06:00
|
|
|
proc retrieveCid(
|
2025-01-21 21:54:46 +01:00
|
|
|
node: CodexNodeRef, cid: Cid, local: bool = true, resp: HttpResponseRef
|
|
|
|
): Future[RestApiResponse] {.async.} =
|
2023-11-20 18:14:06 -06:00
|
|
|
## Download a file from the node in a streaming
|
|
|
|
## manner
|
|
|
|
##
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
var stream: LPStream
|
2023-11-20 18:14:06 -06:00
|
|
|
|
|
|
|
var bytes = 0
|
|
|
|
try:
|
|
|
|
without stream =? (await node.retrieve(cid, local)), error:
|
|
|
|
if error of BlockNotFoundError:
|
2024-10-25 14:43:19 +01:00
|
|
|
resp.status = Http404
|
|
|
|
return await resp.sendBody("")
|
2023-11-20 18:14:06 -06:00
|
|
|
else:
|
2024-10-25 14:43:19 +01:00
|
|
|
resp.status = Http500
|
|
|
|
return await resp.sendBody(error.msg)
|
|
|
|
|
|
|
|
# It is ok to fetch again the manifest because it will hit the cache
|
|
|
|
without manifest =? (await node.fetchManifest(cid)), err:
|
|
|
|
error "Failed to fetch manifest", err = err.msg
|
|
|
|
resp.status = Http404
|
|
|
|
return await resp.sendBody(err.msg)
|
|
|
|
|
|
|
|
if manifest.mimetype.isSome:
|
|
|
|
resp.setHeader("Content-Type", manifest.mimetype.get())
|
|
|
|
else:
|
|
|
|
resp.addHeader("Content-Type", "application/octet-stream")
|
|
|
|
|
|
|
|
if manifest.filename.isSome:
|
2025-01-21 21:54:46 +01:00
|
|
|
resp.setHeader(
|
|
|
|
"Content-Disposition",
|
|
|
|
"attachment; filename=\"" & manifest.filename.get() & "\"",
|
|
|
|
)
|
2025-01-16 14:25:26 +01:00
|
|
|
else:
|
|
|
|
resp.setHeader("Content-Disposition", "attachment")
|
2023-11-20 18:14:06 -06:00
|
|
|
|
|
|
|
await resp.prepareChunked()
|
|
|
|
|
|
|
|
while not stream.atEof:
|
|
|
|
var
|
|
|
|
buff = newSeqUninitialized[byte](DefaultBlockSize.int)
|
|
|
|
len = await stream.readOnce(addr buff[0], buff.len)
|
|
|
|
|
|
|
|
buff.setLen(len)
|
|
|
|
if buff.len <= 0:
|
|
|
|
break
|
|
|
|
|
|
|
|
bytes += buff.len
|
2024-10-25 14:43:19 +01:00
|
|
|
|
2023-11-20 18:14:06 -06:00
|
|
|
await resp.sendChunk(addr buff[0], buff.len)
|
|
|
|
await resp.finish()
|
|
|
|
codex_api_downloads.inc()
|
|
|
|
except CatchableError as exc:
|
2024-05-16 19:06:12 +02:00
|
|
|
warn "Excepting streaming blocks", exc = exc.msg
|
2024-10-25 14:43:19 +01:00
|
|
|
resp.status = Http500
|
|
|
|
return await resp.sendBody("")
|
2023-11-20 18:14:06 -06:00
|
|
|
finally:
|
2024-05-16 19:06:12 +02:00
|
|
|
info "Sent bytes", cid = cid, bytes
|
2023-11-20 18:14:06 -06:00
|
|
|
if not stream.isNil:
|
|
|
|
await stream.close()
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
proc buildCorsHeaders(
|
|
|
|
httpMethod: string, allowedOrigin: Option[string]
|
|
|
|
): seq[(string, string)] =
|
2024-10-10 10:25:07 +02:00
|
|
|
var headers: seq[(string, string)] = newSeq[(string, string)]()
|
|
|
|
|
|
|
|
if corsOrigin =? allowedOrigin:
|
|
|
|
headers.add(("Access-Control-Allow-Origin", corsOrigin))
|
|
|
|
headers.add(("Access-Control-Allow-Methods", httpMethod & ", OPTIONS"))
|
|
|
|
headers.add(("Access-Control-Max-Age", "86400"))
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return headers
|
2024-10-10 10:25:07 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
proc setCorsHeaders(resp: HttpResponseRef, httpMethod: string, origin: string) =
|
2024-10-10 10:25:07 +02:00
|
|
|
resp.setHeader("Access-Control-Allow-Origin", origin)
|
|
|
|
resp.setHeader("Access-Control-Allow-Methods", httpMethod & ", OPTIONS")
|
|
|
|
resp.setHeader("Access-Control-Max-Age", "86400")
|
|
|
|
|
2024-10-25 14:43:19 +01:00
|
|
|
proc getFilenameFromContentDisposition(contentDisposition: string): ?string =
|
2025-01-21 21:54:46 +01:00
|
|
|
if not ("filename=" in contentDisposition):
|
2024-10-25 14:43:19 +01:00
|
|
|
return string.none
|
|
|
|
|
|
|
|
let parts = contentDisposition.split("filename=\"")
|
|
|
|
|
|
|
|
if parts.len < 2:
|
|
|
|
return string.none
|
|
|
|
|
|
|
|
let filename = parts[1].strip()
|
2025-01-21 21:54:46 +01:00
|
|
|
return filename[0 ..^ 2].some
|
2024-10-25 14:43:19 +01:00
|
|
|
|
2023-12-14 11:57:16 +01:00
|
|
|
proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRouter) =
|
2024-06-17 21:33:21 +10:00
|
|
|
let allowedOrigin = router.allowedOrigin # prevents capture inside of api defintion
|
2024-10-25 14:43:19 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodOptions, "/api/codex/v1/data") do(
|
|
|
|
resp: HttpResponseRef
|
|
|
|
) -> RestApiResponse:
|
|
|
|
if corsOrigin =? allowedOrigin:
|
|
|
|
resp.setCorsHeaders("POST", corsOrigin)
|
|
|
|
resp.setHeader(
|
|
|
|
"Access-Control-Allow-Headers", "content-type, content-disposition"
|
|
|
|
)
|
2024-10-25 14:43:19 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
resp.status = Http204
|
|
|
|
await resp.sendBody("")
|
2024-10-25 14:43:19 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.rawApi(MethodPost, "/api/codex/v1/data") do() -> RestApiResponse:
|
|
|
|
## Upload a file in a streaming manner
|
|
|
|
##
|
2022-01-10 09:32:56 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
trace "Handling file upload"
|
|
|
|
var bodyReader = request.getBodyReader()
|
|
|
|
if bodyReader.isErr():
|
2022-01-10 09:32:56 -06:00
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
# Attempt to handle `Expect` header
|
|
|
|
# some clients (curl), wait 1000ms
|
|
|
|
# before giving up
|
|
|
|
#
|
|
|
|
await request.handleExpect()
|
2022-11-14 17:42:57 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
var mimetype = request.headers.getString(ContentTypeHeader).some
|
2024-10-10 10:25:07 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if mimetype.get() != "":
|
|
|
|
var m = newMimetypes()
|
|
|
|
let extension = m.getExt(mimetype.get(), "")
|
|
|
|
if extension == "":
|
|
|
|
return RestApiResponse.error(Http422, "The MIME type is not valid.")
|
|
|
|
else:
|
|
|
|
mimetype = string.none
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
const ContentDispositionHeader = "Content-Disposition"
|
|
|
|
let contentDisposition = request.headers.getString(ContentDispositionHeader)
|
|
|
|
let filename = getFilenameFromContentDisposition(contentDisposition)
|
2024-06-17 21:33:21 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if filename.isSome and not isValidFilename(filename.get()):
|
|
|
|
return RestApiResponse.error(Http422, "The filename is not valid.")
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
# Here we could check if the extension matches the filename if needed
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let reader = bodyReader.get()
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
try:
|
|
|
|
without cid =? (
|
|
|
|
await node.store(
|
|
|
|
AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)),
|
|
|
|
filename = filename,
|
|
|
|
mimetype = mimetype,
|
|
|
|
)
|
|
|
|
), error:
|
|
|
|
error "Error uploading file", exc = error.msg
|
|
|
|
return RestApiResponse.error(Http500, error.msg)
|
|
|
|
|
|
|
|
codex_api_uploads.inc()
|
|
|
|
trace "Uploaded file", cid
|
|
|
|
return RestApiResponse.response($cid)
|
|
|
|
except CancelledError:
|
|
|
|
trace "Upload cancelled error"
|
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
except AsyncStreamError:
|
|
|
|
trace "Async stream error"
|
|
|
|
return RestApiResponse.error(Http500)
|
|
|
|
finally:
|
|
|
|
await reader.closeWait()
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
trace "Something went wrong error"
|
|
|
|
return RestApiResponse.error(Http500)
|
2022-01-10 14:35:52 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/data") do() -> RestApiResponse:
|
|
|
|
let json = await formatManifestBlocks(node)
|
|
|
|
return RestApiResponse.response($json, contentType = "application/json")
|
2024-10-10 10:25:07 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/data/{cid}") do(
|
|
|
|
cid: Cid, resp: HttpResponseRef
|
|
|
|
) -> RestApiResponse:
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2022-01-10 09:32:56 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
## Download a file from the local node in a streaming
|
|
|
|
## manner
|
|
|
|
if cid.isErr:
|
|
|
|
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
|
2024-06-17 21:33:21 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if corsOrigin =? allowedOrigin:
|
|
|
|
resp.setCorsHeaders("GET", corsOrigin)
|
|
|
|
resp.setHeader("Access-Control-Headers", "X-Requested-With")
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
await node.retrieveCid(cid.get(), local = true, resp = resp)
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodPost, "/api/codex/v1/data/{cid}/network") do(
|
|
|
|
cid: Cid, resp: HttpResponseRef
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Download a file from the network to the local node
|
|
|
|
##
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if cid.isErr:
|
|
|
|
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without manifest =? (await node.fetchManifest(cid.get())), err:
|
|
|
|
error "Failed to fetch manifest", err = err.msg
|
|
|
|
return RestApiResponse.error(Http404, err.msg, headers = headers)
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
proc fetchDatasetAsync(): Future[void] {.async.} =
|
|
|
|
try:
|
|
|
|
if err =? (await node.fetchBatched(manifest)).errorOption:
|
|
|
|
error "Unable to fetch dataset", cid = cid.get(), err = err.msg
|
|
|
|
except CatchableError as exc:
|
|
|
|
error "CatchableError when fetching dataset", cid = cid.get(), exc = exc.msg
|
|
|
|
discard
|
2023-12-14 11:57:16 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
asyncSpawn fetchDatasetAsync()
|
2024-09-23 17:08:56 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let json = %formatManifest(cid.get(), manifest)
|
|
|
|
return RestApiResponse.response($json, contentType = "application/json")
|
2024-10-10 10:25:07 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/data/{cid}/network/stream") do(
|
|
|
|
cid: Cid, resp: HttpResponseRef
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Download a file from the network in a streaming
|
|
|
|
## manner
|
|
|
|
##
|
2023-06-20 14:52:15 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2023-06-20 14:52:15 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if cid.isErr:
|
|
|
|
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if corsOrigin =? allowedOrigin:
|
|
|
|
resp.setCorsHeaders("GET", corsOrigin)
|
|
|
|
resp.setHeader("Access-Control-Headers", "X-Requested-With")
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
await node.retrieveCid(cid.get(), local = false, resp = resp)
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/data/{cid}/network/manifest") do(
|
|
|
|
cid: Cid, resp: HttpResponseRef
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Download only the manifest.
|
|
|
|
##
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if cid.isErr:
|
|
|
|
return RestApiResponse.error(Http400, $cid.error(), headers = headers)
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without manifest =? (await node.fetchManifest(cid.get())), err:
|
|
|
|
error "Failed to fetch manifest", err = err.msg
|
|
|
|
return RestApiResponse.error(Http404, err.msg, headers = headers)
|
2022-05-09 16:51:08 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let json = %formatManifest(cid.get(), manifest)
|
|
|
|
return RestApiResponse.response($json, contentType = "application/json")
|
2022-05-09 16:51:08 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/space") do() -> RestApiResponse:
|
|
|
|
let json =
|
|
|
|
%RestRepoStore(
|
|
|
|
totalBlocks: repoStore.totalBlocks,
|
|
|
|
quotaMaxBytes: repoStore.quotaMaxBytes,
|
|
|
|
quotaUsedBytes: repoStore.quotaUsedBytes,
|
|
|
|
quotaReservedBytes: repoStore.quotaReservedBytes,
|
|
|
|
)
|
|
|
|
return RestApiResponse.response($json, contentType = "application/json")
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 17:05:16 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
|
|
|
|
let allowedOrigin = router.allowedOrigin
|
2022-05-09 16:51:08 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/sales/slots") do() -> RestApiResponse:
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2022-04-21 10:12:16 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
## Returns active slots for the host
|
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2024-09-23 17:08:56 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let json = %(await contracts.sales.mySlots())
|
|
|
|
return RestApiResponse.response(
|
|
|
|
$json, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
|
|
|
|
|
|
|
router.api(MethodGet, "/api/codex/v1/sales/slots/{slotId}") do(
|
|
|
|
slotId: SlotId
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Returns active slot with id {slotId} for the host. Returns 404 if the
|
|
|
|
## slot is not active for the host.
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
|
|
|
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
return
|
|
|
|
RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers)
|
|
|
|
|
|
|
|
without slotId =? slotId.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
|
|
|
|
|
|
|
without agent =? await contracts.sales.activeSale(slotId):
|
|
|
|
return
|
|
|
|
RestApiResponse.error(Http404, "Provider not filling slot", headers = headers)
|
|
|
|
|
|
|
|
let restAgent = RestSalesAgent(
|
|
|
|
state: agent.state() |? "none",
|
|
|
|
slotIndex: agent.data.slotIndex,
|
|
|
|
requestId: agent.data.requestId,
|
|
|
|
request: agent.data.request,
|
|
|
|
reservation: agent.data.reservation,
|
|
|
|
)
|
|
|
|
|
|
|
|
return RestApiResponse.response(
|
|
|
|
restAgent.toJson, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
|
|
|
|
router.api(MethodGet, "/api/codex/v1/sales/availability") do() -> RestApiResponse:
|
|
|
|
## Returns storage that is for sale
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
|
|
|
|
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2022-05-09 16:51:08 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without avails =? (await contracts.sales.context.reservations.all(Availability)),
|
|
|
|
err:
|
|
|
|
return RestApiResponse.error(Http500, err.msg, headers = headers)
|
2022-04-21 10:12:16 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let json = %avails
|
|
|
|
return RestApiResponse.response(
|
|
|
|
$json, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
|
|
|
|
|
|
|
router.rawApi(MethodPost, "/api/codex/v1/sales/availability") do() -> RestApiResponse:
|
|
|
|
## Add available storage to sell.
|
|
|
|
## Every time Availability's offer finishes, its capacity is returned to the availability.
|
|
|
|
##
|
|
|
|
## totalSize - size of available storage in bytes
|
|
|
|
## duration - maximum time the storage should be sold for (in seconds)
|
|
|
|
## minPrice - minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration
|
|
|
|
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
|
|
|
|
|
|
|
|
var headers = buildCorsHeaders("POST", allowedOrigin)
|
|
|
|
|
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2022-04-21 10:12:16 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let body = await request.getBody()
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 17:05:16 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without restAv =? RestAvailability.fromJson(body), error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let reservations = contracts.sales.context.reservations
|
[marketplace] Add Reservations Module (#340)
* [marketplace] reservations module
- add de/serialization for Availability
- add markUsed/markUnused in persisted availability
- add query for unused
- add reserve/release
- reservation module tests
- split ContractInteractions into client contracts and host contracts
- remove reservations start/stop as the repo start/stop is being managed by the node
- remove dedicated reservations metadata store and use the metadata store from the repo instead
- Split ContractInteractions into:
- ClientInteractions (with purchasing)
- HostInteractions (with sales and proving)
- compilation fix for nim 1.2
[repostore] fix started flag, add tests
[marketplace] persist slot index
For loading the sales state from chain, the slot index was not previously persisted in the contract. Will retrieve the slot index from the contract when the sales state is loaded.
* Revert repostore changes
In favour of separate PR https://github.com/status-im/nim-codex/pull/374.
* remove warnings
* clean up
* tests: stop repostore during teardown
* change constructor type identifier
Change Contracts constructor to accept Contracts type instead of ContractInteractions.
* change constructor return type to Result instead of Option
* fix and split interactions tests
* clean up, fix tests
* find availability by slot id
* remove duplication in host/client interactions
* add test for finding availability by slotId
* log instead of raiseAssert when failed to mark availability as unused
* move to SaleErrored state instead of raiseAssert
* remove unneeded reverse
It appears that order is not preserved in the repostore, so reversing does not have the intended effect here.
* update open api spec for potential rest endpoint errors
* move functions about available bytes to repostore
* WIP: reserve and release availabilities as needed
WIP: not tested yet
Availabilities are marked as used when matched (just before downloading starts) so that future matching logic does not match an availability currently in use.
As the download progresses, batches of blocks are written to disk, and the equivalent bytes are released from the reservation module. The size of the availability is reduced as well.
During a reserve or release operation, availability updates occur after the repo is manipulated. If the availability update operation fails, the reserve or release is rolled back to maintain correct accounting of bytes.
Finally, once download completes, or if an error occurs, the availability is marked as unused so future matching can occur.
* delete availability when all bytes released
* fix tests + cleanup
* remove availability from SalesContext callbacks
Availability is no longer used past the SaleDownloading state in the state machine. Cleanup of Availability (marking unused) is handled directly in the SaleDownloading state, and no longer in SaleErrored or SaleFinished. Likewise, Availabilities shouldn’t need to be handled on node restart.
Additionally, Availability was being passed in SalesContext callbacks, and now that Availability is only used temporarily in the SaleDownloading state, Availability is contextually irrelevant to the callbacks, except in OnStore possibly, though it was not being consumed.
* test clean up
* - remove availability from callbacks and constructors from previous commit that needed to be removed (oopsie)
- fix integration test that checks availabilities
- there was a bug fixed that crashed the node due to a missing `return success` in onStore
- the test was fixed by ensuring that availabilities are remaining on the node, and the size has been reduced
- change Availability back to non-ref object and constructor back to init
- add trace logging of all state transitions in state machine
- add generally useful trace logging
* fixes after rebase
1. Fix onProve callbacks
2. Use Slot type instead of tuple for retrieving active slot.
3. Bump codex-contracts-eth that exposes getActivceSlot call.
* swap contracts branch to not support slot collateral
Slot collateral changes in the contracts require further changes in the client code, so we’ll skip those changes for now and add in a separate commit.
* modify Interactions and Deployment constructors
- `HostInteractions` and `ClientInteractions` constructors were simplified to take a contract address and no overloads
- `Interactions` prepared simplified so there are no overloads
- `Deployment` constructor updated so that it takes an optional string parameter, instead `Option[string]`
* Move `batchProc` declaration
`batchProc` needs to be consumed by both `node` and `salescontext`, and they can’t reference each other as it creates a circular dependency.
* [reservations] rename `available` to `hasAvailable`
* [reservations] default error message to inner error msg
* add SaleIngored state
When a storage request is handled but the request does match availabilities, the sales agent machine is sent to the SaleIgnored state. In addition, the agent is constructed in a way that if the request is ignored, the sales agent is removed from the list of active agents being tracked in the sales module.
2023-04-04 17:05:16 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if restAv.totalSize == 0:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400, "Total size must be larger then zero", headers = headers
|
|
|
|
)
|
2022-05-09 16:51:08 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if not reservations.hasAvailable(restAv.totalSize.truncate(uint)):
|
|
|
|
return
|
|
|
|
RestApiResponse.error(Http422, "Not enough storage quota", headers = headers)
|
2024-09-23 17:08:56 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without availability =? (
|
|
|
|
await reservations.createAvailability(
|
|
|
|
restAv.totalSize, restAv.duration, restAv.minPrice, restAv.maxCollateral
|
|
|
|
)
|
|
|
|
), error:
|
|
|
|
return RestApiResponse.error(Http500, error.msg, headers = headers)
|
|
|
|
|
|
|
|
return RestApiResponse.response(
|
|
|
|
availability.toJson,
|
|
|
|
Http201,
|
|
|
|
contentType = "application/json",
|
|
|
|
headers = headers,
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
|
|
|
|
|
|
|
router.api(MethodOptions, "/api/codex/v1/sales/availability/{id}") do(
|
|
|
|
id: AvailabilityId, resp: HttpResponseRef
|
|
|
|
) -> RestApiResponse:
|
|
|
|
if corsOrigin =? allowedOrigin:
|
|
|
|
resp.setCorsHeaders("PATCH", corsOrigin)
|
|
|
|
|
|
|
|
resp.status = Http204
|
|
|
|
await resp.sendBody("")
|
|
|
|
|
|
|
|
router.rawApi(MethodPatch, "/api/codex/v1/sales/availability/{id}") do(
|
|
|
|
id: AvailabilityId
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Updates Availability.
|
|
|
|
## The new parameters will be only considered for new requests.
|
|
|
|
## Existing Requests linked to this Availability will continue as is.
|
|
|
|
##
|
|
|
|
## totalSize - size of available storage in bytes. When decreasing the size, then lower limit is the currently `totalSize - freeSize`.
|
|
|
|
## duration - maximum time the storage should be sold for (in seconds)
|
|
|
|
## minPrice - minimum price to be paid (in amount of tokens)
|
|
|
|
## maxCollateral - maximum collateral user is willing to pay per filled Slot (in amount of tokens)
|
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
return RestApiResponse.error(Http503, "Persistence is not enabled")
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without id =? id.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
|
|
|
without keyId =? id.key.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let
|
|
|
|
body = await request.getBody()
|
|
|
|
reservations = contracts.sales.context.reservations
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
type OptRestAvailability = Optionalize(RestAvailability)
|
|
|
|
without restAv =? OptRestAvailability.fromJson(body), error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without availability =? (await reservations.get(keyId, Availability)), error:
|
|
|
|
if error of NotExistsError:
|
|
|
|
return RestApiResponse.error(Http404, "Availability not found")
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.error(Http500, error.msg)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if isSome restAv.freeSize:
|
|
|
|
return RestApiResponse.error(Http400, "Updating freeSize is not allowed")
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if size =? restAv.totalSize:
|
|
|
|
# we don't allow lowering the totalSize bellow currently utilized size
|
|
|
|
if size < (availability.totalSize - availability.freeSize):
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400,
|
|
|
|
"New totalSize must be larger then current totalSize - freeSize, which is currently: " &
|
|
|
|
$(availability.totalSize - availability.freeSize),
|
|
|
|
)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
availability.freeSize += size - availability.totalSize
|
|
|
|
availability.totalSize = size
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if duration =? restAv.duration:
|
|
|
|
availability.duration = duration
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if minPrice =? restAv.minPrice:
|
|
|
|
availability.minPrice = minPrice
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if maxCollateral =? restAv.maxCollateral:
|
|
|
|
availability.maxCollateral = maxCollateral
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if err =? (await reservations.update(availability)).errorOption:
|
|
|
|
return RestApiResponse.error(Http500, err.msg)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.response(Http200)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.rawApi(MethodGet, "/api/codex/v1/sales/availability/{id}/reservations") do(
|
|
|
|
id: AvailabilityId
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Gets Availability's reservations.
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.host:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without id =? id.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
|
|
|
without keyId =? id.key.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let reservations = contracts.sales.context.reservations
|
|
|
|
let market = contracts.sales.context.market
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if error =? (await reservations.get(keyId, Availability)).errorOption:
|
|
|
|
if error of NotExistsError:
|
|
|
|
return
|
|
|
|
RestApiResponse.error(Http404, "Availability not found", headers = headers)
|
|
|
|
else:
|
|
|
|
return RestApiResponse.error(Http500, error.msg, headers = headers)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without availabilitysReservations =? (await reservations.all(Reservation, id)),
|
|
|
|
err:
|
|
|
|
return RestApiResponse.error(Http500, err.msg, headers = headers)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
# TODO: Expand this structure with information about the linked StorageRequest not only RequestID
|
|
|
|
return RestApiResponse.response(
|
|
|
|
availabilitysReservations.toJson,
|
|
|
|
contentType = "application/json",
|
|
|
|
headers = headers,
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2024-03-21 11:53:45 +01:00
|
|
|
|
2023-11-09 09:47:09 +01:00
|
|
|
proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
|
2024-08-16 01:57:50 +02:00
|
|
|
let allowedOrigin = router.allowedOrigin
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.rawApi(MethodPost, "/api/codex/v1/storage/request/{cid}") do(
|
|
|
|
cid: Cid
|
|
|
|
) -> RestApiResponse:
|
|
|
|
var headers = buildCorsHeaders("POST", allowedOrigin)
|
|
|
|
|
|
|
|
## Create a request for storage
|
|
|
|
##
|
|
|
|
## cid - the cid of a previously uploaded dataset
|
|
|
|
## duration - the duration of the request in seconds
|
|
|
|
## proofProbability - how often storage proofs are required
|
|
|
|
## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
|
|
|
|
## expiry - specifies threshold in seconds from now when the request expires if the Request does not find requested amount of nodes to host the data
|
|
|
|
## nodes - number of nodes the content should be stored on
|
|
|
|
## tolerance - allowed number of nodes that can be lost before content is lost
|
|
|
|
## colateral - requested collateral from hosts when they fill slot
|
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.client:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without cid =? cid.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let body = await request.getBody()
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without params =? StorageRequestParams.fromJson(body), error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
2024-09-24 10:37:08 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let nodes = params.nodes |? 3
|
|
|
|
let tolerance = params.tolerance |? 1
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if tolerance == 0:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400, "Tolerance needs to be bigger then zero", headers = headers
|
|
|
|
)
|
2024-06-28 07:26:19 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
# prevent underflow
|
|
|
|
if tolerance > nodes:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400,
|
|
|
|
"Invalid parameters: `tolerance` cannot be greater than `nodes`",
|
|
|
|
headers = headers,
|
|
|
|
)
|
2024-06-28 07:26:19 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let ecK = nodes - tolerance
|
|
|
|
let ecM = tolerance # for readability
|
2024-01-11 10:45:23 -06:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
# ensure leopard constrainst of 1 < K ≥ M
|
|
|
|
if ecK <= 1 or ecK < ecM:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400,
|
|
|
|
"Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`",
|
|
|
|
headers = headers,
|
|
|
|
)
|
2023-11-22 12:35:26 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without expiry =? params.expiry:
|
|
|
|
return RestApiResponse.error(Http400, "Expiry required", headers = headers)
|
2023-12-05 14:25:28 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if expiry <= 0 or expiry >= params.duration:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400,
|
|
|
|
"Expiry needs value bigger then zero and smaller then the request's duration",
|
|
|
|
headers = headers,
|
|
|
|
)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without purchaseId =?
|
|
|
|
await node.requestStorage(
|
|
|
|
cid, params.duration, params.proofProbability, nodes, tolerance,
|
|
|
|
params.reward, params.collateral, expiry,
|
|
|
|
), error:
|
|
|
|
if error of InsufficientBlocksError:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http400,
|
2024-06-28 07:26:19 +10:00
|
|
|
"Dataset too small for erasure parameters, need at least " &
|
2025-01-21 21:54:46 +01:00
|
|
|
$(ref InsufficientBlocksError)(error).minSize.int & " bytes",
|
|
|
|
headers = headers,
|
|
|
|
)
|
2024-06-28 07:26:19 +10:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.error(Http500, error.msg, headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.response(purchaseId.toHex)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2022-05-11 10:51:59 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/storage/purchases/{id}") do(
|
|
|
|
id: PurchaseId
|
|
|
|
) -> RestApiResponse:
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2022-05-11 10:51:59 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.client:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2022-05-11 10:51:59 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
without id =? id.tryGet.catch, error:
|
|
|
|
return RestApiResponse.error(Http400, error.msg, headers = headers)
|
|
|
|
|
|
|
|
without purchase =? contracts.purchasing.getPurchase(id):
|
|
|
|
return RestApiResponse.error(Http404, headers = headers)
|
2022-05-11 10:51:59 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let json =
|
|
|
|
%RestPurchase(
|
2023-11-14 13:05:43 -06:00
|
|
|
state: purchase.state |? "none",
|
2025-01-21 21:54:46 +01:00
|
|
|
error: purchase.error .? msg,
|
2023-11-14 13:05:43 -06:00
|
|
|
request: purchase.request,
|
2025-01-21 21:54:46 +01:00
|
|
|
requestId: purchase.requestId,
|
2023-11-14 13:05:43 -06:00
|
|
|
)
|
2022-05-11 10:51:59 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.response(
|
|
|
|
$json, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2022-05-11 10:51:59 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/storage/purchases") do() -> RestApiResponse:
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2024-10-10 10:25:07 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
try:
|
|
|
|
without contracts =? node.contracts.client:
|
|
|
|
return RestApiResponse.error(
|
|
|
|
Http503, "Persistence is not enabled", headers = headers
|
|
|
|
)
|
2023-11-13 12:30:27 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
let purchaseIds = contracts.purchasing.getPurchaseIds()
|
|
|
|
return RestApiResponse.response(
|
|
|
|
$ %purchaseIds, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2023-11-13 12:30:27 +01:00
|
|
|
|
2024-04-02 14:13:49 +03:00
|
|
|
proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
2024-10-10 10:25:07 +02:00
|
|
|
let allowedOrigin = router.allowedOrigin
|
|
|
|
|
2024-04-02 14:13:49 +03:00
|
|
|
## various node management api's
|
2024-05-06 17:35:46 +02:00
|
|
|
##
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/spr") do() -> RestApiResponse:
|
|
|
|
## Returns node SPR in requested format, json or text.
|
|
|
|
##
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
|
|
|
|
|
|
|
try:
|
|
|
|
without spr =? node.discovery.dhtRecord:
|
|
|
|
return RestApiResponse.response(
|
|
|
|
"", status = Http503, contentType = "application/json", headers = headers
|
|
|
|
)
|
2024-10-10 10:25:07 +02:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if $preferredContentType().get() == "text/plain":
|
|
|
|
return RestApiResponse.response(
|
|
|
|
spr.toURI, contentType = "text/plain", headers = headers
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
return RestApiResponse.response(
|
|
|
|
$ %*{"spr": spr.toURI}, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2024-04-02 14:13:49 +03:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/peerid") do() -> RestApiResponse:
|
|
|
|
## Returns node's peerId in requested format, json or text.
|
|
|
|
##
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2024-04-02 14:13:49 +03:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
try:
|
|
|
|
let id = $node.switch.peerInfo.peerId
|
2024-04-02 14:13:49 +03:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
if $preferredContentType().get() == "text/plain":
|
|
|
|
return
|
|
|
|
RestApiResponse.response(id, contentType = "text/plain", headers = headers)
|
|
|
|
else:
|
|
|
|
return RestApiResponse.response(
|
|
|
|
$ %*{"id": id}, contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
|
|
|
|
|
|
|
router.api(MethodGet, "/api/codex/v1/connect/{peerId}") do(
|
|
|
|
peerId: PeerId, addrs: seq[MultiAddress]
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Connect to a peer
|
|
|
|
##
|
|
|
|
## If `addrs` param is supplied, it will be used to
|
|
|
|
## dial the peer, otherwise the `peerId` is used
|
|
|
|
## to invoke peer discovery, if it succeeds
|
|
|
|
## the returned addresses will be used to dial
|
|
|
|
##
|
|
|
|
## `addrs` the listening addresses of the peers to dial, eg the one specified with `--listen-addrs`
|
|
|
|
##
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
|
|
|
|
|
|
|
if peerId.isErr:
|
|
|
|
return RestApiResponse.error(Http400, $peerId.error(), headers = headers)
|
|
|
|
|
|
|
|
let addresses =
|
|
|
|
if addrs.isOk and addrs.get().len > 0:
|
|
|
|
addrs.get()
|
|
|
|
else:
|
|
|
|
without peerRecord =? (await node.findPeer(peerId.get())):
|
|
|
|
return
|
|
|
|
RestApiResponse.error(Http400, "Unable to find Peer!", headers = headers)
|
|
|
|
peerRecord.addresses.mapIt(it.address)
|
|
|
|
try:
|
|
|
|
await node.connect(peerId.get(), addresses)
|
|
|
|
return
|
|
|
|
RestApiResponse.response("Successfully connected to peer", headers = headers)
|
|
|
|
except DialFailedError:
|
|
|
|
return RestApiResponse.error(Http400, "Unable to dial peer", headers = headers)
|
|
|
|
except CatchableError:
|
|
|
|
return
|
|
|
|
RestApiResponse.error(Http500, "Unknown error dialling peer", headers = headers)
|
2024-04-02 14:13:49 +03:00
|
|
|
|
2023-11-09 09:47:09 +01:00
|
|
|
proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) =
|
2024-10-10 10:25:07 +02:00
|
|
|
let allowedOrigin = router.allowedOrigin
|
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/debug/info") do() -> RestApiResponse:
|
|
|
|
## Print rudimentary node information
|
|
|
|
##
|
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
|
|
|
|
|
|
|
try:
|
|
|
|
let table = RestRoutingTable.init(node.discovery.protocol.routingTable)
|
|
|
|
|
|
|
|
let json =
|
|
|
|
%*{
|
|
|
|
"id": $node.switch.peerInfo.peerId,
|
|
|
|
"addrs": node.switch.peerInfo.addrs.mapIt($it),
|
|
|
|
"repo": $conf.dataDir,
|
|
|
|
"spr":
|
|
|
|
if node.discovery.dhtRecord.isSome:
|
|
|
|
node.discovery.dhtRecord.get.toURI
|
|
|
|
else:
|
|
|
|
"",
|
|
|
|
"announceAddresses": node.discovery.announceAddrs,
|
|
|
|
"table": table,
|
|
|
|
"codex": {"version": $codexVersion, "revision": $codexRevision},
|
|
|
|
}
|
|
|
|
|
|
|
|
# return pretty json for human readability
|
|
|
|
return RestApiResponse.response(
|
|
|
|
json.pretty(), contentType = "application/json", headers = headers
|
|
|
|
)
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
|
|
|
|
|
|
|
router.api(MethodPost, "/api/codex/v1/debug/chronicles/loglevel") do(
|
|
|
|
level: Option[string]
|
|
|
|
) -> RestApiResponse:
|
|
|
|
## Set log level at run time
|
|
|
|
##
|
|
|
|
## e.g. `chronicles/loglevel?level=DEBUG`
|
|
|
|
##
|
|
|
|
## `level` - chronicles log level
|
|
|
|
##
|
|
|
|
var headers = buildCorsHeaders("POST", allowedOrigin)
|
|
|
|
|
|
|
|
try:
|
|
|
|
without res =? level and level =? res:
|
|
|
|
return RestApiResponse.error(Http400, "Missing log level", headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2023-11-14 13:05:43 -06:00
|
|
|
try:
|
2025-01-21 21:54:46 +01:00
|
|
|
{.gcsafe.}:
|
|
|
|
updateLogLevel(level)
|
2023-11-14 13:05:43 -06:00
|
|
|
except CatchableError as exc:
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.error(Http500, exc.msg, headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2025-01-21 21:54:46 +01:00
|
|
|
return RestApiResponse.response("")
|
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
|
|
|
when codex_enable_api_debug_peers:
|
2025-01-21 21:54:46 +01:00
|
|
|
router.api(MethodGet, "/api/codex/v1/debug/peer/{peerId}") do(
|
|
|
|
peerId: PeerId
|
|
|
|
) -> RestApiResponse:
|
2024-10-10 10:25:07 +02:00
|
|
|
var headers = buildCorsHeaders("GET", allowedOrigin)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2023-11-14 13:05:43 -06:00
|
|
|
try:
|
2023-11-09 09:47:09 +01:00
|
|
|
trace "debug/peer start"
|
|
|
|
without peerRecord =? (await node.findPeer(peerId.get())):
|
|
|
|
trace "debug/peer peer not found!"
|
2025-01-21 21:54:46 +01:00
|
|
|
return
|
|
|
|
RestApiResponse.error(Http400, "Unable to find Peer!", headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
|
|
|
let json = %RestPeerRecord.init(peerRecord)
|
|
|
|
trace "debug/peer returning peer record"
|
2024-10-10 10:25:07 +02:00
|
|
|
return RestApiResponse.response($json, headers = headers)
|
2023-11-14 13:05:43 -06:00
|
|
|
except CatchableError as exc:
|
|
|
|
trace "Excepting processing request", exc = exc.msg
|
2024-10-10 10:25:07 +02:00
|
|
|
return RestApiResponse.error(Http500, headers = headers)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2024-06-17 21:33:21 +10:00
|
|
|
proc initRestApi*(
|
2025-01-21 21:54:46 +01:00
|
|
|
node: CodexNodeRef,
|
|
|
|
conf: CodexConf,
|
|
|
|
repoStore: RepoStore,
|
|
|
|
corsAllowedOrigin: ?string,
|
|
|
|
): RestRouter =
|
2024-06-17 21:33:21 +10:00
|
|
|
var router = RestRouter.init(validate, corsAllowedOrigin)
|
2023-11-09 09:47:09 +01:00
|
|
|
|
2023-12-14 11:57:16 +01:00
|
|
|
initDataApi(node, repoStore, router)
|
2023-11-09 09:47:09 +01:00
|
|
|
initSalesApi(node, router)
|
|
|
|
initPurchasingApi(node, router)
|
2024-04-02 14:13:49 +03:00
|
|
|
initNodeApi(node, conf, router)
|
2023-11-09 09:47:09 +01:00
|
|
|
initDebugApi(node, conf, router)
|
|
|
|
|
2022-01-10 09:32:56 -06:00
|
|
|
return router
|