2023-03-27 15:47:25 +02:00
|
|
|
import std/strutils
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2023-09-01 15:44:41 +10:00
|
|
|
from pkg/libp2p import Cid, `$`, init
|
2023-03-27 15:47:25 +02:00
|
|
|
import pkg/stint
|
|
|
|
|
import pkg/questionable/results
|
2025-03-17 17:08:24 -03:00
|
|
|
import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable]
|
feat: create logging proxy (#663)
* implement a logging proxy
The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks
* Rename logging -> logutils to avoid ambiguity with common names
* clean up
* add setProperty for JsonRecord, remove nim-json-serialization conflict
* Allow specifying textlines and json format separately
Not specifying a LogFormat will apply the formatting to both textlines and json sinks.
Specifying a LogFormat will apply the formatting to only that sink.
* remove unneeded usages of std/json
We only need to import utils/json instead of std/json
* move serialization from rest/json to utils/json so it can be shared
* fix NoColors ambiguity
Was causing unit tests to fail on Windows.
* Remove nre usage to fix Windows error
Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)
* Add logutils module doc
* Shorten logutils.formatIt for `NBytes`
Both json and textlines formatIt were not needed, and could be combined into one formatIt
* remove debug integration test config
debug output and logformat of json for integration test logs
* Use ## module doc to support docgen
* bump nim-poseidon2 to export fromBytes
Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.
* fixes to get compiling after rebasing master
* Add support for Result types being logged using formatIt
2024-01-23 18:35:03 +11:00
|
|
|
import pkg/codex/logutils
|
2023-09-01 15:44:41 +10:00
|
|
|
import pkg/codex/rest/json
|
|
|
|
|
import pkg/codex/errors
|
2023-03-27 15:47:25 +02:00
|
|
|
|
2025-12-17 15:20:48 +11:00
|
|
|
export httptable, httpclient
|
2023-12-07 01:16:36 +00:00
|
|
|
|
2023-03-27 15:47:25 +02:00
|
|
|
type CodexClient* = ref object
|
|
|
|
|
baseurl: string
|
2025-03-17 17:08:24 -03:00
|
|
|
session: HttpSessionRef
|
2024-12-17 14:01:41 +01:00
|
|
|
|
2025-11-02 08:32:47 +04:00
|
|
|
type HasBlockResponse = object
|
|
|
|
|
has: bool
|
|
|
|
|
|
2023-03-27 15:47:25 +02:00
|
|
|
proc new*(_: type CodexClient, baseurl: string): CodexClient =
|
2025-03-17 17:08:24 -03:00
|
|
|
CodexClient(session: HttpSessionRef.new(), baseurl: baseurl)
|
|
|
|
|
|
|
|
|
|
proc close*(self: CodexClient): Future[void] {.async: (raises: []).} =
|
|
|
|
|
await self.session.closeWait()
|
|
|
|
|
|
|
|
|
|
proc request(
|
|
|
|
|
self: CodexClient,
|
|
|
|
|
httpMethod: httputils.HttpMethod,
|
|
|
|
|
url: string,
|
|
|
|
|
body: openArray[char] = [],
|
|
|
|
|
headers: openArray[HttpHeaderTuple] = [],
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
HttpClientRequestRef
|
|
|
|
|
.new(
|
|
|
|
|
self.session,
|
|
|
|
|
url,
|
|
|
|
|
httpMethod,
|
|
|
|
|
version = HttpVersion11,
|
|
|
|
|
flags = {},
|
|
|
|
|
maxResponseHeadersSize = HttpMaxHeadersSize,
|
|
|
|
|
headers = headers,
|
|
|
|
|
body = body.toOpenArrayByte(0, len(body) - 1),
|
|
|
|
|
).get
|
|
|
|
|
.send()
|
|
|
|
|
|
2025-03-24 16:47:05 +01:00
|
|
|
proc post*(
|
2025-03-17 17:08:24 -03:00
|
|
|
self: CodexClient,
|
|
|
|
|
url: string,
|
|
|
|
|
body: string = "",
|
|
|
|
|
headers: seq[HttpHeaderTuple] = @[],
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return self.request(MethodPost, url, headers = headers, body = body)
|
|
|
|
|
|
|
|
|
|
proc get(
|
|
|
|
|
self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return self.request(MethodGet, url, headers = headers)
|
|
|
|
|
|
|
|
|
|
proc delete(
|
|
|
|
|
self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return self.request(MethodDelete, url, headers = headers)
|
|
|
|
|
|
2025-03-24 16:47:05 +01:00
|
|
|
proc patch*(
|
2025-03-17 17:08:24 -03:00
|
|
|
self: CodexClient,
|
|
|
|
|
url: string,
|
|
|
|
|
body: string = "",
|
|
|
|
|
headers: seq[HttpHeaderTuple] = @[],
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return self.request(MethodPatch, url, headers = headers, body = body)
|
|
|
|
|
|
|
|
|
|
proc body*(
|
|
|
|
|
response: HttpClientResponseRef
|
|
|
|
|
): Future[string] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
return bytesToString (await response.getBodyBytes())
|
|
|
|
|
|
|
|
|
|
proc getContent(
|
|
|
|
|
client: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[]
|
|
|
|
|
): Future[string] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.get(url, headers)
|
|
|
|
|
return await response.body
|
|
|
|
|
|
|
|
|
|
proc info*(
|
|
|
|
|
client: CodexClient
|
|
|
|
|
): Future[?!JsonNode] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.get(client.baseurl & "/debug/info")
|
|
|
|
|
return JsonNode.parse(await response.body)
|
|
|
|
|
|
|
|
|
|
proc setLogLevel*(
|
|
|
|
|
client: CodexClient, level: string
|
|
|
|
|
): Future[void] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let
|
|
|
|
|
url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
|
|
|
|
|
headers = @[("Content-Type", "text/plain")]
|
|
|
|
|
response = await client.post(url, headers = headers, body = "")
|
|
|
|
|
assert response.status == 200
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc uploadRaw*(
|
|
|
|
|
client: CodexClient, contents: string, headers: seq[HttpHeaderTuple] = @[]
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return client.post(client.baseurl & "/data", body = contents, headers = headers)
|
|
|
|
|
|
|
|
|
|
proc upload*(
|
|
|
|
|
client: CodexClient, contents: string
|
|
|
|
|
): Future[?!Cid] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.uploadRaw(contents)
|
|
|
|
|
assert response.status == 200
|
|
|
|
|
Cid.init(await response.body).mapFailure
|
|
|
|
|
|
|
|
|
|
proc upload*(
|
|
|
|
|
client: CodexClient, bytes: seq[byte]
|
|
|
|
|
): Future[?!Cid] {.async: (raw: true).} =
|
|
|
|
|
return client.upload(string.fromBytes(bytes))
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc downloadRaw*(
|
|
|
|
|
client: CodexClient, cid: string, local = false
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return
|
|
|
|
|
client.get(client.baseurl & "/data/" & cid & (if local: "" else: "/network/stream"))
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc downloadBytes*(
|
|
|
|
|
client: CodexClient, cid: Cid, local = false
|
|
|
|
|
): Future[?!seq[byte]] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.downloadRaw($cid, local = local)
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
if response.status != 200:
|
|
|
|
|
return failure($response.status)
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
success await response.getBodyBytes()
|
2024-10-17 18:54:28 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc download*(
|
|
|
|
|
client: CodexClient, cid: Cid, local = false
|
|
|
|
|
): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
without response =? await client.downloadBytes(cid, local = local), err:
|
|
|
|
|
return failure(err)
|
|
|
|
|
return success bytesToString(response)
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc downloadNoStream*(
|
|
|
|
|
client: CodexClient, cid: Cid
|
|
|
|
|
): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.post(client.baseurl & "/data/" & $cid & "/network")
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
if response.status != 200:
|
|
|
|
|
return failure($response.status)
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
success await response.body
|
2024-06-26 22:02:39 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc downloadManifestOnly*(
|
|
|
|
|
client: CodexClient, cid: Cid
|
|
|
|
|
): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response =
|
|
|
|
|
await client.get(client.baseurl & "/data/" & $cid & "/network/manifest")
|
2024-06-26 22:02:39 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
if response.status != 200:
|
|
|
|
|
return failure($response.status)
|
2024-06-26 22:02:39 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
success await response.body
|
2024-06-26 22:02:39 +02:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc deleteRaw*(
|
|
|
|
|
client: CodexClient, cid: string
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return client.delete(client.baseurl & "/data/" & cid)
|
|
|
|
|
|
|
|
|
|
proc delete*(
|
|
|
|
|
client: CodexClient, cid: Cid
|
|
|
|
|
): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.deleteRaw($cid)
|
2025-02-11 16:00:05 -03:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
if response.status != 204:
|
|
|
|
|
return failure($response.status)
|
2025-02-11 16:00:05 -03:00
|
|
|
|
|
|
|
|
success()
|
|
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc listRaw*(
|
|
|
|
|
client: CodexClient
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
return client.get(client.baseurl & "/data")
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
proc list*(
|
|
|
|
|
client: CodexClient
|
|
|
|
|
): Future[?!RestContentList] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let response = await client.listRaw()
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
if response.status != 200:
|
|
|
|
|
return failure($response.status)
|
2023-11-20 18:14:06 -06:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
RestContentList.fromJson(await response.body)
|
|
|
|
|
|
|
|
|
|
proc space*(
|
|
|
|
|
client: CodexClient
|
|
|
|
|
): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} =
|
2023-12-14 11:57:16 +01:00
|
|
|
let url = client.baseurl & "/space"
|
2025-03-17 17:08:24 -03:00
|
|
|
let response = await client.get(url)
|
2023-12-14 11:57:16 +01:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
if response.status != 200:
|
|
|
|
|
return failure($response.status)
|
2023-12-14 11:57:16 +01:00
|
|
|
|
2025-03-17 17:08:24 -03:00
|
|
|
RestRepoStore.fromJson(await response.body)
|
2023-12-14 11:57:16 +01:00
|
|
|
|
2025-03-24 16:47:05 +01:00
|
|
|
proc buildUrl*(client: CodexClient, path: string): string =
|
|
|
|
|
return client.baseurl & path
|
2025-05-29 08:57:05 +02:00
|
|
|
|
2025-11-02 08:32:47 +04:00
|
|
|
proc hasBlock*(
|
|
|
|
|
client: CodexClient, cid: Cid
|
|
|
|
|
): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} =
|
|
|
|
|
let url = client.baseurl & "/data/" & $cid & "/exists"
|
|
|
|
|
let body = await client.getContent(url)
|
|
|
|
|
let response = HasBlockResponse.fromJson(body)
|
|
|
|
|
if response.isErr:
|
|
|
|
|
return failure "Failed to parse has block response"
|
|
|
|
|
return response.get.has.success
|
|
|
|
|
|
|
|
|
|
proc hasBlockRaw*(
|
|
|
|
|
client: CodexClient, cid: string
|
|
|
|
|
): Future[HttpClientResponseRef] {.
|
|
|
|
|
async: (raw: true, raises: [CancelledError, HttpError])
|
|
|
|
|
.} =
|
|
|
|
|
let url = client.baseurl & "/data/" & cid & "/exists"
|
|
|
|
|
return client.get(url)
|