From f293082ae9b883071d600ad7e11064ff8a0e1ec3 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:24:40 +1100 Subject: [PATCH] Reverts logging-proxy, commit 27f585eb6f380a6de2fbbef721fdf1a16bd5f600 (#660) --- codex.nim | 4 +- codex/blockexchange/engine/discovery.nim | 7 +- codex/blockexchange/engine/engine.nim | 2 +- codex/blockexchange/engine/pendingblocks.nim | 2 +- codex/blockexchange/network/network.nim | 2 +- codex/blockexchange/network/networkpeer.nim | 2 +- codex/blockexchange/peers/peercontext.nim | 2 +- codex/blockexchange/peers/peerctxstore.nim | 3 +- codex/blocktype.nim | 26 +-- codex/chunker.nim | 2 +- codex/codex.nim | 2 +- codex/conf.nim | 9 +- codex/contracts/deployment.nim | 3 +- .../interactions/clientinteractions.nim | 4 +- .../interactions/hostinteractions.nim | 6 +- codex/contracts/market.nim | 2 +- codex/contracts/requests.nim | 24 +- codex/discovery.nim | 3 +- codex/erasure/erasure.nim | 2 +- codex/formats.nim | 28 +++ codex/logutils.nim | 211 ----------------- codex/manifest/coders.nim | 2 +- codex/manifest/manifest.nim | 2 +- codex/merkletree/merkletree.nim | 2 +- codex/node.nim | 5 +- codex/purchasing/purchaseid.nim | 4 - codex/purchasing/states/cancelled.nim | 6 +- codex/purchasing/states/error.nim | 2 +- codex/purchasing/states/finished.nim | 3 +- codex/purchasing/states/started.nim | 3 +- codex/purchasing/states/submitted.nim | 3 +- codex/rest/api.nim | 2 +- codex/rest/json.nim | 21 +- codex/sales.nim | 4 +- codex/sales/reservations.nim | 13 +- codex/sales/salesagent.nim | 2 +- codex/sales/slotqueue.nim | 2 +- codex/sales/states/cancelled.nim | 6 +- codex/sales/states/downloading.nim | 3 +- codex/sales/states/errored.nim | 3 +- codex/sales/states/failed.nim | 4 +- codex/sales/states/filled.nim | 5 +- codex/sales/states/filling.nim | 4 +- codex/sales/states/finished.nim | 5 +- codex/sales/states/ignored.nim | 3 +- codex/sales/states/initialproving.nim | 6 +- codex/sales/states/payout.nim | 4 +- codex/sales/states/preparing.nim | 3 +- codex/sales/states/proving.nim | 13 +- codex/sales/states/provingsimulated.nim | 4 +- codex/sales/states/unknown.nim | 2 +- codex/stores/cachestore.nim | 2 +- codex/stores/maintenance.nim | 2 +- codex/stores/networkstore.nim | 15 +- codex/stores/repostore.nim | 2 +- codex/streams/asyncstreamwrapper.nim | 5 +- codex/streams/seekablestream.nim | 5 +- codex/streams/storestream.nim | 6 +- codex/units.nim | 19 +- codex/utils/asyncstatemachine.nim | 4 +- codex/utils/fileutils.nim | 9 +- codex/utils/json.nim | 22 +- codex/utils/keyutils.nim | 4 +- codex/utils/timer.nim | 3 +- codex/utils/trackedfutures.nim | 3 +- codex/validation.nim | 8 +- tests/codex/sales/testslotqueue.nim | 2 +- tests/codex/testchunking.nim | 2 +- tests/codex/testlogutils.nim | 214 ------------------ tests/codex/testnode.nim | 2 +- tests/codex/utils/testjson.nim | 2 +- tests/contracts/testContracts.nim | 1 + tests/ethertest.nim | 2 +- tests/integration/codexclient.nim | 2 +- tests/integration/multinodes.nim | 3 +- tests/integration/nodes.nim | 10 +- tests/integration/testproofs.nim | 6 +- tests/integration/twonodes.nim | 1 + tests/logging.nim | 2 +- tests/testCodex.nim | 1 - 80 files changed, 231 insertions(+), 615 deletions(-) create mode 100644 codex/formats.nim delete mode 100644 codex/logutils.nim delete mode 100644 tests/codex/testlogutils.nim diff --git a/codex.nim b/codex.nim index f7215db6..073331f1 100644 --- a/codex.nim +++ b/codex.nim @@ -7,6 +7,7 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/confutils @@ -20,11 +21,10 @@ import pkg/libp2p import ./codex/conf import ./codex/codex -import ./codex/logutils import ./codex/units import ./codex/utils/keyutils -export codex, conf, libp2p, chronos, logutils +export codex, conf, libp2p, chronos, chronicles when isMainModule: import std/sequtils diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index 267d8f12..8bb4dd4a 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -10,21 +10,22 @@ import std/sequtils import pkg/chronos +import pkg/chronicles import pkg/libp2p/cid import pkg/metrics import pkg/questionable import pkg/questionable/results -import ./pendingblocks - import ../protobuf/presence + import ../network import ../peers import ../../utils import ../../discovery import ../../stores/blockstore -import ../../logutils + +import ./pendingblocks logScope: topics = "codex discoveryengine" diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index f3423434..ddff3c49 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -14,6 +14,7 @@ import std/algorithm import std/sugar import pkg/chronos +import pkg/chronicles import pkg/libp2p/[cid, switch, multihash, multicodec] import pkg/metrics import pkg/stint @@ -22,7 +23,6 @@ import ../../stores/blockstore import ../../blocktype import ../../utils import ../../merkletree -import ../../logutils import ../protobuf/blockexc import ../protobuf/presence diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index cd4c1ed7..c4763998 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -14,6 +14,7 @@ import pkg/upraises push: {.upraises: [].} +import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/metrics @@ -22,7 +23,6 @@ import pkg/questionable/results import ../protobuf/blockexc import ../../blocktype import ../../merkletree -import ../../logutils logScope: topics = "codex pendingblocks" diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index 1a39364b..0ce0c9ce 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -10,6 +10,7 @@ import std/tables import std/sequtils +import pkg/chronicles import pkg/chronos import pkg/libp2p @@ -18,7 +19,6 @@ import pkg/questionable import pkg/questionable/results import ../../blocktype as bt -import ../../logutils import ../protobuf/blockexc as pb import ../protobuf/payments diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index 69a19386..c3536f61 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -11,12 +11,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos +import pkg/chronicles import pkg/libp2p import ../protobuf/blockexc import ../protobuf/message import ../../errors -import ../../logutils logScope: topics = "codex blockexcnetworkpeer" diff --git a/codex/blockexchange/peers/peercontext.nim b/codex/blockexchange/peers/peercontext.nim index 70b139b7..66418ddd 100644 --- a/codex/blockexchange/peers/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -12,6 +12,7 @@ import std/tables import std/sugar import std/sets +import pkg/chronicles import pkg/libp2p import pkg/chronos import pkg/nitro @@ -22,7 +23,6 @@ import ../protobuf/payments import ../protobuf/presence import ../../blocktype -import ../../logutils export payments, nitro diff --git a/codex/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim index b07265b4..f23415f6 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -16,12 +16,11 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos +import pkg/chronicles import pkg/libp2p import ../protobuf/blockexc import ../../blocktype -import ../../logutils - import ./peercontext export peercontext diff --git a/codex/blocktype.nim b/codex/blocktype.nim index 786320c2..a26a3157 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -19,14 +19,15 @@ import pkg/libp2p/[cid, multicodec, multihash] import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results +import pkg/chronicles +import pkg/json_serialization import ./units import ./utils +import ./formats import ./errors -import ./logutils -import ./utils/json -export errors, logutils, units +export errors, formats, units const # Size of blocks for storage / network exchange, @@ -41,18 +42,11 @@ type BlockAddress* = object case leaf*: bool of true: - treeCid* {.serialize.}: Cid - index* {.serialize.}: Natural + treeCid*: Cid + index*: Natural else: - cid* {.serialize.}: Cid + cid*: Cid -logutils.formatIt(LogFormat.textLines, BlockAddress): - if it.leaf: - "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index - else: - "cid: " & shortLog($it.cid) - -logutils.formatIt(LogFormat.json, BlockAddress): %it proc `==`*(a, b: BlockAddress): bool = a.leaf == b.leaf and @@ -69,6 +63,12 @@ proc `$`*(a: BlockAddress): string = else: "cid: " & $a.cid +proc writeValue*( + writer: var JsonWriter, + value: Cid +) {.upraises:[IOError].} = + writer.writeValue($value) + proc cidOrTreeCid*(a: BlockAddress): Cid = if a.leaf: a.treeCid diff --git a/codex/chunker.nim b/codex/chunker.nim index 36f28f7a..58fcb4a3 100644 --- a/codex/chunker.nim +++ b/codex/chunker.nim @@ -13,13 +13,13 @@ import pkg/upraises push: {.upraises: [].} +import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/chronos import pkg/libp2p except shuffle import ./blocktype -import ./logutils export blocktype diff --git a/codex/codex.nim b/codex/codex.nim index 0b3176e7..2f5cf860 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -12,6 +12,7 @@ import std/strutils import std/os import std/tables +import pkg/chronicles import pkg/chronos import pkg/presto import pkg/libp2p @@ -38,7 +39,6 @@ import ./contracts/clock import ./contracts/deployment import ./utils/addrutils import ./namespaces -import ./logutils logScope: topics = "codex node" diff --git a/codex/conf.nim b/codex/conf.nim index 2ae40458..4a0e70f4 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -18,6 +18,7 @@ import std/strutils import std/typetraits import pkg/chronos +import pkg/chronicles import pkg/chronicles/helpers import pkg/chronicles/topics_registry import pkg/confutils/defs @@ -34,7 +35,6 @@ import pkg/questionable import pkg/questionable/results import ./discovery -import ./logutils import ./stores import ./units import ./utils @@ -53,7 +53,7 @@ type noCommand, initNode - LogKind* {.pure.} = enum + LogKind* = enum Auto = "auto" Colors = "colors" NoColors = "nocolors" @@ -276,9 +276,6 @@ type EthAddress* = ethers.Address -logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog -logutils.formatIt(LogFormat.json, EthAddress): %it - proc getCodexVersion(): string = let tag = strip(staticExec("git tag")) if tag.isEmptyOrWhitespace: @@ -410,7 +407,7 @@ proc completeCmdArg*(T: type Duration; val: string): seq[string] = discard # silly chronicles, colors is a compile-time property -proc stripAnsi*(v: string): string = +proc stripAnsi(v: string): string = var res = newStringOfCap(v.len) i: int diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index 46591100..9e5b9388 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -1,10 +1,11 @@ +import std/json import std/os import std/tables import pkg/ethers import pkg/questionable +import pkg/chronicles import ../conf -import ../logutils import ./marketplace type Deployment* = ref object diff --git a/codex/contracts/interactions/clientinteractions.nim b/codex/contracts/interactions/clientinteractions.nim index 78b3bedf..e2354d85 100644 --- a/codex/contracts/interactions/clientinteractions.nim +++ b/codex/contracts/interactions/clientinteractions.nim @@ -1,13 +1,13 @@ import pkg/ethers +import pkg/chronicles import ../../purchasing -import ../../logutils import ../market import ../clock import ./interactions export purchasing -export logutils +export chronicles except toJson type ClientInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/interactions/hostinteractions.nim b/codex/contracts/interactions/hostinteractions.nim index 2decfa44..e9749df5 100644 --- a/codex/contracts/interactions/hostinteractions.nim +++ b/codex/contracts/interactions/hostinteractions.nim @@ -1,11 +1,11 @@ -import pkg/chronos +import pkg/ethers +import pkg/chronicles -import ../../logutils import ../../sales import ./interactions export sales -export logutils +export chronicles except toJson type HostInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 51b3577b..538b44ac 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,11 +1,11 @@ import std/sequtils import std/strutils import std/sugar +import pkg/chronicles import pkg/ethers import pkg/ethers/testing import pkg/upraises import pkg/questionable -import ../logutils import ../market import ./marketplace diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 1363fb9d..60c3c577 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -1,13 +1,12 @@ import std/hashes -import std/sequtils import std/typetraits import pkg/contractabi import pkg/nimcrypto import pkg/ethers/fields import pkg/questionable/results import pkg/stew/byteutils +import pkg/json_serialization import pkg/upraises -import ../logutils import ../utils/json export contractabi @@ -80,13 +79,6 @@ proc toHex*[T: distinct](id: T): string = type baseType = T.distinctBase baseType(id).toHex -logutils.formatIt(LogFormat.textLines, Nonce): it.short0xHexLog -logutils.formatIt(LogFormat.textLines, RequestId): it.short0xHexLog -logutils.formatIt(LogFormat.textLines, SlotId): it.short0xHexLog -logutils.formatIt(LogFormat.json, Nonce): it.to0xHexLog -logutils.formatIt(LogFormat.json, RequestId): it.to0xHexLog -logutils.formatIt(LogFormat.json, SlotId): it.to0xHexLog - func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = StorageRequest( client: tupl[0], @@ -184,3 +176,17 @@ func price*(request: StorageRequest): UInt256 = func size*(ask: StorageAsk): UInt256 = ask.slots.u256 * ask.slotSize + +proc writeValue*( + writer: var JsonWriter, + value: SlotId | RequestId) {.upraises:[IOError].} = + + mixin writeValue + writer.writeValue value.toArray + +proc readValue*[T: SlotId | RequestId]( + reader: var JsonReader, + value: var T) {.upraises: [SerializationError, IOError].} = + + mixin readValue + value = T reader.readValue(T.distinctBase) diff --git a/codex/discovery.nim b/codex/discovery.nim index ce0b4032..67aacd17 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -11,6 +11,7 @@ import std/algorithm import std/sequtils import pkg/chronos +import pkg/chronicles import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope] import pkg/questionable import pkg/questionable/results @@ -20,7 +21,7 @@ import pkg/codexdht/discv5/protocol as discv5 import ./rng import ./errors -import ./logutils +import ./formats export discv5 diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 26be0473..1bab089a 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -15,10 +15,10 @@ import std/sequtils import std/sugar import pkg/chronos +import pkg/chronicles import pkg/libp2p/[multicodec, cid, multibase, multihash] import pkg/libp2p/protobuf/minprotobuf -import ../logutils import ../manifest import ../merkletree import ../stores diff --git a/codex/formats.nim b/codex/formats.nim new file mode 100644 index 00000000..38881bc9 --- /dev/null +++ b/codex/formats.nim @@ -0,0 +1,28 @@ +## Nim-Codex +## Copyright (c) 2022 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import std/strutils + +import pkg/chronicles +import pkg/libp2p/cid + +func shortLog*(cid: Cid): string = + ## Returns compact string representation of ``pid``. + var scid = $cid + if len(scid) > 10: + scid[3] = '*' + + when (NimMajor, NimMinor) > (1, 4): + scid.delete(4 .. scid.high - 6) + else: + scid.delete(4, scid.high - 6) + + scid + +chronicles.formatIt(Cid): shortLog(it) diff --git a/codex/logutils.nim b/codex/logutils.nim deleted file mode 100644 index b178d38c..00000000 --- a/codex/logutils.nim +++ /dev/null @@ -1,211 +0,0 @@ -## logutils is a module that has several goals: -## 1. Fix json logging output (run with `--log-format=json`) which was -## effectively broken for many types using default Chronicles json -## serialization. -## 2. Ability to specify log output for textlines and json sinks together or -## separately -## - This is useful if consuming json in some kind of log parser and need -## valid json with real values -## - eg a shortened Cid is nice to see in a text log in stdout, but won't -## provide a real Cid when parsed in json -## 4. Remove usages of `nim-json-serialization` from the codebase -## 5. Remove need to declare `writeValue` for new types -## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent -## conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467) -## -## When declaring a new type, one should consider importing the `codex/logutils` -## module, and specifying `formatIt`. If textlines log output and json log output -## need to be different, overload `formatIt` and specify a `LogFormat`. If json -## serialization is needed, it can be declared with a `%` proc. `logutils` -## imports and exports `utils/json` which handles the de/serialization, examples -## below. **Only `codex/logutils` needs to be imported.** -## -## Using `logutils` in the Codex codebase: -## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` -## - most of `chronicles` is exported by `logutils` -## - Instead of importing `std/json`, import `pkg/codex/utils/json` -## - `std/json` is exported by `utils/json` which is exported by `logutils` -## - Instead of importing `pkg/nim-json-serialization`, import -## `pkg/codex/utils/json` -## - one of the goals is to remove the use of `nim-json-serialization` -## -## ```nim -## import pkg/codex/logutils -## -## type -## BlockAddress* = object -## case leaf*: bool -## of true: -## treeCid* {.serialize.}: Cid -## index* {.serialize.}: Natural -## else: -## cid* {.serialize.}: Cid -## -## logutils.formatIt(LogFormat.textLines, BlockAddress): -## if it.leaf: -## "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index -## else: -## "cid: " & shortLog($it.cid) -## -## logutils.formatIt(LogFormat.json, BlockAddress): %it -## -## # chronicles textlines output -## TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0" -## # chronicles json output -## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} -## ``` -## In this case, `BlockAddress` is just an object, so `utils/json` can handle -## serializing it without issue (only fields annotated with `{.serialize.}` will -## serialize (aka opt-in serialization)). -## -## If one so wished, another option for the textlines log output, would be to -## simply `toString` the serialised json: -## ```nim -## logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it -## # or, more succinctly: -## logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson -## ``` -## In that case, both the textlines and json sinks would have the same output, -## so we could reduce this even further by not specifying a `LogFormat`: -## ```nim -## type -## BlockAddress* = object -## case leaf*: bool -## of true: -## treeCid* {.serialize.}: Cid -## index* {.serialize.}: Natural -## else: -## cid* {.serialize.}: Cid -## -## logutils.formatIt(BlockAddress): %it -## -## # chronicles textlines output -## TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}" -## # chronicles json output -## {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} -## ``` - -import std/options -import std/sequtils -import std/strutils -import std/sugar -import std/typetraits - -import pkg/chronicles except toJson, `%` -from pkg/libp2p import Cid, MultiAddress, `$` -import pkg/questionable -import pkg/stew/byteutils -import pkg/stint -import pkg/upraises - -import ./utils/json - -export byteutils -export chronicles except toJson, formatIt, `%` -export questionable -export sequtils -export strutils -export sugar -export upraises -export json - -func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = - ## Returns compact string representation of ``long``. - var short = long - let minLen = start + ellipses.len + stop - if len(short) > minLen: - short.insert(ellipses, start) - - when (NimMajor, NimMinor) > (1, 4): - short.delete(start + ellipses.len .. short.high - stop) - else: - short.delete(start + ellipses.len, short.high - stop) - - short - -func shortHexLog*(long: string): string = - if long[0..1] == "0x": result &= "0x" - result &= long[2..long.high].shortLog("..", 4, 4) - -func short0xHexLog*[N: static[int], T: array[N, byte]](v: T): string = - v.to0xHex.shortHexLog - -func short0xHexLog*[T: distinct](v: T): string = - type BaseType = T.distinctBase - BaseType(v).short0xHexLog - -func short0xHexLog*[U: distinct, T: seq[U]](v: T): string = - type BaseType = U.distinctBase - "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]" - -func to0xHexLog*[T: distinct](v: T): string = - type BaseType = T.distinctBase - BaseType(v).to0xHex - -func to0xHexLog*[U: distinct, T: seq[U]](v: T): string = - type BaseType = U.distinctBase - "@[" & v.map(x => BaseType(x).to0xHex).join(",") & "]" - -proc formatTextLineSeq*(val: seq[string]): string = - "@[" & val.join(", ") & "]" - -template formatIt*(format: LogFormat, T: typedesc, body: untyped) {.dirty.} = - # Provides formatters for logging with Chronicles for the given type and - # `LogFormat`. - # NOTE: `seq[T]`, `Option[T]`, and `seq[Option[T]]` are overriddden - # since the base `setProperty` is generic using `auto` and conflicts with - # providing a generic `seq` and `Option` override. - when format == LogFormat.json: - proc formatJsonOption(val: ?T): JsonNode = - if it =? val: - json.`%`(body) - else: - newJNull() - - proc setProperty*(r: var JsonRecord, key: string, opt: ?T) = - let v = opt.formatJsonOption - setProperty(r, key, v) - - proc setProperty*(r: var JsonRecord, key: string, opts: seq[?T]) = - let v = opts.map(opt => opt.formatJsonOption) - setProperty(r, key, json.`%`(v)) - - proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) = - let v = val.map(it => body) - setProperty(r, key, json.`%`(v)) - - proc setProperty*(r: var JsonRecord, key: string, it: T) {.upraises:[ValueError, IOError].} = - let v = body - setProperty(r, key, json.`%`(v)) - - elif format == LogFormat.textLines: - proc formatTextLineOption*(val: ?T): string = - var v = "none(" & $T & ")" - if it =? val: - v = "some(" & $(body) & ")" # that I used to know :) - v - - proc setProperty*(r: var TextLineRecord, key: string, opt: ?T) = - let v = opt.formatTextLineOption - setProperty(r, key, v) - - proc setProperty*(r: var TextLineRecord, key: string, opts: seq[?T]) = - let v = opts.map(opt => opt.formatTextLineOption) - setProperty(r, key, v.formatTextLineSeq) - - proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) = - let v = val.map(it => body) - setProperty(r, key, v.formatTextLineSeq) - - proc setProperty*(r: var TextLineRecord, key: string, it: T) {.upraises:[ValueError, IOError].} = - let v = body - setProperty(r, key, v) - -template formatIt*(T: type, body: untyped) {.dirty.} = - formatIt(LogFormat.textLines, T): body - formatIt(LogFormat.json, T): body - -formatIt(LogFormat.textLines, Cid): shortLog($it) -formatIt(LogFormat.json, Cid): $it -formatIt(UInt256): $it -formatIt(MultiAddress): $it diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index 24f5199a..7bf173f9 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -19,12 +19,12 @@ import std/sequtils import pkg/libp2p import pkg/questionable import pkg/questionable/results +import pkg/chronicles import pkg/chronos import ./manifest import ../errors import ../blocktype -import ../logutils import ./types proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index f2bf319b..8f8df55c 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -17,13 +17,13 @@ import pkg/libp2p/protobuf/minprotobuf import pkg/libp2p import pkg/questionable import pkg/questionable/results +import pkg/chronicles import ../errors import ../utils import ../utils/json import ../units import ../blocktype -import ../logutils import ./types export types diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index afc49be1..74ddba6d 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -13,6 +13,7 @@ import std/sequtils import std/sugar import std/algorithm +import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/nimcrypto/sha2 @@ -20,7 +21,6 @@ import pkg/libp2p/[cid, multicodec, multihash, vbuffer] import pkg/stew/byteutils import ../errors -import ../logutils logScope: topics = "codex merkletree" diff --git a/codex/node.nim b/codex/node.nim index 56893ca3..60606c36 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -15,6 +15,7 @@ import std/sugar import pkg/questionable import pkg/questionable/results +import pkg/chronicles import pkg/chronos import pkg/libp2p/[switch, multicodec, multihash] @@ -38,10 +39,8 @@ import ./contracts import ./node/batch import ./utils import ./errors -import ./logutils export batch -export logutils logScope: topics = "codex node" @@ -484,7 +483,7 @@ proc start*(node: CodexNodeRef) {.async.} = node.contracts.validator = ValidatorInteractions.none node.networkId = node.switch.peerInfo.peerId - notice "Started codex node", id = node.networkId, addrs = node.switch.peerInfo.addrs + notice "Started codex node", id = $node.networkId, addrs = node.switch.peerInfo.addrs proc stop*(node: CodexNodeRef) {.async.} = trace "Stopping node" diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim index 226fcbee..ee5c3a16 100644 --- a/codex/purchasing/purchaseid.nim +++ b/codex/purchasing/purchaseid.nim @@ -1,12 +1,8 @@ import std/hashes import pkg/nimcrypto -import ../logutils type PurchaseId* = distinct array[32, byte] -logutils.formatIt(LogFormat.textLines, PurchaseId): it.short0xHexLog -logutils.formatIt(LogFormat.json, PurchaseId): it.to0xHexLog - proc hash*(x: PurchaseId): Hash {.borrow.} proc `==`*(x, y: PurchaseId): bool {.borrow.} proc toHex*(x: PurchaseId): string = array[32, byte](x).toHex diff --git a/codex/purchasing/states/cancelled.nim b/codex/purchasing/states/cancelled.nim index f9bb1ece..a4765276 100644 --- a/codex/purchasing/states/cancelled.nim +++ b/codex/purchasing/states/cancelled.nim @@ -1,8 +1,8 @@ import pkg/metrics - -import ../../logutils +import pkg/chronicles import ../statemachine import ./errorhandling +import ./error declareCounter(codex_purchases_cancelled, "codex purchases cancelled") @@ -18,7 +18,7 @@ method run*(state: PurchaseCancelled, machine: Machine): Future[?State] {.async. codex_purchases_cancelled.inc() let purchase = Purchase(machine) - warn "Request cancelled, withdrawing remaining funds", requestId = purchase.requestId + warn "Request cancelled, withdrawing remaining funds", requestId = $purchase.requestId await purchase.market.withdrawFunds(purchase.requestId) let error = newException(Timeout, "Purchase cancelled due to timeout") diff --git a/codex/purchasing/states/error.nim b/codex/purchasing/states/error.nim index 0ebe1dbe..baa0cceb 100644 --- a/codex/purchasing/states/error.nim +++ b/codex/purchasing/states/error.nim @@ -1,7 +1,7 @@ import pkg/metrics +import pkg/chronicles import ../statemachine import ../../utils/exceptions -import ../../logutils declareCounter(codex_purchases_error, "codex purchases error") diff --git a/codex/purchasing/states/finished.nim b/codex/purchasing/states/finished.nim index 0f97150d..4a2662cf 100644 --- a/codex/purchasing/states/finished.nim +++ b/codex/purchasing/states/finished.nim @@ -1,7 +1,6 @@ import pkg/metrics - +import pkg/chronicles import ../statemachine -import ../../logutils declareCounter(codex_purchases_finished, "codex purchases finished") diff --git a/codex/purchasing/states/started.nim b/codex/purchasing/states/started.nim index fff1bbc8..04736a13 100644 --- a/codex/purchasing/states/started.nim +++ b/codex/purchasing/states/started.nim @@ -1,6 +1,5 @@ import pkg/metrics - -import ../../logutils +import pkg/chronicles import ../statemachine import ./errorhandling import ./finished diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim index 391dbf7c..f3a68512 100644 --- a/codex/purchasing/states/submitted.nim +++ b/codex/purchasing/states/submitted.nim @@ -1,6 +1,5 @@ import pkg/metrics - -import ../../logutils +import pkg/chronicles import ../statemachine import ./errorhandling import ./started diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 6100f35c..2f1b192c 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -16,6 +16,7 @@ import std/sequtils import pkg/questionable import pkg/questionable/results +import pkg/chronicles except toJson import pkg/chronos import pkg/presto except toJson import pkg/metrics except toJson @@ -27,7 +28,6 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/codexdht/discv5/spr as spr -import ../logutils import ../node import ../blocktype import ../conf diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 181ea584..078a0124 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -111,4 +111,23 @@ func `%`*(obj: StorageRequest | Slot): JsonNode = return jsonObj -func `%`*(obj: RestNodeId): JsonNode = % $obj.id +func `%`*(obj: Cid): JsonNode = + % $obj + +func `%`*(obj: PeerId): JsonNode = + % $obj + +func `%`*(obj: RestNodeId): JsonNode = + % $obj.id + +func `%`*(obj: SignedPeerRecord): JsonNode = + % $obj + +func `%`*(obj: dn.Address): JsonNode = + % $obj + +func `%`*(obj: AddressInfo): JsonNode = + % $obj.address + +func `%`*(obj: MultiAddress): JsonNode = + % $obj diff --git a/codex/sales.nim b/codex/sales.nim index dba4e18d..17a97f77 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -3,13 +3,13 @@ import std/sugar import pkg/questionable import pkg/questionable/results import pkg/stint +import pkg/chronicles import pkg/datastore import ./market import ./clock import ./stores import ./contracts/requests import ./contracts/marketplace -import ./logutils import ./sales/salescontext import ./sales/salesagent import ./sales/statemachine @@ -165,7 +165,7 @@ proc filled( processing.complete() proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = - debug "processing slot from queue", requestId = item.requestId, + debug "processing slot from queue", requestId = $item.requestId, slot = item.slotIndex let agent = newSalesAgent( diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 6c68853e..611c1ad6 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -28,19 +28,19 @@ push: {.upraises: [].} import std/typetraits import pkg/chronos +import pkg/chronicles except toJson import pkg/datastore import pkg/nimcrypto import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/stew/byteutils -import ../logutils import ../stores import ../contracts/requests import ../utils/json export requests -export logutils +export chronicles except toJson logScope: topics = "sales reservations" @@ -139,8 +139,13 @@ proc toErr[E1: ref CatchableError, E2: ReservationsError]( return newException(E2, msg, e1) -logutils.formatIt(LogFormat.textLines, SomeStorableId): it.short0xHexLog -logutils.formatIt(LogFormat.json, SomeStorableId): it.to0xHexLog +proc writeValue*( + writer: var JsonWriter, + value: SomeStorableId) {.upraises:[IOError].} = + ## used for chronicles' logs + + mixin writeValue + writer.writeValue %value proc `onAvailabilityAdded=`*(self: Reservations, onAvailabilityAdded: OnAvailabilityAdded) = diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 21475426..08d3ab0e 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -1,11 +1,11 @@ import pkg/chronos +import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/upraises import ../contracts/requests import ../errors -import ../logutils import ./statemachine import ./salescontext import ./salesdata diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim index a875f917..fa821864 100644 --- a/codex/sales/slotqueue.nim +++ b/codex/sales/slotqueue.nim @@ -1,11 +1,11 @@ import std/sequtils import std/tables +import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/upraises import ../errors -import ../logutils import ../rng import ../utils import ../contracts/requests diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim index aafed177..f1f75407 100644 --- a/codex/sales/states/cancelled.nim +++ b/codex/sales/states/cancelled.nim @@ -1,4 +1,4 @@ -import ../../logutils +import pkg/chronicles import ../salesagent import ../statemachine import ./errorhandling @@ -21,7 +21,7 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting collateral and partial payout", requestId = data.requestId, slotIndex = data.slotIndex + debug "Collecting collateral and partial payout", requestId = $data.requestId, slotIndex = $data.slotIndex await market.freeSlot(slot.id) if onClear =? agent.context.onClear and @@ -31,4 +31,4 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} = if onCleanUp =? agent.onCleanUp: await onCleanUp(returnBytes = true) - warn "Sale cancelled due to timeout", requestId = data.requestId, slotIndex = data.slotIndex + warn "Sale cancelled due to timeout", requestId = $data.requestId, slotIndex = $data.slotIndex diff --git a/codex/sales/states/downloading.nim b/codex/sales/states/downloading.nim index 7c71f249..f79f0d68 100644 --- a/codex/sales/states/downloading.nim +++ b/codex/sales/states/downloading.nim @@ -1,8 +1,7 @@ +import pkg/chronicles import pkg/questionable import pkg/questionable/results - import ../../blocktype as bt -import ../../logutils import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim index 51f34bc9..59754eca 100644 --- a/codex/sales/states/errored.nim +++ b/codex/sales/states/errored.nim @@ -1,10 +1,9 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises - +import pkg/chronicles import ../statemachine import ../salesagent -import ../../logutils import ../../utils/exceptions logScope: diff --git a/codex/sales/states/failed.nim b/codex/sales/states/failed.nim index e32fbb58..cd954be4 100644 --- a/codex/sales/states/failed.nim +++ b/codex/sales/states/failed.nim @@ -1,4 +1,4 @@ -import ../../logutils +import pkg/chronicles import ../salesagent import ../statemachine import ./errorhandling @@ -21,7 +21,7 @@ method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Removing slot from mySlots", requestId = data.requestId, slotIndex = data.slotIndex + debug "Removing slot from mySlots", requestId = $data.requestId, slotIndex = $data.slotIndex await market.freeSlot(slot.id) let error = newException(SaleFailedError, "Sale failed") diff --git a/codex/sales/states/filled.nim b/codex/sales/states/filled.nim index 6e3e0106..c5bc4c27 100644 --- a/codex/sales/states/filled.nim +++ b/codex/sales/states/filled.nim @@ -1,8 +1,7 @@ import pkg/questionable import pkg/questionable/results - +import pkg/chronicles import ../../conf -import ../../logutils import ../statemachine import ../salesagent import ./errorhandling @@ -37,7 +36,7 @@ method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} = let me = await market.getSigner() if host == me.some: - info "Slot succesfully filled", requestId = data.requestId, slotIndex = data.slotIndex + info "Slot succesfully filled", requestId = $data.requestId, slotIndex = $data.slotIndex without request =? data.request: raiseAssert "no sale request" diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index b6646733..b1e8471c 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -1,4 +1,4 @@ -import ../../logutils +import pkg/chronicles import ../../market import ../statemachine import ../salesagent @@ -32,5 +32,5 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} = without (collateral =? data.request.?ask.?collateral): raiseAssert "Request not set" - debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex + debug "Filling slot", requestId = $data.requestId, slotIndex = $data.slotIndex await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) diff --git a/codex/sales/states/finished.nim b/codex/sales/states/finished.nim index 59e9244c..539cde62 100644 --- a/codex/sales/states/finished.nim +++ b/codex/sales/states/finished.nim @@ -1,6 +1,5 @@ import pkg/chronos - -import ../../logutils +import pkg/chronicles import ../statemachine import ../salesagent import ./errorhandling @@ -28,7 +27,7 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} = without request =? data.request: raiseAssert "no sale request" - info "Slot finished and paid out", requestId = data.requestId, slotIndex = data.slotIndex + info "Slot finished and paid out", requestId = $data.requestId, slotIndex = $data.slotIndex if onCleanUp =? agent.onCleanUp: await onCleanUp() diff --git a/codex/sales/states/ignored.nim b/codex/sales/states/ignored.nim index d757e9c1..fa0641cc 100644 --- a/codex/sales/states/ignored.nim +++ b/codex/sales/states/ignored.nim @@ -1,6 +1,5 @@ +import pkg/chronicles import pkg/chronos - -import ../../logutils import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/initialproving.nim b/codex/sales/states/initialproving.nim index 3496b292..d44b058b 100644 --- a/codex/sales/states/initialproving.nim +++ b/codex/sales/states/initialproving.nim @@ -1,4 +1,4 @@ -import ../../logutils +import pkg/chronicles import ../statemachine import ../salesagent import ./errorhandling @@ -30,11 +30,11 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async without onProve =? context.onProve: raiseAssert "onProve callback not set" - debug "Generating initial proof", requestId = data.requestId + debug "Generating initial proof", requestId = $data.requestId let slot = Slot(request: request, slotIndex: data.slotIndex) challenge = await context.market.getChallenge(slot.id) proof = await onProve(slot, challenge) - debug "Finished proof calculation", requestId = data.requestId + debug "Finished proof calculation", requestId = $data.requestId return some State(SaleFilling(proof: proof)) diff --git a/codex/sales/states/payout.nim b/codex/sales/states/payout.nim index 5c8c2859..a70f7eac 100644 --- a/codex/sales/states/payout.nim +++ b/codex/sales/states/payout.nim @@ -1,4 +1,4 @@ -import ../../logutils +import pkg/chronicles import ../../market import ../statemachine import ../salesagent @@ -29,7 +29,7 @@ method run(state: SalePayout, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting finished slot's reward", requestId = data.requestId, slotIndex = data.slotIndex + debug "Collecting finished slot's reward", requestId = $data.requestId, slotIndex = $data.slotIndex await market.freeSlot(slot.id) return some State(SaleFinished()) diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index 973446e2..bb0fe743 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -1,7 +1,6 @@ +import pkg/chronicles import pkg/questionable import pkg/questionable/results - -import ../../logutils import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim index 6fba9096..367eaa9e 100644 --- a/codex/sales/states/proving.nim +++ b/codex/sales/states/proving.nim @@ -1,7 +1,6 @@ import std/options - +import pkg/chronicles import ../../clock -import ../../logutils import ../statemachine import ../salesagent import ../salescontext @@ -29,7 +28,7 @@ method prove*( ) {.base, async.} = try: let proof = await onProve(slot, challenge) - debug "Submitting proof", currentPeriod = currentPeriod, slotId = slot.id + debug "Submitting proof", currentPeriod = currentPeriod, slotId = $slot.id await market.submitProof(slot.id, proof) except CatchableError as e: error "Submitting proof failed", msg = e.msg @@ -48,9 +47,9 @@ proc proveLoop( logScope: period = currentPeriod - requestId = request.id + requestId = $request.id slotIndex - slotId = slot.id + slotId = $slot.id proc getCurrentPeriod(): Future[Period] {.async.} = let periodicity = await market.periodicity() @@ -107,7 +106,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} = without clock =? context.clock: raiseAssert("clock not set") - debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex + debug "Start proving", requestId = $data.requestId, slotIndex = $data.slotIndex try: let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve) state.loop = loop @@ -119,7 +118,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} = return some State(SaleErrored(error: e)) finally: # Cleanup of the proving loop - debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex + debug "Stopping proving.", requestId = $data.requestId, slotIndex = $data.slotIndex if not state.loop.isNil: if not state.loop.finished: diff --git a/codex/sales/states/provingsimulated.nim b/codex/sales/states/provingsimulated.nim index 78ce5ee5..e3f5b2c2 100644 --- a/codex/sales/states/provingsimulated.nim +++ b/codex/sales/states/provingsimulated.nim @@ -1,12 +1,12 @@ import ../../conf when codex_enable_proof_failures: import std/strutils + import pkg/chronicles import pkg/stint import pkg/ethers import pkg/ethers/testing import ../../contracts/requests - import ../../logutils import ../../market import ../salescontext import ./proving @@ -20,7 +20,7 @@ when codex_enable_proof_failures: proofCount: int proc onSubmitProofError(error: ref CatchableError, period: UInt256, slotId: SlotId) = - error "Submitting invalid proof failed", period = period, slotId, msg = error.msg + error "Submitting invalid proof failed", period = period, slotId = $slotId, msg = error.msg method prove*(state: SaleProvingSimulated, slot: Slot, challenge: ProofChallenge, onProve: OnProve, market: Market, currentPeriod: Period) {.async.} = trace "Processing proving in simulated mode" diff --git a/codex/sales/states/unknown.nim b/codex/sales/states/unknown.nim index db00f517..3672ed96 100644 --- a/codex/sales/states/unknown.nim +++ b/codex/sales/states/unknown.nim @@ -1,4 +1,4 @@ -import ../../logutils +import pkg/chronicles import ../statemachine import ../salesagent import ./filled diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index 7a21ab8a..13308c7a 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -13,6 +13,7 @@ push: {.upraises: [].} import std/options +import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/lrucache @@ -23,7 +24,6 @@ import ./blockstore import ../units import ../chunker import ../errors -import ../logutils import ../manifest import ../merkletree import ../utils diff --git a/codex/stores/maintenance.nim b/codex/stores/maintenance.nim index 343fed8f..98b6fab9 100644 --- a/codex/stores/maintenance.nim +++ b/codex/stores/maintenance.nim @@ -11,6 +11,7 @@ ## Looks for and removes expired blocks from blockstores. import pkg/chronos +import pkg/chronicles import pkg/questionable import pkg/questionable/results @@ -18,7 +19,6 @@ import ./repostore import ../utils/timer import ../utils/asynciter import ../clock -import ../logutils import ../systemclock const diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 2f0f1f87..6da1465c 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -7,22 +7,25 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. - import pkg/upraises + push: {.upraises: [].} +import std/sugar + +import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/questionable/results -import ../clock -import ../blocktype -import ../blockexchange -import ../logutils -import ../merkletree import ../utils/asyncheapqueue import ../utils/asynciter +import ../clock + +import ../blocktype import ./blockstore +import ../blockexchange +import ../merkletree export blockstore, blockexchange, asyncheapqueue diff --git a/codex/stores/repostore.nim b/codex/stores/repostore.nim index b07e148f..db53ae6f 100644 --- a/codex/stores/repostore.nim +++ b/codex/stores/repostore.nim @@ -13,6 +13,7 @@ push: {.upraises: [].} import pkg/chronos import pkg/chronos/futures +import pkg/chronicles import pkg/libp2p/[cid, multicodec, multihash] import pkg/lrucache import pkg/metrics @@ -26,7 +27,6 @@ import ./keyutils import ../blocktype import ../clock import ../systemclock -import ../logutils import ../merkletree import ../utils diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index a8a55955..f2491e8d 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -11,10 +11,9 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos +import pkg/chronicles import pkg/libp2p -import ../logutils - logScope: topics = "libp2p asyncstreamwrapper" @@ -38,7 +37,7 @@ proc new*( writer: AsyncStreamWriter = nil ): AsyncStreamWrapper = ## Create new instance of an asynchronous stream wrapper - ## + ## let stream = C(reader: reader, writer: writer) diff --git a/codex/streams/seekablestream.nim b/codex/streams/seekablestream.nim index b07e1116..785d9afe 100644 --- a/codex/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -9,10 +9,9 @@ import pkg/libp2p/stream/lpstream import pkg/chronos +import pkg/chronicles -import ../logutils - -export lpstream, chronos, logutils +export lpstream, chronos, chronicles logScope: topics = "codex seekablestream" diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index d3b8c036..63406a5f 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -14,12 +14,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos +import pkg/chronicles import pkg/stew/ptrops import ../stores import ../manifest import ../blocktype -import ../logutils import ../utils import ./seekablestream @@ -53,7 +53,7 @@ proc new*( pad = true ): StoreStream = ## Create a new StoreStream instance for a given store and manifest - ## + ## result = StoreStream( store: store, manifest: manifest, @@ -80,7 +80,7 @@ method readOnce*( ## Read `nbytes` from current position in the StoreStream into output buffer pointed by `pbytes`. ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. - ## + ## trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.blocksCount if self.atEof: diff --git a/codex/units.nim b/codex/units.nim index 57b52ed6..dcda8152 100644 --- a/codex/units.nim +++ b/codex/units.nim @@ -6,14 +6,14 @@ ## at your option. ## This file may not be copied, modified, or distributed except according to ## those terms. -## +## import std/hashes import std/strutils import pkg/upraises - -import ./logutils +import pkg/json_serialization +import pkg/json_serialization/std/options type NBytes* = distinct Natural @@ -42,7 +42,6 @@ divMaths(NBytes) proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" proc `'nb`*(n: string): NBytes = parseInt(n).NBytes -logutils.formatIt(NBytes): $it const MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz @@ -54,6 +53,18 @@ func divUp*[T: NBytes](a, b : T): int = assert(b != T(0)) if a==T(0): int(0) else: int( ((a - T(1)) div b) + 1 ) +proc writeValue*( + writer: var JsonWriter, + value: NBytes +) {.upraises:[IOError].} = + writer.writeValue value.int + +proc readValue*( + reader: var JsonReader, + value: var NBytes +) {.upraises: [SerializationError, IOError].} = + value = NBytes reader.readValue(int) + when isMainModule: import unittest2 diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 9aeb3eab..512617ee 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -1,10 +1,10 @@ import std/sugar import pkg/questionable import pkg/chronos +import pkg/chronicles import pkg/upraises -import ../logutils -import ./then import ./trackedfutures +import ./then push: {.upraises:[].} diff --git a/codex/utils/fileutils.nim b/codex/utils/fileutils.nim index 847ce06c..03cffeba 100644 --- a/codex/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -9,17 +9,16 @@ ## Partially taken from nim beacon chain +import std/strutils import pkg/upraises push: {.upraises: [].} -import std/strutils -import pkg/stew/io2 - -import ../logutils +import pkg/chronicles +import stew/io2 export io2 -export logutils +export chronicles except toJson when defined(windows): import stew/[windows/acl] diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 080d4e89..ade6936d 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -6,10 +6,9 @@ import std/strutils import std/strformat import std/tables import std/typetraits -from pkg/ethers import Address -from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` +import pkg/chronicles +from pkg/libp2p import Cid, init import pkg/contractabi -import pkg/codexdht/discv5/node as dn import pkg/stew/byteutils import pkg/stint import pkg/questionable/results @@ -17,6 +16,9 @@ import ../errors export json except `%`, `%*` +logScope: + topics = "json serialization" + type SerializationError = object of CodexError UnexpectedKindError = object of SerializationError @@ -301,20 +303,6 @@ func `%`*[T: distinct](id: T): JsonNode = type baseType = T.distinctBase % baseType(id) -func `%`*(cid: Cid): JsonNode = % $cid - -func `%`*(obj: PeerId): JsonNode = % $obj - -func `%`*(obj: SignedPeerRecord): JsonNode = % $obj - -func `%`*(obj: dn.Address): JsonNode = % $obj - -func `%`*(obj: AddressInfo): JsonNode = % $obj.address - -func `%`*(obj: MultiAddress): JsonNode = % $obj - -func `%`*(address: ethers.Address): JsonNode = % $address - func toJson*[T](item: T): string = $(%item) proc toJsnImpl(x: NimNode): NimNode = diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index c7f76263..ef6f6246 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -10,12 +10,12 @@ import pkg/upraises push: {.upraises: [].} +import pkg/chronicles import pkg/questionable/results import pkg/libp2p/crypto/crypto import ./fileutils import ../errors -import ../logutils import ../rng export crypto @@ -39,6 +39,6 @@ proc setupKey*(path: string): ?!PrivateKey = warn "The network private key file is not safe, aborting" return failure newException( CodexKeyUnsafeError, "The network private key file is not safe") - + let kb = ? path.readAllBytes().mapFailure(CodexKeyError) return PrivateKey.init(kb).mapFailure(CodexKeyError) diff --git a/codex/utils/timer.nim b/codex/utils/timer.nim index 64180dc7..4eefc599 100644 --- a/codex/utils/timer.nim +++ b/codex/utils/timer.nim @@ -11,10 +11,9 @@ ## Used to execute a callback in a loop import pkg/chronos +import pkg/chronicles import pkg/upraises -import ../logutils - type TimerCallback* = proc(): Future[void] {.gcsafe, upraises:[].} Timer* = ref object of RootObj diff --git a/codex/utils/trackedfutures.nim b/codex/utils/trackedfutures.nim index f3fcdb2d..064bf9e3 100644 --- a/codex/utils/trackedfutures.nim +++ b/codex/utils/trackedfutures.nim @@ -1,8 +1,7 @@ import std/sugar import std/tables +import pkg/chronicles import pkg/chronos - -import ../logutils import ../utils/then type diff --git a/codex/validation.nim b/codex/validation.nim index b549997e..2be39975 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -1,9 +1,9 @@ import std/sets import std/sequtils import pkg/chronos +import pkg/chronicles import ./market import ./clock -import ./logutils export market export sets @@ -48,7 +48,7 @@ proc subscribeSlotFilled(validation: Validation) {.async.} = let slotId = slotId(requestId, slotIndex) if slotId notin validation.slots: if validation.slots.len < validation.maxSlots: - trace "Adding slot", slotId + trace "Adding slot", slotId = $slotId validation.slots.incl(slotId) let subscription = await validation.market.subscribeSlotFilled(onSlotFilled) validation.subscriptions.add(subscription) @@ -58,7 +58,7 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = for slotId in validation.slots: let state = await validation.market.slotState(slotId) if state != SlotState.Filled: - trace "Removing slot", slotId + trace "Removing slot", slot = $slotId ended.incl(slotId) validation.slots.excl(ended) @@ -70,7 +70,7 @@ proc markProofAsMissing(validation: Validation, try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): - trace "Marking proof as missing", slotId, periodProofMissed = period + trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period await validation.market.markProofAsMissing(slotId, period) else: let inDowntime {.used.} = await validation.market.inDowntime(slotId) diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 18d1e429..706bcdbd 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -1,11 +1,11 @@ import std/sequtils import pkg/asynctest +import pkg/chronicles import pkg/chronos import pkg/datastore import pkg/questionable import pkg/questionable/results -import pkg/codex/logutils import pkg/codex/sales/slotqueue import ../helpers diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 57babc79..860e52b6 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/stew/byteutils import pkg/codex/chunker -import pkg/codex/logutils +import pkg/chronicles import pkg/chronos import ./helpers diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim deleted file mode 100644 index 76923df6..00000000 --- a/tests/codex/testlogutils.nim +++ /dev/null @@ -1,214 +0,0 @@ -import std/options -import std/strutils -import std/unittest -import pkg/codex/blocktype -import pkg/codex/conf -import pkg/codex/contracts/requests -import pkg/codex/logutils -import pkg/codex/purchasing/purchaseid -import pkg/codex/units -import pkg/libp2p/cid -import pkg/libp2p/multiaddress -import pkg/stew/byteutils -import pkg/stint -import ../checktest - -export logutils - -logStream testlines[textlines[nocolors,notimestamps,dynamic]] -logStream testjson[json[nocolors,notimestamps,dynamic]] - -type - ObjectType = object - a: string - DistinctType {.borrow: `.`.} = distinct ObjectType - RefType = ref object - a: string - AnotherType = object - a: int - -# must be defined at the top-level -proc `$`*(t: ObjectType): string = "used `$`" -func `%`*(t: RefType): JsonNode = % t.a -logutils.formatIt(LogFormat.textLines, ObjectType): "formatted_" & it.a -logutils.formatIt(LogFormat.textLines, RefType): "formatted_" & it.a -logutils.formatIt(LogFormat.textLines, DistinctType): "formatted_" & it.a -logutils.formatIt(LogFormat.json, ObjectType): "formatted_" & it.a -logutils.formatIt(LogFormat.json, RefType): %it -logutils.formatIt(LogFormat.json, DistinctType): "formatted_" & it.a -logutils.formatIt(AnotherType): it.a - -checksuite "Test logging output": - var outputLines: string - var outputJson: string - - proc writeToLines(logLevel: LogLevel, msg: LogOutputStr) = - outputLines &= msg - - proc writeToJson(logLevel: LogLevel, msg: LogOutputStr) = - outputJson &= msg - - setup: - outputLines = "" - outputJson = "" - testlines.outputs[0].writer = writeToLines - testjson.outputs[0].writer = writeToJson - - template logged(prop, expected): auto = - let toFind = prop & "=" & expected - outputLines.contains(toFind) - - template loggedJson(prop, expected): auto = - let json = $ parseJson(outputJson){prop} - json == expected - - template log(val) = - testlines.trace "test", val - testjson.trace "test", val - - test "logs objects": - let t = ObjectType(a: "a") - log t - check logged("t", "formatted_a") - check loggedJson("t", "\"formatted_a\"") - - test "logs sequences of objects": - let t1 = ObjectType(a: "a") - let t2 = ObjectType(a: "b") - let t = @[t1, t2] - log t - check logged("t", "\"@[formatted_a, formatted_b]\"") - check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") - - test "logs ref types": - let t = RefType(a: "a") - log t - check logged("t", "formatted_a") - check loggedJson("t", "\"a\"") - - test "logs sequences of ref types": - let t1 = RefType(a: "a") - let t2 = RefType(a: "b") - let t = @[t1, t2] - log t - check logged("t", "\"@[formatted_a, formatted_b]\"") - check loggedJson("t", "[\"a\",\"b\"]") - - test "logs distinct types": - let t = DistinctType(ObjectType(a: "a")) - log t - check logged("t", "formatted_a") - check loggedJson("t", "\"formatted_a\"") - - test "logs sequences of distinct types": - let t1 = DistinctType(ObjectType(a: "a")) - let t2 = DistinctType(ObjectType(a: "b")) - let t = @[t1, t2] - log t - check logged("t", "\"@[formatted_a, formatted_b]\"") - check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") - - test "formatIt can return non-string types": - let t = AnotherType(a: 1) - log t - check logged("t", "1") - check loggedJson("t", "1") - - test "logs Option types": - let t = some ObjectType(a: "a") - log t - check logged("t", "some(formatted_a)") - check loggedJson("t", "\"formatted_a\"") - - test "logs sequences of Option types": - let t1 = some ObjectType(a: "a") - let t2 = none ObjectType - let t = @[t1, t2] - log t - check logged("t", "\"@[some(formatted_a), none(ObjectType)]\"") - check loggedJson("t", """["formatted_a",null]""") - - test "can define `$` override for T": - let o = ObjectType() - check $o == "used `$`" - - test "logs NByte correctly": - let nb = 12345.NBytes - log nb - check logged("nb", "12345\'NByte") - check loggedJson("nb", "\"12345\'NByte\"") - - test "logs BlockAddress correctly": - let cid = Cid.init("zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1").tryGet - let ba = BlockAddress.init(cid, 0) - log ba - check logged("ba", "\"treeCid: zb2*fndjU1, index: 0\"") - check loggedJson("ba", """{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}""") - - test "logs Cid correctly": - let cid = Cid.init("zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD").tryGet - log cid - check logged("cid", "zb2*A5TgiD") - check loggedJson("cid", "\"zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD\"") - - test "logs StUint correctly": - let stint = 12345678901234.u256 - log stint - check logged("stint", "12345678901234") - check loggedJson("stint", "\"12345678901234\"") - - test "logs int correctly": - let int = 123 - log int - check logged("int", "123") - check loggedJson("int", "123") - - test "logs EthAddress correctly": - let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b") - log address - check logged("address", "0xf75e..1b1b") - check loggedJson("address", "\"0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b\"") - - test "logs PurchaseId correctly": - let id = PurchaseId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") - log id - check logged("id", "0x7120..3dea") - - test "logs RequestId correctly": - let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") - log id - check logged("id", "0x7120..3dea") - check loggedJson("id", "\"0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea\"") - - test "logs seq[RequestId] correctly": - let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") - let id2 = RequestId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") - let ids = @[id, id2] - log ids - check logged("ids", "\"@[0x7120..3dea, 0x9ab2..c282]\"") - check loggedJson("ids", """["0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea","0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282"]""") - - test "logs SlotId correctly": - let id = SlotId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") - log id - check logged("id", "0x9ab2..c282") - check loggedJson("id", "\"0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282\"") - - test "logs Nonce correctly": - let id = SlotId.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea") - log id - check logged("id", "0xce88..d6ea") - check loggedJson("id", "\"0xce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea\"") - - test "logs MultiAddress correctly": - let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet - log ma - check logged("ma", "/ip4/127.0.0.1/tcp/0") - check loggedJson("ma", "\"/ip4/127.0.0.1/tcp/0\"") - - test "logs seq[MultiAddress] correctly": - let ma = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet, - MultiAddress.init("/ip4/127.0.0.2/tcp/1").tryGet] - log ma - check logged("ma", "\"@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]\"") - check loggedJson("ma", "[\"/ip4/127.0.0.1/tcp/0\",\"/ip4/127.0.0.2/tcp/1\"]") diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index 7fb996fe..bb3876e3 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -5,6 +5,7 @@ import std/times import pkg/asynctest import pkg/chronos +import pkg/chronicles import pkg/stew/byteutils import pkg/datastore import pkg/questionable @@ -17,7 +18,6 @@ import pkg/codexdht/discv5/protocol as discv5 import pkg/codex/stores import pkg/codex/clock import pkg/codex/contracts -import pkg/codex/logutils import pkg/codex/systemclock import pkg/codex/blockexchange import pkg/codex/chunker diff --git a/tests/codex/utils/testjson.nim b/tests/codex/utils/testjson.nim index 5ba7fff3..aa3731e5 100644 --- a/tests/codex/utils/testjson.nim +++ b/tests/codex/utils/testjson.nim @@ -3,11 +3,11 @@ import std/options import std/strformat import std/strutils import std/unittest +import pkg/chronicles except toJson import pkg/stew/byteutils import pkg/stint import pkg/codex/contracts/requests from pkg/codex/rest/json import RestPurchase -import pkg/codex/logutils import pkg/codex/utils/json as utilsjson import pkg/questionable import pkg/questionable/results diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index 4864f992..22aaeba0 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -1,3 +1,4 @@ +import std/json import pkg/chronos import pkg/ethers/testing import pkg/ethers/erc20 diff --git a/tests/ethertest.nim b/tests/ethertest.nim index 5c620310..1166b0e0 100644 --- a/tests/ethertest.nim +++ b/tests/ethertest.nim @@ -1,4 +1,4 @@ - +import std/json import pkg/asynctest import pkg/ethers diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 7cb0b50e..dbddc204 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -3,9 +3,9 @@ import std/strutils import std/sequtils from pkg/libp2p import Cid, `$`, init +import pkg/chronicles import pkg/stint import pkg/questionable/results -import pkg/codex/logutils import pkg/codex/rest/json import pkg/codex/purchasing import pkg/codex/errors diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 6c9a8bf8..b601e6db 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -1,7 +1,8 @@ import std/os import std/macros +import std/json import std/httpclient -import pkg/codex/logutils +import pkg/chronicles import ../ethertest import ./codexclient import ./nodes diff --git a/tests/integration/nodes.nim b/tests/integration/nodes.nim index a50f9974..777de80b 100644 --- a/tests/integration/nodes.nim +++ b/tests/integration/nodes.nim @@ -1,12 +1,12 @@ +import pkg/questionable +import pkg/confutils +import pkg/chronicles +import pkg/libp2p import std/osproc import std/os import std/streams import std/strutils -import pkg/codex/conf -import pkg/codex/logutils -import pkg/confutils -import pkg/libp2p -import pkg/questionable +import codex/conf import ./codexclient export codexclient diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index 0af0572a..86905f8f 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -1,9 +1,9 @@ import std/sequtils import std/os from std/times import getTime, toUnix -import pkg/codex/contracts -import pkg/codex/logutils -import pkg/codex/periods +import pkg/chronicles +import codex/contracts +import codex/periods import ../contracts/time import ../contracts/deployment import ./twonodes diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index 9fa11f1f..fafaf567 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -1,5 +1,6 @@ import std/os import std/macros +import std/json import std/httpclient import ../ethertest import ./codexclient diff --git a/tests/logging.nim b/tests/logging.nim index ece9c9b0..cf2633d3 100644 --- a/tests/logging.nim +++ b/tests/logging.nim @@ -1,5 +1,5 @@ when not defined(nimscript): - import pkg/codex/logutils + import pkg/chronicles proc ignoreLogging(level: LogLevel, message: LogOutputStr) = discard diff --git a/tests/testCodex.nim b/tests/testCodex.nim index 745fb6fe..db907684 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -2,7 +2,6 @@ import ./codex/teststores import ./codex/testblockexchange import ./codex/testasyncheapqueue import ./codex/testchunking -import ./codex/testlogutils import ./codex/testmanifest import ./codex/testnode import ./codex/teststorestream