diff --git a/codex.nim b/codex.nim index 073331f1..f7215db6 100644 --- a/codex.nim +++ b/codex.nim @@ -7,7 +7,6 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/confutils @@ -21,10 +20,11 @@ import pkg/libp2p import ./codex/conf import ./codex/codex +import ./codex/logutils import ./codex/units import ./codex/utils/keyutils -export codex, conf, libp2p, chronos, chronicles +export codex, conf, libp2p, chronos, logutils when isMainModule: import std/sequtils diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index 8bb4dd4a..267d8f12 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -10,22 +10,21 @@ import std/sequtils import pkg/chronos -import pkg/chronicles import pkg/libp2p/cid import pkg/metrics import pkg/questionable import pkg/questionable/results -import ../protobuf/presence +import ./pendingblocks +import ../protobuf/presence import ../network import ../peers import ../../utils import ../../discovery import ../../stores/blockstore - -import ./pendingblocks +import ../../logutils logScope: topics = "codex discoveryengine" diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index c24a0104..ec7b7f64 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -14,7 +14,6 @@ import std/algorithm import std/sugar import pkg/chronos -import pkg/chronicles import pkg/libp2p/[cid, switch, multihash, multicodec] import pkg/metrics import pkg/stint @@ -23,6 +22,7 @@ import ../../stores/blockstore import ../../blocktype import ../../utils import ../../merkletree +import ../../logutils import ../protobuf/blockexc import ../protobuf/presence diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index ba92da97..2d517dfc 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -14,13 +14,13 @@ import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/metrics import ../protobuf/blockexc import ../../blocktype +import ../../logutils logScope: topics = "codex pendingblocks" diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index 0ce0c9ce..1a39364b 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -10,7 +10,6 @@ import std/tables import std/sequtils -import pkg/chronicles import pkg/chronos import pkg/libp2p @@ -19,6 +18,7 @@ import pkg/questionable import pkg/questionable/results import ../../blocktype as bt +import ../../logutils import ../protobuf/blockexc as pb import ../protobuf/payments diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index c3536f61..69a19386 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -11,12 +11,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/libp2p import ../protobuf/blockexc import ../protobuf/message import ../../errors +import ../../logutils logScope: topics = "codex blockexcnetworkpeer" diff --git a/codex/blockexchange/peers/peercontext.nim b/codex/blockexchange/peers/peercontext.nim index f3c3050d..ef8fa64c 100644 --- a/codex/blockexchange/peers/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -11,7 +11,6 @@ import std/sequtils import std/tables import std/sets -import pkg/chronicles import pkg/libp2p import pkg/chronos import pkg/nitro @@ -22,6 +21,7 @@ import ../protobuf/payments import ../protobuf/presence import ../../blocktype +import ../../logutils export payments, nitro diff --git a/codex/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim index f23415f6..b07265b4 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -16,11 +16,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/libp2p import ../protobuf/blockexc import ../../blocktype +import ../../logutils + import ./peercontext export peercontext diff --git a/codex/blocktype.nim b/codex/blocktype.nim index 52bb5432..f74295a3 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -20,16 +20,15 @@ import pkg/libp2p/[cid, multicodec, multihash] import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -import pkg/chronicles -import pkg/json_serialization import ./units import ./utils -import ./formats import ./errors +import ./logutils +import ./utils/json import ./codextypes -export errors, formats, units, codextypes +export errors, logutils, units, codextypes type Block* = ref object of RootObj @@ -39,10 +38,18 @@ type BlockAddress* = object case leaf*: bool of true: - treeCid*: Cid - index*: Natural + treeCid* {.serialize.}: Cid + index* {.serialize.}: Natural else: - cid*: Cid + cid* {.serialize.}: Cid + +logutils.formatIt(LogFormat.textLines, BlockAddress): + if it.leaf: + "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index + else: + "cid: " & shortLog($it.cid) + +logutils.formatIt(LogFormat.json, BlockAddress): %it proc `==`*(a, b: BlockAddress): bool = a.leaf == b.leaf and @@ -59,12 +66,6 @@ proc `$`*(a: BlockAddress): string = else: "cid: " & $a.cid -proc writeValue*( - writer: var JsonWriter, - value: Cid -) {.upraises:[IOError].} = - writer.writeValue($value) - proc cidOrTreeCid*(a: BlockAddress): Cid = if a.leaf: a.treeCid diff --git a/codex/chunker.nim b/codex/chunker.nim index 58fcb4a3..36f28f7a 100644 --- a/codex/chunker.nim +++ b/codex/chunker.nim @@ -13,13 +13,13 @@ import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/chronos import pkg/libp2p except shuffle import ./blocktype +import ./logutils export blocktype diff --git a/codex/codex.nim b/codex/codex.nim index 1d77ea9f..ab8b7eb3 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -12,7 +12,6 @@ import std/strutils import std/os import std/tables -import pkg/chronicles import pkg/chronos import pkg/presto import pkg/libp2p @@ -39,6 +38,7 @@ import ./contracts/clock import ./contracts/deployment import ./utils/addrutils import ./namespaces +import ./logutils logScope: topics = "codex node" diff --git a/codex/conf.nim b/codex/conf.nim index e693a4f4..09f17f83 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -18,7 +18,6 @@ import std/strutils import std/typetraits import pkg/chronos -import pkg/chronicles import pkg/chronicles/helpers import pkg/chronicles/topics_registry import pkg/confutils/defs @@ -35,6 +34,7 @@ import pkg/questionable import pkg/questionable/results import ./discovery +import ./logutils import ./stores import ./units import ./utils @@ -58,7 +58,7 @@ type noCommand, initNode - LogKind* = enum + LogKind* {.pure.} = enum Auto = "auto" Colors = "colors" NoColors = "nocolors" @@ -281,6 +281,9 @@ type EthAddress* = ethers.Address +logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog +logutils.formatIt(LogFormat.json, EthAddress): %it + proc getCodexVersion(): string = let tag = strip(staticExec("git tag")) if tag.isEmptyOrWhitespace: @@ -412,7 +415,7 @@ proc completeCmdArg*(T: type Duration; val: string): seq[string] = discard # silly chronicles, colors is a compile-time property -proc stripAnsi(v: string): string = +proc stripAnsi*(v: string): string = var res = newStringOfCap(v.len) i: int diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index 9e5b9388..46591100 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -1,11 +1,10 @@ -import std/json import std/os import std/tables import pkg/ethers import pkg/questionable -import pkg/chronicles import ../conf +import ../logutils import ./marketplace type Deployment* = ref object diff --git a/codex/contracts/interactions/clientinteractions.nim b/codex/contracts/interactions/clientinteractions.nim index e2354d85..78b3bedf 100644 --- a/codex/contracts/interactions/clientinteractions.nim +++ b/codex/contracts/interactions/clientinteractions.nim @@ -1,13 +1,13 @@ import pkg/ethers -import pkg/chronicles import ../../purchasing +import ../../logutils import ../market import ../clock import ./interactions export purchasing -export chronicles except toJson +export logutils type ClientInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/interactions/hostinteractions.nim b/codex/contracts/interactions/hostinteractions.nim index e9749df5..2decfa44 100644 --- a/codex/contracts/interactions/hostinteractions.nim +++ b/codex/contracts/interactions/hostinteractions.nim @@ -1,11 +1,11 @@ -import pkg/ethers -import pkg/chronicles +import pkg/chronos +import ../../logutils import ../../sales import ./interactions export sales -export chronicles except toJson +export logutils type HostInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 538b44ac..51b3577b 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,11 +1,11 @@ import std/sequtils import std/strutils import std/sugar -import pkg/chronicles import pkg/ethers import pkg/ethers/testing import pkg/upraises import pkg/questionable +import ../logutils import ../market import ./marketplace diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 60c3c577..1363fb9d 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -1,12 +1,13 @@ import std/hashes +import std/sequtils import std/typetraits import pkg/contractabi import pkg/nimcrypto import pkg/ethers/fields import pkg/questionable/results import pkg/stew/byteutils -import pkg/json_serialization import pkg/upraises +import ../logutils import ../utils/json export contractabi @@ -79,6 +80,13 @@ proc toHex*[T: distinct](id: T): string = type baseType = T.distinctBase baseType(id).toHex +logutils.formatIt(LogFormat.textLines, Nonce): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, RequestId): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, SlotId): it.short0xHexLog +logutils.formatIt(LogFormat.json, Nonce): it.to0xHexLog +logutils.formatIt(LogFormat.json, RequestId): it.to0xHexLog +logutils.formatIt(LogFormat.json, SlotId): it.to0xHexLog + func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = StorageRequest( client: tupl[0], @@ -176,17 +184,3 @@ func price*(request: StorageRequest): UInt256 = func size*(ask: StorageAsk): UInt256 = ask.slots.u256 * ask.slotSize - -proc writeValue*( - writer: var JsonWriter, - value: SlotId | RequestId) {.upraises:[IOError].} = - - mixin writeValue - writer.writeValue value.toArray - -proc readValue*[T: SlotId | RequestId]( - reader: var JsonReader, - value: var T) {.upraises: [SerializationError, IOError].} = - - mixin readValue - value = T reader.readValue(T.distinctBase) diff --git a/codex/discovery.nim b/codex/discovery.nim index 67aacd17..ce0b4032 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -11,7 +11,6 @@ import std/algorithm import std/sequtils import pkg/chronos -import pkg/chronicles import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope] import pkg/questionable import pkg/questionable/results @@ -21,7 +20,7 @@ import pkg/codexdht/discv5/protocol as discv5 import ./rng import ./errors -import ./formats +import ./logutils export discv5 diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 43eb20be..e7735641 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -15,10 +15,10 @@ import std/sequtils import std/sugar import pkg/chronos -import pkg/chronicles import pkg/libp2p/[multicodec, cid, multihash] import pkg/libp2p/protobuf/minprotobuf +import ../logutils import ../manifest import ../merkletree import ../stores diff --git a/codex/formats.nim b/codex/formats.nim deleted file mode 100644 index 38881bc9..00000000 --- a/codex/formats.nim +++ /dev/null @@ -1,28 +0,0 @@ -## Nim-Codex -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/strutils - -import pkg/chronicles -import pkg/libp2p/cid - -func shortLog*(cid: Cid): string = - ## Returns compact string representation of ``pid``. - var scid = $cid - if len(scid) > 10: - scid[3] = '*' - - when (NimMajor, NimMinor) > (1, 4): - scid.delete(4 .. scid.high - 6) - else: - scid.delete(4, scid.high - 6) - - scid - -chronicles.formatIt(Cid): shortLog(it) diff --git a/codex/logutils.nim b/codex/logutils.nim new file mode 100644 index 00000000..1c002b70 --- /dev/null +++ b/codex/logutils.nim @@ -0,0 +1,241 @@ +## logutils is a module that has several goals: +## 1. Fix json logging output (run with `--log-format=json`) which was +## effectively broken for many types using default Chronicles json +## serialization. +## 2. Ability to specify log output for textlines and json sinks together or +## separately +## - This is useful if consuming json in some kind of log parser and need +## valid json with real values +## - eg a shortened Cid is nice to see in a text log in stdout, but won't +## provide a real Cid when parsed in json +## 4. Remove usages of `nim-json-serialization` from the codebase +## 5. Remove need to declare `writeValue` for new types +## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent +## conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467) +## +## When declaring a new type, one should consider importing the `codex/logutils` +## module, and specifying `formatIt`. If textlines log output and json log output +## need to be different, overload `formatIt` and specify a `LogFormat`. If json +## serialization is needed, it can be declared with a `%` proc. `logutils` +## imports and exports `utils/json` which handles the de/serialization, examples +## below. **Only `codex/logutils` needs to be imported.** +## +## Using `logutils` in the Codex codebase: +## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` +## - most of `chronicles` is exported by `logutils` +## - Instead of importing `std/json`, import `pkg/codex/utils/json` +## - `std/json` is exported by `utils/json` which is exported by `logutils` +## - Instead of importing `pkg/nim-json-serialization`, import +## `pkg/codex/utils/json` +## - one of the goals is to remove the use of `nim-json-serialization` +## +## ```nim +## import pkg/codex/logutils +## +## type +## BlockAddress* = object +## case leaf*: bool +## of true: +## treeCid* {.serialize.}: Cid +## index* {.serialize.}: Natural +## else: +## cid* {.serialize.}: Cid +## +## logutils.formatIt(LogFormat.textLines, BlockAddress): +## if it.leaf: +## "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index +## else: +## "cid: " & shortLog($it.cid) +## +## logutils.formatIt(LogFormat.json, BlockAddress): %it +## +## # chronicles textlines output +## TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0" +## # chronicles json output +## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} +## ``` +## In this case, `BlockAddress` is just an object, so `utils/json` can handle +## serializing it without issue (only fields annotated with `{.serialize.}` will +## serialize (aka opt-in serialization)). +## +## If one so wished, another option for the textlines log output, would be to +## simply `toString` the serialised json: +## ```nim +## logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it +## # or, more succinctly: +## logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson +## ``` +## In that case, both the textlines and json sinks would have the same output, +## so we could reduce this even further by not specifying a `LogFormat`: +## ```nim +## type +## BlockAddress* = object +## case leaf*: bool +## of true: +## treeCid* {.serialize.}: Cid +## index* {.serialize.}: Natural +## else: +## cid* {.serialize.}: Cid +## +## logutils.formatIt(BlockAddress): %it +## +## # chronicles textlines output +## TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}" +## # chronicles json output +## {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} +## ``` + +import std/options +import std/sequtils +import std/strutils +import std/sugar +import std/typetraits + +import pkg/chronicles except toJson, `%` +from pkg/libp2p import Cid, MultiAddress, `$` +import pkg/questionable +import pkg/questionable/results +import pkg/stew/byteutils +import pkg/stint +import pkg/upraises + +import ./utils/json + +export byteutils +export chronicles except toJson, formatIt, `%` +export questionable +export sequtils +export strutils +export sugar +export upraises +export json +export results + +func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = + ## Returns compact string representation of ``long``. + var short = long + let minLen = start + ellipses.len + stop + if len(short) > minLen: + short.insert(ellipses, start) + + when (NimMajor, NimMinor) > (1, 4): + short.delete(start + ellipses.len .. short.high - stop) + else: + short.delete(start + ellipses.len, short.high - stop) + + short + +func shortHexLog*(long: string): string = + if long[0..1] == "0x": result &= "0x" + result &= long[2..long.high].shortLog("..", 4, 4) + +func short0xHexLog*[N: static[int], T: array[N, byte]](v: T): string = + v.to0xHex.shortHexLog + +func short0xHexLog*[T: distinct](v: T): string = + type BaseType = T.distinctBase + BaseType(v).short0xHexLog + +func short0xHexLog*[U: distinct, T: seq[U]](v: T): string = + type BaseType = U.distinctBase + "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]" + +func to0xHexLog*[T: distinct](v: T): string = + type BaseType = T.distinctBase + BaseType(v).to0xHex + +func to0xHexLog*[U: distinct, T: seq[U]](v: T): string = + type BaseType = U.distinctBase + "@[" & v.map(x => BaseType(x).to0xHex).join(",") & "]" + +proc formatTextLineSeq*(val: seq[string]): string = + "@[" & val.join(", ") & "]" + +template formatIt*(format: LogFormat, T: typedesc, body: untyped) = + # Provides formatters for logging with Chronicles for the given type and + # `LogFormat`. + # NOTE: `seq[T]`, `Option[T]`, and `seq[Option[T]]` are overriddden + # since the base `setProperty` is generic using `auto` and conflicts with + # providing a generic `seq` and `Option` override. + when format == LogFormat.json: + proc formatJsonOption(val: ?T): JsonNode = + if it =? val: + json.`%`(body) + else: + newJNull() + + proc formatJsonResult*(val: ?!T): JsonNode = + without it =? val, error: + let jObj = newJObject() + jObj["error"] = newJString(error.msg) + return jObj + json.`%`(body) + + proc setProperty*(r: var JsonRecord, key: string, res: ?!T) = + var it {.inject, used.}: T + setProperty(r, key, res.formatJsonResult) + + proc setProperty*(r: var JsonRecord, key: string, opt: ?T) = + var it {.inject, used.}: T + let v = opt.formatJsonOption + setProperty(r, key, v) + + proc setProperty*(r: var JsonRecord, key: string, opts: seq[?T]) = + var it {.inject, used.}: T + let v = opts.map(opt => opt.formatJsonOption) + setProperty(r, key, json.`%`(v)) + + proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) = + var it {.inject, used.}: T + let v = val.map(it => body) + setProperty(r, key, json.`%`(v)) + + proc setProperty*(r: var JsonRecord, key: string, val: T) {.upraises:[ValueError, IOError].} = + var it {.inject, used.}: T = val + let v = body + setProperty(r, key, json.`%`(v)) + + elif format == LogFormat.textLines: + proc formatTextLineOption*(val: ?T): string = + var v = "none(" & $T & ")" + if it =? val: + v = "some(" & $(body) & ")" # that I used to know :) + v + + proc formatTextLineResult*(val: ?!T): string = + without it =? val, error: + return "Error: " & error.msg + $(body) + + proc setProperty*(r: var TextLineRecord, key: string, res: ?!T) = + var it {.inject, used.}: T + setProperty(r, key, res.formatTextLineResult) + + proc setProperty*(r: var TextLineRecord, key: string, opt: ?T) = + var it {.inject, used.}: T + let v = opt.formatTextLineOption + setProperty(r, key, v) + + proc setProperty*(r: var TextLineRecord, key: string, opts: seq[?T]) = + var it {.inject, used.}: T + let v = opts.map(opt => opt.formatTextLineOption) + setProperty(r, key, v.formatTextLineSeq) + + proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) = + var it {.inject, used.}: T + let v = val.map(it => body) + setProperty(r, key, v.formatTextLineSeq) + + proc setProperty*(r: var TextLineRecord, key: string, val: T) {.upraises:[ValueError, IOError].} = + var it {.inject, used.}: T = val + let v = body + setProperty(r, key, v) + +template formatIt*(T: type, body: untyped) {.dirty.} = + formatIt(LogFormat.textLines, T): body + formatIt(LogFormat.json, T): body + +formatIt(LogFormat.textLines, Cid): shortLog($it) +formatIt(LogFormat.json, Cid): $it +formatIt(UInt256): $it +formatIt(MultiAddress): $it diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index aef9d452..1106c9bc 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -19,12 +19,12 @@ import std/sequtils import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/chronicles import pkg/chronos import ./manifest import ../errors import ../blocktype +import ../logutils proc encode*(manifest: Manifest): ?!seq[byte] = ## Encode the manifest into a ``ManifestCodec`` diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index a3dbd7cc..8b491a17 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -22,6 +22,7 @@ import ../utils import ../utils/json import ../units import ../blocktype +import ../logutils type Manifest* = ref object of RootObj diff --git a/codex/node.nim b/codex/node.nim index 40e8ee26..e3c6baca 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -16,7 +16,6 @@ import std/sugar import pkg/questionable import pkg/questionable/results -import pkg/chronicles import pkg/chronos import pkg/poseidon2 @@ -41,6 +40,9 @@ import ./discovery import ./contracts import ./utils import ./errors +import ./logutils + +export logutils logScope: topics = "codex node" @@ -658,7 +660,7 @@ proc start*(self: CodexNodeRef) {.async.} = self.contracts.validator = ValidatorInteractions.none self.networkId = self.switch.peerInfo.peerId - notice "Started codex node", id = $self.networkId, addrs = self.switch.peerInfo.addrs + notice "Started codex node", id = self.networkId, addrs = self.switch.peerInfo.addrs proc stop*(self: CodexNodeRef) {.async.} = trace "Stopping node" diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim index ee5c3a16..226fcbee 100644 --- a/codex/purchasing/purchaseid.nim +++ b/codex/purchasing/purchaseid.nim @@ -1,8 +1,12 @@ import std/hashes import pkg/nimcrypto +import ../logutils type PurchaseId* = distinct array[32, byte] +logutils.formatIt(LogFormat.textLines, PurchaseId): it.short0xHexLog +logutils.formatIt(LogFormat.json, PurchaseId): it.to0xHexLog + proc hash*(x: PurchaseId): Hash {.borrow.} proc `==`*(x, y: PurchaseId): bool {.borrow.} proc toHex*(x: PurchaseId): string = array[32, byte](x).toHex diff --git a/codex/purchasing/states/cancelled.nim b/codex/purchasing/states/cancelled.nim index ad54f6a0..f9bb1ece 100644 --- a/codex/purchasing/states/cancelled.nim +++ b/codex/purchasing/states/cancelled.nim @@ -1,5 +1,6 @@ import pkg/metrics -import pkg/chronicles + +import ../../logutils import ../statemachine import ./errorhandling @@ -17,7 +18,7 @@ method run*(state: PurchaseCancelled, machine: Machine): Future[?State] {.async. codex_purchases_cancelled.inc() let purchase = Purchase(machine) - warn "Request cancelled, withdrawing remaining funds", requestId = $purchase.requestId + warn "Request cancelled, withdrawing remaining funds", requestId = purchase.requestId await purchase.market.withdrawFunds(purchase.requestId) let error = newException(Timeout, "Purchase cancelled due to timeout") diff --git a/codex/purchasing/states/error.nim b/codex/purchasing/states/error.nim index baa0cceb..0ebe1dbe 100644 --- a/codex/purchasing/states/error.nim +++ b/codex/purchasing/states/error.nim @@ -1,7 +1,7 @@ import pkg/metrics -import pkg/chronicles import ../statemachine import ../../utils/exceptions +import ../../logutils declareCounter(codex_purchases_error, "codex purchases error") diff --git a/codex/purchasing/states/finished.nim b/codex/purchasing/states/finished.nim index 4a2662cf..0f97150d 100644 --- a/codex/purchasing/states/finished.nim +++ b/codex/purchasing/states/finished.nim @@ -1,6 +1,7 @@ import pkg/metrics -import pkg/chronicles + import ../statemachine +import ../../logutils declareCounter(codex_purchases_finished, "codex purchases finished") diff --git a/codex/purchasing/states/started.nim b/codex/purchasing/states/started.nim index 04736a13..fff1bbc8 100644 --- a/codex/purchasing/states/started.nim +++ b/codex/purchasing/states/started.nim @@ -1,5 +1,6 @@ import pkg/metrics -import pkg/chronicles + +import ../../logutils import ../statemachine import ./errorhandling import ./finished diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim index f3a68512..391dbf7c 100644 --- a/codex/purchasing/states/submitted.nim +++ b/codex/purchasing/states/submitted.nim @@ -1,5 +1,6 @@ import pkg/metrics -import pkg/chronicles + +import ../../logutils import ../statemachine import ./errorhandling import ./started diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 69254ee9..ccb213ab 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -16,7 +16,6 @@ import std/sequtils import pkg/questionable import pkg/questionable/results -import pkg/chronicles except toJson import pkg/chronos import pkg/presto except toJson import pkg/metrics except toJson @@ -28,6 +27,7 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/codexdht/discv5/spr as spr +import ../logutils import ../node import ../blocktype import ../conf diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 078a0124..181ea584 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -111,23 +111,4 @@ func `%`*(obj: StorageRequest | Slot): JsonNode = return jsonObj -func `%`*(obj: Cid): JsonNode = - % $obj - -func `%`*(obj: PeerId): JsonNode = - % $obj - -func `%`*(obj: RestNodeId): JsonNode = - % $obj.id - -func `%`*(obj: SignedPeerRecord): JsonNode = - % $obj - -func `%`*(obj: dn.Address): JsonNode = - % $obj - -func `%`*(obj: AddressInfo): JsonNode = - % $obj.address - -func `%`*(obj: MultiAddress): JsonNode = - % $obj +func `%`*(obj: RestNodeId): JsonNode = % $obj.id diff --git a/codex/sales.nim b/codex/sales.nim index 7994db89..3e04228c 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -3,13 +3,13 @@ import std/sugar import pkg/questionable import pkg/questionable/results import pkg/stint -import pkg/chronicles import pkg/datastore import ./market import ./clock import ./stores import ./contracts/requests import ./contracts/marketplace +import ./logutils import ./sales/salescontext import ./sales/salesagent import ./sales/statemachine @@ -166,7 +166,7 @@ proc filled( processing.complete() proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = - debug "processing slot from queue", requestId = $item.requestId, + debug "processing slot from queue", requestId = item.requestId, slot = item.slotIndex let agent = newSalesAgent( diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 611c1ad6..6c68853e 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -28,19 +28,19 @@ push: {.upraises: [].} import std/typetraits import pkg/chronos -import pkg/chronicles except toJson import pkg/datastore import pkg/nimcrypto import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/stew/byteutils +import ../logutils import ../stores import ../contracts/requests import ../utils/json export requests -export chronicles except toJson +export logutils logScope: topics = "sales reservations" @@ -139,13 +139,8 @@ proc toErr[E1: ref CatchableError, E2: ReservationsError]( return newException(E2, msg, e1) -proc writeValue*( - writer: var JsonWriter, - value: SomeStorableId) {.upraises:[IOError].} = - ## used for chronicles' logs - - mixin writeValue - writer.writeValue %value +logutils.formatIt(LogFormat.textLines, SomeStorableId): it.short0xHexLog +logutils.formatIt(LogFormat.json, SomeStorableId): it.to0xHexLog proc `onAvailabilityAdded=`*(self: Reservations, onAvailabilityAdded: OnAvailabilityAdded) = diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 08d3ab0e..21475426 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -1,11 +1,11 @@ import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/upraises import ../contracts/requests import ../errors +import ../logutils import ./statemachine import ./salescontext import ./salesdata diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim index fa821864..a875f917 100644 --- a/codex/sales/slotqueue.nim +++ b/codex/sales/slotqueue.nim @@ -1,11 +1,11 @@ import std/sequtils import std/tables -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/upraises import ../errors +import ../logutils import ../rng import ../utils import ../contracts/requests diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim index 45718b4e..8464a61b 100644 --- a/codex/sales/states/cancelled.nim +++ b/codex/sales/states/cancelled.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logutils import ../salesagent import ../statemachine import ./errorhandling @@ -20,7 +20,7 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting collateral and partial payout", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Collecting collateral and partial payout", requestId = data.requestId, slotIndex = data.slotIndex await market.freeSlot(slot.id) if onClear =? agent.context.onClear and @@ -30,4 +30,4 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} = if onCleanUp =? agent.onCleanUp: await onCleanUp(returnBytes = true) - warn "Sale cancelled due to timeout", requestId = $data.requestId, slotIndex = $data.slotIndex + warn "Sale cancelled due to timeout", requestId = data.requestId, slotIndex = data.slotIndex diff --git a/codex/sales/states/downloading.nim b/codex/sales/states/downloading.nim index 741cdaee..118f97cf 100644 --- a/codex/sales/states/downloading.nim +++ b/codex/sales/states/downloading.nim @@ -1,7 +1,8 @@ -import pkg/chronicles import pkg/questionable import pkg/questionable/results + import ../../blocktype as bt +import ../../logutils import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim index 59754eca..51f34bc9 100644 --- a/codex/sales/states/errored.nim +++ b/codex/sales/states/errored.nim @@ -1,9 +1,10 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises -import pkg/chronicles + import ../statemachine import ../salesagent +import ../../logutils import ../../utils/exceptions logScope: diff --git a/codex/sales/states/failed.nim b/codex/sales/states/failed.nim index cd954be4..e32fbb58 100644 --- a/codex/sales/states/failed.nim +++ b/codex/sales/states/failed.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logutils import ../salesagent import ../statemachine import ./errorhandling @@ -21,7 +21,7 @@ method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Removing slot from mySlots", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Removing slot from mySlots", requestId = data.requestId, slotIndex = data.slotIndex await market.freeSlot(slot.id) let error = newException(SaleFailedError, "Sale failed") diff --git a/codex/sales/states/filled.nim b/codex/sales/states/filled.nim index c5bc4c27..6e3e0106 100644 --- a/codex/sales/states/filled.nim +++ b/codex/sales/states/filled.nim @@ -1,7 +1,8 @@ import pkg/questionable import pkg/questionable/results -import pkg/chronicles + import ../../conf +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling @@ -36,7 +37,7 @@ method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} = let me = await market.getSigner() if host == me.some: - info "Slot succesfully filled", requestId = $data.requestId, slotIndex = $data.slotIndex + info "Slot succesfully filled", requestId = data.requestId, slotIndex = data.slotIndex without request =? data.request: raiseAssert "no sale request" diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index b1e8471c..b6646733 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logutils import ../../market import ../statemachine import ../salesagent @@ -32,5 +32,5 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} = without (collateral =? data.request.?ask.?collateral): raiseAssert "Request not set" - debug "Filling slot", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) diff --git a/codex/sales/states/finished.nim b/codex/sales/states/finished.nim index 539cde62..59e9244c 100644 --- a/codex/sales/states/finished.nim +++ b/codex/sales/states/finished.nim @@ -1,5 +1,6 @@ import pkg/chronos -import pkg/chronicles + +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling @@ -27,7 +28,7 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} = without request =? data.request: raiseAssert "no sale request" - info "Slot finished and paid out", requestId = $data.requestId, slotIndex = $data.slotIndex + info "Slot finished and paid out", requestId = data.requestId, slotIndex = data.slotIndex if onCleanUp =? agent.onCleanUp: await onCleanUp() diff --git a/codex/sales/states/ignored.nim b/codex/sales/states/ignored.nim index fa0641cc..d757e9c1 100644 --- a/codex/sales/states/ignored.nim +++ b/codex/sales/states/ignored.nim @@ -1,5 +1,6 @@ -import pkg/chronicles import pkg/chronos + +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/initialproving.nim b/codex/sales/states/initialproving.nim index eeb1592f..8f3a7287 100644 --- a/codex/sales/states/initialproving.nim +++ b/codex/sales/states/initialproving.nim @@ -1,5 +1,5 @@ -import pkg/chronicles import pkg/questionable/results +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling @@ -32,7 +32,7 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async without onProve =? context.onProve: raiseAssert "onProve callback not set" - debug "Generating initial proof", requestId = $data.requestId + debug "Generating initial proof", requestId = data.requestId let slot = Slot(request: request, slotIndex: data.slotIndex) challenge = await context.market.getChallenge(slot.id) @@ -40,6 +40,6 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async error "Failed to generate initial proof", error = err.msg return some State(SaleErrored(error: err)) - debug "Finished proof calculation", requestId = $data.requestId + debug "Finished proof calculation", requestId = data.requestId return some State(SaleFilling(proof: proof)) diff --git a/codex/sales/states/payout.nim b/codex/sales/states/payout.nim index a70f7eac..5c8c2859 100644 --- a/codex/sales/states/payout.nim +++ b/codex/sales/states/payout.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logutils import ../../market import ../statemachine import ../salesagent @@ -29,7 +29,7 @@ method run(state: SalePayout, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting finished slot's reward", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Collecting finished slot's reward", requestId = data.requestId, slotIndex = data.slotIndex await market.freeSlot(slot.id) return some State(SaleFinished()) diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index bb0fe743..973446e2 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -1,6 +1,7 @@ -import pkg/chronicles import pkg/questionable import pkg/questionable/results + +import ../../logutils import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim index dffb8368..4593fd19 100644 --- a/codex/sales/states/proving.nim +++ b/codex/sales/states/proving.nim @@ -1,7 +1,7 @@ import std/options -import pkg/chronicles import pkg/questionable/results import ../../clock +import ../../logutils import ../statemachine import ../salesagent import ../salescontext @@ -32,7 +32,7 @@ method prove*( error "Failed to generate proof", error = err.msg # In this state, there's nothing we can do except try again next time. return - debug "Submitting proof", currentPeriod = currentPeriod, slotId = $slot.id + debug "Submitting proof", currentPeriod = currentPeriod, slotId = slot.id await market.submitProof(slot.id, proof) except CatchableError as e: error "Submitting proof failed", msg = e.msg @@ -51,9 +51,9 @@ proc proveLoop( logScope: period = currentPeriod - requestId = $request.id + requestId = request.id slotIndex - slotId = $slot.id + slotId = slot.id proc getCurrentPeriod(): Future[Period] {.async.} = let periodicity = await market.periodicity() @@ -110,7 +110,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} = without clock =? context.clock: raiseAssert("clock not set") - debug "Start proving", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex try: let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve) state.loop = loop @@ -122,7 +122,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} = return some State(SaleErrored(error: e)) finally: # Cleanup of the proving loop - debug "Stopping proving.", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex if not state.loop.isNil: if not state.loop.finished: diff --git a/codex/sales/states/provingsimulated.nim b/codex/sales/states/provingsimulated.nim index e3f5b2c2..78ce5ee5 100644 --- a/codex/sales/states/provingsimulated.nim +++ b/codex/sales/states/provingsimulated.nim @@ -1,12 +1,12 @@ import ../../conf when codex_enable_proof_failures: import std/strutils - import pkg/chronicles import pkg/stint import pkg/ethers import pkg/ethers/testing import ../../contracts/requests + import ../../logutils import ../../market import ../salescontext import ./proving @@ -20,7 +20,7 @@ when codex_enable_proof_failures: proofCount: int proc onSubmitProofError(error: ref CatchableError, period: UInt256, slotId: SlotId) = - error "Submitting invalid proof failed", period = period, slotId = $slotId, msg = error.msg + error "Submitting invalid proof failed", period = period, slotId, msg = error.msg method prove*(state: SaleProvingSimulated, slot: Slot, challenge: ProofChallenge, onProve: OnProve, market: Market, currentPeriod: Period) {.async.} = trace "Processing proving in simulated mode" diff --git a/codex/sales/states/unknown.nim b/codex/sales/states/unknown.nim index 3672ed96..db00f517 100644 --- a/codex/sales/states/unknown.nim +++ b/codex/sales/states/unknown.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logutils import ../statemachine import ../salesagent import ./filled diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index 1f15c3a1..3488fb58 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -15,13 +15,13 @@ import std/sugar import pkg/libp2p import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/poseidon2 import pkg/poseidon2/io import pkg/constantine/math/arithmetic/finite_fields +import ../../logutils import ../../indexingstrategy import ../../merkletree import ../../stores diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index 08dd8ef9..f93bbec4 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -10,7 +10,6 @@ import std/sugar import std/sequtils -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results @@ -20,6 +19,7 @@ import pkg/poseidon2/types import pkg/poseidon2/io import pkg/stew/arrayops +import ../../logutils import ../../market import ../../blocktype as bt import ../../merkletree diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index 8fde35a1..e670a093 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -13,7 +13,6 @@ push: {.upraises: [].} import std/options -import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/lrucache @@ -24,6 +23,7 @@ import ./blockstore import ../units import ../chunker import ../errors +import ../logutils import ../manifest import ../merkletree import ../utils diff --git a/codex/stores/maintenance.nim b/codex/stores/maintenance.nim index 98b6fab9..343fed8f 100644 --- a/codex/stores/maintenance.nim +++ b/codex/stores/maintenance.nim @@ -11,7 +11,6 @@ ## Looks for and removes expired blocks from blockstores. import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results @@ -19,6 +18,7 @@ import ./repostore import ../utils/timer import ../utils/asynciter import ../clock +import ../logutils import ../systemclock const diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 59e4bb94..82a1428d 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -7,23 +7,22 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises +import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/questionable/results +import ../clock +import ../blocktype +import ../blockexchange +import ../logutils +import ../merkletree import ../utils/asyncheapqueue import ../utils/asynciter -import ../clock - -import ../blocktype import ./blockstore -import ../blockexchange -import ../merkletree export blockstore, blockexchange, asyncheapqueue diff --git a/codex/stores/repostore.nim b/codex/stores/repostore.nim index 45faefbd..dc63f4cd 100644 --- a/codex/stores/repostore.nim +++ b/codex/stores/repostore.nim @@ -13,7 +13,6 @@ push: {.upraises: [].} import pkg/chronos import pkg/chronos/futures -import pkg/chronicles import pkg/libp2p/[cid, multicodec, multihash] import pkg/lrucache import pkg/metrics @@ -27,6 +26,7 @@ import ./keyutils import ../blocktype import ../clock import ../systemclock +import ../logutils import ../merkletree import ../utils diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index f2491e8d..a8a55955 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -11,9 +11,10 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/libp2p +import ../logutils + logScope: topics = "libp2p asyncstreamwrapper" @@ -37,7 +38,7 @@ proc new*( writer: AsyncStreamWriter = nil ): AsyncStreamWrapper = ## Create new instance of an asynchronous stream wrapper - ## + ## let stream = C(reader: reader, writer: writer) diff --git a/codex/streams/seekablestream.nim b/codex/streams/seekablestream.nim index 785d9afe..b07e1116 100644 --- a/codex/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -9,9 +9,10 @@ import pkg/libp2p/stream/lpstream import pkg/chronos -import pkg/chronicles -export lpstream, chronos, chronicles +import ../logutils + +export lpstream, chronos, logutils logScope: topics = "codex seekablestream" diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 63406a5f..d3b8c036 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -14,12 +14,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/stew/ptrops import ../stores import ../manifest import ../blocktype +import ../logutils import ../utils import ./seekablestream @@ -53,7 +53,7 @@ proc new*( pad = true ): StoreStream = ## Create a new StoreStream instance for a given store and manifest - ## + ## result = StoreStream( store: store, manifest: manifest, @@ -80,7 +80,7 @@ method readOnce*( ## Read `nbytes` from current position in the StoreStream into output buffer pointed by `pbytes`. ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. - ## + ## trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.blocksCount if self.atEof: diff --git a/codex/units.nim b/codex/units.nim index d3a4b9f8..63921d7d 100644 --- a/codex/units.nim +++ b/codex/units.nim @@ -12,8 +12,8 @@ import std/hashes import std/strutils import pkg/upraises -import pkg/json_serialization -import pkg/json_serialization/std/options + +import ./logutils type NBytes* = distinct Natural @@ -43,6 +43,7 @@ divMaths(NBytes) proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" proc `'nb`*(n: string): NBytes = parseInt(n).NBytes +logutils.formatIt(NBytes): $it const MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz @@ -54,18 +55,6 @@ func divUp*[T: NBytes](a, b : T): int = assert(b != T(0)) if a==T(0): int(0) else: int( ((a - T(1)) div b) + 1 ) -proc writeValue*( - writer: var JsonWriter, - value: NBytes -) {.upraises:[IOError].} = - writer.writeValue value.int - -proc readValue*( - reader: var JsonReader, - value: var NBytes -) {.upraises: [SerializationError, IOError].} = - value = NBytes reader.readValue(int) - when isMainModule: import unittest2 diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 512617ee..9aeb3eab 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -1,10 +1,10 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/chronicles import pkg/upraises -import ./trackedfutures +import ../logutils import ./then +import ./trackedfutures push: {.upraises:[].} diff --git a/codex/utils/fileutils.nim b/codex/utils/fileutils.nim index 03cffeba..847ce06c 100644 --- a/codex/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -9,16 +9,17 @@ ## Partially taken from nim beacon chain -import std/strutils import pkg/upraises push: {.upraises: [].} -import pkg/chronicles -import stew/io2 +import std/strutils +import pkg/stew/io2 + +import ../logutils export io2 -export chronicles except toJson +export logutils when defined(windows): import stew/[windows/acl] diff --git a/codex/utils/json.nim b/codex/utils/json.nim index ade6936d..080d4e89 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -6,9 +6,10 @@ import std/strutils import std/strformat import std/tables import std/typetraits -import pkg/chronicles -from pkg/libp2p import Cid, init +from pkg/ethers import Address +from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` import pkg/contractabi +import pkg/codexdht/discv5/node as dn import pkg/stew/byteutils import pkg/stint import pkg/questionable/results @@ -16,9 +17,6 @@ import ../errors export json except `%`, `%*` -logScope: - topics = "json serialization" - type SerializationError = object of CodexError UnexpectedKindError = object of SerializationError @@ -303,6 +301,20 @@ func `%`*[T: distinct](id: T): JsonNode = type baseType = T.distinctBase % baseType(id) +func `%`*(cid: Cid): JsonNode = % $cid + +func `%`*(obj: PeerId): JsonNode = % $obj + +func `%`*(obj: SignedPeerRecord): JsonNode = % $obj + +func `%`*(obj: dn.Address): JsonNode = % $obj + +func `%`*(obj: AddressInfo): JsonNode = % $obj.address + +func `%`*(obj: MultiAddress): JsonNode = % $obj + +func `%`*(address: ethers.Address): JsonNode = % $address + func toJson*[T](item: T): string = $(%item) proc toJsnImpl(x: NimNode): NimNode = diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index ef6f6246..c7f76263 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -10,12 +10,12 @@ import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/questionable/results import pkg/libp2p/crypto/crypto import ./fileutils import ../errors +import ../logutils import ../rng export crypto @@ -39,6 +39,6 @@ proc setupKey*(path: string): ?!PrivateKey = warn "The network private key file is not safe, aborting" return failure newException( CodexKeyUnsafeError, "The network private key file is not safe") - + let kb = ? path.readAllBytes().mapFailure(CodexKeyError) return PrivateKey.init(kb).mapFailure(CodexKeyError) diff --git a/codex/utils/timer.nim b/codex/utils/timer.nim index dbde437f..9361d07b 100644 --- a/codex/utils/timer.nim +++ b/codex/utils/timer.nim @@ -15,7 +15,8 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles + +import ../logutils type TimerCallback* = proc(): Future[void] {.gcsafe, upraises:[].} diff --git a/codex/utils/trackedfutures.nim b/codex/utils/trackedfutures.nim index 064bf9e3..f3fcdb2d 100644 --- a/codex/utils/trackedfutures.nim +++ b/codex/utils/trackedfutures.nim @@ -1,7 +1,8 @@ import std/sugar import std/tables -import pkg/chronicles import pkg/chronos + +import ../logutils import ../utils/then type diff --git a/codex/validation.nim b/codex/validation.nim index 2be39975..b549997e 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -1,9 +1,9 @@ import std/sets import std/sequtils import pkg/chronos -import pkg/chronicles import ./market import ./clock +import ./logutils export market export sets @@ -48,7 +48,7 @@ proc subscribeSlotFilled(validation: Validation) {.async.} = let slotId = slotId(requestId, slotIndex) if slotId notin validation.slots: if validation.slots.len < validation.maxSlots: - trace "Adding slot", slotId = $slotId + trace "Adding slot", slotId validation.slots.incl(slotId) let subscription = await validation.market.subscribeSlotFilled(onSlotFilled) validation.subscriptions.add(subscription) @@ -58,7 +58,7 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = for slotId in validation.slots: let state = await validation.market.slotState(slotId) if state != SlotState.Filled: - trace "Removing slot", slot = $slotId + trace "Removing slot", slotId ended.incl(slotId) validation.slots.excl(ended) @@ -70,7 +70,7 @@ proc markProofAsMissing(validation: Validation, try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): - trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period + trace "Marking proof as missing", slotId, periodProofMissed = period await validation.market.markProofAsMissing(slotId, period) else: let inDowntime {.used.} = await validation.market.inDowntime(slotId) diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index 05e5a047..601912f1 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -7,7 +7,6 @@ import std/importutils import pkg/asynctest import pkg/chronos -import pkg/chronicles import pkg/stew/byteutils import pkg/datastore import pkg/questionable @@ -19,6 +18,7 @@ import pkg/poseidon2/io import pkg/nitro import pkg/codexdht/discv5/protocol as discv5 +import pkg/codex/logutils import pkg/codex/stores import pkg/codex/clock import pkg/codex/contracts diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 4827d530..51dd7e0d 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -7,7 +7,6 @@ import std/importutils import pkg/asynctest import pkg/chronos -import pkg/chronicles import pkg/stew/byteutils import pkg/datastore import pkg/questionable @@ -19,6 +18,7 @@ import pkg/poseidon2/io import pkg/nitro import pkg/codexdht/discv5/protocol as discv5 +import pkg/codex/logutils import pkg/codex/stores import pkg/codex/clock import pkg/codex/contracts diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 706bcdbd..18d1e429 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -1,11 +1,11 @@ import std/sequtils import pkg/asynctest -import pkg/chronicles import pkg/chronos import pkg/datastore import pkg/questionable import pkg/questionable/results +import pkg/codex/logutils import pkg/codex/sales/slotqueue import ../helpers diff --git a/tests/codex/slots/testsampler.nim b/tests/codex/slots/testsampler.nim index 2ca2b40f..6cf687c5 100644 --- a/tests/codex/slots/testsampler.nim +++ b/tests/codex/slots/testsampler.nim @@ -10,6 +10,7 @@ import pkg/poseidon2/io import pkg/poseidon2 import pkg/chronos import pkg/asynctest +import pkg/nimcrypto import pkg/codex/stores/cachestore import pkg/codex/chunker import pkg/codex/stores @@ -113,6 +114,6 @@ asyncchecksuite "Test DataSampler": toStr(input.samples[2].cellProof) == expectedCellBlockProofs[2] # # cell data - toHex(input.samples[0].data) == expectedCellData[0] - toHex(input.samples[1].data) == expectedCellData[1] - toHex(input.samples[2].data) == expectedCellData[2] + nimcrypto.toHex(input.samples[0].data) == expectedCellData[0] + nimcrypto.toHex(input.samples[1].data) == expectedCellData[1] + nimcrypto.toHex(input.samples[2].data) == expectedCellData[2] diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 99eec1c1..e8186c01 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/stew/byteutils import pkg/codex/chunker -import pkg/chronicles +import pkg/codex/logutils import pkg/chronos import ./helpers diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim new file mode 100644 index 00000000..9f8e42ac --- /dev/null +++ b/tests/codex/testlogutils.nim @@ -0,0 +1,234 @@ +import std/options +import std/strutils +import std/unittest +import pkg/codex/blocktype +import pkg/codex/conf +import pkg/codex/contracts/requests +import pkg/codex/logutils +import pkg/codex/purchasing/purchaseid +import pkg/codex/units +import pkg/libp2p/cid +import pkg/libp2p/multiaddress +import pkg/questionable +import pkg/questionable/results +import pkg/stew/byteutils +import pkg/stint +import ../checktest + +export logutils + +logStream testlines[textlines[nocolors,notimestamps,dynamic]] +logStream testjson[json[nocolors,notimestamps,dynamic]] + +type + ObjectType = object + a: string + DistinctType {.borrow: `.`.} = distinct ObjectType + RefType = ref object + a: string + AnotherType = object + a: int + +# must be defined at the top-level +proc `$`*(t: ObjectType): string = "used `$`" +func `%`*(t: RefType): JsonNode = % t.a +logutils.formatIt(LogFormat.textLines, ObjectType): "formatted_" & it.a +logutils.formatIt(LogFormat.textLines, RefType): "formatted_" & it.a +logutils.formatIt(LogFormat.textLines, DistinctType): "formatted_" & it.a +logutils.formatIt(LogFormat.json, ObjectType): "formatted_" & it.a +logutils.formatIt(LogFormat.json, RefType): %it +logutils.formatIt(LogFormat.json, DistinctType): "formatted_" & it.a +logutils.formatIt(AnotherType): it.a + +checksuite "Test logging output": + var outputLines: string + var outputJson: string + + proc writeToLines(logLevel: LogLevel, msg: LogOutputStr) = + outputLines &= msg + + proc writeToJson(logLevel: LogLevel, msg: LogOutputStr) = + outputJson &= msg + + setup: + outputLines = "" + outputJson = "" + testlines.outputs[0].writer = writeToLines + testjson.outputs[0].writer = writeToJson + + template logged(prop, expected): auto = + let toFind = prop & "=" & expected + outputLines.contains(toFind) + + template loggedJson(prop, expected): auto = + let json = $ parseJson(outputJson){prop} + json == expected + + template log(val) = + testlines.trace "test", val + testjson.trace "test", val + + test "logs objects": + let t = ObjectType(a: "a") + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"formatted_a\"") + + test "logs sequences of objects": + let t1 = ObjectType(a: "a") + let t2 = ObjectType(a: "b") + let t = @[t1, t2] + log t + check logged("t", "\"@[formatted_a, formatted_b]\"") + check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + + test "logs ref types": + let t = RefType(a: "a") + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"a\"") + + test "logs sequences of ref types": + let t1 = RefType(a: "a") + let t2 = RefType(a: "b") + let t = @[t1, t2] + log t + check logged("t", "\"@[formatted_a, formatted_b]\"") + check loggedJson("t", "[\"a\",\"b\"]") + + test "logs distinct types": + let t = DistinctType(ObjectType(a: "a")) + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"formatted_a\"") + + test "logs sequences of distinct types": + let t1 = DistinctType(ObjectType(a: "a")) + let t2 = DistinctType(ObjectType(a: "b")) + let t = @[t1, t2] + log t + check logged("t", "\"@[formatted_a, formatted_b]\"") + check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + + test "formatIt can return non-string types": + let t = AnotherType(a: 1) + log t + check logged("t", "1") + check loggedJson("t", "1") + + test "logs Option types": + let t = some ObjectType(a: "a") + log t + check logged("t", "some(formatted_a)") + check loggedJson("t", "\"formatted_a\"") + + test "logs sequences of Option types": + let t1 = some ObjectType(a: "a") + let t2 = none ObjectType + let t = @[t1, t2] + log t + check logged("t", "\"@[some(formatted_a), none(ObjectType)]\"") + check loggedJson("t", """["formatted_a",null]""") + + test "logs Result types -- success with string property": + let t: ?!ObjectType = success ObjectType(a: "a") + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"formatted_a\"") + + test "logs Result types -- success with int property": + let t: ?!AnotherType = success AnotherType(a: 1) + log t + check logged("t", "1") + check loggedJson("t", "1") + + test "logs Result types -- failure": + let t: ?!ObjectType = ObjectType.failure newException(ValueError, "some error") + log t + check logged("t", "\"Error: some error\"") + check loggedJson("t", """{"error":"some error"}""") + + test "can define `$` override for T": + let o = ObjectType() + check $o == "used `$`" + + test "logs NByte correctly": + let nb = 12345.NBytes + log nb + check logged("nb", "12345\'NByte") + check loggedJson("nb", "\"12345\'NByte\"") + + test "logs BlockAddress correctly": + let cid = Cid.init("zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1").tryGet + let ba = BlockAddress.init(cid, 0) + log ba + check logged("ba", "\"treeCid: zb2*fndjU1, index: 0\"") + check loggedJson("ba", """{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}""") + + test "logs Cid correctly": + let cid = Cid.init("zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD").tryGet + log cid + check logged("cid", "zb2*A5TgiD") + check loggedJson("cid", "\"zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD\"") + + test "logs StUint correctly": + let stint = 12345678901234.u256 + log stint + check logged("stint", "12345678901234") + check loggedJson("stint", "\"12345678901234\"") + + test "logs int correctly": + let int = 123 + log int + check logged("int", "123") + check loggedJson("int", "123") + + test "logs EthAddress correctly": + let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b") + log address + check logged("address", "0xf75e..1b1b") + check loggedJson("address", "\"0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b\"") + + test "logs PurchaseId correctly": + let id = PurchaseId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") + log id + check logged("id", "0x7120..3dea") + + test "logs RequestId correctly": + let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") + log id + check logged("id", "0x7120..3dea") + check loggedJson("id", "\"0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea\"") + + test "logs seq[RequestId] correctly": + let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") + let id2 = RequestId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") + let ids = @[id, id2] + log ids + check logged("ids", "\"@[0x7120..3dea, 0x9ab2..c282]\"") + check loggedJson("ids", """["0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea","0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282"]""") + + test "logs SlotId correctly": + let id = SlotId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") + log id + check logged("id", "0x9ab2..c282") + check loggedJson("id", "\"0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282\"") + + test "logs Nonce correctly": + let n = Nonce.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea") + log n + check logged("n", "0xce88..d6ea") + check loggedJson("n", "\"0xce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea\"") + + test "logs MultiAddress correctly": + let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet + log ma + check logged("ma", "/ip4/127.0.0.1/tcp/0") + check loggedJson("ma", "\"/ip4/127.0.0.1/tcp/0\"") + + test "logs seq[MultiAddress] correctly": + let ma = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet, + MultiAddress.init("/ip4/127.0.0.2/tcp/1").tryGet] + log ma + check logged("ma", "\"@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]\"") + check loggedJson("ma", "[\"/ip4/127.0.0.1/tcp/0\",\"/ip4/127.0.0.2/tcp/1\"]") diff --git a/tests/codex/utils/testjson.nim b/tests/codex/utils/testjson.nim index aa3731e5..5ba7fff3 100644 --- a/tests/codex/utils/testjson.nim +++ b/tests/codex/utils/testjson.nim @@ -3,11 +3,11 @@ import std/options import std/strformat import std/strutils import std/unittest -import pkg/chronicles except toJson import pkg/stew/byteutils import pkg/stint import pkg/codex/contracts/requests from pkg/codex/rest/json import RestPurchase +import pkg/codex/logutils import pkg/codex/utils/json as utilsjson import pkg/questionable import pkg/questionable/results diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index 22aaeba0..4864f992 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -1,4 +1,3 @@ -import std/json import pkg/chronos import pkg/ethers/testing import pkg/ethers/erc20 diff --git a/tests/ethertest.nim b/tests/ethertest.nim index 1166b0e0..5c620310 100644 --- a/tests/ethertest.nim +++ b/tests/ethertest.nim @@ -1,4 +1,4 @@ -import std/json + import pkg/asynctest import pkg/ethers diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index dbddc204..7cb0b50e 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -3,9 +3,9 @@ import std/strutils import std/sequtils from pkg/libp2p import Cid, `$`, init -import pkg/chronicles import pkg/stint import pkg/questionable/results +import pkg/codex/logutils import pkg/codex/rest/json import pkg/codex/purchasing import pkg/codex/errors diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index b601e6db..6c9a8bf8 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -1,8 +1,7 @@ import std/os import std/macros -import std/json import std/httpclient -import pkg/chronicles +import pkg/codex/logutils import ../ethertest import ./codexclient import ./nodes diff --git a/tests/integration/nodes.nim b/tests/integration/nodes.nim index 777de80b..a50f9974 100644 --- a/tests/integration/nodes.nim +++ b/tests/integration/nodes.nim @@ -1,12 +1,12 @@ -import pkg/questionable -import pkg/confutils -import pkg/chronicles -import pkg/libp2p import std/osproc import std/os import std/streams import std/strutils -import codex/conf +import pkg/codex/conf +import pkg/codex/logutils +import pkg/confutils +import pkg/libp2p +import pkg/questionable import ./codexclient export codexclient diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index 86905f8f..0af0572a 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -1,9 +1,9 @@ import std/sequtils import std/os from std/times import getTime, toUnix -import pkg/chronicles -import codex/contracts -import codex/periods +import pkg/codex/contracts +import pkg/codex/logutils +import pkg/codex/periods import ../contracts/time import ../contracts/deployment import ./twonodes diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index fafaf567..9fa11f1f 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -1,6 +1,5 @@ import std/os import std/macros -import std/json import std/httpclient import ../ethertest import ./codexclient diff --git a/tests/logging.nim b/tests/logging.nim index cf2633d3..ece9c9b0 100644 --- a/tests/logging.nim +++ b/tests/logging.nim @@ -1,5 +1,5 @@ when not defined(nimscript): - import pkg/chronicles + import pkg/codex/logutils proc ignoreLogging(level: LogLevel, message: LogOutputStr) = discard diff --git a/tests/testCodex.nim b/tests/testCodex.nim index 7fb34ff3..6a9b107e 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -2,6 +2,7 @@ import ./codex/teststores import ./codex/testblockexchange import ./codex/testasyncheapqueue import ./codex/testchunking +import ./codex/testlogutils import ./codex/testmanifest import ./codex/testnode import ./codex/teststorestream diff --git a/vendor/nim-poseidon2 b/vendor/nim-poseidon2 index eef2603c..0346982f 160000 --- a/vendor/nim-poseidon2 +++ b/vendor/nim-poseidon2 @@ -1 +1 @@ -Subproject commit eef2603c11eeb5069eaed6c0296f3a1c5d23aad1 +Subproject commit 0346982f2c6891bcedd03d552af3a3bd57b2c1f9