feat: create logging proxy (#663)

* implement a logging proxy

The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks

* Rename logging -> logutils to avoid ambiguity with common names

* clean up

* add setProperty for JsonRecord, remove nim-json-serialization conflict

* Allow specifying textlines and json format separately

Not specifying a LogFormat will apply the formatting to both textlines and json sinks.

Specifying a LogFormat will apply the formatting to only that sink.

* remove unneeded usages of std/json

We only need to import utils/json instead of std/json

* move serialization from rest/json to utils/json so it can be shared

* fix NoColors ambiguity

Was causing unit tests to fail on Windows.

* Remove nre usage to fix Windows error

Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)

* Add logutils module doc

* Shorten logutils.formatIt for `NBytes`

Both json and textlines formatIt were not needed, and could be combined into one formatIt

* remove debug integration test config

debug output and logformat of json for integration test logs

* Use ## module doc to support docgen

* bump nim-poseidon2 to export fromBytes

Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.

* fixes to get compiling after rebasing master

* Add support for Result types being logged using formatIt

codex.nim

@@ -7,7 +7,6 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/questionable
 import pkg/confutils
@@ -21,10 +20,11 @@ import pkg/libp2p
 
 import ./codex/conf
 import ./codex/codex
+import ./codex/logutils
 import ./codex/units
 import ./codex/utils/keyutils
 
-export codex, conf, libp2p, chronos, chronicles
+export codex, conf, libp2p, chronos, logutils
 
 when isMainModule:
   import std/sequtils

codex/blockexchange/engine/discovery.nim

@@ -10,22 +10,21 @@
 import std/sequtils
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p/cid
 import pkg/metrics
 import pkg/questionable
 import pkg/questionable/results
 
-import ../protobuf/presence
+import ./pendingblocks
 
+import ../protobuf/presence
 import ../network
 import ../peers
 
 import ../../utils
 import ../../discovery
 import ../../stores/blockstore
-
+import ../../logutils
-import ./pendingblocks
 
 logScope:
   topics = "codex discoveryengine"

codex/blockexchange/engine/engine.nim

@@ -14,7 +14,6 @@ import std/algorithm
 import std/sugar
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p/[cid, switch, multihash, multicodec]
 import pkg/metrics
 import pkg/stint
@@ -23,6 +22,7 @@ import ../../stores/blockstore
 import ../../blocktype
 import ../../utils
 import ../../merkletree
+import ../../logutils
 
 import ../protobuf/blockexc
 import ../protobuf/presence

codex/blockexchange/engine/pendingblocks.nim

@@ -14,13 +14,13 @@ import pkg/upraises
 
 push: {.upraises: [].}
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/libp2p
 import pkg/metrics
 
 import ../protobuf/blockexc
 import ../../blocktype
+import ../../logutils
 
 logScope:
   topics = "codex pendingblocks"

codex/blockexchange/network/network.nim

@@ -10,7 +10,6 @@
 import std/tables
 import std/sequtils
 
-import pkg/chronicles
 import pkg/chronos
 
 import pkg/libp2p
@@ -19,6 +18,7 @@ import pkg/questionable
 import pkg/questionable/results
 
 import ../../blocktype as bt
+import ../../logutils
 import ../protobuf/blockexc as pb
 import ../protobuf/payments
 

codex/blockexchange/network/networkpeer.nim

@@ -11,12 +11,12 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p
 
 import ../protobuf/blockexc
 import ../protobuf/message
 import ../../errors
+import ../../logutils
 
 logScope:
   topics = "codex blockexcnetworkpeer"

codex/blockexchange/peers/peercontext.nim

@@ -11,7 +11,6 @@ import std/sequtils
 import std/tables
 import std/sets
 
-import pkg/chronicles
 import pkg/libp2p
 import pkg/chronos
 import pkg/nitro
@@ -22,6 +21,7 @@ import ../protobuf/payments
 import ../protobuf/presence
 
 import ../../blocktype
+import ../../logutils
 
 export payments, nitro
 

codex/blockexchange/peers/peerctxstore.nim

@@ -16,11 +16,12 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p
 
 import ../protobuf/blockexc
 import ../../blocktype
+import ../../logutils
+
 
 import ./peercontext
 export peercontext

codex/blocktype.nim

@@ -20,16 +20,15 @@ import pkg/libp2p/[cid, multicodec, multihash]
 import pkg/stew/byteutils
 import pkg/questionable
 import pkg/questionable/results
-import pkg/chronicles
-import pkg/json_serialization
 
 import ./units
 import ./utils
-import ./formats
 import ./errors
+import ./logutils
+import ./utils/json
 import ./codextypes
 
-export errors, formats, units, codextypes
+export errors, logutils, units, codextypes
 
 type
   Block* = ref object of RootObj
@@ -39,10 +38,18 @@ type
   BlockAddress* = object
     case leaf*: bool
     of true:
-      treeCid*: Cid
+      treeCid* {.serialize.}: Cid
-      index*: Natural
+      index* {.serialize.}: Natural
     else:
-      cid*: Cid
+      cid* {.serialize.}: Cid
+
+logutils.formatIt(LogFormat.textLines, BlockAddress):
+  if it.leaf:
+    "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index
+  else:
+    "cid: " & shortLog($it.cid)
+
+logutils.formatIt(LogFormat.json, BlockAddress): %it
 
 proc `==`*(a, b: BlockAddress): bool =
   a.leaf == b.leaf and
@@ -59,12 +66,6 @@ proc `$`*(a: BlockAddress): string =
   else:
     "cid: " & $a.cid
 
-proc writeValue*(
-  writer: var JsonWriter,
-  value: Cid
-) {.upraises:[IOError].} =
-  writer.writeValue($value)
-
 proc cidOrTreeCid*(a: BlockAddress): Cid =
   if a.leaf:
     a.treeCid

codex/chunker.nim

@@ -13,13 +13,13 @@ import pkg/upraises
 
 push: {.upraises: [].}
 
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
 import pkg/chronos
 import pkg/libp2p except shuffle
 
 import ./blocktype
+import ./logutils
 
 export blocktype
 

codex/codex.nim

@@ -12,7 +12,6 @@ import std/strutils
 import std/os
 import std/tables
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/presto
 import pkg/libp2p
@@ -39,6 +38,7 @@ import ./contracts/clock
 import ./contracts/deployment
 import ./utils/addrutils
 import ./namespaces
+import ./logutils
 
 logScope:
   topics = "codex node"

codex/conf.nim

@@ -18,7 +18,6 @@ import std/strutils
 import std/typetraits
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/chronicles/helpers
 import pkg/chronicles/topics_registry
 import pkg/confutils/defs
@@ -35,6 +34,7 @@ import pkg/questionable
 import pkg/questionable/results
 
 import ./discovery
+import ./logutils
 import ./stores
 import ./units
 import ./utils
@@ -58,7 +58,7 @@ type
     noCommand,
     initNode
 
-  LogKind* = enum
+  LogKind* {.pure.} = enum
     Auto = "auto"
     Colors = "colors"
     NoColors = "nocolors"
@@ -281,6 +281,9 @@ type
 
   EthAddress* = ethers.Address
 
+logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
+logutils.formatIt(LogFormat.json, EthAddress): %it
+
 proc getCodexVersion(): string =
   let tag = strip(staticExec("git tag"))
   if tag.isEmptyOrWhitespace:
@@ -412,7 +415,7 @@ proc completeCmdArg*(T: type Duration; val: string): seq[string] =
   discard
 
 # silly chronicles, colors is a compile-time property
-proc stripAnsi(v: string): string =
+proc stripAnsi*(v: string): string =
   var
     res = newStringOfCap(v.len)
     i: int

codex/contracts/deployment.nim

@@ -1,11 +1,10 @@
-import std/json
 import std/os
 import std/tables
 import pkg/ethers
 import pkg/questionable
-import pkg/chronicles
 
 import ../conf
+import ../logutils
 import ./marketplace
 
 type Deployment* = ref object

codex/contracts/interactions/clientinteractions.nim

@@ -1,13 +1,13 @@
 import pkg/ethers
-import pkg/chronicles
 
 import ../../purchasing
+import ../../logutils
 import ../market
 import ../clock
 import ./interactions
 
 export purchasing
-export chronicles except toJson
+export logutils
 
 type
   ClientInteractions* = ref object of ContractInteractions

codex/contracts/interactions/hostinteractions.nim

@@ -1,11 +1,11 @@
-import pkg/ethers
+import pkg/chronos
-import pkg/chronicles
 
+import ../../logutils
 import ../../sales
 import ./interactions
 
 export sales
-export chronicles except toJson
+export logutils
 
 type
   HostInteractions* = ref object of ContractInteractions

codex/contracts/market.nim

@@ -1,11 +1,11 @@
 import std/sequtils
 import std/strutils
 import std/sugar
-import pkg/chronicles
 import pkg/ethers
 import pkg/ethers/testing
 import pkg/upraises
 import pkg/questionable
+import ../logutils
 import ../market
 import ./marketplace
 

codex/contracts/requests.nim

@@ -1,12 +1,13 @@
 import std/hashes
+import std/sequtils
 import std/typetraits
 import pkg/contractabi
 import pkg/nimcrypto
 import pkg/ethers/fields
 import pkg/questionable/results
 import pkg/stew/byteutils
-import pkg/json_serialization
 import pkg/upraises
+import ../logutils
 import ../utils/json
 
 export contractabi
@@ -79,6 +80,13 @@ proc toHex*[T: distinct](id: T): string =
   type baseType = T.distinctBase
   baseType(id).toHex
 
+logutils.formatIt(LogFormat.textLines, Nonce): it.short0xHexLog
+logutils.formatIt(LogFormat.textLines, RequestId): it.short0xHexLog
+logutils.formatIt(LogFormat.textLines, SlotId): it.short0xHexLog
+logutils.formatIt(LogFormat.json, Nonce): it.to0xHexLog
+logutils.formatIt(LogFormat.json, RequestId): it.to0xHexLog
+logutils.formatIt(LogFormat.json, SlotId): it.to0xHexLog
+
 func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest =
   StorageRequest(
     client: tupl[0],
@@ -176,17 +184,3 @@ func price*(request: StorageRequest): UInt256 =
 
 func size*(ask: StorageAsk): UInt256 =
   ask.slots.u256 * ask.slotSize
-
-proc writeValue*(
-  writer: var JsonWriter,
-  value: SlotId | RequestId) {.upraises:[IOError].} =
-
-  mixin writeValue
-  writer.writeValue value.toArray
-
-proc readValue*[T: SlotId | RequestId](
-  reader: var JsonReader,
-  value: var T) {.upraises: [SerializationError, IOError].} =
-
-  mixin readValue
-  value = T reader.readValue(T.distinctBase)

codex/discovery.nim

@@ -11,7 +11,6 @@ import std/algorithm
 import std/sequtils
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope]
 import pkg/questionable
 import pkg/questionable/results
@@ -21,7 +20,7 @@ import pkg/codexdht/discv5/protocol as discv5
 
 import ./rng
 import ./errors
-import ./formats
+import ./logutils
 
 export discv5
 

codex/erasure/erasure.nim

@@ -15,10 +15,10 @@ import std/sequtils
 import std/sugar
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p/[multicodec, cid, multihash]
 import pkg/libp2p/protobuf/minprotobuf
 
+import ../logutils
 import ../manifest
 import ../merkletree
 import ../stores

codex/formats.nim

@@ -1,28 +0,0 @@
-## Nim-Codex
-## Copyright (c) 2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
-
-import std/strutils
-
-import pkg/chronicles
-import pkg/libp2p/cid
-
-func shortLog*(cid: Cid): string =
-  ## Returns compact string representation of ``pid``.
-  var scid = $cid
-  if len(scid) > 10:
-    scid[3] = '*'
-
-    when (NimMajor, NimMinor) > (1, 4):
-      scid.delete(4 .. scid.high - 6)
-    else:
-      scid.delete(4, scid.high - 6)
-
-  scid
-
-chronicles.formatIt(Cid): shortLog(it)

codex/logutils.nim

@@ -0,0 +1,241 @@
+## logutils is a module that has several goals:
+## 1. Fix json logging output (run with `--log-format=json`) which was
+##    effectively broken for many types using default Chronicles json
+##    serialization.
+## 2. Ability to specify log output for textlines and json sinks together or
+##    separately
+##     - This is useful if consuming json in some kind of log parser and need
+##       valid json with real values
+##     - eg a shortened Cid is nice to see in a text log in stdout, but won't
+##       provide a real Cid when parsed in json
+## 4. Remove usages of `nim-json-serialization` from the codebase
+## 5. Remove need to declare `writeValue` for new types
+## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent
+##    conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467)
+##
+## When declaring a new type, one should consider importing the `codex/logutils`
+## module, and specifying `formatIt`. If textlines log output and json log output
+## need to be different, overload `formatIt` and specify a `LogFormat`. If json
+## serialization is needed, it can be declared with a `%` proc. `logutils`
+## imports and exports `utils/json` which handles the de/serialization, examples
+## below. **Only `codex/logutils` needs to be imported.**
+##
+## Using `logutils` in the Codex codebase:
+## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils`
+##     - most of `chronicles` is exported by `logutils`
+## - Instead of importing `std/json`, import `pkg/codex/utils/json`
+##     - `std/json` is exported by `utils/json` which is exported by `logutils`
+## - Instead of importing `pkg/nim-json-serialization`, import
+##   `pkg/codex/utils/json`
+##     - one of the goals is to remove the use of `nim-json-serialization`
+##
+## ```nim
+## import pkg/codex/logutils
+##
+## type
+##   BlockAddress* = object
+##     case leaf*: bool
+##     of true:
+##       treeCid* {.serialize.}: Cid
+##       index* {.serialize.}: Natural
+##     else:
+##       cid* {.serialize.}: Cid
+##
+## logutils.formatIt(LogFormat.textLines, BlockAddress):
+##   if it.leaf:
+##     "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index
+##   else:
+##     "cid: " & shortLog($it.cid)
+##
+## logutils.formatIt(LogFormat.json, BlockAddress): %it
+##
+## # chronicles textlines output
+## TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0"
+## # chronicles json output
+## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
+## ```
+## In this case, `BlockAddress` is just an object, so `utils/json` can handle
+## serializing it without issue (only fields annotated with `{.serialize.}` will
+## serialize (aka opt-in serialization)).
+##
+## If one so wished, another option for the textlines log output, would be to
+## simply `toString` the serialised json:
+## ```nim
+## logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it
+## # or, more succinctly:
+## logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson
+## ```
+## In that case, both the textlines and json sinks would have the same output,
+## so we could reduce this even further by not specifying a `LogFormat`:
+## ```nim
+## type
+##   BlockAddress* = object
+##     case leaf*: bool
+##     of true:
+##       treeCid* {.serialize.}: Cid
+##       index* {.serialize.}: Natural
+##     else:
+##       cid* {.serialize.}: Cid
+##
+## logutils.formatIt(BlockAddress): %it
+##
+## # chronicles textlines output
+## TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}"
+## # chronicles json output
+## {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
+## ```
+
+import std/options
+import std/sequtils
+import std/strutils
+import std/sugar
+import std/typetraits
+
+import pkg/chronicles except toJson, `%`
+from pkg/libp2p import Cid, MultiAddress, `$`
+import pkg/questionable
+import pkg/questionable/results
+import pkg/stew/byteutils
+import pkg/stint
+import pkg/upraises
+
+import ./utils/json
+
+export byteutils
+export chronicles except toJson, formatIt, `%`
+export questionable
+export sequtils
+export strutils
+export sugar
+export upraises
+export json
+export results
+
+func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string =
+  ## Returns compact string representation of ``long``.
+  var short = long
+  let minLen = start + ellipses.len + stop
+  if len(short) > minLen:
+    short.insert(ellipses, start)
+
+    when (NimMajor, NimMinor) > (1, 4):
+      short.delete(start + ellipses.len .. short.high - stop)
+    else:
+      short.delete(start + ellipses.len, short.high - stop)
+
+  short
+
+func shortHexLog*(long: string): string =
+  if long[0..1] == "0x": result &= "0x"
+  result &= long[2..long.high].shortLog("..", 4, 4)
+
+func short0xHexLog*[N: static[int], T: array[N, byte]](v: T): string =
+  v.to0xHex.shortHexLog
+
+func short0xHexLog*[T: distinct](v: T): string =
+  type BaseType = T.distinctBase
+  BaseType(v).short0xHexLog
+
+func short0xHexLog*[U: distinct, T: seq[U]](v: T): string =
+  type BaseType = U.distinctBase
+  "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]"
+
+func to0xHexLog*[T: distinct](v: T): string =
+  type BaseType = T.distinctBase
+  BaseType(v).to0xHex
+
+func to0xHexLog*[U: distinct, T: seq[U]](v: T): string =
+  type BaseType = U.distinctBase
+  "@[" & v.map(x => BaseType(x).to0xHex).join(",") & "]"
+
+proc formatTextLineSeq*(val: seq[string]): string =
+  "@[" & val.join(", ") & "]"
+
+template formatIt*(format: LogFormat, T: typedesc, body: untyped) =
+  # Provides formatters for logging with Chronicles for the given type and
+  # `LogFormat`.
+  # NOTE: `seq[T]`, `Option[T]`, and `seq[Option[T]]` are overriddden
+  # since the base `setProperty` is generic using `auto` and conflicts with
+  # providing a generic `seq` and `Option` override.
+  when format == LogFormat.json:
+    proc formatJsonOption(val: ?T): JsonNode =
+      if it =? val:
+        json.`%`(body)
+      else:
+        newJNull()
+
+    proc formatJsonResult*(val: ?!T): JsonNode =
+      without it =? val, error:
+        let jObj = newJObject()
+        jObj["error"] = newJString(error.msg)
+        return jObj
+      json.`%`(body)
+
+    proc setProperty*(r: var JsonRecord, key: string, res: ?!T) =
+      var it {.inject, used.}: T
+      setProperty(r, key, res.formatJsonResult)
+
+    proc setProperty*(r: var JsonRecord, key: string, opt: ?T) =
+      var it {.inject, used.}: T
+      let v = opt.formatJsonOption
+      setProperty(r, key, v)
+
+    proc setProperty*(r: var JsonRecord, key: string, opts: seq[?T]) =
+      var it {.inject, used.}: T
+      let v = opts.map(opt => opt.formatJsonOption)
+      setProperty(r, key, json.`%`(v))
+
+    proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) =
+      var it {.inject, used.}: T
+      let v = val.map(it => body)
+      setProperty(r, key, json.`%`(v))
+
+    proc setProperty*(r: var JsonRecord, key: string, val: T) {.upraises:[ValueError, IOError].} =
+      var it {.inject, used.}: T = val
+      let v = body
+      setProperty(r, key, json.`%`(v))
+
+  elif format == LogFormat.textLines:
+    proc formatTextLineOption*(val: ?T): string =
+      var v = "none(" & $T & ")"
+      if it =? val:
+        v = "some(" & $(body) & ")" # that I used to know :)
+      v
+
+    proc formatTextLineResult*(val: ?!T): string =
+      without it =? val, error:
+        return "Error: " & error.msg
+      $(body)
+
+    proc setProperty*(r: var TextLineRecord, key: string, res: ?!T) =
+      var it {.inject, used.}: T
+      setProperty(r, key, res.formatTextLineResult)
+
+    proc setProperty*(r: var TextLineRecord, key: string, opt: ?T) =
+      var it {.inject, used.}: T
+      let v = opt.formatTextLineOption
+      setProperty(r, key, v)
+
+    proc setProperty*(r: var TextLineRecord, key: string, opts: seq[?T]) =
+      var it {.inject, used.}: T
+      let v = opts.map(opt => opt.formatTextLineOption)
+      setProperty(r, key, v.formatTextLineSeq)
+
+    proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) =
+      var it {.inject, used.}: T
+      let v = val.map(it => body)
+      setProperty(r, key, v.formatTextLineSeq)
+
+    proc setProperty*(r: var TextLineRecord, key: string, val: T) {.upraises:[ValueError, IOError].} =
+      var it {.inject, used.}: T = val
+      let v = body
+      setProperty(r, key, v)
+
+template formatIt*(T: type, body: untyped) {.dirty.} =
+  formatIt(LogFormat.textLines, T): body
+  formatIt(LogFormat.json, T): body
+
+formatIt(LogFormat.textLines, Cid): shortLog($it)
+formatIt(LogFormat.json, Cid): $it
+formatIt(UInt256): $it
+formatIt(MultiAddress): $it

codex/manifest/coders.nim

@@ -19,12 +19,12 @@ import std/sequtils
 import pkg/libp2p
 import pkg/questionable
 import pkg/questionable/results
-import pkg/chronicles
 import pkg/chronos
 
 import ./manifest
 import ../errors
 import ../blocktype
+import ../logutils
 
 proc encode*(manifest: Manifest): ?!seq[byte] =
   ## Encode the manifest into a ``ManifestCodec``

codex/manifest/manifest.nim

@@ -22,6 +22,7 @@ import ../utils
 import ../utils/json
 import ../units
 import ../blocktype
+import ../logutils
 
 type
   Manifest* = ref object of RootObj

codex/node.nim

@@ -16,7 +16,6 @@ import std/sugar
 
 import pkg/questionable
 import pkg/questionable/results
-import pkg/chronicles
 import pkg/chronos
 import pkg/poseidon2
 
@@ -41,6 +40,9 @@ import ./discovery
 import ./contracts
 import ./utils
 import ./errors
+import ./logutils
+
+export logutils
 
 logScope:
   topics = "codex node"
@@ -658,7 +660,7 @@ proc start*(self: CodexNodeRef) {.async.} =
       self.contracts.validator = ValidatorInteractions.none
 
   self.networkId = self.switch.peerInfo.peerId
-  notice "Started codex node", id = $self.networkId, addrs = self.switch.peerInfo.addrs
+  notice "Started codex node", id = self.networkId, addrs = self.switch.peerInfo.addrs
 
 proc stop*(self: CodexNodeRef) {.async.} =
   trace "Stopping node"

codex/purchasing/purchaseid.nim

@@ -1,8 +1,12 @@
 import std/hashes
 import pkg/nimcrypto
+import ../logutils
 
 type PurchaseId* = distinct array[32, byte]
 
+logutils.formatIt(LogFormat.textLines, PurchaseId): it.short0xHexLog
+logutils.formatIt(LogFormat.json, PurchaseId): it.to0xHexLog
+
 proc hash*(x: PurchaseId): Hash {.borrow.}
 proc `==`*(x, y: PurchaseId): bool {.borrow.}
 proc toHex*(x: PurchaseId): string = array[32, byte](x).toHex

codex/purchasing/states/cancelled.nim

@@ -1,5 +1,6 @@
 import pkg/metrics
-import pkg/chronicles
+
+import ../../logutils
 import ../statemachine
 import ./errorhandling
 
@@ -17,7 +18,7 @@ method run*(state: PurchaseCancelled, machine: Machine): Future[?State] {.async.
   codex_purchases_cancelled.inc()
   let purchase = Purchase(machine)
 
-  warn "Request cancelled, withdrawing remaining funds",  requestId = $purchase.requestId
+  warn "Request cancelled, withdrawing remaining funds",  requestId = purchase.requestId
   await purchase.market.withdrawFunds(purchase.requestId)
 
   let error = newException(Timeout, "Purchase cancelled due to timeout")

codex/purchasing/states/error.nim

@@ -1,7 +1,7 @@
 import pkg/metrics
-import pkg/chronicles
 import ../statemachine
 import ../../utils/exceptions
+import ../../logutils
 
 declareCounter(codex_purchases_error, "codex purchases error")
 

codex/purchasing/states/finished.nim

@@ -1,6 +1,7 @@
 import pkg/metrics
-import pkg/chronicles
+
 import ../statemachine
+import ../../logutils
 
 declareCounter(codex_purchases_finished, "codex purchases finished")
 

codex/purchasing/states/started.nim

@@ -1,5 +1,6 @@
 import pkg/metrics
-import pkg/chronicles
+
+import ../../logutils
 import ../statemachine
 import ./errorhandling
 import ./finished

codex/purchasing/states/submitted.nim

@@ -1,5 +1,6 @@
 import pkg/metrics
-import pkg/chronicles
+
+import ../../logutils
 import ../statemachine
 import ./errorhandling
 import ./started

codex/rest/api.nim

@@ -16,7 +16,6 @@ import std/sequtils
 
 import pkg/questionable
 import pkg/questionable/results
-import pkg/chronicles except toJson
 import pkg/chronos
 import pkg/presto except toJson
 import pkg/metrics except toJson
@@ -28,6 +27,7 @@ import pkg/libp2p
 import pkg/libp2p/routing_record
 import pkg/codexdht/discv5/spr as spr
 
+import ../logutils
 import ../node
 import ../blocktype
 import ../conf

codex/rest/json.nim

@@ -111,23 +111,4 @@ func `%`*(obj: StorageRequest | Slot): JsonNode =
 
   return jsonObj
 
-func `%`*(obj: Cid): JsonNode =
+func `%`*(obj: RestNodeId): JsonNode = % $obj.id
-  % $obj
-
-func `%`*(obj: PeerId): JsonNode =
-  % $obj
-
-func `%`*(obj: RestNodeId): JsonNode =
-  % $obj.id
-
-func `%`*(obj: SignedPeerRecord): JsonNode =
-  % $obj
-
-func `%`*(obj: dn.Address): JsonNode =
-  % $obj
-
-func `%`*(obj: AddressInfo): JsonNode =
-  % $obj.address
-
-func `%`*(obj: MultiAddress): JsonNode =
-  % $obj

codex/sales.nim

@@ -3,13 +3,13 @@ import std/sugar
 import pkg/questionable
 import pkg/questionable/results
 import pkg/stint
-import pkg/chronicles
 import pkg/datastore
 import ./market
 import ./clock
 import ./stores
 import ./contracts/requests
 import ./contracts/marketplace
+import ./logutils
 import ./sales/salescontext
 import ./sales/salesagent
 import ./sales/statemachine
@@ -166,7 +166,7 @@ proc filled(
     processing.complete()
 
 proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) =
-  debug "processing slot from queue", requestId = $item.requestId,
+  debug "processing slot from queue", requestId = item.requestId,
     slot = item.slotIndex
 
   let agent = newSalesAgent(

codex/sales/reservations.nim

@@ -28,19 +28,19 @@ push: {.upraises: [].}
 
 import std/typetraits
 import pkg/chronos
-import pkg/chronicles except toJson
 import pkg/datastore
 import pkg/nimcrypto
 import pkg/questionable
 import pkg/questionable/results
 import pkg/stint
 import pkg/stew/byteutils
+import ../logutils
 import ../stores
 import ../contracts/requests
 import ../utils/json
 
 export requests
-export chronicles except toJson
+export logutils
 
 logScope:
   topics = "sales reservations"
@@ -139,13 +139,8 @@ proc toErr[E1: ref CatchableError, E2: ReservationsError](
 
   return newException(E2, msg, e1)
 
-proc writeValue*(
+logutils.formatIt(LogFormat.textLines, SomeStorableId): it.short0xHexLog
-  writer: var JsonWriter,
+logutils.formatIt(LogFormat.json, SomeStorableId): it.to0xHexLog
-  value: SomeStorableId) {.upraises:[IOError].} =
-  ## used for chronicles' logs
-
-  mixin writeValue
-  writer.writeValue %value
 
 proc `onAvailabilityAdded=`*(self: Reservations,
                             onAvailabilityAdded: OnAvailabilityAdded) =

codex/sales/salesagent.nim

@@ -1,11 +1,11 @@
 import pkg/chronos
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
 import pkg/stint
 import pkg/upraises
 import ../contracts/requests
 import ../errors
+import ../logutils
 import ./statemachine
 import ./salescontext
 import ./salesdata

codex/sales/slotqueue.nim

@@ -1,11 +1,11 @@
 import std/sequtils
 import std/tables
-import pkg/chronicles
 import pkg/chronos
 import pkg/questionable
 import pkg/questionable/results
 import pkg/upraises
 import ../errors
+import ../logutils
 import ../rng
 import ../utils
 import ../contracts/requests

codex/sales/states/cancelled.nim

@@ -1,4 +1,4 @@
-import pkg/chronicles
+import ../../logutils
 import ../salesagent
 import ../statemachine
 import ./errorhandling
@@ -20,7 +20,7 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
     raiseAssert "no sale request"
 
   let slot = Slot(request: request, slotIndex: data.slotIndex)
-  debug "Collecting collateral and partial payout",  requestId = $data.requestId, slotIndex = $data.slotIndex
+  debug "Collecting collateral and partial payout",  requestId = data.requestId, slotIndex = data.slotIndex
   await market.freeSlot(slot.id)
 
   if onClear =? agent.context.onClear and
@@ -30,4 +30,4 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
   if onCleanUp =? agent.onCleanUp:
     await onCleanUp(returnBytes = true)
 
-  warn "Sale cancelled due to timeout",  requestId = $data.requestId, slotIndex = $data.slotIndex
+  warn "Sale cancelled due to timeout",  requestId = data.requestId, slotIndex = data.slotIndex

codex/sales/states/downloading.nim

@@ -1,7 +1,8 @@
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
+
 import ../../blocktype as bt
+import ../../logutils
 import ../../market
 import ../salesagent
 import ../statemachine

codex/sales/states/errored.nim

@@ -1,9 +1,10 @@
 import pkg/questionable
 import pkg/questionable/results
 import pkg/upraises
-import pkg/chronicles
+
 import ../statemachine
 import ../salesagent
+import ../../logutils
 import ../../utils/exceptions
 
 logScope:

codex/sales/states/failed.nim

@@ -1,4 +1,4 @@
-import pkg/chronicles
+import ../../logutils
 import ../salesagent
 import ../statemachine
 import ./errorhandling
@@ -21,7 +21,7 @@ method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} =
     raiseAssert "no sale request"
 
   let slot = Slot(request: request, slotIndex: data.slotIndex)
-  debug "Removing slot from mySlots",  requestId = $data.requestId, slotIndex = $data.slotIndex
+  debug "Removing slot from mySlots",  requestId = data.requestId, slotIndex = data.slotIndex
   await market.freeSlot(slot.id)
 
   let error = newException(SaleFailedError, "Sale failed")

codex/sales/states/filled.nim

@@ -1,7 +1,8 @@
 import pkg/questionable
 import pkg/questionable/results
-import pkg/chronicles
+
 import ../../conf
+import ../../logutils
 import ../statemachine
 import ../salesagent
 import ./errorhandling
@@ -36,7 +37,7 @@ method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} =
   let me = await market.getSigner()
 
   if host == me.some:
-    info "Slot succesfully filled", requestId = $data.requestId, slotIndex = $data.slotIndex
+    info "Slot succesfully filled", requestId = data.requestId, slotIndex = data.slotIndex
 
     without request =? data.request:
       raiseAssert "no sale request"

codex/sales/states/filling.nim

@@ -1,4 +1,4 @@
-import pkg/chronicles
+import ../../logutils
 import ../../market
 import ../statemachine
 import ../salesagent
@@ -32,5 +32,5 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
   without (collateral =? data.request.?ask.?collateral):
     raiseAssert "Request not set"
 
-  debug "Filling slot", requestId = $data.requestId, slotIndex = $data.slotIndex
+  debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex
   await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)

codex/sales/states/finished.nim

@@ -1,5 +1,6 @@
 import pkg/chronos
-import pkg/chronicles
+
+import ../../logutils
 import ../statemachine
 import ../salesagent
 import ./errorhandling
@@ -27,7 +28,7 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
   without request =? data.request:
     raiseAssert "no sale request"
 
-  info "Slot finished and paid out", requestId = $data.requestId, slotIndex = $data.slotIndex
+  info "Slot finished and paid out", requestId = data.requestId, slotIndex = data.slotIndex
 
   if onCleanUp =? agent.onCleanUp:
     await onCleanUp()

codex/sales/states/ignored.nim

@@ -1,5 +1,6 @@
-import pkg/chronicles
 import pkg/chronos
+
+import ../../logutils
 import ../statemachine
 import ../salesagent
 import ./errorhandling

codex/sales/states/initialproving.nim

@@ -1,5 +1,5 @@
-import pkg/chronicles
 import pkg/questionable/results
+import ../../logutils
 import ../statemachine
 import ../salesagent
 import ./errorhandling
@@ -32,7 +32,7 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async
   without onProve =? context.onProve:
     raiseAssert "onProve callback not set"
 
-  debug "Generating initial proof", requestId = $data.requestId
+  debug "Generating initial proof", requestId = data.requestId
   let
     slot = Slot(request: request, slotIndex: data.slotIndex)
     challenge = await context.market.getChallenge(slot.id)
@@ -40,6 +40,6 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async
     error "Failed to generate initial proof", error = err.msg
     return some State(SaleErrored(error: err))
 
-  debug "Finished proof calculation", requestId = $data.requestId
+  debug "Finished proof calculation", requestId = data.requestId
 
   return some State(SaleFilling(proof: proof))

codex/sales/states/payout.nim

@@ -1,4 +1,4 @@
-import pkg/chronicles
+import ../../logutils
 import ../../market
 import ../statemachine
 import ../salesagent
@@ -29,7 +29,7 @@ method run(state: SalePayout, machine: Machine): Future[?State] {.async.} =
     raiseAssert "no sale request"
 
   let slot = Slot(request: request, slotIndex: data.slotIndex)
-  debug "Collecting finished slot's reward",  requestId = $data.requestId, slotIndex = $data.slotIndex
+  debug "Collecting finished slot's reward",  requestId = data.requestId, slotIndex = data.slotIndex
   await market.freeSlot(slot.id)
 
   return some State(SaleFinished())

codex/sales/states/preparing.nim

@@ -1,6 +1,7 @@
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
+
+import ../../logutils
 import ../../market
 import ../salesagent
 import ../statemachine

codex/sales/states/proving.nim

@@ -1,7 +1,7 @@
 import std/options
-import pkg/chronicles
 import pkg/questionable/results
 import ../../clock
+import ../../logutils
 import ../statemachine
 import ../salesagent
 import ../salescontext
@@ -32,7 +32,7 @@ method prove*(
       error "Failed to generate proof", error = err.msg
       # In this state, there's nothing we can do except try again next time.
       return
-    debug "Submitting proof", currentPeriod = currentPeriod, slotId = $slot.id
+    debug "Submitting proof", currentPeriod = currentPeriod, slotId = slot.id
     await market.submitProof(slot.id, proof)
   except CatchableError as e:
     error "Submitting proof failed", msg = e.msg
@@ -51,9 +51,9 @@ proc proveLoop(
 
   logScope:
     period = currentPeriod
-    requestId = $request.id
+    requestId = request.id
     slotIndex
-    slotId = $slot.id
+    slotId = slot.id
 
   proc getCurrentPeriod(): Future[Period] {.async.} =
     let periodicity = await market.periodicity()
@@ -110,7 +110,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
   without clock =? context.clock:
     raiseAssert("clock not set")
 
-  debug "Start proving", requestId = $data.requestId, slotIndex = $data.slotIndex
+  debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex
   try:
     let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve)
     state.loop = loop
@@ -122,7 +122,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
     return some State(SaleErrored(error: e))
   finally:
     # Cleanup of the proving loop
-    debug "Stopping proving.", requestId = $data.requestId, slotIndex = $data.slotIndex
+    debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex
 
     if not state.loop.isNil:
         if not state.loop.finished:

codex/sales/states/provingsimulated.nim

@@ -1,12 +1,12 @@
 import ../../conf
 when codex_enable_proof_failures:
   import std/strutils
-  import pkg/chronicles
   import pkg/stint
   import pkg/ethers
   import pkg/ethers/testing
 
   import ../../contracts/requests
+  import ../../logutils
   import ../../market
   import ../salescontext
   import ./proving
@@ -20,7 +20,7 @@ when codex_enable_proof_failures:
       proofCount: int
 
   proc onSubmitProofError(error: ref CatchableError, period: UInt256, slotId: SlotId) =
-    error "Submitting invalid proof failed", period = period, slotId = $slotId, msg = error.msg
+    error "Submitting invalid proof failed", period = period, slotId, msg = error.msg
 
   method prove*(state: SaleProvingSimulated, slot: Slot, challenge: ProofChallenge, onProve: OnProve, market: Market, currentPeriod: Period) {.async.} =
     trace "Processing proving in simulated mode"

codex/sales/states/unknown.nim

@@ -1,4 +1,4 @@
-import pkg/chronicles
+import ../../logutils
 import ../statemachine
 import ../salesagent
 import ./filled

codex/slots/builder/builder.nim

@@ -15,13 +15,13 @@ import std/sugar
 
 import pkg/libp2p
 import pkg/chronos
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
 import pkg/poseidon2
 import pkg/poseidon2/io
 import pkg/constantine/math/arithmetic/finite_fields
 
+import ../../logutils
 import ../../indexingstrategy
 import ../../merkletree
 import ../../stores

codex/slots/sampler/sampler.nim

@@ -10,7 +10,6 @@
 import std/sugar
 import std/sequtils
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/questionable
 import pkg/questionable/results
@@ -20,6 +19,7 @@ import pkg/poseidon2/types
 import pkg/poseidon2/io
 import pkg/stew/arrayops
 
+import ../../logutils
 import ../../market
 import ../../blocktype as bt
 import ../../merkletree

codex/stores/cachestore.nim

@@ -13,7 +13,6 @@ push: {.upraises: [].}
 
 import std/options
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/libp2p
 import pkg/lrucache
@@ -24,6 +23,7 @@ import ./blockstore
 import ../units
 import ../chunker
 import ../errors
+import ../logutils
 import ../manifest
 import ../merkletree
 import ../utils

codex/stores/maintenance.nim

@@ -11,7 +11,6 @@
 ## Looks for and removes expired blocks from blockstores.
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
 
@@ -19,6 +18,7 @@ import ./repostore
 import ../utils/timer
 import ../utils/asynciter
 import ../clock
+import ../logutils
 import ../systemclock
 
 const

codex/stores/networkstore.nim

@@ -7,23 +7,22 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-import pkg/upraises
 
+import pkg/upraises
 push: {.upraises: [].}
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/libp2p
 import pkg/questionable/results
 
+import ../clock
+import ../blocktype
+import ../blockexchange
+import ../logutils
+import ../merkletree
 import ../utils/asyncheapqueue
 import ../utils/asynciter
-import ../clock
-
-import ../blocktype
 import ./blockstore
-import ../blockexchange
-import ../merkletree
 
 export blockstore, blockexchange, asyncheapqueue
 

codex/stores/repostore.nim

@@ -13,7 +13,6 @@ push: {.upraises: [].}
 
 import pkg/chronos
 import pkg/chronos/futures
-import pkg/chronicles
 import pkg/libp2p/[cid, multicodec, multihash]
 import pkg/lrucache
 import pkg/metrics
@@ -27,6 +26,7 @@ import ./keyutils
 import ../blocktype
 import ../clock
 import ../systemclock
+import ../logutils
 import ../merkletree
 import ../utils
 

codex/streams/asyncstreamwrapper.nim

@@ -11,9 +11,10 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/libp2p
 
+import ../logutils
+
 logScope:
   topics = "libp2p asyncstreamwrapper"
 

codex/streams/seekablestream.nim

@@ -9,9 +9,10 @@
 
 import pkg/libp2p/stream/lpstream
 import pkg/chronos
-import pkg/chronicles
 
-export lpstream, chronos, chronicles
+import ../logutils
+
+export lpstream, chronos, logutils
 
 logScope:
   topics = "codex seekablestream"

codex/streams/storestream.nim

@@ -14,12 +14,12 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/chronos
-import pkg/chronicles
 import pkg/stew/ptrops
 
 import ../stores
 import ../manifest
 import ../blocktype
+import ../logutils
 import ../utils
 
 import ./seekablestream

codex/units.nim

@@ -12,8 +12,8 @@ import std/hashes
 import std/strutils
 
 import pkg/upraises
-import pkg/json_serialization
+
-import pkg/json_serialization/std/options
+import ./logutils
 
 type
   NBytes* = distinct Natural
@@ -43,6 +43,7 @@ divMaths(NBytes)
 proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte"
 proc `'nb`*(n: string): NBytes = parseInt(n).NBytes
 
+logutils.formatIt(NBytes): $it
 
 const
   MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz
@@ -54,18 +55,6 @@ func divUp*[T: NBytes](a, b : T): int =
   assert(b != T(0))
   if a==T(0):  int(0) else: int( ((a - T(1)) div b) + 1 )
 
-proc writeValue*(
-    writer: var JsonWriter,
-    value: NBytes
-) {.upraises:[IOError].} =
-  writer.writeValue value.int
-
-proc readValue*(
-    reader: var JsonReader,
-    value: var NBytes
-) {.upraises: [SerializationError, IOError].} =
-  value = NBytes reader.readValue(int)
-
 when isMainModule:
 
   import unittest2

codex/utils/asyncstatemachine.nim

@@ -1,10 +1,10 @@
 import std/sugar
 import pkg/questionable
 import pkg/chronos
-import pkg/chronicles
 import pkg/upraises
-import ./trackedfutures
+import ../logutils
 import ./then
+import ./trackedfutures
 
 push: {.upraises:[].}
 

codex/utils/fileutils.nim

@@ -9,16 +9,17 @@
 
 ## Partially taken from nim beacon chain
 
-import std/strutils
 import pkg/upraises
 
 push: {.upraises: [].}
 
-import pkg/chronicles
+import std/strutils
-import stew/io2
+import pkg/stew/io2
+
+import ../logutils
 
 export io2
-export chronicles except toJson
+export logutils
 
 when defined(windows):
   import stew/[windows/acl]

codex/utils/json.nim

@@ -6,9 +6,10 @@ import std/strutils
 import std/strformat
 import std/tables
 import std/typetraits
-import pkg/chronicles
+from pkg/ethers import Address
-from pkg/libp2p import Cid, init
+from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
 import pkg/contractabi
+import pkg/codexdht/discv5/node as dn
 import pkg/stew/byteutils
 import pkg/stint
 import pkg/questionable/results
@@ -16,9 +17,6 @@ import ../errors
 
 export json except `%`, `%*`
 
-logScope:
-  topics = "json serialization"
-
 type
   SerializationError = object of CodexError
   UnexpectedKindError = object of SerializationError
@@ -303,6 +301,20 @@ func `%`*[T: distinct](id: T): JsonNode =
   type baseType = T.distinctBase
   % baseType(id)
 
+func `%`*(cid: Cid): JsonNode = % $cid
+
+func `%`*(obj: PeerId): JsonNode = % $obj
+
+func `%`*(obj: SignedPeerRecord): JsonNode = % $obj
+
+func `%`*(obj: dn.Address): JsonNode = % $obj
+
+func `%`*(obj: AddressInfo): JsonNode = % $obj.address
+
+func `%`*(obj: MultiAddress): JsonNode = % $obj
+
+func `%`*(address: ethers.Address): JsonNode = % $address
+
 func toJson*[T](item: T): string = $(%item)
 
 proc toJsnImpl(x: NimNode): NimNode =

codex/utils/keyutils.nim

@@ -10,12 +10,12 @@
 import pkg/upraises
 push: {.upraises: [].}
 
-import pkg/chronicles
 import pkg/questionable/results
 import pkg/libp2p/crypto/crypto
 
 import ./fileutils
 import ../errors
+import ../logutils
 import ../rng
 
 export crypto

codex/utils/timer.nim

@@ -15,7 +15,8 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/chronos
-import pkg/chronicles
+
+import ../logutils
 
 type
   TimerCallback* = proc(): Future[void] {.gcsafe, upraises:[].}

codex/utils/trackedfutures.nim

@@ -1,7 +1,8 @@
 import std/sugar
 import std/tables
-import pkg/chronicles
 import pkg/chronos
+
+import ../logutils
 import ../utils/then
 
 type

codex/validation.nim

@@ -1,9 +1,9 @@
 import std/sets
 import std/sequtils
 import pkg/chronos
-import pkg/chronicles
 import ./market
 import ./clock
+import ./logutils
 
 export market
 export sets
@@ -48,7 +48,7 @@ proc subscribeSlotFilled(validation: Validation) {.async.} =
     let slotId = slotId(requestId, slotIndex)
     if slotId notin validation.slots:
       if validation.slots.len < validation.maxSlots:
-        trace "Adding slot", slotId = $slotId
+        trace "Adding slot", slotId
         validation.slots.incl(slotId)
   let subscription = await validation.market.subscribeSlotFilled(onSlotFilled)
   validation.subscriptions.add(subscription)
@@ -58,7 +58,7 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} =
   for slotId in validation.slots:
     let state = await validation.market.slotState(slotId)
     if state != SlotState.Filled:
-      trace "Removing slot", slot = $slotId
+      trace "Removing slot", slotId
       ended.incl(slotId)
   validation.slots.excl(ended)
 
@@ -70,7 +70,7 @@ proc markProofAsMissing(validation: Validation,
 
   try:
     if await validation.market.canProofBeMarkedAsMissing(slotId, period):
-      trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period
+      trace "Marking proof as missing", slotId, periodProofMissed = period
       await validation.market.markProofAsMissing(slotId, period)
     else:
       let inDowntime {.used.} = await validation.market.inDowntime(slotId)

tests/codex/node/testcontracts.nim

@@ -7,7 +7,6 @@ import std/importutils
 
 import pkg/asynctest
 import pkg/chronos
-import pkg/chronicles
 import pkg/stew/byteutils
 import pkg/datastore
 import pkg/questionable
@@ -19,6 +18,7 @@ import pkg/poseidon2/io
 import pkg/nitro
 import pkg/codexdht/discv5/protocol as discv5
 
+import pkg/codex/logutils
 import pkg/codex/stores
 import pkg/codex/clock
 import pkg/codex/contracts

tests/codex/node/testnode.nim

@@ -7,7 +7,6 @@ import std/importutils
 
 import pkg/asynctest
 import pkg/chronos
-import pkg/chronicles
 import pkg/stew/byteutils
 import pkg/datastore
 import pkg/questionable
@@ -19,6 +18,7 @@ import pkg/poseidon2/io
 import pkg/nitro
 import pkg/codexdht/discv5/protocol as discv5
 
+import pkg/codex/logutils
 import pkg/codex/stores
 import pkg/codex/clock
 import pkg/codex/contracts

tests/codex/sales/testslotqueue.nim

@@ -1,11 +1,11 @@
 import std/sequtils
 import pkg/asynctest
-import pkg/chronicles
 import pkg/chronos
 import pkg/datastore
 import pkg/questionable
 import pkg/questionable/results
 
+import pkg/codex/logutils
 import pkg/codex/sales/slotqueue
 
 import ../helpers

tests/codex/slots/testsampler.nim

@@ -10,6 +10,7 @@ import pkg/poseidon2/io
 import pkg/poseidon2
 import pkg/chronos
 import pkg/asynctest
+import pkg/nimcrypto
 import pkg/codex/stores/cachestore
 import pkg/codex/chunker
 import pkg/codex/stores
@@ -113,6 +114,6 @@ asyncchecksuite "Test DataSampler":
       toStr(input.samples[2].cellProof) == expectedCellBlockProofs[2]
 
       # # cell data
-      toHex(input.samples[0].data) == expectedCellData[0]
+      nimcrypto.toHex(input.samples[0].data) == expectedCellData[0]
-      toHex(input.samples[1].data) == expectedCellData[1]
+      nimcrypto.toHex(input.samples[1].data) == expectedCellData[1]
-      toHex(input.samples[2].data) == expectedCellData[2]
+      nimcrypto.toHex(input.samples[2].data) == expectedCellData[2]

tests/codex/testchunking.nim

@@ -2,7 +2,7 @@
 import pkg/asynctest
 import pkg/stew/byteutils
 import pkg/codex/chunker
-import pkg/chronicles
+import pkg/codex/logutils
 import pkg/chronos
 
 import ./helpers

tests/codex/testlogutils.nim

@@ -0,0 +1,234 @@
+import std/options
+import std/strutils
+import std/unittest
+import pkg/codex/blocktype
+import pkg/codex/conf
+import pkg/codex/contracts/requests
+import pkg/codex/logutils
+import pkg/codex/purchasing/purchaseid
+import pkg/codex/units
+import pkg/libp2p/cid
+import pkg/libp2p/multiaddress
+import pkg/questionable
+import pkg/questionable/results
+import pkg/stew/byteutils
+import pkg/stint
+import ../checktest
+
+export logutils
+
+logStream testlines[textlines[nocolors,notimestamps,dynamic]]
+logStream testjson[json[nocolors,notimestamps,dynamic]]
+
+type
+  ObjectType = object
+    a: string
+  DistinctType {.borrow: `.`.} = distinct ObjectType
+  RefType = ref object
+    a: string
+  AnotherType = object
+    a: int
+
+# must be defined at the top-level
+proc `$`*(t: ObjectType): string = "used `$`"
+func `%`*(t: RefType): JsonNode = % t.a
+logutils.formatIt(LogFormat.textLines, ObjectType): "formatted_" & it.a
+logutils.formatIt(LogFormat.textLines, RefType): "formatted_" & it.a
+logutils.formatIt(LogFormat.textLines, DistinctType): "formatted_" & it.a
+logutils.formatIt(LogFormat.json, ObjectType): "formatted_" & it.a
+logutils.formatIt(LogFormat.json, RefType): %it
+logutils.formatIt(LogFormat.json, DistinctType): "formatted_" & it.a
+logutils.formatIt(AnotherType): it.a
+
+checksuite "Test logging output":
+  var outputLines: string
+  var outputJson: string
+
+  proc writeToLines(logLevel: LogLevel, msg: LogOutputStr) =
+    outputLines &= msg
+
+  proc writeToJson(logLevel: LogLevel, msg: LogOutputStr) =
+    outputJson &= msg
+
+  setup:
+    outputLines = ""
+    outputJson = ""
+    testlines.outputs[0].writer = writeToLines
+    testjson.outputs[0].writer = writeToJson
+
+  template logged(prop, expected): auto =
+    let toFind = prop & "=" & expected
+    outputLines.contains(toFind)
+
+  template loggedJson(prop, expected): auto =
+    let json = $ parseJson(outputJson){prop}
+    json == expected
+
+  template log(val) =
+    testlines.trace "test", val
+    testjson.trace "test", val
+
+  test "logs objects":
+    let t = ObjectType(a: "a")
+    log t
+    check logged("t", "formatted_a")
+    check loggedJson("t", "\"formatted_a\"")
+
+  test "logs sequences of objects":
+    let t1 = ObjectType(a: "a")
+    let t2 = ObjectType(a: "b")
+    let t = @[t1, t2]
+    log t
+    check logged("t", "\"@[formatted_a, formatted_b]\"")
+    check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]")
+
+  test "logs ref types":
+    let t = RefType(a: "a")
+    log t
+    check logged("t", "formatted_a")
+    check loggedJson("t", "\"a\"")
+
+  test "logs sequences of ref types":
+    let t1 = RefType(a: "a")
+    let t2 = RefType(a: "b")
+    let t = @[t1, t2]
+    log t
+    check logged("t", "\"@[formatted_a, formatted_b]\"")
+    check loggedJson("t", "[\"a\",\"b\"]")
+
+  test "logs distinct types":
+    let t = DistinctType(ObjectType(a: "a"))
+    log t
+    check logged("t", "formatted_a")
+    check loggedJson("t", "\"formatted_a\"")
+
+  test "logs sequences of distinct types":
+    let t1 = DistinctType(ObjectType(a: "a"))
+    let t2 = DistinctType(ObjectType(a: "b"))
+    let t = @[t1, t2]
+    log t
+    check logged("t", "\"@[formatted_a, formatted_b]\"")
+    check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]")
+
+  test "formatIt can return non-string types":
+    let t = AnotherType(a: 1)
+    log t
+    check logged("t", "1")
+    check loggedJson("t", "1")
+
+  test "logs Option types":
+    let t = some ObjectType(a: "a")
+    log t
+    check logged("t", "some(formatted_a)")
+    check loggedJson("t", "\"formatted_a\"")
+
+  test "logs sequences of Option types":
+    let t1 = some ObjectType(a: "a")
+    let t2 = none ObjectType
+    let t = @[t1, t2]
+    log t
+    check logged("t", "\"@[some(formatted_a), none(ObjectType)]\"")
+    check loggedJson("t", """["formatted_a",null]""")
+
+  test "logs Result types -- success with string property":
+    let t: ?!ObjectType = success ObjectType(a: "a")
+    log t
+    check logged("t", "formatted_a")
+    check loggedJson("t", "\"formatted_a\"")
+
+  test "logs Result types -- success with int property":
+    let t: ?!AnotherType = success AnotherType(a: 1)
+    log t
+    check logged("t", "1")
+    check loggedJson("t", "1")
+
+  test "logs Result types -- failure":
+    let t: ?!ObjectType = ObjectType.failure newException(ValueError, "some error")
+    log t
+    check logged("t", "\"Error: some error\"")
+    check loggedJson("t", """{"error":"some error"}""")
+
+  test "can define `$` override for T":
+    let o = ObjectType()
+    check $o == "used `$`"
+
+  test "logs NByte correctly":
+    let nb = 12345.NBytes
+    log nb
+    check logged("nb", "12345\'NByte")
+    check loggedJson("nb", "\"12345\'NByte\"")
+
+  test "logs BlockAddress correctly":
+    let cid = Cid.init("zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1").tryGet
+    let ba = BlockAddress.init(cid, 0)
+    log ba
+    check logged("ba", "\"treeCid: zb2*fndjU1, index: 0\"")
+    check loggedJson("ba", """{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}""")
+
+  test "logs Cid correctly":
+    let cid = Cid.init("zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD").tryGet
+    log cid
+    check logged("cid", "zb2*A5TgiD")
+    check loggedJson("cid", "\"zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD\"")
+
+  test "logs StUint correctly":
+    let stint = 12345678901234.u256
+    log stint
+    check logged("stint", "12345678901234")
+    check loggedJson("stint", "\"12345678901234\"")
+
+  test "logs int correctly":
+    let int = 123
+    log int
+    check logged("int", "123")
+    check loggedJson("int", "123")
+
+  test "logs EthAddress correctly":
+    let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b")
+    log address
+    check logged("address", "0xf75e..1b1b")
+    check loggedJson("address", "\"0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b\"")
+
+  test "logs PurchaseId correctly":
+    let id = PurchaseId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea")
+    log id
+    check logged("id", "0x7120..3dea")
+
+  test "logs RequestId correctly":
+    let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea")
+    log id
+    check logged("id", "0x7120..3dea")
+    check loggedJson("id", "\"0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea\"")
+
+  test "logs seq[RequestId] correctly":
+    let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea")
+    let id2 = RequestId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282")
+    let ids = @[id, id2]
+    log ids
+    check logged("ids", "\"@[0x7120..3dea, 0x9ab2..c282]\"")
+    check loggedJson("ids", """["0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea","0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282"]""")
+
+  test "logs SlotId correctly":
+    let id = SlotId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282")
+    log id
+    check logged("id", "0x9ab2..c282")
+    check loggedJson("id", "\"0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282\"")
+
+  test "logs Nonce correctly":
+    let n = Nonce.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea")
+    log n
+    check logged("n", "0xce88..d6ea")
+    check loggedJson("n", "\"0xce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea\"")
+
+  test "logs MultiAddress correctly":
+    let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet
+    log ma
+    check logged("ma", "/ip4/127.0.0.1/tcp/0")
+    check loggedJson("ma", "\"/ip4/127.0.0.1/tcp/0\"")
+
+  test "logs seq[MultiAddress] correctly":
+    let ma = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet,
+               MultiAddress.init("/ip4/127.0.0.2/tcp/1").tryGet]
+    log ma
+    check logged("ma", "\"@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]\"")
+    check loggedJson("ma", "[\"/ip4/127.0.0.1/tcp/0\",\"/ip4/127.0.0.2/tcp/1\"]")

tests/codex/utils/testjson.nim

@@ -3,11 +3,11 @@ import std/options
 import std/strformat
 import std/strutils
 import std/unittest
-import pkg/chronicles except toJson
 import pkg/stew/byteutils
 import pkg/stint
 import pkg/codex/contracts/requests
 from pkg/codex/rest/json import RestPurchase
+import pkg/codex/logutils
 import pkg/codex/utils/json as utilsjson
 import pkg/questionable
 import pkg/questionable/results

tests/contracts/testContracts.nim

@@ -1,4 +1,3 @@
-import std/json
 import pkg/chronos
 import pkg/ethers/testing
 import pkg/ethers/erc20

tests/ethertest.nim

@@ -1,4 +1,4 @@
-import std/json
+
 import pkg/asynctest
 import pkg/ethers
 

tests/integration/codexclient.nim

@@ -3,9 +3,9 @@ import std/strutils
 import std/sequtils
 
 from pkg/libp2p import Cid, `$`, init
-import pkg/chronicles
 import pkg/stint
 import pkg/questionable/results
+import pkg/codex/logutils
 import pkg/codex/rest/json
 import pkg/codex/purchasing
 import pkg/codex/errors

tests/integration/multinodes.nim

@@ -1,8 +1,7 @@
 import std/os
 import std/macros
-import std/json
 import std/httpclient
-import pkg/chronicles
+import pkg/codex/logutils
 import ../ethertest
 import ./codexclient
 import ./nodes

tests/integration/nodes.nim

@@ -1,12 +1,12 @@
-import pkg/questionable
-import pkg/confutils
-import pkg/chronicles
-import pkg/libp2p
 import std/osproc
 import std/os
 import std/streams
 import std/strutils
-import codex/conf
+import pkg/codex/conf
+import pkg/codex/logutils
+import pkg/confutils
+import pkg/libp2p
+import pkg/questionable
 import ./codexclient
 
 export codexclient

tests/integration/testproofs.nim

@@ -1,9 +1,9 @@
 import std/sequtils
 import std/os
 from std/times import getTime, toUnix
-import pkg/chronicles
+import pkg/codex/contracts
-import codex/contracts
+import pkg/codex/logutils
-import codex/periods
+import pkg/codex/periods
 import ../contracts/time
 import ../contracts/deployment
 import ./twonodes

tests/integration/twonodes.nim

@@ -1,6 +1,5 @@
 import std/os
 import std/macros
-import std/json
 import std/httpclient
 import ../ethertest
 import ./codexclient

tests/logging.nim

@@ -1,5 +1,5 @@
 when not defined(nimscript):
-  import pkg/chronicles
+  import pkg/codex/logutils
 
   proc ignoreLogging(level: LogLevel, message: LogOutputStr) =
     discard

tests/testCodex.nim

@@ -2,6 +2,7 @@ import ./codex/teststores
 import ./codex/testblockexchange
 import ./codex/testasyncheapqueue
 import ./codex/testchunking
+import ./codex/testlogutils
 import ./codex/testmanifest
 import ./codex/testnode
 import ./codex/teststorestream

vendor/nim-poseidon2

@@ -1 +1 @@
-Subproject commit eef2603c11eeb5069eaed6c0296f3a1c5d23aad1
+Subproject commit 0346982f2c6891bcedd03d552af3a3bd57b2c1f9