Various dev-ops things (#85)

* Add metrics server
* Add --version
* Fixes for chronicles=trace + ci

.github/workflows/ci.yml

@@ -215,3 +215,5 @@ jobs:
             echo ldd build/testDagger
             ldd build/testDagger
           fi
+          echo "Testing TRACE log level"
+          ./env.sh nim c -d:chronicles_log_level=TRACE dagger.nim

config.nims

@@ -68,6 +68,9 @@ switch("warning", "ObservableStores:off")
 switch("warning", "LockLevel:off")
 
 switch("define", "libp2p_pki_schemes=secp256k1")
+#TODO this infects everything in this folder, ideally it would only
+# apply to dagger.nim, but since dagger.nims is used for other purpose
+# we can't use it. And dagger.cfg doesn't work
 switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic]")
 
 # begin Nimble config (version 1)

dagger.nim

@@ -27,8 +27,11 @@ when isMainModule:
   when defined(posix):
     import system/ansi_c
 
-  let config = DaggerConf.load()
+  let config = DaggerConf.load(
+    version = daggerFullVersion
+  )
   config.setupLogging()
+  config.setupMetrics()
 
   case config.cmd:
   of StartUpCommand.noCommand:
@@ -38,7 +41,7 @@ when isMainModule:
       # permissions are insecure.
       quit QuitFailure
 
-    trace "Data dir initialized", dir = config.dataDir
+    trace "Data dir initialized", dir = $config.dataDir
 
     if not(checkAndCreateDataDir((config.dataDir / "repo").string)):
       # We are unable to access/create data folder or data folder's

dagger/blockexchange/engine.nim

@@ -280,7 +280,8 @@ proc requestBlock*(
 
   assert discovery.provides.len > 0
 
-  trace "Requesting block from peer", peer = blockPeer.id, cid
+  debug "Requesting block from peer", providerCount = discovery.provides.len,
+    peer = discovery.provides[0], cid
   # request block
   b.network.request.sendWantList(
     discovery.provides[0],

dagger/conf.nim

@@ -14,12 +14,15 @@ push: {.upraises: [].}
 import std/os
 import std/terminal
 import std/options
+import std/strutils
 import std/typetraits
 
 import pkg/chronicles
 import pkg/chronicles/topics_registry
 import pkg/confutils/defs
 import pkg/confutils/std/net
+import pkg/metrics
+import pkg/metrics/chronos_httpserver
 import pkg/stew/shims/net as stewnet
 import pkg/libp2p
 
@@ -53,6 +56,21 @@ type
       defaultValue: LogKind.Auto
       name: "log-format" }: LogKind
 
+    metricsEnabled* {.
+      desc: "Enable the metrics server"
+      defaultValue: false
+      name: "metrics" }: bool
+
+    metricsAddress* {.
+      desc: "Listening address of the metrics server"
+      defaultValue: ValidIpAddress.init("127.0.0.1")
+      defaultValueDesc: "127.0.0.1"
+      name: "metrics-address" }: ValidIpAddress
+
+    metricsPort* {.
+      desc: "Listening HTTP port of the metrics server"
+      defaultValue: 8008
+      name: "metrics-port" }: Port
 
     dataDir* {.
       desc: "The directory where dagger will store configuration and data."
@@ -97,7 +115,7 @@ type
       bootstrapNodes* {.
         desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
         abbr: "b"
-        name: "bootstrap-nodes" }: seq[SignedPeerRecord]
+        name: "bootstrap-node" }: seq[SignedPeerRecord]
 
       maxPeers* {.
         desc: "The maximum number of peers to connect to"
@@ -126,6 +144,19 @@ type
     of initNode:
       discard
 
+const
+  gitRevision* = strip(staticExec("git rev-parse --short HEAD"))[0..5]
+
+  nimBanner* = staticExec("nim --version | grep Version")
+
+  #TODO add versionMajor, Minor & Fix when we switch to semver
+  daggerVersion* = gitRevision
+
+  daggerFullVersion* =
+    "Dagger build " & daggerVersion & "\p" &
+    nimBanner
+
+
 proc defaultDataDir*(): string =
   let dataDir = when defined(windows):
     "AppData" / "Roaming" / "Dagger"
@@ -223,3 +254,15 @@ proc setupLogging*(conf: DaggerConf) =
         noOutput
 
     setLogLevel(conf.logLevel)
+
+proc setupMetrics*(config: DaggerConf) =
+  if config.metricsEnabled:
+    let metricsAddress = config.metricsAddress
+    notice "Starting metrics HTTP server",
+      url = "http://" & $metricsAddress & ":" & $config.metricsPort & "/metrics"
+    try:
+      startMetricsHttpServer($metricsAddress, config.metricsPort)
+    except CatchableError as exc:
+      raiseAssert exc.msg
+    except Exception as exc:
+      raiseAssert exc.msg # TODO fix metrics

dagger/node.nim

@@ -86,7 +86,7 @@ proc retrieve*(
 
   # if we got a manifest, stream the blocks
   if $mc in ManifestContainers:
-    trace "Retrieving data set", cid, mc
+    trace "Retrieving data set", cid, mc = $mc
 
     without manifest =? Manifest.decode(blk.data, ManifestContainers[$mc]):
       return failure("Unable to construct manifest!")
@@ -203,7 +203,7 @@ proc requestStorage*(
 
   # if we got a manifest, stream the blocks
   if $mc notin ManifestContainers:
-    trace "Not a manifest type!", cid, mc
+    trace "Not a manifest type!", cid, mc = $mc
     return failure("Not a manifest type!")
 
   without var manifest =? Manifest.decode(blk.data), error:

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit b2d980f258ebeabdaa0eedfe3722ebf949f6dadb
+Subproject commit eeb3c210a37408716b6a8b45f578adf87610cef2

vendor/nim-libp2p-dht

@@ -1 +1 @@
-Subproject commit 6c4ce518f55007a861fe577b48cabdca7eaa32df
+Subproject commit 9a872518d621bf8b390f88cd65617bca6aca1d2d