From c06c9b578c262da1e1347062995f1aae08296820 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 14 Apr 2022 12:49:03 +0200 Subject: [PATCH] Various dev-ops things (#85) * Add metrics server * Add --version * Fixes for chronicles=trace + ci --- .github/workflows/ci.yml | 2 ++ config.nims | 3 +++ dagger.nim | 7 +++-- dagger/blockexchange/engine.nim | 3 ++- dagger/conf.nim | 45 ++++++++++++++++++++++++++++++++- dagger/node.nim | 4 +-- vendor/nim-libp2p | 2 +- vendor/nim-libp2p-dht | 2 +- 8 files changed, 60 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e3c4371..209629b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,3 +215,5 @@ jobs: echo ldd build/testDagger ldd build/testDagger fi + echo "Testing TRACE log level" + ./env.sh nim c -d:chronicles_log_level=TRACE dagger.nim diff --git a/config.nims b/config.nims index 068353f1..35023252 100644 --- a/config.nims +++ b/config.nims @@ -68,6 +68,9 @@ switch("warning", "ObservableStores:off") switch("warning", "LockLevel:off") switch("define", "libp2p_pki_schemes=secp256k1") +#TODO this infects everything in this folder, ideally it would only +# apply to dagger.nim, but since dagger.nims is used for other purpose +# we can't use it. And dagger.cfg doesn't work switch("define", "chronicles_sinks=textlines[dynamic],json[dynamic]") # begin Nimble config (version 1) diff --git a/dagger.nim b/dagger.nim index 3c7c574e..72249041 100644 --- a/dagger.nim +++ b/dagger.nim @@ -27,8 +27,11 @@ when isMainModule: when defined(posix): import system/ansi_c - let config = DaggerConf.load() + let config = DaggerConf.load( + version = daggerFullVersion + ) config.setupLogging() + config.setupMetrics() case config.cmd: of StartUpCommand.noCommand: @@ -38,7 +41,7 @@ when isMainModule: # permissions are insecure. quit QuitFailure - trace "Data dir initialized", dir = config.dataDir + trace "Data dir initialized", dir = $config.dataDir if not(checkAndCreateDataDir((config.dataDir / "repo").string)): # We are unable to access/create data folder or data folder's diff --git a/dagger/blockexchange/engine.nim b/dagger/blockexchange/engine.nim index 4b5a2203..7be6167e 100644 --- a/dagger/blockexchange/engine.nim +++ b/dagger/blockexchange/engine.nim @@ -280,7 +280,8 @@ proc requestBlock*( assert discovery.provides.len > 0 - trace "Requesting block from peer", peer = blockPeer.id, cid + debug "Requesting block from peer", providerCount = discovery.provides.len, + peer = discovery.provides[0], cid # request block b.network.request.sendWantList( discovery.provides[0], diff --git a/dagger/conf.nim b/dagger/conf.nim index 6e07aad9..58ed2c01 100644 --- a/dagger/conf.nim +++ b/dagger/conf.nim @@ -14,12 +14,15 @@ push: {.upraises: [].} import std/os import std/terminal import std/options +import std/strutils import std/typetraits import pkg/chronicles import pkg/chronicles/topics_registry import pkg/confutils/defs import pkg/confutils/std/net +import pkg/metrics +import pkg/metrics/chronos_httpserver import pkg/stew/shims/net as stewnet import pkg/libp2p @@ -53,6 +56,21 @@ type defaultValue: LogKind.Auto name: "log-format" }: LogKind + metricsEnabled* {. + desc: "Enable the metrics server" + defaultValue: false + name: "metrics" }: bool + + metricsAddress* {. + desc: "Listening address of the metrics server" + defaultValue: ValidIpAddress.init("127.0.0.1") + defaultValueDesc: "127.0.0.1" + name: "metrics-address" }: ValidIpAddress + + metricsPort* {. + desc: "Listening HTTP port of the metrics server" + defaultValue: 8008 + name: "metrics-port" }: Port dataDir* {. desc: "The directory where dagger will store configuration and data." @@ -97,7 +115,7 @@ type bootstrapNodes* {. desc: "Specifies one or more bootstrap nodes to use when connecting to the network." abbr: "b" - name: "bootstrap-nodes" }: seq[SignedPeerRecord] + name: "bootstrap-node" }: seq[SignedPeerRecord] maxPeers* {. desc: "The maximum number of peers to connect to" @@ -126,6 +144,19 @@ type of initNode: discard +const + gitRevision* = strip(staticExec("git rev-parse --short HEAD"))[0..5] + + nimBanner* = staticExec("nim --version | grep Version") + + #TODO add versionMajor, Minor & Fix when we switch to semver + daggerVersion* = gitRevision + + daggerFullVersion* = + "Dagger build " & daggerVersion & "\p" & + nimBanner + + proc defaultDataDir*(): string = let dataDir = when defined(windows): "AppData" / "Roaming" / "Dagger" @@ -223,3 +254,15 @@ proc setupLogging*(conf: DaggerConf) = noOutput setLogLevel(conf.logLevel) + +proc setupMetrics*(config: DaggerConf) = + if config.metricsEnabled: + let metricsAddress = config.metricsAddress + notice "Starting metrics HTTP server", + url = "http://" & $metricsAddress & ":" & $config.metricsPort & "/metrics" + try: + startMetricsHttpServer($metricsAddress, config.metricsPort) + except CatchableError as exc: + raiseAssert exc.msg + except Exception as exc: + raiseAssert exc.msg # TODO fix metrics diff --git a/dagger/node.nim b/dagger/node.nim index 5ef9d1c1..eb849f33 100644 --- a/dagger/node.nim +++ b/dagger/node.nim @@ -86,7 +86,7 @@ proc retrieve*( # if we got a manifest, stream the blocks if $mc in ManifestContainers: - trace "Retrieving data set", cid, mc + trace "Retrieving data set", cid, mc = $mc without manifest =? Manifest.decode(blk.data, ManifestContainers[$mc]): return failure("Unable to construct manifest!") @@ -203,7 +203,7 @@ proc requestStorage*( # if we got a manifest, stream the blocks if $mc notin ManifestContainers: - trace "Not a manifest type!", cid, mc + trace "Not a manifest type!", cid, mc = $mc return failure("Not a manifest type!") without var manifest =? Manifest.decode(blk.data), error: diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index b2d980f2..eeb3c210 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit b2d980f258ebeabdaa0eedfe3722ebf949f6dadb +Subproject commit eeb3c210a37408716b6a8b45f578adf87610cef2 diff --git a/vendor/nim-libp2p-dht b/vendor/nim-libp2p-dht index 6c4ce518..9a872518 160000 --- a/vendor/nim-libp2p-dht +++ b/vendor/nim-libp2p-dht @@ -1 +1 @@ -Subproject commit 6c4ce518f55007a861fe577b48cabdca7eaa32df +Subproject commit 9a872518d621bf8b390f88cd65617bca6aca1d2d