Merge 4bf72b938978b44376d40b69cc197aeefcaf2018 into bd360322515addd487c8f97fc4447bf25eda6c99

.github/workflows/docker-dist-tests.yml

@@ -52,7 +52,7 @@ jobs:
     uses: ./.github/workflows/docker-reusable.yml
     needs: get-contracts-hash
     with:
-      nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true'
+      nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true -d:chronosProfiling=true'
       nat_ip_auto: true
       tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
       tag_suffix: dist-tests

.gitmodules

@@ -75,9 +75,9 @@
 	branch = master
 [submodule "vendor/nim-chronos"]
 	path = vendor/nim-chronos
-	url = https://github.com/status-im/nim-chronos.git
+	url = https://github.com/codex-storage/nim-chronos.git
 	ignore = untracked
-	branch = master
+	branch = feature/profiler-v4
 [submodule "vendor/nim-faststreams"]
 	path = vendor/nim-faststreams
 	url = https://github.com/status-im/nim-faststreams.git
@@ -221,6 +221,9 @@
 [submodule "vendor/nph"]
 	path = vendor/nph
 	url = https://github.com/arnetheduck/nph.git
+[submodule "vendor/nim-chroprof"]
+	path = vendor/nim-chroprof
+	url = https://github.com/codex-storage/nim-chroprof.git
 [submodule "vendor/nim-quic"]
 	path = vendor/nim-quic
 	url = https://github.com/vacp2p/nim-quic.git

build.nims

@@ -55,7 +55,7 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
 
 task codex, "build codex binary":
   buildBinary "codex",
-    params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
+    params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronosProfiling=true"
 
 task toolsCirdl, "build tools/cirdl binary":
   buildBinary "tools/cirdl/cirdl"

codex/conf.nim

@@ -20,6 +20,7 @@ import std/strutils
 import std/typetraits
 
 import pkg/chronos
+import pkg/chronos/config
 import pkg/chronicles/helpers
 import pkg/chronicles/topics_registry
 import pkg/confutils/defs
@@ -36,6 +37,10 @@ import pkg/questionable
 import pkg/questionable/results
 import pkg/stew/base64
 
+when chronosProfiling:
+  import pkg/chroprof
+  import pkg/chroprof/collector
+
 import ./codextypes
 import ./discovery
 import ./logutils
@@ -144,6 +149,12 @@ type
       name: "data-dir"
     .}: OutDir
 
+    profilerMaxMetrics* {.
+      desc: "Maximum number of metrics to export to Prometheus.",
+      defaultValue: 100,
+      name: "profiler-max-metrics"
+    .}: int
+
     listenAddrs* {.
       desc: "Multi Addresses to listen on",
       defaultValue:
@@ -286,6 +297,14 @@ type
       desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden
     .}: Option[string]
 
+    execTimeThreshold* {.
+      desc: "Threshold for execution time",
+      defaultValue: 200,
+      defaultValueDesc: "200 milliseconds",
+      name: "exec-time-threshold",
+      abbr: "et"
+    .}: int
+
     case cmd* {.defaultValue: noCmd, command.}: StartUpCmd
     of persistence:
       ethProvider* {.
@@ -815,11 +834,20 @@ proc setupLogging*(conf: CodexConf) =
       defaultChroniclesStream.outputs[0].writer = writer
 
 proc setupMetrics*(config: CodexConf) =
+  let threshold = config.execTimeThreshold.milliseconds
+  when chronosProfiling:
+    notice "Enabling profiling without metrics", execTimeThreshold = threshold
+    enableProfiling(threshold)
+
   if config.metricsEnabled:
     let metricsAddress = config.metricsAddress
     notice "Starting metrics HTTP server",
       url = "http://" & $metricsAddress & ":" & $config.metricsPort & "/metrics"
     try:
+      when chronosProfiling:
+        notice "Enabling profiling with metrics", execTimeThreshold = threshold
+        enableProfilerMetrics(k = config.profilerMaxMetrics, threshold)
+
       startMetricsHttpServer($metricsAddress, config.metricsPort)
     except CatchableError as exc:
       raiseAssert exc.msg

config.nims

@@ -156,6 +156,8 @@ switch("define", "nimOldCaseObjects")
 # Enable compat mode for Chronos V4
 switch("define", "chronosHandleException")
 
+switch("define", "chronosProfiling")
+
 # begin Nimble config (version 1)
 when system.fileExists("nimble.paths"):
   include "nimble.paths"

tests/config.nims

@@ -12,3 +12,6 @@ when not defined(chronicles_log_level):
     "chronicles_sinks:textlines[dynamic]" # allow logs to be filtered at runtime
   --"import":
     "logging" # ensure that logging is ignored at runtime
+
+--undef:
+  chronosProfiling

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit c04576d829b8a0a1b12baaa8bc92037501b3a4a0
+Subproject commit f85a3b43c2606d8966bb1eb6e36de10c7b4aab94

vendor/nim-chroprof

@@ -0,0 +1 @@
+Subproject commit f12bded51e2a60513b414a93642b44f1156a964e