finished rework

codexcrawler/components/dhtmetrics.nim

@@ -31,8 +31,6 @@ proc handleCheckEvent(
 
   d.metrics.setOkNodes(d.ok.len)
   d.metrics.setNokNodes(d.nok.len)
-
-  trace "metrics updated", ok = d.ok.len, nok = d.nok.len
   return success()
 
 method start*(d: DhtMetrics): Future[?!void] {.async.} =
@@ -44,6 +42,13 @@ method start*(d: DhtMetrics): Future[?!void] {.async.} =
     await d.handleCheckEvent(event)
 
   d.sub = d.state.events.dhtNodeCheck.subscribe(onCheck)
+
+  proc logDhtMetrics(): Future[?!void] {.async: (raises: []), gcsafe.} =
+    trace "Metrics", ok = d.ok.len, nok = d.nok.len
+    return success()
+
+  await d.state.whileRunning(logDhtMetrics, 1.minutes)
+
   return success()
 
 method stop*(d: DhtMetrics): Future[?!void] {.async.} =

codexcrawler/components/nodestore.nim

@@ -81,12 +81,11 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} =
   for nid in nids:
     without isNew =? (await s.storeNodeIsNew(nid)), err:
       return failure(err)
-
     if isNew:
       newNodes.add(nid)
 
-  trace "Processed found nodes", total = nids.len, numNew = newNodes.len
   if newNodes.len > 0:
+    trace "Discovered new nodes", newNodes = newNodes.len
     ?await s.fireNewNodesDiscovered(newNodes)
   return success()
 

codexcrawler/components/timetracker.nim

@@ -33,6 +33,7 @@ proc checkForExpiredNodes(t: TimeTracker): Future[?!void] {.async: (raises: []).
   return success()
 
 proc raiseRoutingTableNodes(t: TimeTracker): Future[?!void] {.async: (raises: []).} =
+  trace "Raising routing table nodes..."
   let nids = t.dht.getRoutingTableNodeIds()
   if err =? (await t.state.events.nodesFound.fire(nids)).errorOption:
     return failure(err)

codexcrawler/components/todolist.nim

@@ -26,8 +26,8 @@ proc addNodes(t: TodoList, nids: seq[Nid]) =
   for nid in nids:
     t.nids.add(nid)
 
-  trace "Nodes added", nodes = nids.len
   if s =? t.emptySignal:
+    trace "Nodes added, resuming...", nodes = nids.len
     s.complete()
     t.emptySignal = Future[void].none
 

codexcrawler/config.nim

@@ -14,14 +14,14 @@ Usage:
 
 Options:
   --logLevel=<l>          Sets log level [default: INFO]
-  --publicIp=<a>          Public IP address where this instance is reachable. [default: 45.82.185.194]
+  --publicIp=<a>          Public IP address where this instance is reachable.
   --metricsAddress=<ip>   Listen address of the metrics server [default: 0.0.0.0]
   --metricsPort=<p>       Listen HTTP port of the metrics server [default: 8008]
   --dataDir=<dir>         Directory for storing data [default: crawler_data]
   --discoveryPort=<p>     Port used for DHT [default: 8090]
   --bootNodes=<n>         Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
   --stepDelay=<ms>        Delay in milliseconds per crawl step [default: 1000]
-  --revisitDelay=<m>      Delay in minutes after which a node can be revisited [default: 1] (24h)
+  --revisitDelay=<m>      Delay in minutes after which a node can be revisited [default: 10] (24h)
 """
 
 import strutils

codexcrawler/installer.nim

@@ -27,11 +27,11 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
     return failure(err)
 
+  components.add(dht)
   components.add(todoList)
   components.add(nodeStore)
-  components.add(dht)
   components.add(Crawler.new(state, dht, todoList))
-  components.add(TimeTracker.new(state, nodeStore))
+  components.add(TimeTracker.new(state, nodeStore, dht))
   components.add(dhtMetrics)
 
   return success(components)