From 1a05ecc88f5a24a1132c589e464a193ac04ff76d Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 14:43:12 +0100 Subject: [PATCH] finished rework --- codexcrawler/components/dhtmetrics.nim | 9 +++++++-- codexcrawler/components/nodestore.nim | 3 +-- codexcrawler/components/timetracker.nim | 1 + codexcrawler/components/todolist.nim | 2 +- codexcrawler/config.nim | 4 ++-- codexcrawler/installer.nim | 4 ++-- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim index 46ed115..cbf2a9c 100644 --- a/codexcrawler/components/dhtmetrics.nim +++ b/codexcrawler/components/dhtmetrics.nim @@ -31,8 +31,6 @@ proc handleCheckEvent( d.metrics.setOkNodes(d.ok.len) d.metrics.setNokNodes(d.nok.len) - - trace "metrics updated", ok = d.ok.len, nok = d.nok.len return success() method start*(d: DhtMetrics): Future[?!void] {.async.} = @@ -44,6 +42,13 @@ method start*(d: DhtMetrics): Future[?!void] {.async.} = await d.handleCheckEvent(event) d.sub = d.state.events.dhtNodeCheck.subscribe(onCheck) + + proc logDhtMetrics(): Future[?!void] {.async: (raises: []), gcsafe.} = + trace "Metrics", ok = d.ok.len, nok = d.nok.len + return success() + + await d.state.whileRunning(logDhtMetrics, 1.minutes) + return success() method stop*(d: DhtMetrics): Future[?!void] {.async.} = diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index e303ced..1a7509e 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -81,12 +81,11 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = for nid in nids: without isNew =? (await s.storeNodeIsNew(nid)), err: return failure(err) - if isNew: newNodes.add(nid) - trace "Processed found nodes", total = nids.len, numNew = newNodes.len if newNodes.len > 0: + trace "Discovered new nodes", newNodes = newNodes.len ?await s.fireNewNodesDiscovered(newNodes) return success() diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index afbfca0..67eb48c 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -33,6 +33,7 @@ proc checkForExpiredNodes(t: TimeTracker): Future[?!void] {.async: (raises: []). return success() proc raiseRoutingTableNodes(t: TimeTracker): Future[?!void] {.async: (raises: []).} = + trace "Raising routing table nodes..." let nids = t.dht.getRoutingTableNodeIds() if err =? (await t.state.events.nodesFound.fire(nids)).errorOption: return failure(err) diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim index 19211b0..f84e2c1 100644 --- a/codexcrawler/components/todolist.nim +++ b/codexcrawler/components/todolist.nim @@ -26,8 +26,8 @@ proc addNodes(t: TodoList, nids: seq[Nid]) = for nid in nids: t.nids.add(nid) - trace "Nodes added", nodes = nids.len if s =? t.emptySignal: + trace "Nodes added, resuming...", nodes = nids.len s.complete() t.emptySignal = Future[void].none diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index 01d9c5e..71e183b 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -14,14 +14,14 @@ Usage: Options: --logLevel= Sets log level [default: INFO] - --publicIp= Public IP address where this instance is reachable. [default: 45.82.185.194] + --publicIp= Public IP address where this instance is reachable. --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] --metricsPort=

Listen HTTP port of the metrics server [default: 8008] --dataDir=

Directory for storing data [default: crawler_data] --discoveryPort=

Port used for DHT [default: 8090] --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] --stepDelay= Delay in milliseconds per crawl step [default: 1000] - --revisitDelay= Delay in minutes after which a node can be revisited [default: 1] (24h) + --revisitDelay= Delay in minutes after which a node can be revisited [default: 10] (24h) """ import strutils diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index 38cf0d6..0dbd1fb 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -27,11 +27,11 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = without dhtMetrics =? createDhtMetrics(state, metrics), err: return failure(err) + components.add(dht) components.add(todoList) components.add(nodeStore) - components.add(dht) components.add(Crawler.new(state, dht, todoList)) - components.add(TimeTracker.new(state, nodeStore)) + components.add(TimeTracker.new(state, nodeStore, dht)) components.add(dhtMetrics) return success(components)