diff --git a/codexcrawler/crawler.nim b/codexcrawler/crawler.nim index 97f98f2..5673e94 100644 --- a/codexcrawler/crawler.nim +++ b/codexcrawler/crawler.nim @@ -54,16 +54,20 @@ proc step(c: Crawler) {.async.} = target.lastVisit = Moment.now().epochSeconds.uint64 without receivedNodes =? (await c.dht.getNeighbors(target.id)), err: - trace "Call failed", node = $target.id, err = err.msg await c.handleNodeNotOk(target) return let newNodes = receivedNodes.filterIt(isNew(c, it)) + if newNodes.len > 0: + trace "Discovered new nodes", newNodes = newNodes.len - trace "Received nodes", receivedNodes = receivedNodes.len, newNodes = newNodes.len await c.handleNodeOk(target) await c.addNewTodoNodes(newNodes) + # Don't log the status every loop: + if (c.todoNodes.len mod 10) == 0: + trace "Status" + proc worker(c: Crawler) {.async.} = try: while true: @@ -82,7 +86,7 @@ proc start*(c: Crawler): Future[?!void] {.async.} = error "Failed to add routing-table node to todo-list", err = err.msg return failure(err) - info "Starting crawler..." + info "Starting crawler...", stepDelayMs = $c.config.stepDelayMs asyncSpawn c.worker() return success() diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index 3596f8a..8227304 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -29,6 +29,7 @@ type store: TypedDatastore items: seq[NodeEntry] onMetric: OnUpdateMetric + emptySignal: ?Future[void] proc encode(s: NodeEntry): seq[byte] = s.toBytes() @@ -80,6 +81,11 @@ proc add*(this: List, item: NodeEntry): Future[?!void] {.async.} = this.items.add(item) this.onMetric(this.items.len.int64) + if isSome(this.emptySignal): + trace "List no longer empty.", name = this.name + this.emptySignal.get().complete() + this.emptySignal = Future[void].none + if err =? (await this.saveItem(item)).errorOption: return failure(err) return success() @@ -97,7 +103,11 @@ proc remove*(this: List, item: NodeEntry): Future[?!void] {.async.} = proc pop*(this: List): Future[?!NodeEntry] {.async.} = if this.items.len < 1: - return failure(this.name & "List is empty.") + trace "List is empty. Waiting for new items...", name = this.name + this.emptySignal = some(newFuture[void]("list.emptySignal")) + await this.emptySignal.get().wait(1.hours) + if this.items.len < 1: + return failure(this.name & "List is empty.") let item = this.items[0] diff --git a/codexcrawler/timetracker.nim b/codexcrawler/timetracker.nim index e839490..4c91893 100644 --- a/codexcrawler/timetracker.nim +++ b/codexcrawler/timetracker.nim @@ -8,8 +8,6 @@ import ./list import ./nodeentry import ./config -import std/sequtils - logScope: topics = "timetracker" @@ -24,11 +22,13 @@ proc processList(t: TimeTracker, list: List, expiry: uint64) {.async.} = var toMove = newSeq[NodeEntry]() proc onItem(item: NodeEntry) = if item.lastVisit < expiry: - trace "expired node, moving to todo" toMove.add(item) await list.iterateAll(onItem) + if toMove.len > 0: + trace "expired node, moving to todo", nodes = $toMove.len + for item in toMove: if err =? (await t.todoNodes.add(item)).errorOption: error "Failed to add expired node to todo list", err = err.msg @@ -51,7 +51,7 @@ proc worker(t: TimeTracker) {.async.} = quit QuitFailure proc start*(t: TimeTracker): Future[?!void] {.async.} = - info "Starting timetracker..." + info "Starting timetracker...", revisitDelayMins = $t.workerDelay asyncSpawn t.worker() return success() diff --git a/config.nims b/config.nims index 3e5bfb2..0c6f69f 100644 --- a/config.nims +++ b/config.nims @@ -7,6 +7,7 @@ switch("define", "libp2p_pki_schemes=secp256k1") # Sets TRACE logging for everything except DHT switch("define", "chronicles_log_level=TRACE") switch("define", "chronicles_disabled_topics:discv5") +switch("define", "chronicles_disable_thread_id") when (NimMajor, NimMinor) >= (2, 0): --mm: