logos-storage-network-crawler/codexcrawler/components/crawler.nim

import pkg/chronicles
import pkg/chronos
import pkg/questionable
import pkg/questionable/results

import ../services/dht
import ./todolist
import ../config
import ../types
import ../component
import ../state
import ../utils/asyncdataevent

logScope:
  topics = "crawler"

type Crawler* = ref object of Component
  state: State
  dht: Dht
  todo: TodoList

proc raiseCheckEvent(c: Crawler, nid: Nid, success: bool): Future[?!void] {.async: (raises: []).} =
  let event = DhtNodeCheckEventData(
    id: nid,
    isOk: success
  )
  if err =? (await c.state.events.dhtNodeCheck.fire(event)).errorOption:
    return failure(err)
  return success()

proc step(c: Crawler): Future[?!void] {.async: (raises: []).} =
  without nid =? (await c.todo.pop()), err:
    return failure(err)

  without response =? await c.dht.getNeighbors(nid), err:
    return failure(err)

  if err =? (await c.raiseCheckEvent(nid, response.isResponsive)).errorOption:
    return failure(err)

  if err =? (await c.state.events.nodesFound.fire(response.nodeIds)).errorOption:
    return failure(err)

  return success()

method start*(c: Crawler): Future[?!void] {.async.} =
  info "Starting crawler..."

  proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
    await c.step()
  await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds)

  return success()

method stop*(c: Crawler): Future[?!void] {.async.} =
  return success()

proc new*(
    T: type Crawler,
    state: State,
    dht: Dht,
    todo: TodoList
): Crawler =
  Crawler(
    state: state,
    dht: dht,
    todo: todo
  )