From b5b29572098d6a7d986fd9d422ffe22dd6769cef Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Feb 2025 13:51:26 +0100 Subject: [PATCH 01/20] Sets up asyncdataevent --- codexcrawler/utils/asyncdataevent.nim | 64 ++++++++++++++ tests/codexcrawler/exampletest.nim | 7 -- tests/codexcrawler/testutils.nim | 3 + .../codexcrawler/utils/testasyncdataevent.nim | 86 +++++++++++++++++++ tests/test.nim | 2 +- 5 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 codexcrawler/utils/asyncdataevent.nim delete mode 100644 tests/codexcrawler/exampletest.nim create mode 100644 tests/codexcrawler/testutils.nim create mode 100644 tests/codexcrawler/utils/testasyncdataevent.nim diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim new file mode 100644 index 0000000..2605a30 --- /dev/null +++ b/codexcrawler/utils/asyncdataevent.nim @@ -0,0 +1,64 @@ +import pkg/questionable +import pkg/questionable/results +import pkg/chronos + +type + AsyncDataEventSubscription* = ref object + key: EventQueueKey + isRunning: bool + fireEvent: AsyncEvent + stopEvent: AsyncEvent + lastResult: ?!void + + AsyncDataEvent*[T] = ref object + queue: AsyncEventQueue[?T] + subscriptions: seq[AsyncDataEventSubscription] + + AsyncDataEventHandler*[T] = proc(data: T): Future[?!void] + +proc newAsyncDataEvent*[T](): AsyncDataEvent[T] = + AsyncDataEvent[T]( + queue: newAsyncEventQueue[?T](), + subscriptions: newSeq[AsyncDataEventSubscription]() + ) + +proc subscribe*[T](event: AsyncDataEvent[T], handler: AsyncDataEventHandler[T]): AsyncDataEventSubscription = + let subscription = AsyncDataEventSubscription( + key: event.queue.register(), + isRunning: true, + fireEvent: newAsyncEvent(), + stopEvent: newAsyncEvent() + ) + + proc listener() {.async.} = + while subscription.isRunning: + let items = await event.queue.waitEvents(subscription.key) + for item in items: + if data =? item: + subscription.lastResult = (await handler(data)) + subscription.fireEvent.fire() + subscription.stopEvent.fire() + + asyncSpawn listener() + + event.subscriptions.add(subscription) + subscription + +proc fire*[T](event: AsyncDataEvent[T], data: T): Future[?!void] {.async.} = + event.queue.emit(data.some) + for subscription in event.subscriptions: + await subscription.fireEvent.wait() + if err =? subscription.lastResult.errorOption: + return failure(err) + success() + +proc unsubscribe*[T](event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription) {.async.} = + subscription.isRunning = false + event.queue.emit(T.none) + await subscription.stopEvent.wait() + event.subscriptions.delete(event.subscriptions.find(subscription)) + +proc unsubscribeAll*[T](event: AsyncDataEvent[T]) {.async.} = + let all = event.subscriptions + for subscription in all: + await event.unsubscribe(subscription) diff --git a/tests/codexcrawler/exampletest.nim b/tests/codexcrawler/exampletest.nim deleted file mode 100644 index ca0d9b4..0000000 --- a/tests/codexcrawler/exampletest.nim +++ /dev/null @@ -1,7 +0,0 @@ -import pkg/asynctest/chronos/unittest - -suite "Example tests": - test "Example": - echo "Woo!" - check: - 1 == 1 diff --git a/tests/codexcrawler/testutils.nim b/tests/codexcrawler/testutils.nim new file mode 100644 index 0000000..eaec989 --- /dev/null +++ b/tests/codexcrawler/testutils.nim @@ -0,0 +1,3 @@ +import ./utils/testasyncdataevent + +{.warning[UnusedImport]: off.} diff --git a/tests/codexcrawler/utils/testasyncdataevent.nim b/tests/codexcrawler/utils/testasyncdataevent.nim new file mode 100644 index 0000000..9b034c8 --- /dev/null +++ b/tests/codexcrawler/utils/testasyncdataevent.nim @@ -0,0 +1,86 @@ +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/asynctest/chronos/unittest + +import ../../../codexcrawler/utils/asyncdataevent + +type + ExampleData = object + s: string + +suite "AsyncDataEvent": + var event: AsyncDataEvent[ExampleData] + let msg = "Yeah!" + + setup: + event = newAsyncDataEvent[ExampleData]() + + teardown: + await event.unsubscribeAll() + + test "Successful event": + var data = "" + proc eventHandler(e: ExampleData): Future[?!void] {.async.} = + data = e.s + success() + + let s = event.subscribe(eventHandler) + + check: + isOK(await event.fire(ExampleData( + s: msg + ))) + data == msg + + await event.unsubscribe(s) + + test "Failed event preserves error message": + proc eventHandler(e: ExampleData): Future[?!void] {.async.} = + failure(msg) + + let s = event.subscribe(eventHandler) + let fireResult = await event.fire(ExampleData( + s: "a" + )) + + check: + fireResult.isErr + fireResult.error.msg == msg + + await event.unsubscribe(s) + + test "Emits data to multiple subscribers": + var + data1 = "" + data2 = "" + data3 = "" + + proc handler1(e: ExampleData): Future[?!void] {.async.} = + data1 = e.s + success() + proc handler2(e: ExampleData): Future[?!void] {.async.} = + data2 = e.s + success() + proc handler3(e: ExampleData): Future[?!void] {.async.} = + data3 = e.s + success() + + let + s1 = event.subscribe(handler1) + s2 = event.subscribe(handler2) + s3 = event.subscribe(handler3) + + let fireResult = await event.fire(ExampleData( + s: msg + )) + + check: + fireResult.isOK + data1 == msg + data2 == msg + data3 == msg + + await event.unsubscribe(s1) + await event.unsubscribe(s2) + await event.unsubscribe(s3) diff --git a/tests/test.nim b/tests/test.nim index d7c2006..ca1588f 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -1,3 +1,3 @@ -import ./codexcrawler/exampletest +import ./codexcrawler/testutils {.warning[UnusedImport]: off.} From ed26070d24c987e323d889167e25899d472113f5 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Feb 2025 13:54:59 +0100 Subject: [PATCH 02/20] move utils to utils folder --- codexcrawler/application.nim | 5 ++--- codexcrawler/config.nim | 9 ++++++--- codexcrawler/dht.nim | 2 +- codexcrawler/{ => utils}/keyutils.nim | 0 codexcrawler/{ => utils}/logging.nim | 0 codexcrawler/{ => utils}/rng.nim | 0 codexcrawler/{utils.nim => utils/timeutils.nim} | 0 codexcrawler/{ => utils}/version.nim | 0 8 files changed, 9 insertions(+), 7 deletions(-) rename codexcrawler/{ => utils}/keyutils.nim (100%) rename codexcrawler/{ => utils}/logging.nim (100%) rename codexcrawler/{ => utils}/rng.nim (100%) rename codexcrawler/{utils.nim => utils/timeutils.nim} (100%) rename codexcrawler/{ => utils}/version.nim (100%) diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index 8909816..975ab65 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -1,5 +1,4 @@ import std/os -import std/sequtils import pkg/chronicles import pkg/chronos import pkg/questionable @@ -10,11 +9,11 @@ import pkg/datastore/typedds import pkg/metrics import ./config -import ./logging +import ./utils/logging import ./metrics import ./list import ./dht -import ./keyutils +import ./utils/keyutils import ./crawler import ./timetracker diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index 4692670..2f0e137 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -3,7 +3,7 @@ import std/sequtils import pkg/chronicles import pkg/libp2p import pkg/codexdht -import ./version +import ./utils/version let doc = """ @@ -13,15 +13,15 @@ Usage: codexcrawler [--logLevel=] [--publicIp=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] [--bootNodes=] [--stepDelay=] [--revisitDelay=] Options: - --publicIp= Public IP address where this instance is reachable. --logLevel= Sets log level [default: INFO] + --publicIp= Public IP address where this instance is reachable. [default: 45.82.185.194] --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] --metricsPort=

Listen HTTP port of the metrics server [default: 8008] --dataDir=

Directory for storing data [default: crawler_data] --discoveryPort=

Port used for DHT [default: 8090] --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] --stepDelay= Delay in milliseconds per crawl step [default: 1000] - --revisitDelay= Delay in minutes after which a node can be revisited [default: 1440] (24h) + --revisitDelay= Delay in minutes after which a node can be revisited [default: 1] (24h) """ import strutils @@ -54,6 +54,9 @@ proc getDefaultTestnetBootNodes(): seq[string] = "spr:CiUIAhIhAzZn3JmJab46BNjadVnLNQKbhnN3eYxwqpteKYY32SbOEgIDARo8CicAJQgCEiEDNmfcmYlpvjoE2Np1Wcs1ApuGc3d5jHCqm14phjfZJs4QrvWesAYaCwoJBKpA-TaRAnViKkcwRQIhANuMmZDD2c25xzTbKSirEpkZYoxbq-FU_lpI0K0e4mIVAiBfQX4yR47h1LCnHznXgDs6xx5DLO5q3lUcicqUeaqGeg", "spr:CiUIAhIhAgybmRwboqDdUJjeZrzh43sn5mp8jt6ENIb08tLn4x01EgIDARo8CicAJQgCEiECDJuZHBuioN1QmN5mvOHjeyfmanyO3oQ0hvTy0ufjHTUQh4ifsAYaCwoJBI_0zSiRAnVsKkcwRQIhAJCb_z0E3RsnQrEePdJzMSQrmn_ooHv6mbw1DOh5IbVNAiBbBJrWR8eBV6ftzMd6ofa5khNA2h88OBhMqHCIzSjCeA", "spr:CiUIAhIhAntGLadpfuBCD9XXfiN_43-V3L5VWgFCXxg4a8uhDdnYEgIDARo8CicAJQgCEiECe0Ytp2l-4EIP1dd-I3_jf5XcvlVaAUJfGDhry6EN2dgQsIufsAYaCwoJBNEmoCiRAnV2KkYwRAIgXO3bzd5VF8jLZG8r7dcLJ_FnQBYp1BcxrOvovEa40acCIDhQ14eJRoPwJ6GKgqOkXdaFAsoszl-HIRzYcXKeb7D9", + "spr:CiUIAhIhA2AEPzVj1Z_pshWAwvTp0xvRZTigIkYphXGZdiYGmYRwEgIDARo8CicAJQgCEiEDYAQ_NWPVn-myFYDC9OnTG9FlOKAiRimFcZl2JgaZhHAQvKCXugYaCwoJBES3CuORAnd-KkYwRAIgNwrc7n8A107pYUoWfJxL8X0f-flfUKeA6bFrjVKzEo0CID_0q-KO5ZAGf65VsK-d9rV3S0PbFg7Hj3Cv4aVX2Lnn", + "spr:CiUIAhIhAuhggJhkjeRoR7MHjZ_L_naZKnjF541X0GXTI7LEwXi_EgIDARo8CicAJQgCEiEC6GCAmGSN5GhHsweNn8v-dpkqeMXnjVfQZdMjssTBeL8Qop2quwYaCwoJBJK-4V-RAncuKkYwRAIgaXWoxvKkzrjUZ5K_ayQHKNlYhUEzBXhGviujxfJiGXkCICbsYFivi6Ny1FT6tbofVBRj7lnaR3K9_3j5pUT4862k", + "spr:CiUIAhIhA-pnA5sLGDVbqEXsRxDUjQEpiSAximHNbyqr2DwLmTq8EgIDARo8CicAJQgCEiED6mcDmwsYNVuoRexHENSNASmJIDGKYc1vKqvYPAuZOrwQyrekvAYaCwoJBIDHOw-RAnc4KkcwRQIhAJtKNeTykcE5bkKwe-vhSmqyBwc2AnexqFX1tAQGLQJ4AiBJOPseqvI3PyEM8l3hY3zvelZU9lT03O7MA_8cUfF4Uw", ] proc getBootNodeStrings(input: string): seq[string] = diff --git a/codexcrawler/dht.nim b/codexcrawler/dht.nim index a649ffe..3e8c2fb 100644 --- a/codexcrawler/dht.nim +++ b/codexcrawler/dht.nim @@ -7,7 +7,7 @@ import pkg/questionable/results import pkg/codexdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 -import ./rng +import ./utils/rng export discv5 diff --git a/codexcrawler/keyutils.nim b/codexcrawler/utils/keyutils.nim similarity index 100% rename from codexcrawler/keyutils.nim rename to codexcrawler/utils/keyutils.nim diff --git a/codexcrawler/logging.nim b/codexcrawler/utils/logging.nim similarity index 100% rename from codexcrawler/logging.nim rename to codexcrawler/utils/logging.nim diff --git a/codexcrawler/rng.nim b/codexcrawler/utils/rng.nim similarity index 100% rename from codexcrawler/rng.nim rename to codexcrawler/utils/rng.nim diff --git a/codexcrawler/utils.nim b/codexcrawler/utils/timeutils.nim similarity index 100% rename from codexcrawler/utils.nim rename to codexcrawler/utils/timeutils.nim diff --git a/codexcrawler/version.nim b/codexcrawler/utils/version.nim similarity index 100% rename from codexcrawler/version.nim rename to codexcrawler/utils/version.nim From 50962d9a915385562d348895a526e9e88236366c Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Feb 2025 14:49:30 +0100 Subject: [PATCH 03/20] setup component abstraction --- codexcrawler/application.nim | 102 +++++------------- codexcrawler/component.nim | 14 +++ codexcrawler/{ => components}/crawler.nim | 29 ++--- codexcrawler/{ => components}/dht.nim | 11 +- codexcrawler/{ => components}/timetracker.nim | 43 ++++---- codexcrawler/config.nim | 10 +- codexcrawler/installer.nim | 54 ++++++++++ codexcrawler/state.nim | 15 +++ codexcrawler/utils/datastoreutils.nim | 15 +++ 9 files changed, 179 insertions(+), 114 deletions(-) create mode 100644 codexcrawler/component.nim rename codexcrawler/{ => components}/crawler.nim (86%) rename codexcrawler/{ => components}/dht.nim (94%) rename codexcrawler/{ => components}/timetracker.nim (73%) create mode 100644 codexcrawler/installer.nim create mode 100644 codexcrawler/state.nim create mode 100644 codexcrawler/utils/datastoreutils.nim diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index 975ab65..5eef739 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -4,18 +4,16 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results -import pkg/datastore -import pkg/datastore/typedds import pkg/metrics import ./config import ./utils/logging import ./metrics import ./list -import ./dht -import ./utils/keyutils -import ./crawler -import ./timetracker +import ./utils/datastoreutils +import ./installer +import ./state +import ./component declareGauge(todoNodesGauge, "DHT nodes to be visited") declareGauge(okNodesGauge, "DHT nodes successfully contacted") @@ -29,27 +27,13 @@ type Application* = ref object status: ApplicationStatus - config*: CrawlerConfig + config*: Config todoNodes*: List okNodes*: List nokNodes*: List - dht*: Dht - crawler*: Crawler - timeTracker*: TimeTracker - -proc createDatastore(app: Application, path: string): ?!Datastore = - without store =? LevelDbDatastore.new(path), err: - error "Failed to create datastore" - return failure(err) - return success(Datastore(store)) - -proc createTypedDatastore(app: Application, path: string): ?!TypedDatastore = - without store =? app.createDatastore(path), err: - return failure(err) - return success(TypedDatastore.init(store)) proc initializeLists(app: Application): Future[?!void] {.async.} = - without store =? app.createTypedDatastore(app.config.dataDir / "lists"), err: + without store =? createTypedDatastore(app.config.dataDir / "lists"), err: return failure(err) # We can't extract this into a function because gauges cannot be passed as argument. @@ -76,71 +60,41 @@ proc initializeLists(app: Application): Future[?!void] {.async.} = return success() -proc initializeDht(app: Application): Future[?!void] {.async.} = - without dhtStore =? app.createDatastore(app.config.dataDir / "dht"), err: - return failure(err) - let keyPath = app.config.dataDir / "privatekey" - without privateKey =? setupKey(keyPath), err: - return failure(err) - - var listenAddresses = newSeq[MultiAddress]() - # TODO: when p2p connections are supported: - # let aaa = MultiAddress.init("/ip4/" & app.config.publicIp & "/tcp/53678").expect("Should init multiaddress") - # listenAddresses.add(aaa) - - var discAddresses = newSeq[MultiAddress]() - let bbb = MultiAddress - .init("/ip4/" & app.config.publicIp & "/udp/" & $app.config.discPort) - .expect("Should init multiaddress") - discAddresses.add(bbb) - - app.dht = Dht.new( - privateKey, - bindPort = app.config.discPort, - announceAddrs = listenAddresses, - bootstrapNodes = app.config.bootNodes, - store = dhtStore, - ) - - app.dht.updateAnnounceRecord(listenAddresses) - app.dht.updateDhtRecord(discAddresses) - - await app.dht.start() - - return success() - -proc initializeCrawler(app: Application): Future[?!void] {.async.} = - app.crawler = - Crawler.new(app.dht, app.todoNodes, app.okNodes, app.nokNodes, app.config) - return await app.crawler.start() - -proc initializeTimeTracker(app: Application): Future[?!void] {.async.} = - app.timeTracker = - TimeTracker.new(app.todoNodes, app.okNodes, app.nokNodes, app.config) - return await app.timeTracker.start() - proc initializeApp(app: Application): Future[?!void] {.async.} = if err =? (await app.initializeLists()).errorOption: error "Failed to initialize lists", err = err.msg return failure(err) - if err =? (await app.initializeDht()).errorOption: - error "Failed to initialize DHT", err = err.msg + # if err =? (await app.initializeDht()).errorOption: + # error "Failed to initialize DHT", err = err.msg + # return failure(err) + + # if err =? (await app.initializeCrawler()).errorOption: + # error "Failed to initialize crawler", err = err.msg + # return failure(err) + + # if err =? (await app.initializeTimeTracker()).errorOption: + # error "Failed to initialize timetracker", err = err.msg + # return failure(err) + + without components =? (await createComponents(app.config)), err: + error "Failed to create componenents", err = err.msg return failure(err) - if err =? (await app.initializeCrawler()).errorOption: - error "Failed to initialize crawler", err = err.msg - return failure(err) + # todo move this + let state = State( + config: app.config + ) - if err =? (await app.initializeTimeTracker()).errorOption: - error "Failed to initialize timetracker", err = err.msg - return failure(err) + for c in components: + if err =? (await c.start(state)).errorOption: + error "Failed to start component", err = err.msg return success() proc stop*(app: Application) = app.status = ApplicationStatus.Stopping - waitFor app.dht.stop() + # waitFor app.dht.stop() proc run*(app: Application) = app.config = parseConfig() diff --git a/codexcrawler/component.nim b/codexcrawler/component.nim new file mode 100644 index 0000000..29acc8c --- /dev/null +++ b/codexcrawler/component.nim @@ -0,0 +1,14 @@ +import pkg/chronos +import pkg/questionable/results + +import ./state + +type + Component* = ref object of RootObj + +method start*(c: Component, state: State): Future[?!void] {.async, base.} = + raiseAssert("call to abstract method") + +method stop*(c: Component): Future[?!void] {.async, base.} = + raiseAssert("call to abstract method") + diff --git a/codexcrawler/crawler.nim b/codexcrawler/components/crawler.nim similarity index 86% rename from codexcrawler/crawler.nim rename to codexcrawler/components/crawler.nim index 5673e94..1a03b56 100644 --- a/codexcrawler/crawler.nim +++ b/codexcrawler/components/crawler.nim @@ -4,18 +4,19 @@ import pkg/questionable import pkg/questionable/results import ./dht -import ./list -import ./nodeentry -import ./config +import ../list +import ../nodeentry +import ../config +import ../component import std/sequtils logScope: topics = "crawler" -type Crawler* = ref object +type Crawler* = ref object of Component dht: Dht - config: CrawlerConfig + config: Config todoNodes: List okNodes: List nokNodes: List @@ -90,14 +91,18 @@ proc start*(c: Crawler): Future[?!void] {.async.} = asyncSpawn c.worker() return success() +proc stop*(c: Crawler): Future[?!void] {.async.} = + return success() + proc new*( T: type Crawler, dht: Dht, - todoNodes: List, - okNodes: List, - nokNodes: List, - config: CrawlerConfig, + # todoNodes: List, + # okNodes: List, + # nokNodes: List, + config: Config, ): Crawler = - Crawler( - dht: dht, todoNodes: todoNodes, okNodes: okNodes, nokNodes: nokNodes, config: config - ) + raiseAssert("todo") + # Crawler( + # dht: dht, todoNodes: todoNodes, okNodes: okNodes, nokNodes: nokNodes, config: config + # ) diff --git a/codexcrawler/dht.nim b/codexcrawler/components/dht.nim similarity index 94% rename from codexcrawler/dht.nim rename to codexcrawler/components/dht.nim index 3e8c2fb..d46f13e 100644 --- a/codexcrawler/dht.nim +++ b/codexcrawler/components/dht.nim @@ -7,14 +7,15 @@ import pkg/questionable/results import pkg/codexdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 -import ./utils/rng +import ../utils/rng +import ../component export discv5 logScope: topics = "dht" -type Dht* = ref object +type Dht* = ref object of Component protocol*: discv5.Protocol key: PrivateKey peerId: PeerId @@ -96,12 +97,14 @@ proc updateDhtRecord*(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") -proc start*(d: Dht) {.async.} = +proc start*(d: Dht): Future[?!void] {.async.} = d.protocol.open() await d.protocol.start() + return success() -proc stop*(d: Dht) {.async.} = +proc stop*(d: Dht): Future[?!void] {.async.} = await d.protocol.closeWait() + return success() proc new*( T: type Dht, diff --git a/codexcrawler/timetracker.nim b/codexcrawler/components/timetracker.nim similarity index 73% rename from codexcrawler/timetracker.nim rename to codexcrawler/components/timetracker.nim index 4c91893..da348ff 100644 --- a/codexcrawler/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -4,15 +4,16 @@ import pkg/questionable import pkg/questionable/results import ./dht -import ./list -import ./nodeentry -import ./config +import ../list +import ../nodeentry +import ../config +import ../component logScope: topics = "timetracker" -type TimeTracker* = ref object - config: CrawlerConfig +type TimeTracker* = ref object of Component + config: Config todoNodes: List okNodes: List nokNodes: List @@ -55,21 +56,25 @@ proc start*(t: TimeTracker): Future[?!void] {.async.} = asyncSpawn t.worker() return success() +proc stop*(t: TimeTracker): Future[?!void] {.async.} = + return success() + proc new*( T: type TimeTracker, - todoNodes: List, - okNodes: List, - nokNodes: List, - config: CrawlerConfig, + # todoNodes: List, + # okNodes: List, + # nokNodes: List, + config: Config, ): TimeTracker = - var delay = config.revisitDelayMins div 10 - if delay < 1: - delay = 1 + raiseAssert("todo") + # var delay = config.revisitDelayMins div 10 + # if delay < 1: + # delay = 1 - TimeTracker( - todoNodes: todoNodes, - okNodes: okNodes, - nokNodes: nokNodes, - config: config, - workerDelay: delay, - ) + # TimeTracker( + # todoNodes: todoNodes, + # okNodes: okNodes, + # nokNodes: nokNodes, + # config: config, + # workerDelay: delay, + # ) diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index 2f0e137..01d9c5e 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -27,7 +27,7 @@ Options: import strutils import docopt -type CrawlerConfig* = ref object +type Config* = ref object logLevel*: string publicIp*: string metricsAddress*: IpAddress @@ -38,8 +38,8 @@ type CrawlerConfig* = ref object stepDelayMs*: int revisitDelayMins*: int -proc `$`*(config: CrawlerConfig): string = - "CrawlerConfig:" & " logLevel=" & config.logLevel & " publicIp=" & config.publicIp & +proc `$`*(config: Config): string = + "Crawler:" & " logLevel=" & config.logLevel & " publicIp=" & config.publicIp & " metricsAddress=" & $config.metricsAddress & " metricsPort=" & $config.metricsPort & " dataDir=" & config.dataDir & " discPort=" & $config.discPort & " bootNodes=" & config.bootNodes.mapIt($it).join(";") & " stepDelay=" & $config.stepDelayMs & @@ -81,13 +81,13 @@ proc stringToSpr(uri: string): SignedPeerRecord = proc getBootNodes(input: string): seq[SignedPeerRecord] = getBootNodeStrings(input).mapIt(stringToSpr(it)) -proc parseConfig*(): CrawlerConfig = +proc parseConfig*(): Config = let args = docopt(doc, version = crawlerFullVersion) proc get(name: string): string = $args[name] - return CrawlerConfig( + return Config( logLevel: get("--logLevel"), publicIp: get("--publicIp"), metricsAddress: parseIpAddress(get("--metricsAddress")), diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim new file mode 100644 index 0000000..02f6a53 --- /dev/null +++ b/codexcrawler/installer.nim @@ -0,0 +1,54 @@ +import std/os +import pkg/chronos +import pkg/chronicles +import pkg/questionable/results + +import ./config +import ./component +import ./components/dht +import ./components/crawler +import ./components/timetracker +import ./utils/keyutils +import ./utils/datastoreutils + +proc initializeDht(config: Config): Future[?!Dht] {.async.} = + without dhtStore =? createDatastore(config.dataDir / "dht"), err: + return failure(err) + let keyPath = config.dataDir / "privatekey" + without privateKey =? setupKey(keyPath), err: + return failure(err) + + var listenAddresses = newSeq[MultiAddress]() + # TODO: when p2p connections are supported: + # let aaa = MultiAddress.init("/ip4/" & config.publicIp & "/tcp/53678").expect("Should init multiaddress") + # listenAddresses.add(aaa) + + var discAddresses = newSeq[MultiAddress]() + let bbb = MultiAddress + .init("/ip4/" & config.publicIp & "/udp/" & $config.discPort) + .expect("Should init multiaddress") + discAddresses.add(bbb) + + let dht = Dht.new( + privateKey, + bindPort = config.discPort, + announceAddrs = listenAddresses, + bootstrapNodes = config.bootNodes, + store = dhtStore, + ) + + dht.updateAnnounceRecord(listenAddresses) + dht.updateDhtRecord(discAddresses) + + return success(dht) + +proc createComponents*(config: Config): Future[?!seq[Component]] {.async.} = + var components: seq[Component] = newSeq[Component]() + + without dht =? (await initializeDht(config)), err: + return failure(err) + + components.add(dht) + components.add(Crawler.new(dht, config)) + components.add(TimeTracker.new(config)) + return success(components) diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim new file mode 100644 index 0000000..5a37cf0 --- /dev/null +++ b/codexcrawler/state.nim @@ -0,0 +1,15 @@ +import pkg/chronos +import pkg/questionable/results + +import ./config + +type + OnStep = proc(): Future[?!void] {.async: (raises: []), gcsafe.} + State* = ref object + config*: Config + # events + # appstate + +proc whileRunning*(this: State, step: OnStep, delay: Duration) = + discard + #todo: while status == running, step(), asyncsleep duration diff --git a/codexcrawler/utils/datastoreutils.nim b/codexcrawler/utils/datastoreutils.nim new file mode 100644 index 0000000..8754d63 --- /dev/null +++ b/codexcrawler/utils/datastoreutils.nim @@ -0,0 +1,15 @@ +import pkg/chronicles +import pkg/questionable/results +import pkg/datastore +import pkg/datastore/typedds + +proc createDatastore*(path: string): ?!Datastore = + without store =? LevelDbDatastore.new(path), err: + error "Failed to create datastore" + return failure(err) + return success(Datastore(store)) + +proc createTypedDatastore*(path: string): ?!TypedDatastore = + without store =? createDatastore(path), err: + return failure(err) + return success(TypedDatastore.init(store)) From 218443ebe43009831dbf5640309d5064c2dae1a3 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Feb 2025 15:34:41 +0100 Subject: [PATCH 04/20] working example of event --- codexcrawler/application.nim | 15 ++++++- codexcrawler/component.nim | 8 ++-- codexcrawler/components/crawler.nim | 38 ++++++++++------- codexcrawler/components/dht.nim | 42 +++++++++++++++++-- codexcrawler/components/timetracker.nim | 28 ++++++------- codexcrawler/installer.nim | 37 +--------------- codexcrawler/list.nim | 20 +++++---- codexcrawler/nodeentry.nim | 10 ++--- codexcrawler/state.nim | 16 ++++++- codexcrawler/types.nim | 12 ++++++ codexcrawler/utils/asyncdataevent.nim | 13 +++--- .../codexcrawler/utils/testasyncdataevent.nim | 19 ++++----- 12 files changed, 152 insertions(+), 106 deletions(-) create mode 100644 codexcrawler/types.nim diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index 5eef739..20e8e72 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -11,9 +11,11 @@ import ./utils/logging import ./metrics import ./list import ./utils/datastoreutils +import ./utils/asyncdataevent import ./installer import ./state import ./component +import ./types declareGauge(todoNodesGauge, "DHT nodes to be visited") declareGauge(okNodesGauge, "DHT nodes successfully contacted") @@ -83,13 +85,24 @@ proc initializeApp(app: Application): Future[?!void] {.async.} = # todo move this let state = State( - config: app.config + config: app.config, + events: Events( + nodesFound: newAsyncDataEvent[seq[Nid]](), + newNodesDiscovered: newAsyncDataEvent[seq[Nid]](), + dhtNodeCheck: newAsyncDataEvent[DhtNodeCheckEventData](), + nodesExpired: newAsyncDataEvent[seq[Nid]](), + ), ) for c in components: if err =? (await c.start(state)).errorOption: error "Failed to start component", err = err.msg + # test raise newnodes + let nodes: seq[Nid] = newSeq[Nid]() + if err =? (await state.events.nodesFound.fire(nodes)).errorOption: + return failure(err) + return success() proc stop*(app: Application) = diff --git a/codexcrawler/component.nim b/codexcrawler/component.nim index 29acc8c..479cdfc 100644 --- a/codexcrawler/component.nim +++ b/codexcrawler/component.nim @@ -3,12 +3,10 @@ import pkg/questionable/results import ./state -type - Component* = ref object of RootObj +type Component* = ref object of RootObj method start*(c: Component, state: State): Future[?!void] {.async, base.} = - raiseAssert("call to abstract method") + raiseAssert("call to abstract method: component.start") method stop*(c: Component): Future[?!void] {.async, base.} = - raiseAssert("call to abstract method") - + raiseAssert("call to abstract method: component.stop") diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/crawler.nim index 1a03b56..73bdbac 100644 --- a/codexcrawler/components/crawler.nim +++ b/codexcrawler/components/crawler.nim @@ -8,6 +8,9 @@ import ../list import ../nodeentry import ../config import ../component +import ../types +import ../state +import ../utils/asyncdataevent import std/sequtils @@ -72,26 +75,33 @@ proc step(c: Crawler) {.async.} = proc worker(c: Crawler) {.async.} = try: while true: - await c.step() + # await c.step() await sleepAsync(c.config.stepDelayMs.millis) except Exception as exc: error "Exception in crawler worker", msg = exc.msg quit QuitFailure -proc start*(c: Crawler): Future[?!void] {.async.} = - if c.todoNodes.len < 1: - let nodeIds = c.dht.getRoutingTableNodeIds() - info "Loading routing-table nodes to todo-list...", nodes = nodeIds.len - for id in nodeIds: - if err =? (await c.addNewTodoNode(id)).errorOption: - error "Failed to add routing-table node to todo-list", err = err.msg - return failure(err) +method start*(c: Crawler, state: State): Future[?!void] {.async.} = + # if c.todoNodes.len < 1: + # let nodeIds = c.dht.getRoutingTableNodeIds() + # info "Loading routing-table nodes to todo-list...", nodes = nodeIds.len + # for id in nodeIds: + # if err =? (await c.addNewTodoNode(id)).errorOption: + # error "Failed to add routing-table node to todo-list", err = err.msg + # return failure(err) info "Starting crawler...", stepDelayMs = $c.config.stepDelayMs asyncSpawn c.worker() + + proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = + info "Crawler sees nodes found!", num = nids.len + return success() + + let handle = state.events.nodesFound.subscribe(onNodesFound) + return success() -proc stop*(c: Crawler): Future[?!void] {.async.} = +method stop*(c: Crawler): Future[?!void] {.async.} = return success() proc new*( @@ -102,7 +112,7 @@ proc new*( # nokNodes: List, config: Config, ): Crawler = - raiseAssert("todo") - # Crawler( - # dht: dht, todoNodes: todoNodes, okNodes: okNodes, nokNodes: nokNodes, config: config - # ) + Crawler( + dht: dht, + config: config, # todoNodes: todoNodes, okNodes: okNodes, nokNodes: nokNodes, + ) diff --git a/codexcrawler/components/dht.nim b/codexcrawler/components/dht.nim index d46f13e..ab1203b 100644 --- a/codexcrawler/components/dht.nim +++ b/codexcrawler/components/dht.nim @@ -1,3 +1,4 @@ +import std/os import std/net import pkg/chronicles import pkg/chronos @@ -7,8 +8,12 @@ import pkg/questionable/results import pkg/codexdht/discv5/[routing_table, protocol as discv5] from pkg/nimcrypto import keccak256 +import ../utils/keyutils +import ../utils/datastoreutils import ../utils/rng import ../component +import ../config +import ../state export discv5 @@ -97,16 +102,16 @@ proc updateDhtRecord*(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") -proc start*(d: Dht): Future[?!void] {.async.} = +method start*(d: Dht, state: State): Future[?!void] {.async.} = d.protocol.open() await d.protocol.start() return success() -proc stop*(d: Dht): Future[?!void] {.async.} = +method stop*(d: Dht): Future[?!void] {.async.} = await d.protocol.closeWait() return success() -proc new*( +proc new( T: type Dht, key: PrivateKey, bindIp = IPv4_any(), @@ -138,3 +143,34 @@ proc new*( ) self + +proc createDht*(config: Config): Future[?!Dht] {.async.} = + without dhtStore =? createDatastore(config.dataDir / "dht"), err: + return failure(err) + let keyPath = config.dataDir / "privatekey" + without privateKey =? setupKey(keyPath), err: + return failure(err) + + var listenAddresses = newSeq[MultiAddress]() + # TODO: when p2p connections are supported: + # let aaa = MultiAddress.init("/ip4/" & config.publicIp & "/tcp/53678").expect("Should init multiaddress") + # listenAddresses.add(aaa) + + var discAddresses = newSeq[MultiAddress]() + let bbb = MultiAddress + .init("/ip4/" & config.publicIp & "/udp/" & $config.discPort) + .expect("Should init multiaddress") + discAddresses.add(bbb) + + let dht = Dht.new( + privateKey, + bindPort = config.discPort, + announceAddrs = listenAddresses, + bootstrapNodes = config.bootNodes, + store = dhtStore, + ) + + dht.updateAnnounceRecord(listenAddresses) + dht.updateDhtRecord(discAddresses) + + return success(dht) diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index da348ff..755cc5d 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -8,6 +8,7 @@ import ../list import ../nodeentry import ../config import ../component +import ../state logScope: topics = "timetracker" @@ -45,18 +46,18 @@ proc step(t: TimeTracker) {.async.} = proc worker(t: TimeTracker) {.async.} = try: while true: - await t.step() + # await t.step() await sleepAsync(t.workerDelay.minutes) except Exception as exc: error "Exception in timetracker worker", msg = exc.msg quit QuitFailure -proc start*(t: TimeTracker): Future[?!void] {.async.} = +method start*(t: TimeTracker, state: State): Future[?!void] {.async.} = info "Starting timetracker...", revisitDelayMins = $t.workerDelay asyncSpawn t.worker() return success() -proc stop*(t: TimeTracker): Future[?!void] {.async.} = +method stop*(t: TimeTracker): Future[?!void] {.async.} = return success() proc new*( @@ -66,15 +67,14 @@ proc new*( # nokNodes: List, config: Config, ): TimeTracker = - raiseAssert("todo") - # var delay = config.revisitDelayMins div 10 - # if delay < 1: - # delay = 1 + var delay = config.revisitDelayMins div 10 + if delay < 1: + delay = 1 - # TimeTracker( - # todoNodes: todoNodes, - # okNodes: okNodes, - # nokNodes: nokNodes, - # config: config, - # workerDelay: delay, - # ) + TimeTracker( + # todoNodes: todoNodes, + # okNodes: okNodes, + # nokNodes: nokNodes, + config: config, + workerDelay: delay, + ) diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index 02f6a53..247f7ee 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -1,6 +1,4 @@ -import std/os import pkg/chronos -import pkg/chronicles import pkg/questionable/results import ./config @@ -8,44 +6,11 @@ import ./component import ./components/dht import ./components/crawler import ./components/timetracker -import ./utils/keyutils -import ./utils/datastoreutils - -proc initializeDht(config: Config): Future[?!Dht] {.async.} = - without dhtStore =? createDatastore(config.dataDir / "dht"), err: - return failure(err) - let keyPath = config.dataDir / "privatekey" - without privateKey =? setupKey(keyPath), err: - return failure(err) - - var listenAddresses = newSeq[MultiAddress]() - # TODO: when p2p connections are supported: - # let aaa = MultiAddress.init("/ip4/" & config.publicIp & "/tcp/53678").expect("Should init multiaddress") - # listenAddresses.add(aaa) - - var discAddresses = newSeq[MultiAddress]() - let bbb = MultiAddress - .init("/ip4/" & config.publicIp & "/udp/" & $config.discPort) - .expect("Should init multiaddress") - discAddresses.add(bbb) - - let dht = Dht.new( - privateKey, - bindPort = config.discPort, - announceAddrs = listenAddresses, - bootstrapNodes = config.bootNodes, - store = dhtStore, - ) - - dht.updateAnnounceRecord(listenAddresses) - dht.updateDhtRecord(discAddresses) - - return success(dht) proc createComponents*(config: Config): Future[?!seq[Component]] {.async.} = var components: seq[Component] = newSeq[Component]() - without dht =? (await initializeDht(config)), err: + without dht =? (await createDht(config)), err: return failure(err) components.add(dht) diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index 8227304..935d4ae 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -8,14 +8,13 @@ import pkg/stew/endians2 import pkg/questionable import pkg/questionable/results import pkg/stint -import pkg/codexdht import std/sets -import std/strutils import std/sequtils import std/os import ./nodeentry +import ./types logScope: topics = "list" @@ -36,7 +35,7 @@ proc encode(s: NodeEntry): seq[byte] = proc decode(T: type NodeEntry, bytes: seq[byte]): ?!T = if bytes.len < 1: - return success(NodeEntry(id: UInt256.fromHex("0"), lastVisit: 0.uint64)) + return success(NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64)) return NodeEntry.fromBytes(bytes) proc saveItem(this: List, item: NodeEntry): Future[?!void] {.async.} = @@ -46,6 +45,10 @@ proc saveItem(this: List, item: NodeEntry): Future[?!void] {.async.} = return success() proc load*(this: List): Future[?!void] {.async.} = + let id = Nid.fromStr("0") + let bytes = newSeq[byte]() + let ne = NodeEntry.fromBytes(bytes) + without queryKey =? Key.init(this.name), err: return failure(err) without iter =? (await query[NodeEntry](this.store, Query.init(queryKey))), err: @@ -68,7 +71,7 @@ proc new*( ): List = List(name: name, store: store, onMetric: onMetric) -proc contains*(this: List, nodeId: NodeId): bool = +proc contains*(this: List, nodeId: Nid): bool = this.items.anyIt(it.id == nodeId) proc contains*(this: List, item: NodeEntry): bool = @@ -81,9 +84,9 @@ proc add*(this: List, item: NodeEntry): Future[?!void] {.async.} = this.items.add(item) this.onMetric(this.items.len.int64) - if isSome(this.emptySignal): + if s =? this.emptySignal: trace "List no longer empty.", name = this.name - this.emptySignal.get().complete() + s.complete() this.emptySignal = Future[void].none if err =? (await this.saveItem(item)).errorOption: @@ -104,8 +107,9 @@ proc remove*(this: List, item: NodeEntry): Future[?!void] {.async.} = proc pop*(this: List): Future[?!NodeEntry] {.async.} = if this.items.len < 1: trace "List is empty. Waiting for new items...", name = this.name - this.emptySignal = some(newFuture[void]("list.emptySignal")) - await this.emptySignal.get().wait(1.hours) + let signal = newFuture[void]("list.emptySignal") + this.emptySignal = some(signal) + await signal.wait(1.hours) if this.items.len < 1: return failure(this.name & "List is empty.") diff --git a/codexcrawler/nodeentry.nim b/codexcrawler/nodeentry.nim index e5b6721..6f80411 100644 --- a/codexcrawler/nodeentry.nim +++ b/codexcrawler/nodeentry.nim @@ -1,13 +1,11 @@ -import pkg/stew/byteutils -import pkg/stew/endians2 -import pkg/questionable import pkg/questionable/results -import pkg/codexdht import pkg/chronos import pkg/libp2p +import ./types + type NodeEntry* = object - id*: NodeId + id*: Nid lastVisit*: uint64 proc `$`*(entry: NodeEntry): string = @@ -32,4 +30,4 @@ proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry = if buffer.getField(2, lastVisit).isErr: return failure("Unable to decode `lastVisit`") - return success(NodeEntry(id: UInt256.fromHex(idStr), lastVisit: lastVisit)) + return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit)) diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index 5a37cf0..f7cd928 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -2,13 +2,25 @@ import pkg/chronos import pkg/questionable/results import ./config +import ./utils/asyncdataevent +import ./types type OnStep = proc(): Future[?!void] {.async: (raises: []), gcsafe.} + + DhtNodeCheckEventData* = object + id*: Nid + isOk*: bool + + Events* = ref object + nodesFound*: AsyncDataEvent[seq[Nid]] + newNodesDiscovered*: AsyncDataEvent[seq[Nid]] + dhtNodeCheck*: AsyncDataEvent[DhtNodeCheckEventData] + nodesExpired*: AsyncDataEvent[seq[Nid]] + State* = ref object config*: Config - # events - # appstate + events*: Events # appstate proc whileRunning*(this: State, step: OnStep, delay: Duration) = discard diff --git a/codexcrawler/types.nim b/codexcrawler/types.nim new file mode 100644 index 0000000..b4fe39c --- /dev/null +++ b/codexcrawler/types.nim @@ -0,0 +1,12 @@ +import pkg/stew/byteutils +import pkg/stew/endians2 +import pkg/questionable +import pkg/codexdht + +type Nid* = NodeId + +proc `$`*(nid: Nid): string = + $(NodeId(nid)) + +proc fromStr*(T: type Nid, s: string): Nid = + Nid(UInt256.fromHex(s)) diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim index 2605a30..cb2ec75 100644 --- a/codexcrawler/utils/asyncdataevent.nim +++ b/codexcrawler/utils/asyncdataevent.nim @@ -18,16 +18,17 @@ type proc newAsyncDataEvent*[T](): AsyncDataEvent[T] = AsyncDataEvent[T]( - queue: newAsyncEventQueue[?T](), - subscriptions: newSeq[AsyncDataEventSubscription]() + queue: newAsyncEventQueue[?T](), subscriptions: newSeq[AsyncDataEventSubscription]() ) -proc subscribe*[T](event: AsyncDataEvent[T], handler: AsyncDataEventHandler[T]): AsyncDataEventSubscription = +proc subscribe*[T]( + event: AsyncDataEvent[T], handler: AsyncDataEventHandler[T] +): AsyncDataEventSubscription = let subscription = AsyncDataEventSubscription( key: event.queue.register(), isRunning: true, fireEvent: newAsyncEvent(), - stopEvent: newAsyncEvent() + stopEvent: newAsyncEvent(), ) proc listener() {.async.} = @@ -52,7 +53,9 @@ proc fire*[T](event: AsyncDataEvent[T], data: T): Future[?!void] {.async.} = return failure(err) success() -proc unsubscribe*[T](event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription) {.async.} = +proc unsubscribe*[T]( + event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription +) {.async.} = subscription.isRunning = false event.queue.emit(T.none) await subscription.stopEvent.wait() diff --git a/tests/codexcrawler/utils/testasyncdataevent.nim b/tests/codexcrawler/utils/testasyncdataevent.nim index 9b034c8..7e47a7b 100644 --- a/tests/codexcrawler/utils/testasyncdataevent.nim +++ b/tests/codexcrawler/utils/testasyncdataevent.nim @@ -5,9 +5,8 @@ import pkg/asynctest/chronos/unittest import ../../../codexcrawler/utils/asyncdataevent -type - ExampleData = object - s: string +type ExampleData = object + s: string suite "AsyncDataEvent": var event: AsyncDataEvent[ExampleData] @@ -28,9 +27,7 @@ suite "AsyncDataEvent": let s = event.subscribe(eventHandler) check: - isOK(await event.fire(ExampleData( - s: msg - ))) + isOK(await event.fire(ExampleData(s: msg))) data == msg await event.unsubscribe(s) @@ -40,9 +37,7 @@ suite "AsyncDataEvent": failure(msg) let s = event.subscribe(eventHandler) - let fireResult = await event.fire(ExampleData( - s: "a" - )) + let fireResult = await event.fire(ExampleData(s: "a")) check: fireResult.isErr @@ -59,9 +54,11 @@ suite "AsyncDataEvent": proc handler1(e: ExampleData): Future[?!void] {.async.} = data1 = e.s success() + proc handler2(e: ExampleData): Future[?!void] {.async.} = data2 = e.s success() + proc handler3(e: ExampleData): Future[?!void] {.async.} = data3 = e.s success() @@ -71,9 +68,7 @@ suite "AsyncDataEvent": s2 = event.subscribe(handler2) s3 = event.subscribe(handler3) - let fireResult = await event.fire(ExampleData( - s: msg - )) + let fireResult = await event.fire(ExampleData(s: msg)) check: fireResult.isOK From 14e74d6380ef2781e4d6f9c01146adc8e7e4c3dd Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Feb 2025 16:02:47 +0100 Subject: [PATCH 05/20] Setting up for the nodestore --- codexcrawler/application.nim | 14 +---- codexcrawler/components/crawler.nim | 80 ++++++++++--------------- codexcrawler/components/dht.nim | 19 ++++-- codexcrawler/components/nodestore.nim | 40 +++++++++++++ codexcrawler/components/timetracker.nim | 37 ++++++------ codexcrawler/list.nim | 53 ++++++++-------- codexcrawler/nodeentry.nim | 33 ---------- codexcrawler/types.nim | 19 +++++- 8 files changed, 149 insertions(+), 146 deletions(-) create mode 100644 codexcrawler/components/nodestore.nim diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index 20e8e72..c6dab72 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -66,19 +66,7 @@ proc initializeApp(app: Application): Future[?!void] {.async.} = if err =? (await app.initializeLists()).errorOption: error "Failed to initialize lists", err = err.msg return failure(err) - - # if err =? (await app.initializeDht()).errorOption: - # error "Failed to initialize DHT", err = err.msg - # return failure(err) - - # if err =? (await app.initializeCrawler()).errorOption: - # error "Failed to initialize crawler", err = err.msg - # return failure(err) - - # if err =? (await app.initializeTimeTracker()).errorOption: - # error "Failed to initialize timetracker", err = err.msg - # return failure(err) - + without components =? (await createComponents(app.config)), err: error "Failed to create componenents", err = err.msg return failure(err) diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/crawler.nim index 73bdbac..5f4c296 100644 --- a/codexcrawler/components/crawler.nim +++ b/codexcrawler/components/crawler.nim @@ -5,7 +5,6 @@ import pkg/questionable/results import ./dht import ../list -import ../nodeentry import ../config import ../component import ../types @@ -29,48 +28,48 @@ proc isNew(c: Crawler, node: Node): bool = not c.todoNodes.contains(node.id) and not c.okNodes.contains(node.id) and not c.nokNodes.contains(node.id) -proc handleNodeNotOk(c: Crawler, target: NodeEntry) {.async.} = - if err =? (await c.nokNodes.add(target)).errorOption: - error "Failed to add not-OK-node to list", err = err.msg +# proc handleNodeNotOk(c: Crawler, target: NodeEntry) {.async.} = +# if err =? (await c.nokNodes.add(target)).errorOption: +# error "Failed to add not-OK-node to list", err = err.msg -proc handleNodeOk(c: Crawler, target: NodeEntry) {.async.} = - if err =? (await c.okNodes.add(target)).errorOption: - error "Failed to add OK-node to list", err = err.msg +# proc handleNodeOk(c: Crawler, target: NodeEntry) {.async.} = +# if err =? (await c.okNodes.add(target)).errorOption: +# error "Failed to add OK-node to list", err = err.msg -proc addNewTodoNode(c: Crawler, nodeId: NodeId): Future[?!void] {.async.} = - let entry = NodeEntry(id: nodeId, lastVisit: 0) - return await c.todoNodes.add(entry) +# proc addNewTodoNode(c: Crawler, nodeId: NodeId): Future[?!void] {.async.} = +# let entry = NodeEntry(id: nodeId, lastVisit: 0) +# return await c.todoNodes.add(entry) -proc addNewTodoNodes(c: Crawler, newNodes: seq[Node]) {.async.} = - for node in newNodes: - if err =? (await c.addNewTodoNode(node.id)).errorOption: - error "Failed to add todo-node to list", err = err.msg +# proc addNewTodoNodes(c: Crawler, newNodes: seq[Node]) {.async.} = +# for node in newNodes: +# if err =? (await c.addNewTodoNode(node.id)).errorOption: +# error "Failed to add todo-node to list", err = err.msg -proc step(c: Crawler) {.async.} = - logScope: - todo = $c.todoNodes.len - ok = $c.okNodes.len - nok = $c.nokNodes.len +# proc step(c: Crawler) {.async.} = +# logScope: +# todo = $c.todoNodes.len +# ok = $c.okNodes.len +# nok = $c.nokNodes.len - without var target =? (await c.todoNodes.pop()), err: - error "Failed to get todo node", err = err.msg +# without var target =? (await c.todoNodes.pop()), err: +# error "Failed to get todo node", err = err.msg - target.lastVisit = Moment.now().epochSeconds.uint64 +# target.lastVisit = Moment.now().epochSeconds.uint64 - without receivedNodes =? (await c.dht.getNeighbors(target.id)), err: - await c.handleNodeNotOk(target) - return +# without receivedNodes =? (await c.dht.getNeighbors(target.id)), err: +# await c.handleNodeNotOk(target) +# return - let newNodes = receivedNodes.filterIt(isNew(c, it)) - if newNodes.len > 0: - trace "Discovered new nodes", newNodes = newNodes.len +# let newNodes = receivedNodes.filterIt(isNew(c, it)) +# if newNodes.len > 0: +# trace "Discovered new nodes", newNodes = newNodes.len - await c.handleNodeOk(target) - await c.addNewTodoNodes(newNodes) +# await c.handleNodeOk(target) +# await c.addNewTodoNodes(newNodes) - # Don't log the status every loop: - if (c.todoNodes.len mod 10) == 0: - trace "Status" +# # Don't log the status every loop: +# if (c.todoNodes.len mod 10) == 0: +# trace "Status" proc worker(c: Crawler) {.async.} = try: @@ -82,23 +81,8 @@ proc worker(c: Crawler) {.async.} = quit QuitFailure method start*(c: Crawler, state: State): Future[?!void] {.async.} = - # if c.todoNodes.len < 1: - # let nodeIds = c.dht.getRoutingTableNodeIds() - # info "Loading routing-table nodes to todo-list...", nodes = nodeIds.len - # for id in nodeIds: - # if err =? (await c.addNewTodoNode(id)).errorOption: - # error "Failed to add routing-table node to todo-list", err = err.msg - # return failure(err) - info "Starting crawler...", stepDelayMs = $c.config.stepDelayMs asyncSpawn c.worker() - - proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = - info "Crawler sees nodes found!", num = nids.len - return success() - - let handle = state.events.nodesFound.subscribe(onNodesFound) - return success() method stop*(c: Crawler): Future[?!void] {.async.} = diff --git a/codexcrawler/components/dht.nim b/codexcrawler/components/dht.nim index ab1203b..2cef9ec 100644 --- a/codexcrawler/components/dht.nim +++ b/codexcrawler/components/dht.nim @@ -11,6 +11,7 @@ from pkg/nimcrypto import keccak256 import ../utils/keyutils import ../utils/datastoreutils import ../utils/rng +import ../utils/asyncdataevent import ../component import ../config import ../state @@ -44,9 +45,9 @@ proc getNode*(d: Dht, nodeId: NodeId): ?!Node = let node = d.protocol.getNode(nodeId) if node.isSome(): return success(node.get()) - return failure("Node not found for id: " & $nodeId) + return failure("Node not found for id: " & $(NodeId(nodeId))) -proc getRoutingTableNodeIds*(d: Dht): seq[NodeId] = +proc getRoutingTableNodeIds(d: Dht): seq[NodeId] = var ids = newSeq[NodeId]() for bucket in d.protocol.routingTable.buckets: for node in bucket.nodes: @@ -82,7 +83,7 @@ method removeProvider*(d: Dht, peerId: PeerId): Future[void] {.base, gcsafe.} = trace "Removing provider", peerId d.protocol.removeProvidersLocal(peerId) -proc updateAnnounceRecord*(d: Dht, addrs: openArray[MultiAddress]) = +proc updateAnnounceRecord(d: Dht, addrs: openArray[MultiAddress]) = d.announceAddrs = @addrs trace "Updating announce record", addrs = d.announceAddrs @@ -93,7 +94,7 @@ proc updateAnnounceRecord*(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.providerRecord).expect("Should update SPR") -proc updateDhtRecord*(d: Dht, addrs: openArray[MultiAddress]) = +proc updateDhtRecord(d: Dht, addrs: openArray[MultiAddress]) = trace "Updating Dht record", addrs = addrs d.dhtRecord = SignedPeerRecord .init(d.key, PeerRecord.init(d.peerId, @addrs)) @@ -102,9 +103,19 @@ proc updateDhtRecord*(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") +proc findRoutingTableNodes(d: Dht, state: State) {.async.} = + await sleepAsync(5.seconds) + let nodes = d.getRoutingTableNodeIds() + + if err =? (await state.events.nodesFound.fire(nodes)).errorOption: + error "Failed to raise routing-table nodes as found nodes", err = err.msg + else: + trace "Routing table nodes raise as found nodes", num = nodes.len + method start*(d: Dht, state: State): Future[?!void] {.async.} = d.protocol.open() await d.protocol.start() + asyncSpawn d.findRoutingTableNodes(state) return success() method stop*(d: Dht): Future[?!void] {.async.} = diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim new file mode 100644 index 0000000..5f96cf0 --- /dev/null +++ b/codexcrawler/components/nodestore.nim @@ -0,0 +1,40 @@ +import pkg/datastore +import pkg/datastore/typedds +import pkg/questionable/results +import pkg/chronos +import pkg/libp2p + +import ../types +import + +type + NodeEntry* = object + id*: Nid + lastVisit*: uint64 + + NodeStore* = ref object + store: TypedDatastore + +proc `$`*(entry: NodeEntry): string = + $entry.id & ":" & $entry.lastVisit + +proc toBytes*(entry: NodeEntry): seq[byte] = + var buffer = initProtoBuffer() + buffer.write(1, $entry.id) + buffer.write(2, entry.lastVisit) + buffer.finish() + return buffer.buffer + +proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry = + var + buffer = initProtoBuffer(data) + idStr: string + lastVisit: uint64 + + if buffer.getField(1, idStr).isErr: + return failure("Unable to decode `idStr`") + + if buffer.getField(2, lastVisit).isErr: + return failure("Unable to decode `lastVisit`") + + return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit)) diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index 755cc5d..092143a 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -5,7 +5,6 @@ import pkg/questionable/results import ./dht import ../list -import ../nodeentry import ../config import ../component import ../state @@ -20,28 +19,28 @@ type TimeTracker* = ref object of Component nokNodes: List workerDelay: int -proc processList(t: TimeTracker, list: List, expiry: uint64) {.async.} = - var toMove = newSeq[NodeEntry]() - proc onItem(item: NodeEntry) = - if item.lastVisit < expiry: - toMove.add(item) +# # proc processList(t: TimeTracker, list: List, expiry: uint64) {.async.} = +# # var toMove = newSeq[NodeEntry]() +# # proc onItem(item: NodeEntry) = +# # if item.lastVisit < expiry: +# # toMove.add(item) - await list.iterateAll(onItem) +# # await list.iterateAll(onItem) - if toMove.len > 0: - trace "expired node, moving to todo", nodes = $toMove.len +# # if toMove.len > 0: +# # trace "expired node, moving to todo", nodes = $toMove.len - for item in toMove: - if err =? (await t.todoNodes.add(item)).errorOption: - error "Failed to add expired node to todo list", err = err.msg - return - if err =? (await list.remove(item)).errorOption: - error "Failed to remove expired node to source list", err = err.msg +# # for item in toMove: +# # if err =? (await t.todoNodes.add(item)).errorOption: +# # error "Failed to add expired node to todo list", err = err.msg +# # return +# # if err =? (await list.remove(item)).errorOption: +# # error "Failed to remove expired node to source list", err = err.msg -proc step(t: TimeTracker) {.async.} = - let expiry = (Moment.now().epochSeconds - (t.config.revisitDelayMins * 60)).uint64 - await t.processList(t.okNodes, expiry) - await t.processList(t.nokNodes, expiry) +# proc step(t: TimeTracker) {.async.} = +# let expiry = (Moment.now().epochSeconds - (t.config.revisitDelayMins * 60)).uint64 +# await t.processList(t.okNodes, expiry) +# await t.processList(t.nokNodes, expiry) proc worker(t: TimeTracker) {.async.} = try: diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index 935d4ae..aebebe2 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -13,7 +13,6 @@ import std/sets import std/sequtils import std/os -import ./nodeentry import ./types logScope: @@ -21,25 +20,25 @@ logScope: type OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].} - OnItem = proc(item: NodeEntry): void {.gcsafe, raises: [].} + OnItem = proc(item: Nid): void {.gcsafe, raises: [].} List* = ref object name: string store: TypedDatastore - items: seq[NodeEntry] + items: HashSet[Nid] onMetric: OnUpdateMetric emptySignal: ?Future[void] -proc encode(s: NodeEntry): seq[byte] = +proc encode(s: Nid): seq[byte] = s.toBytes() -proc decode(T: type NodeEntry, bytes: seq[byte]): ?!T = +proc decode(T: type Nid, bytes: seq[byte]): ?!T = if bytes.len < 1: - return success(NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64)) - return NodeEntry.fromBytes(bytes) + return success(Nid.fromStr("0")) + return Nid.fromBytes(bytes) -proc saveItem(this: List, item: NodeEntry): Future[?!void] {.async.} = - without itemKey =? Key.init(this.name / $item.id), err: +proc saveItem(this: List, item: Nid): Future[?!void] {.async.} = + without itemKey =? Key.init(this.name / $item), err: return failure(err) ?await this.store.put(itemKey, item) return success() @@ -47,11 +46,11 @@ proc saveItem(this: List, item: NodeEntry): Future[?!void] {.async.} = proc load*(this: List): Future[?!void] {.async.} = let id = Nid.fromStr("0") let bytes = newSeq[byte]() - let ne = NodeEntry.fromBytes(bytes) + let ne = Nid.fromBytes(bytes) without queryKey =? Key.init(this.name), err: return failure(err) - without iter =? (await query[NodeEntry](this.store, Query.init(queryKey))), err: + without iter =? (await query[Nid](this.store, Query.init(queryKey))), err: return failure(err) while not iter.finished: @@ -59,8 +58,8 @@ proc load*(this: List): Future[?!void] {.async.} = return failure(err) without value =? item.value, err: return failure(err) - if value.id > 0 or value.lastVisit > 0: - this.items.add(value) + if value > 0: + this.items.incl(value) this.onMetric(this.items.len.int64) info "Loaded list", name = this.name, items = this.items.len @@ -71,40 +70,38 @@ proc new*( ): List = List(name: name, store: store, onMetric: onMetric) -proc contains*(this: List, nodeId: Nid): bool = - this.items.anyIt(it.id == nodeId) +proc contains*(this: List, nid: Nid): bool = + this.items.anyIt(it == nid) -proc contains*(this: List, item: NodeEntry): bool = - this.contains(item.id) - -proc add*(this: List, item: NodeEntry): Future[?!void] {.async.} = - if this.contains(item): +proc add*(this: List, nid: Nid): Future[?!void] {.async.} = + if this.contains(nid): return success() - this.items.add(item) + this.items.incl(nid) this.onMetric(this.items.len.int64) + if err =? (await this.saveItem(nid)).errorOption: + return failure(err) + if s =? this.emptySignal: trace "List no longer empty.", name = this.name s.complete() this.emptySignal = Future[void].none - if err =? (await this.saveItem(item)).errorOption: - return failure(err) return success() -proc remove*(this: List, item: NodeEntry): Future[?!void] {.async.} = +proc remove*(this: List, nid: Nid): Future[?!void] {.async.} = if this.items.len < 1: return failure(this.name & "List is empty.") - this.items.keepItIf(item.id != it.id) - without itemKey =? Key.init(this.name / $item.id), err: + this.items.excl(nid) + without itemKey =? Key.init(this.name / $nid), err: return failure(err) ?await this.store.delete(itemKey) this.onMetric(this.items.len.int64) return success() -proc pop*(this: List): Future[?!NodeEntry] {.async.} = +proc pop*(this: List): Future[?!Nid] {.async.} = if this.items.len < 1: trace "List is empty. Waiting for new items...", name = this.name let signal = newFuture[void]("list.emptySignal") @@ -113,7 +110,7 @@ proc pop*(this: List): Future[?!NodeEntry] {.async.} = if this.items.len < 1: return failure(this.name & "List is empty.") - let item = this.items[0] + let item = this.items.pop() if err =? (await this.remove(item)).errorOption: return failure(err) diff --git a/codexcrawler/nodeentry.nim b/codexcrawler/nodeentry.nim index 6f80411..e69de29 100644 --- a/codexcrawler/nodeentry.nim +++ b/codexcrawler/nodeentry.nim @@ -1,33 +0,0 @@ -import pkg/questionable/results -import pkg/chronos -import pkg/libp2p - -import ./types - -type NodeEntry* = object - id*: Nid - lastVisit*: uint64 - -proc `$`*(entry: NodeEntry): string = - $entry.id & ":" & $entry.lastVisit - -proc toBytes*(entry: NodeEntry): seq[byte] = - var buffer = initProtoBuffer() - buffer.write(1, $entry.id) - buffer.write(2, entry.lastVisit) - buffer.finish() - return buffer.buffer - -proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry = - var - buffer = initProtoBuffer(data) - idStr: string - lastVisit: uint64 - - if buffer.getField(1, idStr).isErr: - return failure("Unable to decode `idStr`") - - if buffer.getField(2, lastVisit).isErr: - return failure("Unable to decode `lastVisit`") - - return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit)) diff --git a/codexcrawler/types.nim b/codexcrawler/types.nim index b4fe39c..1f1d2d9 100644 --- a/codexcrawler/types.nim +++ b/codexcrawler/types.nim @@ -1,7 +1,8 @@ import pkg/stew/byteutils import pkg/stew/endians2 -import pkg/questionable +import pkg/questionable/results import pkg/codexdht +import pkg/libp2p type Nid* = NodeId @@ -10,3 +11,19 @@ proc `$`*(nid: Nid): string = proc fromStr*(T: type Nid, s: string): Nid = Nid(UInt256.fromHex(s)) + +proc toBytes*(nid: Nid): seq[byte] = + var buffer = initProtoBuffer() + buffer.write(1, $nid) + buffer.finish() + return buffer.buffer + +proc fromBytes*(_: type Nid, data: openArray[byte]): ?!Nid = + var + buffer = initProtoBuffer(data) + idStr: string + + if buffer.getField(1, idStr).isErr: + return failure("Unable to decode `idStr`") + + return success(Nid.fromStr(idStr)) From 5fcd7a7a653a5b8a8c0a337ebde20b9ac145d3c6 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Feb 2025 16:24:54 +0100 Subject: [PATCH 06/20] sets up empty nodestore --- codexcrawler/application.nim | 63 +++++++++++-------------- codexcrawler/component.nim | 2 +- codexcrawler/components/crawler.nim | 2 +- codexcrawler/components/dht.nim | 30 +++++++----- codexcrawler/components/nodestore.nim | 43 ++++++++++++++--- codexcrawler/components/timetracker.nim | 2 +- codexcrawler/installer.nim | 15 ++++-- codexcrawler/nodeentry.nim | 0 codexcrawler/state.nim | 2 +- 9 files changed, 95 insertions(+), 64 deletions(-) delete mode 100644 codexcrawler/nodeentry.nim diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index c6dab72..d894aa6 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -34,43 +34,35 @@ type okNodes*: List nokNodes*: List -proc initializeLists(app: Application): Future[?!void] {.async.} = - without store =? createTypedDatastore(app.config.dataDir / "lists"), err: - return failure(err) +# proc initializeLists(app: Application): Future[?!void] {.async.} = +# without store =? createTypedDatastore(app.config.dataDir / "lists"), err: +# return failure(err) - # We can't extract this into a function because gauges cannot be passed as argument. - # The use of global state in nim-metrics is not pleasant. - proc onTodoMetric(value: int64) = - todoNodesGauge.set(value) +# # We can't extract this into a function because gauges cannot be passed as argument. +# # The use of global state in nim-metrics is not pleasant. +# proc onTodoMetric(value: int64) = +# todoNodesGauge.set(value) - proc onOkMetric(value: int64) = - okNodesGauge.set(value) +# proc onOkMetric(value: int64) = +# okNodesGauge.set(value) - proc onNokMetric(value: int64) = - nokNodesGauge.set(value) +# proc onNokMetric(value: int64) = +# nokNodesGauge.set(value) - app.todoNodes = List.new("todo", store, onTodoMetric) - app.okNodes = List.new("ok", store, onOkMetric) - app.nokNodes = List.new("nok", store, onNokMetric) +# app.todoNodes = List.new("todo", store, onTodoMetric) +# app.okNodes = List.new("ok", store, onOkMetric) +# app.nokNodes = List.new("nok", store, onNokMetric) - if err =? (await app.todoNodes.load()).errorOption: - return failure(err) - if err =? (await app.okNodes.load()).errorOption: - return failure(err) - if err =? (await app.nokNodes.load()).errorOption: - return failure(err) +# if err =? (await app.todoNodes.load()).errorOption: +# return failure(err) +# if err =? (await app.okNodes.load()).errorOption: +# return failure(err) +# if err =? (await app.nokNodes.load()).errorOption: +# return failure(err) - return success() +# return success() proc initializeApp(app: Application): Future[?!void] {.async.} = - if err =? (await app.initializeLists()).errorOption: - error "Failed to initialize lists", err = err.msg - return failure(err) - - without components =? (await createComponents(app.config)), err: - error "Failed to create componenents", err = err.msg - return failure(err) - # todo move this let state = State( config: app.config, @@ -82,15 +74,14 @@ proc initializeApp(app: Application): Future[?!void] {.async.} = ), ) - for c in components: - if err =? (await c.start(state)).errorOption: - error "Failed to start component", err = err.msg - - # test raise newnodes - let nodes: seq[Nid] = newSeq[Nid]() - if err =? (await state.events.nodesFound.fire(nodes)).errorOption: + without components =? (await createComponents(state)), err: + error "Failed to create componenents", err = err.msg return failure(err) + for c in components: + if err =? (await c.start()).errorOption: + error "Failed to start component", err = err.msg + return success() proc stop*(app: Application) = diff --git a/codexcrawler/component.nim b/codexcrawler/component.nim index 479cdfc..f36b0a2 100644 --- a/codexcrawler/component.nim +++ b/codexcrawler/component.nim @@ -5,7 +5,7 @@ import ./state type Component* = ref object of RootObj -method start*(c: Component, state: State): Future[?!void] {.async, base.} = +method start*(c: Component): Future[?!void] {.async, base.} = raiseAssert("call to abstract method: component.start") method stop*(c: Component): Future[?!void] {.async, base.} = diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/crawler.nim index 5f4c296..e86de85 100644 --- a/codexcrawler/components/crawler.nim +++ b/codexcrawler/components/crawler.nim @@ -80,7 +80,7 @@ proc worker(c: Crawler) {.async.} = error "Exception in crawler worker", msg = exc.msg quit QuitFailure -method start*(c: Crawler, state: State): Future[?!void] {.async.} = +method start*(c: Crawler): Future[?!void] {.async.} = info "Starting crawler...", stepDelayMs = $c.config.stepDelayMs asyncSpawn c.worker() return success() diff --git a/codexcrawler/components/dht.nim b/codexcrawler/components/dht.nim index 2cef9ec..565af1e 100644 --- a/codexcrawler/components/dht.nim +++ b/codexcrawler/components/dht.nim @@ -13,7 +13,6 @@ import ../utils/datastoreutils import ../utils/rng import ../utils/asyncdataevent import ../component -import ../config import ../state export discv5 @@ -22,6 +21,7 @@ logScope: topics = "dht" type Dht* = ref object of Component + state: State protocol*: discv5.Protocol key: PrivateKey peerId: PeerId @@ -103,19 +103,19 @@ proc updateDhtRecord(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") -proc findRoutingTableNodes(d: Dht, state: State) {.async.} = +proc findRoutingTableNodes(d: Dht) {.async.} = await sleepAsync(5.seconds) let nodes = d.getRoutingTableNodeIds() - if err =? (await state.events.nodesFound.fire(nodes)).errorOption: + if err =? (await d.state.events.nodesFound.fire(nodes)).errorOption: error "Failed to raise routing-table nodes as found nodes", err = err.msg else: trace "Routing table nodes raise as found nodes", num = nodes.len -method start*(d: Dht, state: State): Future[?!void] {.async.} = +method start*(d: Dht): Future[?!void] {.async.} = d.protocol.open() await d.protocol.start() - asyncSpawn d.findRoutingTableNodes(state) + asyncSpawn d.findRoutingTableNodes() return success() method stop*(d: Dht): Future[?!void] {.async.} = @@ -124,6 +124,7 @@ method stop*(d: Dht): Future[?!void] {.async.} = proc new( T: type Dht, + state: State, key: PrivateKey, bindIp = IPv4_any(), bindPort = 0.Port, @@ -131,7 +132,9 @@ proc new( bootstrapNodes: openArray[SignedPeerRecord] = [], store: Datastore = SQLiteDatastore.new(Memory).expect("Should not fail!"), ): Dht = - var self = Dht(key: key, peerId: PeerId.init(key).expect("Should construct PeerId")) + var self = Dht( + state: state, key: key, peerId: PeerId.init(key).expect("Should construct PeerId") + ) self.updateAnnounceRecord(announceAddrs) @@ -155,29 +158,30 @@ proc new( self -proc createDht*(config: Config): Future[?!Dht] {.async.} = - without dhtStore =? createDatastore(config.dataDir / "dht"), err: +proc createDht*(state: State): Future[?!Dht] {.async.} = + without dhtStore =? createDatastore(state.config.dataDir / "dht"), err: return failure(err) - let keyPath = config.dataDir / "privatekey" + let keyPath = state.config.dataDir / "privatekey" without privateKey =? setupKey(keyPath), err: return failure(err) var listenAddresses = newSeq[MultiAddress]() # TODO: when p2p connections are supported: - # let aaa = MultiAddress.init("/ip4/" & config.publicIp & "/tcp/53678").expect("Should init multiaddress") + # let aaa = MultiAddress.init("/ip4/" & state.config.publicIp & "/tcp/53678").expect("Should init multiaddress") # listenAddresses.add(aaa) var discAddresses = newSeq[MultiAddress]() let bbb = MultiAddress - .init("/ip4/" & config.publicIp & "/udp/" & $config.discPort) + .init("/ip4/" & state.config.publicIp & "/udp/" & $state.config.discPort) .expect("Should init multiaddress") discAddresses.add(bbb) let dht = Dht.new( + state, privateKey, - bindPort = config.discPort, + bindPort = state.config.discPort, announceAddrs = listenAddresses, - bootstrapNodes = config.bootNodes, + bootstrapNodes = state.config.bootNodes, store = dhtStore, ) diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index 5f96cf0..8d0b042 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -1,19 +1,25 @@ -import pkg/datastore +import std/os import pkg/datastore/typedds import pkg/questionable/results +import pkg/chronicles import pkg/chronos import pkg/libp2p import ../types -import +import ../component +import ../state +import ../utils/datastoreutils +import ../utils/asyncdataevent type - NodeEntry* = object - id*: Nid - lastVisit*: uint64 + NodeEntry = object + id: Nid + lastVisit: uint64 - NodeStore* = ref object + NodeStore* = ref object of Component + state: State store: TypedDatastore + sub: AsyncDataEventSubscription proc `$`*(entry: NodeEntry): string = $entry.id & ":" & $entry.lastVisit @@ -38,3 +44,28 @@ proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry = return failure("Unable to decode `lastVisit`") return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit)) + +proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = + # put the nodes in the store. + # track all new ones, if any, raise newNodes event. + return success() + +method start*(s: NodeStore): Future[?!void] {.async.} = + info "Starting nodestore..." + + proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = + return await s.processFoundNodes(nids) + + s.sub = s.state.events.nodesFound.subscribe(onNodesFound) + return success() + +method stop*(s: NodeStore): Future[?!void] {.async.} = + await s.state.events.nodesFound.unsubscribe(s.sub) + return success() + +proc createNodeStore*(state: State): ?!NodeStore = + without ds =? createTypedDatastore(state.config.dataDir / "nodestore"), err: + error "Failed to create typed datastore for node store", err = err.msg + return failure(err) + + return success(NodeStore(state: state, store: ds)) diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index 092143a..e811481 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -51,7 +51,7 @@ proc worker(t: TimeTracker) {.async.} = error "Exception in timetracker worker", msg = exc.msg quit QuitFailure -method start*(t: TimeTracker, state: State): Future[?!void] {.async.} = +method start*(t: TimeTracker): Future[?!void] {.async.} = info "Starting timetracker...", revisitDelayMins = $t.workerDelay asyncSpawn t.worker() return success() diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index 247f7ee..3642b8f 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -1,19 +1,24 @@ import pkg/chronos import pkg/questionable/results -import ./config +import ./state import ./component import ./components/dht import ./components/crawler import ./components/timetracker +import ./components/nodestore -proc createComponents*(config: Config): Future[?!seq[Component]] {.async.} = +proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = var components: seq[Component] = newSeq[Component]() - without dht =? (await createDht(config)), err: + without dht =? (await createDht(state)), err: return failure(err) + without nodeStore =? createNodeStore(state), err: + return failure(err) + + components.add(nodeStore) components.add(dht) - components.add(Crawler.new(dht, config)) - components.add(TimeTracker.new(config)) + components.add(Crawler.new(dht, state.config)) + components.add(TimeTracker.new(state.config)) return success(components) diff --git a/codexcrawler/nodeentry.nim b/codexcrawler/nodeentry.nim deleted file mode 100644 index e69de29..0000000 diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index f7cd928..ead6430 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -20,7 +20,7 @@ type State* = ref object config*: Config - events*: Events # appstate + events*: Events proc whileRunning*(this: State, step: OnStep, delay: Duration) = discard From b6b7624a0559028dc59ed48c9cdb42dbc72259de Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 10:54:58 +0100 Subject: [PATCH 07/20] Tests for nodestore --- codexcrawler/components/nodestore.nim | 35 ++++- codexcrawler/list.nim | 5 - codexcrawler/state.nim | 2 +- codexcrawler/types.nim | 4 +- .../codexcrawler/components/testnodestore.nim | 127 ++++++++++++++++++ tests/codexcrawler/helpers.nim | 6 + tests/codexcrawler/mockstate.nim | 24 ++++ tests/codexcrawler/testcomponents.nim | 3 + tests/codexcrawler/testtypes.nim | 25 ++++ tests/test.nim | 2 + 10 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 tests/codexcrawler/components/testnodestore.nim create mode 100644 tests/codexcrawler/helpers.nim create mode 100644 tests/codexcrawler/mockstate.nim create mode 100644 tests/codexcrawler/testcomponents.nim create mode 100644 tests/codexcrawler/testtypes.nim diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index 8d0b042..1eeb0d8 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -12,9 +12,11 @@ import ../utils/datastoreutils import ../utils/asyncdataevent type - NodeEntry = object - id: Nid - lastVisit: uint64 + OnNodeId = proc(item: Nid): Future[?!void] {.async: (raises: []), gcsafe.} + + NodeEntry* = object + id*: Nid + lastVisit*: uint64 NodeStore* = ref object of Component state: State @@ -45,11 +47,26 @@ proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry = return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit)) +proc encode*(e: NodeEntry): seq[byte] = + e.toBytes() + +proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T = + if bytes.len < 1: + return success(NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64)) + return NodeEntry.fromBytes(bytes) + proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = # put the nodes in the store. # track all new ones, if any, raise newNodes event. return success() +proc iterateAll*(s: NodeStore, onNodeId: OnNodeId) {.async.} = + discard + # query iterator, yield items to callback. + # for item in this.items: + # onItem(item) + # await sleepAsync(1.millis) + method start*(s: NodeStore): Future[?!void] {.async.} = info "Starting nodestore..." @@ -63,9 +80,19 @@ method stop*(s: NodeStore): Future[?!void] {.async.} = await s.state.events.nodesFound.unsubscribe(s.sub) return success() +proc new*( + T: type NodeStore, + state: State, + store: TypedDatastore +): NodeStore = + NodeStore( + state: state, + store: store + ) + proc createNodeStore*(state: State): ?!NodeStore = without ds =? createTypedDatastore(state.config.dataDir / "nodestore"), err: error "Failed to create typed datastore for node store", err = err.msg return failure(err) - return success(NodeStore(state: state, store: ds)) + return success(NodeStore.new(state, ds)) diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index aebebe2..638042b 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -20,7 +20,6 @@ logScope: type OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].} - OnItem = proc(item: Nid): void {.gcsafe, raises: [].} List* = ref object name: string @@ -119,7 +118,3 @@ proc pop*(this: List): Future[?!Nid] {.async.} = proc len*(this: List): int = this.items.len -proc iterateAll*(this: List, onItem: OnItem) {.async.} = - for item in this.items: - onItem(item) - await sleepAsync(1.millis) diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index ead6430..9d8d4be 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -18,7 +18,7 @@ type dhtNodeCheck*: AsyncDataEvent[DhtNodeCheckEventData] nodesExpired*: AsyncDataEvent[seq[Nid]] - State* = ref object + State* = ref object of RootObj config*: Config events*: Events diff --git a/codexcrawler/types.nim b/codexcrawler/types.nim index 1f1d2d9..9895ed1 100644 --- a/codexcrawler/types.nim +++ b/codexcrawler/types.nim @@ -1,5 +1,5 @@ import pkg/stew/byteutils -import pkg/stew/endians2 +import pkg/stint/io import pkg/questionable/results import pkg/codexdht import pkg/libp2p @@ -7,7 +7,7 @@ import pkg/libp2p type Nid* = NodeId proc `$`*(nid: Nid): string = - $(NodeId(nid)) + nid.toHex() proc fromStr*(T: type Nid, s: string): Nid = Nid(UInt256.fromHex(s)) diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim new file mode 100644 index 0000000..2034e9f --- /dev/null +++ b/tests/codexcrawler/components/testnodestore.nim @@ -0,0 +1,127 @@ +import std/os +import pkg/chronos +import pkg/questionable/results +import pkg/asynctest/chronos/unittest +import pkg/datastore/typedds + +import ../../../codexcrawler/components/nodestore +import ../../../codexcrawler/utils/datastoreutils +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../mockstate +import ../helpers + +suite "Nodestore": + let + dsPath = getTempDir() / "testds" + nodestoreName = "nodestore" + + var + ds: TypedDatastore + state: MockState + store: NodeStore + + setup: + ds = createTypedDatastore(dsPath).tryGet() + state = createMockState() + + store = NodeStore.new( + state, ds + ) + + teardown: + (await ds.close()).tryGet() + # state.cleanupMock() + removeDir(dsPath) + + test "nodeEntry encoding": + let entry = NodeEntry( + id: genNid(), + lastVisit: 123.uint64 + ) + + let + bytes = entry.encode() + decoded = NodeEntry.decode(bytes).tryGet() + + check: + entry.id == decoded.id + entry.lastVisit == decoded.lastVisit + + test "nodesFound event should store nodes": + let + nid = genNid() + expectedKey = Key.init(nodestoreName / $nid).tryGet() + + (await state.events.nodesFound.fire(@[nid])).tryGet() + + check: + (await ds.has(expectedKey)).tryGet() + + let entry = (await get[NodeEntry](ds, expectedKey)).tryGet() + check: + entry.id == nid + + test "nodesFound event should fire newNodesDiscovered": + var newNodes = newSeq[Nid]() + proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} = + newNodes = nids + return success() + + let + sub = state.events.newNodesDiscovered.subscribe(onNewNodes) + nid = genNid() + + (await state.events.nodesFound.fire(@[nid])).tryGet() + + check: + newNodes == @[nid] + + await state.events.newNodesDiscovered.unsubscribe(sub) + + test "nodesFound event should not fire newNodesDiscovered for previously seen nodes": + let + nid = genNid() + + # Make nid known first. Then subscribe. + (await state.events.nodesFound.fire(@[nid])).tryGet() + + var + newNodes = newSeq[Nid]() + count = 0 + proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} = + newNodes = nids + inc count + return success() + + let + sub = state.events.newNodesDiscovered.subscribe(onNewNodes) + + # Firing the event again should not trigger newNodesDiscovered for nid + (await state.events.nodesFound.fire(@[nid])).tryGet() + + check: + newNodes.len == 0 + count == 0 + + await state.events.newNodesDiscovered.unsubscribe(sub) + + test "iterateAll yields all known nids": + let + nid1 = genNid() + nid2 = genNid() + nid3 = genNid() + + (await state.events.nodesFound.fire(@[nid1, nid2, nid3])).tryGet() + + var iterNodes = newSeq[Nid]() + proc onNodeId(nid: Nid): Future[?!void] {.async: (raises: []), gcsafe.} = + iterNodes.add(nid) + return success() + + await store.iterateAll(onNodeId) + + check: + nid1 in iterNodes + nid2 in iterNodes + nid3 in iterNodes diff --git a/tests/codexcrawler/helpers.nim b/tests/codexcrawler/helpers.nim new file mode 100644 index 0000000..5feb345 --- /dev/null +++ b/tests/codexcrawler/helpers.nim @@ -0,0 +1,6 @@ +import std/random +import pkg/stint +import ../../codexcrawler/types + +proc genNid*(): Nid = + Nid(rand(uint64).u256) diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim new file mode 100644 index 0000000..af302b7 --- /dev/null +++ b/tests/codexcrawler/mockstate.nim @@ -0,0 +1,24 @@ +import ../../codexcrawler/state +import ../../codexcrawler/utils/asyncdataevent +import ../../codexcrawler/types +import ../../codexcrawler/config + +type + MockState* = ref object of State + # config*: Config + # events*: Events + + +proc createMockState*(): MockState = + MockState( + config: Config(), + events: Events( + nodesFound: newAsyncDataEvent[seq[Nid]](), + newNodesDiscovered: newAsyncDataEvent[seq[Nid]](), + dhtNodeCheck: newAsyncDataEvent[DhtNodeCheckEventData](), + nodesExpired: newAsyncDataEvent[seq[Nid]](), + ), + ) + +proc cleanupMock*(this: MockState) = + discard diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim new file mode 100644 index 0000000..1363c5a --- /dev/null +++ b/tests/codexcrawler/testcomponents.nim @@ -0,0 +1,3 @@ +import ./components/testnodestore + +{.warning[UnusedImport]: off.} diff --git a/tests/codexcrawler/testtypes.nim b/tests/codexcrawler/testtypes.nim new file mode 100644 index 0000000..7332651 --- /dev/null +++ b/tests/codexcrawler/testtypes.nim @@ -0,0 +1,25 @@ +import pkg/chronos +import pkg/asynctest/chronos/unittest +import pkg/questionable/results + +import ../../codexcrawler/types +import ./helpers + +suite "Types": + test "nid string encoding": + let + nid = genNid() + str = $nid + + check: + nid == Nid.fromStr(str) + + test "nid byte encoding": + let + nid = genNid() + bytes = nid.toBytes() + + check: + nid == Nid.fromBytes(bytes).tryGet() + + \ No newline at end of file diff --git a/tests/test.nim b/tests/test.nim index ca1588f..a310fc6 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -1,3 +1,5 @@ import ./codexcrawler/testutils +import ./codexcrawler/testcomponents +import ./codexcrawler/testtypes {.warning[UnusedImport]: off.} From 5fa90c5c2fb546b37423039c3fdbb4b1bc06d0ce Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 12:42:20 +0100 Subject: [PATCH 08/20] Implements and tests nodestore --- codexcrawler/components/nodestore.nim | 59 +++++++++++++++---- codexcrawler/list.nim | 4 -- codexcrawler/utils/asyncdataevent.nim | 48 ++++++++++----- .../codexcrawler/components/testnodestore.nim | 11 ++-- tests/codexcrawler/mockstate.nim | 9 ++- .../codexcrawler/utils/testasyncdataevent.nim | 22 +++++++ tests/config.nims | 1 + 7 files changed, 119 insertions(+), 35 deletions(-) create mode 100644 tests/config.nims diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index 1eeb0d8..37ccb7e 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -1,4 +1,5 @@ import std/os +import pkg/datastore import pkg/datastore/typedds import pkg/questionable/results import pkg/chronicles @@ -11,13 +12,16 @@ import ../state import ../utils/datastoreutils import ../utils/asyncdataevent -type - OnNodeId = proc(item: Nid): Future[?!void] {.async: (raises: []), gcsafe.} +const + nodestoreName = "nodestore" +type NodeEntry* = object id*: Nid lastVisit*: uint64 + OnNodeEntry = proc(item: NodeEntry): Future[?!void] {.async: (raises: []), gcsafe.} + NodeStore* = ref object of Component state: State store: TypedDatastore @@ -55,18 +59,53 @@ proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T = return success(NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64)) return NodeEntry.fromBytes(bytes) +proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async.} = + without key =? Key.init(nodestoreName / $nid), err: + return failure(err) + without exists =? (await s.store.has(key)), err: + return failure(err) + + if not exists: + let entry = NodeEntry( + id: nid, + lastVisit: 0 + ) + ?await s.store.put(key, entry) + + return success(not exists) + +proc fireNewNodesDiscovered(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = + await s.state.events.newNodesDiscovered.fire(nids) + proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = - # put the nodes in the store. - # track all new ones, if any, raise newNodes event. + var newNodes = newSeq[Nid]() + + for nid in nids: + without isNew =? (await s.storeNodeIsNew(nid)), err: + return failure(err) + + if isNew: + newNodes.add(nid) + + if newNodes.len > 0: + ? await s.fireNewNodesDiscovered(newNodes) return success() -proc iterateAll*(s: NodeStore, onNodeId: OnNodeId) {.async.} = - discard - # query iterator, yield items to callback. - # for item in this.items: - # onItem(item) - # await sleepAsync(1.millis) +proc iterateAll*(s: NodeStore, onNode: OnNodeEntry): Future[?!void] {.async.} = + without queryKey =? Key.init(nodestoreName), err: + return failure(err) + without iter =? (await query[NodeEntry](s.store, Query.init(queryKey))), err: + return failure(err) + while not iter.finished: + without item =? (await iter.next()), err: + return failure(err) + without value =? item.value, err: + return failure(err) + + ?await onNode(value) + return success() + method start*(s: NodeStore): Future[?!void] {.async.} = info "Starting nodestore..." diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index 638042b..e7cc444 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -43,10 +43,6 @@ proc saveItem(this: List, item: Nid): Future[?!void] {.async.} = return success() proc load*(this: List): Future[?!void] {.async.} = - let id = Nid.fromStr("0") - let bytes = newSeq[byte]() - let ne = Nid.fromBytes(bytes) - without queryKey =? Key.init(this.name), err: return failure(err) without iter =? (await query[Nid](this.store, Query.init(queryKey))), err: diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim index cb2ec75..86aec1d 100644 --- a/codexcrawler/utils/asyncdataevent.nim +++ b/codexcrawler/utils/asyncdataevent.nim @@ -5,10 +5,11 @@ import pkg/chronos type AsyncDataEventSubscription* = ref object key: EventQueueKey - isRunning: bool + listenFuture: Future[void] fireEvent: AsyncEvent - stopEvent: AsyncEvent lastResult: ?!void + inHandler: bool + delayedUnsubscribe: bool AsyncDataEvent*[T] = ref object queue: AsyncEventQueue[?T] @@ -21,47 +22,64 @@ proc newAsyncDataEvent*[T](): AsyncDataEvent[T] = queue: newAsyncEventQueue[?T](), subscriptions: newSeq[AsyncDataEventSubscription]() ) +proc performUnsubscribe[T](event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription) {.async.} = + if subscription in event.subscriptions: + await subscription.listenFuture.cancelAndWait() + event.subscriptions.delete(event.subscriptions.find(subscription)) + proc subscribe*[T]( event: AsyncDataEvent[T], handler: AsyncDataEventHandler[T] ): AsyncDataEventSubscription = - let subscription = AsyncDataEventSubscription( + var subscription = AsyncDataEventSubscription( key: event.queue.register(), - isRunning: true, + listenFuture: newFuture[void](), fireEvent: newAsyncEvent(), - stopEvent: newAsyncEvent(), + inHandler: false, + delayedUnsubscribe: false ) proc listener() {.async.} = - while subscription.isRunning: + while true: let items = await event.queue.waitEvents(subscription.key) for item in items: if data =? item: + subscription.inHandler = true subscription.lastResult = (await handler(data)) + subscription.inHandler = false subscription.fireEvent.fire() - subscription.stopEvent.fire() - asyncSpawn listener() + subscription.listenFuture = listener() event.subscriptions.add(subscription) subscription proc fire*[T](event: AsyncDataEvent[T], data: T): Future[?!void] {.async.} = event.queue.emit(data.some) - for subscription in event.subscriptions: - await subscription.fireEvent.wait() - if err =? subscription.lastResult.errorOption: + var toUnsubscribe = newSeq[AsyncDataEventSubscription]() + for sub in event.subscriptions: + await sub.fireEvent.wait() + if err =? sub.lastResult.errorOption: return failure(err) + if sub.delayedUnsubscribe: + toUnsubscribe.add(sub) + + for sub in toUnsubscribe: + await event.unsubscribe(sub) + success() proc unsubscribe*[T]( event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription ) {.async.} = - subscription.isRunning = false - event.queue.emit(T.none) - await subscription.stopEvent.wait() - event.subscriptions.delete(event.subscriptions.find(subscription)) + if subscription.inHandler: + subscription.delayedUnsubscribe = true + else: + await event.performUnsubscribe(subscription) proc unsubscribeAll*[T](event: AsyncDataEvent[T]) {.async.} = let all = event.subscriptions for subscription in all: await event.unsubscribe(subscription) + +proc listeners*[T](event: AsyncDataEvent[T]): int = + event.subscriptions.len diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim index 2034e9f..554a966 100644 --- a/tests/codexcrawler/components/testnodestore.nim +++ b/tests/codexcrawler/components/testnodestore.nim @@ -29,9 +29,12 @@ suite "Nodestore": state, ds ) + (await store.start()).tryGet() + teardown: + (await store.stop()).tryGet() (await ds.close()).tryGet() - # state.cleanupMock() + state.checkAllUnsubscribed() removeDir(dsPath) test "nodeEntry encoding": @@ -115,11 +118,11 @@ suite "Nodestore": (await state.events.nodesFound.fire(@[nid1, nid2, nid3])).tryGet() var iterNodes = newSeq[Nid]() - proc onNodeId(nid: Nid): Future[?!void] {.async: (raises: []), gcsafe.} = - iterNodes.add(nid) + proc onNode(entry: NodeEntry): Future[?!void] {.async: (raises: []), gcsafe.} = + iterNodes.add(entry.id) return success() - await store.iterateAll(onNodeId) + (await store.iterateAll(onNode)).tryGet() check: nid1 in iterNodes diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim index af302b7..1974069 100644 --- a/tests/codexcrawler/mockstate.nim +++ b/tests/codexcrawler/mockstate.nim @@ -1,3 +1,4 @@ +import pkg/asynctest/chronos/unittest import ../../codexcrawler/state import ../../codexcrawler/utils/asyncdataevent import ../../codexcrawler/types @@ -20,5 +21,9 @@ proc createMockState*(): MockState = ), ) -proc cleanupMock*(this: MockState) = - discard +proc checkAllUnsubscribed*(this: MockState) = + check: + this.events.nodesFound.listeners == 0 + this.events.newNodesDiscovered.listeners == 0 + this.events.dhtNodeCheck.listeners == 0 + this.events.nodesExpired.listeners == 0 diff --git a/tests/codexcrawler/utils/testasyncdataevent.nim b/tests/codexcrawler/utils/testasyncdataevent.nim index 7e47a7b..8c1efd8 100644 --- a/tests/codexcrawler/utils/testasyncdataevent.nim +++ b/tests/codexcrawler/utils/testasyncdataevent.nim @@ -79,3 +79,25 @@ suite "AsyncDataEvent": await event.unsubscribe(s1) await event.unsubscribe(s2) await event.unsubscribe(s3) + + test "Can unsubscribe in handler": + proc doNothing() {.async, closure.} = + await sleepAsync(1.millis) + + var callback = doNothing + + proc eventHandler(e: ExampleData): Future[?!void] {.async.} = + await callback() + success() + + let s = event.subscribe(eventHandler) + + proc doUnsubscribe() {.async.} = + await event.unsubscribe(s) + + callback = doUnsubscribe + + check: + isOK(await event.fire(ExampleData(s: msg))) + + await event.unsubscribe(s) diff --git a/tests/config.nims b/tests/config.nims new file mode 100644 index 0000000..da6438a --- /dev/null +++ b/tests/config.nims @@ -0,0 +1 @@ +switch("define", "chronicles_log_level=ERROR") From c1f25f10cc3ab7924c457e63450e060bf505d80d Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 12:43:55 +0100 Subject: [PATCH 09/20] formatting --- codexcrawler/components/nodestore.nim | 29 ++++++----------- codexcrawler/list.nim | 1 - codexcrawler/utils/asyncdataevent.nim | 8 +++-- .../codexcrawler/components/testnodestore.nim | 31 +++++++------------ tests/codexcrawler/mockstate.nim | 6 +--- tests/codexcrawler/testtypes.nim | 2 -- 6 files changed, 27 insertions(+), 50 deletions(-) diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index 37ccb7e..f977507 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -12,8 +12,7 @@ import ../state import ../utils/datastoreutils import ../utils/asyncdataevent -const - nodestoreName = "nodestore" +const nodestoreName = "nodestore" type NodeEntry* = object @@ -66,12 +65,9 @@ proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async.} = return failure(err) if not exists: - let entry = NodeEntry( - id: nid, - lastVisit: 0 - ) + let entry = NodeEntry(id: nid, lastVisit: 0) ?await s.store.put(key, entry) - + return success(not exists) proc fireNewNodesDiscovered(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = @@ -83,12 +79,12 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = for nid in nids: without isNew =? (await s.storeNodeIsNew(nid)), err: return failure(err) - + if isNew: newNodes.add(nid) if newNodes.len > 0: - ? await s.fireNewNodesDiscovered(newNodes) + ?await s.fireNewNodesDiscovered(newNodes) return success() proc iterateAll*(s: NodeStore, onNode: OnNodeEntry): Future[?!void] {.async.} = @@ -102,10 +98,10 @@ proc iterateAll*(s: NodeStore, onNode: OnNodeEntry): Future[?!void] {.async.} = return failure(err) without value =? item.value, err: return failure(err) - + ?await onNode(value) return success() - + method start*(s: NodeStore): Future[?!void] {.async.} = info "Starting nodestore..." @@ -119,15 +115,8 @@ method stop*(s: NodeStore): Future[?!void] {.async.} = await s.state.events.nodesFound.unsubscribe(s.sub) return success() -proc new*( - T: type NodeStore, - state: State, - store: TypedDatastore -): NodeStore = - NodeStore( - state: state, - store: store - ) +proc new*(T: type NodeStore, state: State, store: TypedDatastore): NodeStore = + NodeStore(state: state, store: store) proc createNodeStore*(state: State): ?!NodeStore = without ds =? createTypedDatastore(state.config.dataDir / "nodestore"), err: diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index e7cc444..f9b4a7e 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -113,4 +113,3 @@ proc pop*(this: List): Future[?!Nid] {.async.} = proc len*(this: List): int = this.items.len - diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim index 86aec1d..52a0ea5 100644 --- a/codexcrawler/utils/asyncdataevent.nim +++ b/codexcrawler/utils/asyncdataevent.nim @@ -22,7 +22,9 @@ proc newAsyncDataEvent*[T](): AsyncDataEvent[T] = queue: newAsyncEventQueue[?T](), subscriptions: newSeq[AsyncDataEventSubscription]() ) -proc performUnsubscribe[T](event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription) {.async.} = +proc performUnsubscribe[T]( + event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription +) {.async.} = if subscription in event.subscriptions: await subscription.listenFuture.cancelAndWait() event.subscriptions.delete(event.subscriptions.find(subscription)) @@ -35,7 +37,7 @@ proc subscribe*[T]( listenFuture: newFuture[void](), fireEvent: newAsyncEvent(), inHandler: false, - delayedUnsubscribe: false + delayedUnsubscribe: false, ) proc listener() {.async.} = @@ -62,7 +64,7 @@ proc fire*[T](event: AsyncDataEvent[T], data: T): Future[?!void] {.async.} = return failure(err) if sub.delayedUnsubscribe: toUnsubscribe.add(sub) - + for sub in toUnsubscribe: await event.unsubscribe(sub) diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim index 554a966..47eed00 100644 --- a/tests/codexcrawler/components/testnodestore.nim +++ b/tests/codexcrawler/components/testnodestore.nim @@ -16,7 +16,7 @@ suite "Nodestore": dsPath = getTempDir() / "testds" nodestoreName = "nodestore" - var + var ds: TypedDatastore state: MockState store: NodeStore @@ -25,9 +25,7 @@ suite "Nodestore": ds = createTypedDatastore(dsPath).tryGet() state = createMockState() - store = NodeStore.new( - state, ds - ) + store = NodeStore.new(state, ds) (await store.start()).tryGet() @@ -38,10 +36,7 @@ suite "Nodestore": removeDir(dsPath) test "nodeEntry encoding": - let entry = NodeEntry( - id: genNid(), - lastVisit: 123.uint64 - ) + let entry = NodeEntry(id: genNid(), lastVisit: 123.uint64) let bytes = entry.encode() @@ -52,7 +47,7 @@ suite "Nodestore": entry.lastVisit == decoded.lastVisit test "nodesFound event should store nodes": - let + let nid = genNid() expectedKey = Key.init(nodestoreName / $nid).tryGet() @@ -60,7 +55,7 @@ suite "Nodestore": check: (await ds.has(expectedKey)).tryGet() - + let entry = (await get[NodeEntry](ds, expectedKey)).tryGet() check: entry.id == nid @@ -71,7 +66,7 @@ suite "Nodestore": newNodes = nids return success() - let + let sub = state.events.newNodesDiscovered.subscribe(onNewNodes) nid = genNid() @@ -81,10 +76,9 @@ suite "Nodestore": newNodes == @[nid] await state.events.newNodesDiscovered.unsubscribe(sub) - + test "nodesFound event should not fire newNodesDiscovered for previously seen nodes": - let - nid = genNid() + let nid = genNid() # Make nid known first. Then subscribe. (await state.events.nodesFound.fire(@[nid])).tryGet() @@ -97,9 +91,8 @@ suite "Nodestore": inc count return success() - let - sub = state.events.newNodesDiscovered.subscribe(onNewNodes) - + let sub = state.events.newNodesDiscovered.subscribe(onNewNodes) + # Firing the event again should not trigger newNodesDiscovered for nid (await state.events.nodesFound.fire(@[nid])).tryGet() @@ -110,11 +103,11 @@ suite "Nodestore": await state.events.newNodesDiscovered.unsubscribe(sub) test "iterateAll yields all known nids": - let + let nid1 = genNid() nid2 = genNid() nid3 = genNid() - + (await state.events.nodesFound.fire(@[nid1, nid2, nid3])).tryGet() var iterNodes = newSeq[Nid]() diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim index 1974069..1a305a9 100644 --- a/tests/codexcrawler/mockstate.nim +++ b/tests/codexcrawler/mockstate.nim @@ -4,11 +4,7 @@ import ../../codexcrawler/utils/asyncdataevent import ../../codexcrawler/types import ../../codexcrawler/config -type - MockState* = ref object of State - # config*: Config - # events*: Events - +type MockState* = ref object of State proc createMockState*(): MockState = MockState( diff --git a/tests/codexcrawler/testtypes.nim b/tests/codexcrawler/testtypes.nim index 7332651..5c75ec9 100644 --- a/tests/codexcrawler/testtypes.nim +++ b/tests/codexcrawler/testtypes.nim @@ -21,5 +21,3 @@ suite "Types": check: nid == Nid.fromBytes(bytes).tryGet() - - \ No newline at end of file From a2e9d4fac888ecec1df064db24ff377a3758861a Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 14:02:30 +0100 Subject: [PATCH 10/20] sets up dht-metrics component --- codexcrawler/application.nim | 5 -- codexcrawler/components/dhtmetrics.nim | 51 ++++++++++++ codexcrawler/installer.nim | 3 + codexcrawler/list.nim | 47 ++++------- codexcrawler/metrics.nim | 43 +++++++++- .../components/testdhtmetrics.nim | 79 +++++++++++++++++++ tests/codexcrawler/mocklist.nim | 37 +++++++++ tests/codexcrawler/testcomponents.nim | 1 + 8 files changed, 229 insertions(+), 37 deletions(-) create mode 100644 codexcrawler/components/dhtmetrics.nim create mode 100644 tests/codexcrawler/components/testdhtmetrics.nim create mode 100644 tests/codexcrawler/mocklist.nim diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index d894aa6..e207544 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -17,10 +17,6 @@ import ./state import ./component import ./types -declareGauge(todoNodesGauge, "DHT nodes to be visited") -declareGauge(okNodesGauge, "DHT nodes successfully contacted") -declareGauge(nokNodesGauge, "DHT nodes failed to contact") - type ApplicationStatus* {.pure.} = enum Stopped @@ -99,7 +95,6 @@ proc run*(app: Application) = if not existsDir(app.config.dataDir): createDir(app.config.dataDir) - setupMetrics(app.config.metricsAddress, app.config.metricsPort) info "Metrics endpoint initialized" info "Starting application" diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim new file mode 100644 index 0000000..8bffa25 --- /dev/null +++ b/codexcrawler/components/dhtmetrics.nim @@ -0,0 +1,51 @@ +import pkg/chronicles +import pkg/chronos +import pkg/questionable +import pkg/questionable/results + +import ./dht +import ../list +import ../state +import ../component +import ../types +import ../utils/asyncdataevent +import ../metrics + +logScope: + topics = "dhtmetrics" + +type DhtMetrics* = ref object of Component + state: State + ok: List + nok: List + +method start*(d: DhtMetrics): Future[?!void] {.async.} = + info "Starting DhtMetrics..." + return success() + +method stop*(d: DhtMetrics): Future[?!void] {.async.} = + return success() + +proc new*( + T: type DhtMetrics, + state: State, + okList: List, + nokList: List +): DhtMetrics = + DhtMetrics( + state: state, + ok: okList, + nok: nokList + ) + +proc createDhtMetrics*(state: State): ?!DhtMetrics = + without okList =? createList(state.config.dataDir, "dhtok"), err: + return failure(err) + without nokList =? createList(state.config.dataDir, "dhtnok"), err: + return failure(err) + + success(DhtMetrics.new( + state, + okList, + nokList + )) diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index 3642b8f..48c2e9e 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -2,6 +2,7 @@ import pkg/chronos import pkg/questionable/results import ./state +import ./metrics import ./component import ./components/dht import ./components/crawler @@ -17,6 +18,8 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = without nodeStore =? createNodeStore(state), err: return failure(err) + let metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort) + components.add(nodeStore) components.add(dht) components.add(Crawler.new(dht, state.config)) diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index f9b4a7e..fdd1f5e 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -1,6 +1,6 @@ +import std/os import pkg/chronos import pkg/chronicles -import pkg/metrics import pkg/datastore import pkg/datastore/typedds import pkg/stew/byteutils @@ -14,18 +14,16 @@ import std/sequtils import std/os import ./types +import ./utils/datastoreutils logScope: topics = "list" type - OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].} - - List* = ref object + List* = ref object of RootObj name: string store: TypedDatastore items: HashSet[Nid] - onMetric: OnUpdateMetric emptySignal: ?Future[void] proc encode(s: Nid): seq[byte] = @@ -42,7 +40,7 @@ proc saveItem(this: List, item: Nid): Future[?!void] {.async.} = ?await this.store.put(itemKey, item) return success() -proc load*(this: List): Future[?!void] {.async.} = +method load*(this: List): Future[?!void] {.async, base.} = without queryKey =? Key.init(this.name), err: return failure(err) without iter =? (await query[Nid](this.store, Query.init(queryKey))), err: @@ -56,24 +54,17 @@ proc load*(this: List): Future[?!void] {.async.} = if value > 0: this.items.incl(value) - this.onMetric(this.items.len.int64) info "Loaded list", name = this.name, items = this.items.len return success() -proc new*( - _: type List, name: string, store: TypedDatastore, onMetric: OnUpdateMetric -): List = - List(name: name, store: store, onMetric: onMetric) - proc contains*(this: List, nid: Nid): bool = this.items.anyIt(it == nid) -proc add*(this: List, nid: Nid): Future[?!void] {.async.} = +method add*(this: List, nid: Nid): Future[?!void] {.async, base.} = if this.contains(nid): return success() this.items.incl(nid) - this.onMetric(this.items.len.int64) if err =? (await this.saveItem(nid)).errorOption: return failure(err) @@ -85,7 +76,7 @@ proc add*(this: List, nid: Nid): Future[?!void] {.async.} = return success() -proc remove*(this: List, nid: Nid): Future[?!void] {.async.} = +method remove*(this: List, nid: Nid): Future[?!void] {.async, base.} = if this.items.len < 1: return failure(this.name & "List is empty.") @@ -93,23 +84,17 @@ proc remove*(this: List, nid: Nid): Future[?!void] {.async.} = without itemKey =? Key.init(this.name / $nid), err: return failure(err) ?await this.store.delete(itemKey) - this.onMetric(this.items.len.int64) return success() -proc pop*(this: List): Future[?!Nid] {.async.} = - if this.items.len < 1: - trace "List is empty. Waiting for new items...", name = this.name - let signal = newFuture[void]("list.emptySignal") - this.emptySignal = some(signal) - await signal.wait(1.hours) - if this.items.len < 1: - return failure(this.name & "List is empty.") - - let item = this.items.pop() - - if err =? (await this.remove(item)).errorOption: - return failure(err) - return success(item) - proc len*(this: List): int = this.items.len + +proc new*( + _: type List, name: string, store: TypedDatastore +): List = + List(name: name, store: store) + +proc createList*(dataDir: string, name: string): ?!List = + without store =? createTypedDatastore(dataDir / name), err: + return failure(err) + success(List.new(name, store)) diff --git a/codexcrawler/metrics.nim b/codexcrawler/metrics.nim index 7f6b54e..ea713c8 100644 --- a/codexcrawler/metrics.nim +++ b/codexcrawler/metrics.nim @@ -2,7 +2,19 @@ import pkg/chronicles import pkg/metrics import pkg/metrics/chronos_httpserver -proc setupMetrics*(metricsAddress: IpAddress, metricsPort: Port) = +declareGauge(todoNodesGauge, "DHT nodes to be visited") +declareGauge(okNodesGauge, "DHT nodes successfully contacted") +declareGauge(nokNodesGauge, "DHT nodes failed to contact") + +type + OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].} + + Metrics* = ref object + todoNodes: OnUpdateMetric + okNodes: OnUpdateMetric + nokNodes: OnUpdateMetric + +proc startServer(metricsAddress: IpAddress, metricsPort: Port) = let metricsAddress = metricsAddress notice "Starting metrics HTTP server", url = "http://" & $metricsAddress & ":" & $metricsPort & "/metrics" @@ -12,3 +24,32 @@ proc setupMetrics*(metricsAddress: IpAddress, metricsPort: Port) = raiseAssert exc.msg except Exception as exc: raiseAssert exc.msg # TODO fix metrics + +method setTodoNodes*(m: Metrics, value: int) {.base.} = + m.todoNodes(value.int64) + +method setOkNodes*(m: Metrics, value: int) {.base.} = + m.okNodes(value.int64) + +method setNokNodes*(m: Metrics, value: int) {.base.} = + m.nokNodes(value.int64) + +proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics = + startServer(metricsAddress, metricsPort) + + # We can't extract this into a function because gauges cannot be passed as argument. + # The use of global state in nim-metrics is not pleasant. + proc onTodo(value: int64) = + todoNodesGauge.set(value) + + proc onOk(value: int64) = + okNodesGauge.set(value) + + proc onNok(value: int64) = + nokNodesGauge.set(value) + + return Metrics( + todoNodes: onTodo, + okNodes: onOk, + nokNodes: onNok + ) diff --git a/tests/codexcrawler/components/testdhtmetrics.nim b/tests/codexcrawler/components/testdhtmetrics.nim new file mode 100644 index 0000000..0c0d613 --- /dev/null +++ b/tests/codexcrawler/components/testdhtmetrics.nim @@ -0,0 +1,79 @@ +import std/os +import pkg/chronos +import pkg/questionable/results +import pkg/asynctest/chronos/unittest +import pkg/datastore/typedds + +import ../../../codexcrawler/components/dhtmetrics +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../../../codexcrawler/state +import ../mockstate +import ../mocklist +import ../helpers + +suite "DhtMetrics": + var + nid: Nid + state: MockState + okList: MockList + nokList: MockList + dhtmetrics: DhtMetrics + + setup: + nid = genNid() + state = createMockState() + okList = createMockList() + nokList = createMockList() + + dhtmetrics = DhtMetrics.new( + state, + okList, + nokList + ) + + (await dhtmetrics.start()).tryGet() + + teardown: + (await dhtmetrics.stop()).tryGet() + state.checkAllUnsubscribed() + + proc fireDhtNodeCheckEvent(isOk: bool) {.async.} = + let + event = DhtNodeCheckEventData( + id: nid, + isOk: isOk + ) + + (await state.events.dhtNodeCheck.fire(event)).tryGet() + + test "dhtmetrics start should load both lists": + (await dhtmetrics.start()).tryGet() + + check: + okList.loadCalled + nokList.loadCalled + + test "dhtNodeCheck event should add node to okList if check is successful": + await fireDhtNodeCheckEvent(true) + + check: + nid in okList.added + + test "dhtNodeCheck event should add node to nokList if check has failed": + await fireDhtNodeCheckEvent(false) + + check: + nid in nokList.added + + test "dhtNodeCheck event should remove node from nokList if check is successful": + await fireDhtNodeCheckEvent(true) + + check: + nid in nokList.removed + + test "dhtNodeCheck event should remove node from okList if check has failed": + await fireDhtNodeCheckEvent(false) + + check: + nid in okList.removed diff --git a/tests/codexcrawler/mocklist.nim b/tests/codexcrawler/mocklist.nim new file mode 100644 index 0000000..001e9f1 --- /dev/null +++ b/tests/codexcrawler/mocklist.nim @@ -0,0 +1,37 @@ +import pkg/chronos +import pkg/questionable/results + +import ../../codexcrawler/types +import ../../codexcrawler/list + +type + MockList* = ref object of List + loadCalled*: bool + added*: seq[Nid] + addSuccess*: bool + removed*: seq[Nid] + removeSuccess*: bool + +method load*(this: MockList): Future[?!void] {.async.} = + this.loadCalled = true + +method add*(this: MockList, nid: Nid): Future[?!void] {.async.} = + this.added.add(nid) + if this.addSuccess: + return success() + return failure("test failure") + +method remove*(this: MockList, nid: Nid): Future[?!void] {.async.} = + this.removed.add(nid) + if this.removeSuccess: + return success() + return failure("test failure") + +proc createMockList*(): MockList = + MockList( + loadCalled: false, + added: newSeq[Nid](), + addSuccess: true, + removed: newSeq[Nid](), + removeSuccess: true + ) diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim index 1363c5a..f013640 100644 --- a/tests/codexcrawler/testcomponents.nim +++ b/tests/codexcrawler/testcomponents.nim @@ -1,3 +1,4 @@ import ./components/testnodestore +import ./components/testdhtmetrics {.warning[UnusedImport]: off.} From 3f697acefca7afb860904dcc4e6ed90380737a4b Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 14:29:41 +0100 Subject: [PATCH 11/20] implements dht-metrics --- codexcrawler/components/dhtmetrics.nim | 49 ++++++++++++------- codexcrawler/list.nim | 17 +++---- codexcrawler/metrics.nim | 16 +++--- .../components/testdhtmetrics.nim | 43 ++++++++++------ tests/codexcrawler/mocklist.nim | 21 +++++--- tests/codexcrawler/mockmetrics.nim | 18 +++++++ 6 files changed, 102 insertions(+), 62 deletions(-) create mode 100644 tests/codexcrawler/mockmetrics.nim diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim index 8bffa25..0ba6ca8 100644 --- a/codexcrawler/components/dhtmetrics.nim +++ b/codexcrawler/components/dhtmetrics.nim @@ -6,10 +6,9 @@ import pkg/questionable/results import ./dht import ../list import ../state -import ../component -import ../types -import ../utils/asyncdataevent import ../metrics +import ../component +import ../utils/asyncdataevent logScope: topics = "dhtmetrics" @@ -18,34 +17,48 @@ type DhtMetrics* = ref object of Component state: State ok: List nok: List + sub: AsyncDataEventSubscription + metrics: Metrics + +proc handleCheckEvent( + d: DhtMetrics, event: DhtNodeCheckEventData +): Future[?!void] {.async.} = + if event.isOk: + ?await d.ok.add(event.id) + ?await d.nok.remove(event.id) + else: + ?await d.ok.remove(event.id) + ?await d.nok.add(event.id) + + d.metrics.setOkNodes(d.ok.len) + d.metrics.setNokNodes(d.nok.len) + + return success() method start*(d: DhtMetrics): Future[?!void] {.async.} = info "Starting DhtMetrics..." + ?await d.ok.load() + ?await d.nok.load() + + proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} = + await d.handleCheckEvent(event) + + d.sub = d.state.events.dhtNodeCheck.subscribe(onCheck) return success() method stop*(d: DhtMetrics): Future[?!void] {.async.} = + await d.state.events.dhtNodeCheck.unsubscribe(d.sub) return success() proc new*( - T: type DhtMetrics, - state: State, - okList: List, - nokList: List + T: type DhtMetrics, state: State, okList: List, nokList: List, metrics: Metrics ): DhtMetrics = - DhtMetrics( - state: state, - ok: okList, - nok: nokList - ) + DhtMetrics(state: state, ok: okList, nok: nokList, metrics: metrics) -proc createDhtMetrics*(state: State): ?!DhtMetrics = +proc createDhtMetrics*(state: State, metrics: Metrics): ?!DhtMetrics = without okList =? createList(state.config.dataDir, "dhtok"), err: return failure(err) without nokList =? createList(state.config.dataDir, "dhtnok"), err: return failure(err) - success(DhtMetrics.new( - state, - okList, - nokList - )) + success(DhtMetrics.new(state, okList, nokList, metrics)) diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index fdd1f5e..5eeb280 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -19,12 +19,11 @@ import ./utils/datastoreutils logScope: topics = "list" -type - List* = ref object of RootObj - name: string - store: TypedDatastore - items: HashSet[Nid] - emptySignal: ?Future[void] +type List* = ref object of RootObj + name: string + store: TypedDatastore + items: HashSet[Nid] + emptySignal: ?Future[void] proc encode(s: Nid): seq[byte] = s.toBytes() @@ -86,12 +85,10 @@ method remove*(this: List, nid: Nid): Future[?!void] {.async, base.} = ?await this.store.delete(itemKey) return success() -proc len*(this: List): int = +method len*(this: List): int {.base, gcsafe, raises: [].} = this.items.len -proc new*( - _: type List, name: string, store: TypedDatastore -): List = +proc new*(_: type List, name: string, store: TypedDatastore): List = List(name: name, store: store) proc createList*(dataDir: string, name: string): ?!List = diff --git a/codexcrawler/metrics.nim b/codexcrawler/metrics.nim index ea713c8..39d0b63 100644 --- a/codexcrawler/metrics.nim +++ b/codexcrawler/metrics.nim @@ -9,7 +9,7 @@ declareGauge(nokNodesGauge, "DHT nodes failed to contact") type OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].} - Metrics* = ref object + Metrics* = ref object of RootObj todoNodes: OnUpdateMetric okNodes: OnUpdateMetric nokNodes: OnUpdateMetric @@ -25,13 +25,13 @@ proc startServer(metricsAddress: IpAddress, metricsPort: Port) = except Exception as exc: raiseAssert exc.msg # TODO fix metrics -method setTodoNodes*(m: Metrics, value: int) {.base.} = +method setTodoNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} = m.todoNodes(value.int64) -method setOkNodes*(m: Metrics, value: int) {.base.} = +method setOkNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} = m.okNodes(value.int64) -method setNokNodes*(m: Metrics, value: int) {.base.} = +method setNokNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} = m.nokNodes(value.int64) proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics = @@ -47,9 +47,5 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics = proc onNok(value: int64) = nokNodesGauge.set(value) - - return Metrics( - todoNodes: onTodo, - okNodes: onOk, - nokNodes: onNok - ) + + return Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok) diff --git a/tests/codexcrawler/components/testdhtmetrics.nim b/tests/codexcrawler/components/testdhtmetrics.nim index 0c0d613..f3c5f3c 100644 --- a/tests/codexcrawler/components/testdhtmetrics.nim +++ b/tests/codexcrawler/components/testdhtmetrics.nim @@ -1,8 +1,6 @@ -import std/os import pkg/chronos import pkg/questionable/results import pkg/asynctest/chronos/unittest -import pkg/datastore/typedds import ../../../codexcrawler/components/dhtmetrics import ../../../codexcrawler/utils/asyncdataevent @@ -10,6 +8,7 @@ import ../../../codexcrawler/types import ../../../codexcrawler/state import ../mockstate import ../mocklist +import ../mockmetrics import ../helpers suite "DhtMetrics": @@ -18,6 +17,7 @@ suite "DhtMetrics": state: MockState okList: MockList nokList: MockList + metrics: MockMetrics dhtmetrics: DhtMetrics setup: @@ -25,12 +25,9 @@ suite "DhtMetrics": state = createMockState() okList = createMockList() nokList = createMockList() + metrics = createMockMetrics() - dhtmetrics = DhtMetrics.new( - state, - okList, - nokList - ) + dhtmetrics = DhtMetrics.new(state, okList, nokList, metrics) (await dhtmetrics.start()).tryGet() @@ -39,17 +36,11 @@ suite "DhtMetrics": state.checkAllUnsubscribed() proc fireDhtNodeCheckEvent(isOk: bool) {.async.} = - let - event = DhtNodeCheckEventData( - id: nid, - isOk: isOk - ) + let event = DhtNodeCheckEventData(id: nid, isOk: isOk) (await state.events.dhtNodeCheck.fire(event)).tryGet() test "dhtmetrics start should load both lists": - (await dhtmetrics.start()).tryGet() - check: okList.loadCalled nokList.loadCalled @@ -58,13 +49,13 @@ suite "DhtMetrics": await fireDhtNodeCheckEvent(true) check: - nid in okList.added + nid in okList.added test "dhtNodeCheck event should add node to nokList if check has failed": await fireDhtNodeCheckEvent(false) check: - nid in nokList.added + nid in nokList.added test "dhtNodeCheck event should remove node from nokList if check is successful": await fireDhtNodeCheckEvent(true) @@ -77,3 +68,23 @@ suite "DhtMetrics": check: nid in okList.removed + + test "dhtNodeCheck event should set okList length as dht-ok metric": + let length = 123 + + okList.length = length + + await fireDhtNodeCheckEvent(true) + + check: + metrics.ok == length + + test "dhtNodeCheck event should set nokList length as dht-nok metric": + let length = 234 + + nokList.length = length + + await fireDhtNodeCheckEvent(true) + + check: + metrics.nok == length diff --git a/tests/codexcrawler/mocklist.nim b/tests/codexcrawler/mocklist.nim index 001e9f1..e077a0c 100644 --- a/tests/codexcrawler/mocklist.nim +++ b/tests/codexcrawler/mocklist.nim @@ -4,16 +4,17 @@ import pkg/questionable/results import ../../codexcrawler/types import ../../codexcrawler/list -type - MockList* = ref object of List - loadCalled*: bool - added*: seq[Nid] - addSuccess*: bool - removed*: seq[Nid] - removeSuccess*: bool +type MockList* = ref object of List + loadCalled*: bool + added*: seq[Nid] + addSuccess*: bool + removed*: seq[Nid] + removeSuccess*: bool + length*: int method load*(this: MockList): Future[?!void] {.async.} = this.loadCalled = true + return success() method add*(this: MockList, nid: Nid): Future[?!void] {.async.} = this.added.add(nid) @@ -27,11 +28,15 @@ method remove*(this: MockList, nid: Nid): Future[?!void] {.async.} = return success() return failure("test failure") +method len*(this: MockList): int = + return this.length + proc createMockList*(): MockList = MockList( loadCalled: false, added: newSeq[Nid](), addSuccess: true, removed: newSeq[Nid](), - removeSuccess: true + removeSuccess: true, + length: 0, ) diff --git a/tests/codexcrawler/mockmetrics.nim b/tests/codexcrawler/mockmetrics.nim new file mode 100644 index 0000000..021a8b0 --- /dev/null +++ b/tests/codexcrawler/mockmetrics.nim @@ -0,0 +1,18 @@ +import ../../codexcrawler/metrics + +type MockMetrics* = ref object of Metrics + todo*: int + ok*: int + nok*: int + +method setTodoNodes*(m: MockMetrics, value: int) = + m.todo = value + +method setOkNodes*(m: MockMetrics, value: int) = + m.ok = value + +method setNokNodes*(m: MockMetrics, value: int) = + m.nok = value + +proc createMockMetrics*(): MockMetrics = + MockMetrics() From 58b3d9679c7ae4ba0aa03dfc162026811019d116 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 15:03:56 +0100 Subject: [PATCH 12/20] Implements todo list --- codexcrawler/components/todolist.nim | 67 +++++++++++++++++++ codexcrawler/list.nim | 6 -- .../codexcrawler/components/testtodolist.nim | 59 ++++++++++++++++ tests/codexcrawler/testcomponents.nim | 1 + .../codexcrawler/utils/testasyncdataevent.nim | 4 ++ 5 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 codexcrawler/components/todolist.nim create mode 100644 tests/codexcrawler/components/testtodolist.nim diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim new file mode 100644 index 0000000..7eac856 --- /dev/null +++ b/codexcrawler/components/todolist.nim @@ -0,0 +1,67 @@ +import pkg/chronos +import pkg/chronicles +import pkg/datastore +import pkg/datastore/typedds +import pkg/questionable +import pkg/questionable/results + +import std/sets + +import ../state +import ../types +import ../component +import ../utils/asyncdataevent + +logScope: + topics = "todolist" + +type TodoList* = ref object of Component + nids: seq[Nid] + state: State + subNew: AsyncDataEventSubscription + subExp: AsyncDataEventSubscription + emptySignal: ?Future[void] + +proc addNodes(t: TodoList, nids: seq[Nid]) = + for nid in nids: + t.nids.add(nid) + + if s =? t.emptySignal: + s.complete() + t.emptySignal = Future[void].none + +proc pop*(t: TodoList): Future[?!Nid] {.async.} = + if t.nids.len < 1: + trace "List is empty. Waiting for new items..." + let signal = newFuture[void]("list.emptySignal") + t.emptySignal = some(signal) + await signal.wait(1.hours) + if t.nids.len < 1: + return failure("TodoList is empty.") + + let item = t.nids[0] + t.nids.del(0) + + return success(item) + +method start*(t: TodoList): Future[?!void] {.async.} = + info "Starting TodoList..." + + proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} = + t.addNodes(nids) + return success() + + t.subNew = t.state.events.newNodesDiscovered.subscribe(onNewNodes) + t.subExp = t.state.events.nodesExpired.subscribe(onNewNodes) + return success() + +method stop*(t: TodoList): Future[?!void] {.async.} = + await t.state.events.newNodesDiscovered.unsubscribe(t.subNew) + await t.state.events.nodesExpired.unsubscribe(t.subExp) + return success() + +proc new*(_: type TodoList, state: State): TodoList = + TodoList(nids: newSeq[Nid](), state: state, emptySignal: Future[void].none) + +proc createTodoList*(state: State): ?!TodoList = + success(TodoList.new(state)) diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index 5eeb280..dd68314 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -23,7 +23,6 @@ type List* = ref object of RootObj name: string store: TypedDatastore items: HashSet[Nid] - emptySignal: ?Future[void] proc encode(s: Nid): seq[byte] = s.toBytes() @@ -68,11 +67,6 @@ method add*(this: List, nid: Nid): Future[?!void] {.async, base.} = if err =? (await this.saveItem(nid)).errorOption: return failure(err) - if s =? this.emptySignal: - trace "List no longer empty.", name = this.name - s.complete() - this.emptySignal = Future[void].none - return success() method remove*(this: List, nid: Nid): Future[?!void] {.async, base.} = diff --git a/tests/codexcrawler/components/testtodolist.nim b/tests/codexcrawler/components/testtodolist.nim new file mode 100644 index 0000000..8960859 --- /dev/null +++ b/tests/codexcrawler/components/testtodolist.nim @@ -0,0 +1,59 @@ +import pkg/chronos +import pkg/questionable/results +import pkg/asynctest/chronos/unittest + +import ../../../codexcrawler/components/todolist +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../../../codexcrawler/state +import ../mockstate +import ../helpers + +suite "TodoList": + var + nid: Nid + state: MockState + todo: TodoList + + setup: + nid = genNid() + state = createMockState() + + todo = TodoList.new(state) + + (await todo.start()).tryGet() + + teardown: + (await todo.stop()).tryGet() + state.checkAllUnsubscribed() + + proc fireNewNodesDiscoveredEvent(nids: seq[Nid]) {.async.} = + (await state.events.newNodesDiscovered.fire(nids)).tryGet() + + proc fireNodesExpiredEvent(nids: seq[Nid]) {.async.} = + (await state.events.nodesExpired.fire(nids)).tryGet() + + test "discovered nodes are added to todo list": + await fireNewNodesDiscoveredEvent(@[nid]) + let item = (await todo.pop).tryGet() + + check: + item == nid + + test "expired nodes are added to todo list": + await fireNodesExpiredEvent(@[nid]) + let item = (await todo.pop).tryGet() + + check: + item == nid + + test "pop on empty todo list waits until item is added": + let popFuture = todo.pop() + check: + not popFuture.finished + + await fireNewNodesDiscoveredEvent(@[nid]) + + check: + popFuture.finished + popFuture.value.tryGet() == nid diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim index f013640..4226e95 100644 --- a/tests/codexcrawler/testcomponents.nim +++ b/tests/codexcrawler/testcomponents.nim @@ -1,4 +1,5 @@ import ./components/testnodestore import ./components/testdhtmetrics +import ./components/testtodolist {.warning[UnusedImport]: off.} diff --git a/tests/codexcrawler/utils/testasyncdataevent.nim b/tests/codexcrawler/utils/testasyncdataevent.nim index 8c1efd8..c5a47c5 100644 --- a/tests/codexcrawler/utils/testasyncdataevent.nim +++ b/tests/codexcrawler/utils/testasyncdataevent.nim @@ -80,6 +80,10 @@ suite "AsyncDataEvent": await event.unsubscribe(s2) await event.unsubscribe(s3) + test "Can fire and event without subscribers": + check: + isOK(await event.fire(ExampleData(s: msg))) + test "Can unsubscribe in handler": proc doNothing() {.async, closure.} = await sleepAsync(1.millis) From 4fff78903df907354166a8a771af32b6a1945ea7 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 15:33:40 +0100 Subject: [PATCH 13/20] sets up main loop --- codexcrawler/application.nim | 72 +++++++------------------------- codexcrawler/state.nim | 22 ++++++++-- tests/codexcrawler/mockstate.nim | 1 + tests/codexcrawler/teststate.nim | 32 ++++++++++++++ tests/test.nim | 1 + 5 files changed, 68 insertions(+), 60 deletions(-) create mode 100644 tests/codexcrawler/teststate.nim diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index e207544..3cd17cf 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -8,60 +8,19 @@ import pkg/metrics import ./config import ./utils/logging -import ./metrics -import ./list -import ./utils/datastoreutils import ./utils/asyncdataevent import ./installer import ./state import ./component import ./types -type - ApplicationStatus* {.pure.} = enum - Stopped - Stopping - Running +type Application* = ref object + state: State - Application* = ref object - status: ApplicationStatus - config*: Config - todoNodes*: List - okNodes*: List - nokNodes*: List - -# proc initializeLists(app: Application): Future[?!void] {.async.} = -# without store =? createTypedDatastore(app.config.dataDir / "lists"), err: -# return failure(err) - -# # We can't extract this into a function because gauges cannot be passed as argument. -# # The use of global state in nim-metrics is not pleasant. -# proc onTodoMetric(value: int64) = -# todoNodesGauge.set(value) - -# proc onOkMetric(value: int64) = -# okNodesGauge.set(value) - -# proc onNokMetric(value: int64) = -# nokNodesGauge.set(value) - -# app.todoNodes = List.new("todo", store, onTodoMetric) -# app.okNodes = List.new("ok", store, onOkMetric) -# app.nokNodes = List.new("nok", store, onNokMetric) - -# if err =? (await app.todoNodes.load()).errorOption: -# return failure(err) -# if err =? (await app.okNodes.load()).errorOption: -# return failure(err) -# if err =? (await app.nokNodes.load()).errorOption: -# return failure(err) - -# return success() - -proc initializeApp(app: Application): Future[?!void] {.async.} = - # todo move this +proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} = let state = State( - config: app.config, + status: ApplicationStatus.Running, + config: config, events: Events( nodesFound: newAsyncDataEvent[seq[Nid]](), newNodesDiscovered: newAsyncDataEvent[seq[Nid]](), @@ -81,30 +40,29 @@ proc initializeApp(app: Application): Future[?!void] {.async.} = return success() proc stop*(app: Application) = - app.status = ApplicationStatus.Stopping - # waitFor app.dht.stop() + app.state.status = ApplicationStatus.Stopping proc run*(app: Application) = - app.config = parseConfig() - info "Loaded configuration", config = app.config + let config = parseConfig() + info "Loaded configuration", config = $config # Configure loglevel - updateLogLevel(app.config.logLevel) + updateLogLevel(config.logLevel) # Ensure datadir path exists: - if not existsDir(app.config.dataDir): - createDir(app.config.dataDir) + if not existsDir(config.dataDir): + createDir(config.dataDir) info "Metrics endpoint initialized" info "Starting application" - app.status = ApplicationStatus.Running - if err =? (waitFor app.initializeApp()).errorOption: - app.status = ApplicationStatus.Stopping + app.state.status = ApplicationStatus.Running + if err =? (waitFor app.initializeApp(config)).errorOption: + app.state.status = ApplicationStatus.Stopping error "Failed to start application", err = err.msg return - while app.status == ApplicationStatus.Running: + while app.state.status == ApplicationStatus.Running: try: chronos.poll() except Exception as exc: diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index 9d8d4be..e32e856 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -1,10 +1,14 @@ import pkg/chronos +import pkg/chronicles import pkg/questionable/results import ./config import ./utils/asyncdataevent import ./types +logScope: + topics = "state" + type OnStep = proc(): Future[?!void] {.async: (raises: []), gcsafe.} @@ -18,10 +22,22 @@ type dhtNodeCheck*: AsyncDataEvent[DhtNodeCheckEventData] nodesExpired*: AsyncDataEvent[seq[Nid]] + ApplicationStatus* {.pure.} = enum + Stopped + Stopping + Running + State* = ref object of RootObj + status*: ApplicationStatus config*: Config events*: Events -proc whileRunning*(this: State, step: OnStep, delay: Duration) = - discard - #todo: while status == running, step(), asyncsleep duration +proc whileRunning*(s: State, step: OnStep, delay: Duration) {.async.} = + proc worker(): Future[void] {.async.} = + while s.status == ApplicationStatus.Running: + if err =? (await step()).errorOption: + error "Failure-result caught in main loop. Stopping...", err = err.msg + s.status = ApplicationStatus.Stopping + await sleepAsync(delay) + + asyncSpawn worker() diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim index 1a305a9..7721b08 100644 --- a/tests/codexcrawler/mockstate.nim +++ b/tests/codexcrawler/mockstate.nim @@ -8,6 +8,7 @@ type MockState* = ref object of State proc createMockState*(): MockState = MockState( + status: ApplicationStatus.Running, config: Config(), events: Events( nodesFound: newAsyncDataEvent[seq[Nid]](), diff --git a/tests/codexcrawler/teststate.nim b/tests/codexcrawler/teststate.nim new file mode 100644 index 0000000..3a1493e --- /dev/null +++ b/tests/codexcrawler/teststate.nim @@ -0,0 +1,32 @@ +import pkg/chronos +import pkg/questionable/results +import pkg/asynctest/chronos/unittest + +import ../../codexcrawler/state +import ./mockstate + +suite "State": + var state: State + + setup: + # The behavior we're testing is the same for the mock + state = createMockState() + + test "whileRunning": + var counter = 0 + + proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} = + inc counter + return success() + + await state.whileRunning(onStep, 1.milliseconds) + + while counter < 5: + await sleepAsync(1.milliseconds) + + state.status = ApplicationStatus.Stopped + + await sleepAsync(10.milliseconds) + + check: + counter == 5 diff --git a/tests/test.nim b/tests/test.nim index a310fc6..1df02d1 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -1,5 +1,6 @@ import ./codexcrawler/testutils import ./codexcrawler/testcomponents import ./codexcrawler/testtypes +import ./codexcrawler/teststate {.warning[UnusedImport]: off.} From da1d82a4cda5a7558d704dffc8220ddac74f3555 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 11 Feb 2025 16:31:23 +0100 Subject: [PATCH 14/20] Implements timetracker and tests --- codexcrawler/components/nodestore.nim | 26 +++--- codexcrawler/components/timetracker.nim | 86 ++++++------------- codexcrawler/installer.nim | 7 +- codexcrawler/state.nim | 4 +- codexcrawler/utils/asyncdataevent.nim | 14 ++- .../components/testtimetracker.nim | 72 ++++++++++++++++ tests/codexcrawler/mocknodestore.nim | 24 ++++++ tests/codexcrawler/mockstate.nim | 14 +-- tests/codexcrawler/testcomponents.nim | 1 + tests/codexcrawler/teststate.nim | 16 +++- 10 files changed, 182 insertions(+), 82 deletions(-) create mode 100644 tests/codexcrawler/components/testtimetracker.nim create mode 100644 tests/codexcrawler/mocknodestore.nim diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index f977507..51ae946 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -19,7 +19,7 @@ type id*: Nid lastVisit*: uint64 - OnNodeEntry = proc(item: NodeEntry): Future[?!void] {.async: (raises: []), gcsafe.} + OnNodeEntry* = proc(item: NodeEntry): Future[?!void] {.async: (raises: []), gcsafe.} NodeStore* = ref object of Component state: State @@ -87,19 +87,25 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = ?await s.fireNewNodesDiscovered(newNodes) return success() -proc iterateAll*(s: NodeStore, onNode: OnNodeEntry): Future[?!void] {.async.} = +method iterateAll*( + s: NodeStore, onNode: OnNodeEntry +): Future[?!void] {.async: (raises: []), base.} = without queryKey =? Key.init(nodestoreName), err: return failure(err) - without iter =? (await query[NodeEntry](s.store, Query.init(queryKey))), err: - return failure(err) - - while not iter.finished: - without item =? (await iter.next()), err: - return failure(err) - without value =? item.value, err: + try: + without iter =? (await query[NodeEntry](s.store, Query.init(queryKey))), err: return failure(err) - ?await onNode(value) + while not iter.finished: + without item =? (await iter.next()), err: + return failure(err) + without value =? item.value, err: + return failure(err) + + ?await onNode(value) + except CatchableError as exc: + return failure(exc.msg) + return success() method start*(s: NodeStore): Future[?!void] {.async.} = diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index e811481..a89821a 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -1,79 +1,49 @@ import pkg/chronicles import pkg/chronos -import pkg/questionable import pkg/questionable/results -import ./dht -import ../list -import ../config +import ./nodestore import ../component import ../state +import ../types +import ../utils/asyncdataevent logScope: topics = "timetracker" type TimeTracker* = ref object of Component - config: Config - todoNodes: List - okNodes: List - nokNodes: List - workerDelay: int + state: State + nodestore: NodeStore -# # proc processList(t: TimeTracker, list: List, expiry: uint64) {.async.} = -# # var toMove = newSeq[NodeEntry]() -# # proc onItem(item: NodeEntry) = -# # if item.lastVisit < expiry: -# # toMove.add(item) +proc step(t: TimeTracker): Future[?!void] {.async: (raises: []).} = + let expiry = + (Moment.now().epochSeconds - (t.state.config.revisitDelayMins * 60)).uint64 -# # await list.iterateAll(onItem) + var expired = newSeq[Nid]() + proc checkNode(item: NodeEntry): Future[?!void] {.async: (raises: []), gcsafe.} = + if item.lastVisit < expiry: + expired.add(item.id) + return success() -# # if toMove.len > 0: -# # trace "expired node, moving to todo", nodes = $toMove.len - -# # for item in toMove: -# # if err =? (await t.todoNodes.add(item)).errorOption: -# # error "Failed to add expired node to todo list", err = err.msg -# # return -# # if err =? (await list.remove(item)).errorOption: -# # error "Failed to remove expired node to source list", err = err.msg - -# proc step(t: TimeTracker) {.async.} = -# let expiry = (Moment.now().epochSeconds - (t.config.revisitDelayMins * 60)).uint64 -# await t.processList(t.okNodes, expiry) -# await t.processList(t.nokNodes, expiry) - -proc worker(t: TimeTracker) {.async.} = - try: - while true: - # await t.step() - await sleepAsync(t.workerDelay.minutes) - except Exception as exc: - error "Exception in timetracker worker", msg = exc.msg - quit QuitFailure + ?await t.nodestore.iterateAll(checkNode) + ?await t.state.events.nodesExpired.fire(expired) + return success() method start*(t: TimeTracker): Future[?!void] {.async.} = - info "Starting timetracker...", revisitDelayMins = $t.workerDelay - asyncSpawn t.worker() + info "Starting timetracker..." + + proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} = + await t.step() + + var delay = t.state.config.revisitDelayMins div 100 + if delay < 1: + delay = 1 + + await t.state.whileRunning(onStep, delay.minutes) return success() method stop*(t: TimeTracker): Future[?!void] {.async.} = return success() -proc new*( - T: type TimeTracker, - # todoNodes: List, - # okNodes: List, - # nokNodes: List, - config: Config, -): TimeTracker = - var delay = config.revisitDelayMins div 10 - if delay < 1: - delay = 1 - - TimeTracker( - # todoNodes: todoNodes, - # okNodes: okNodes, - # nokNodes: nokNodes, - config: config, - workerDelay: delay, - ) +proc new*(T: type TimeTracker, state: State, nodestore: NodeStore): TimeTracker = + TimeTracker(state: state, nodestore: nodestore) diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index 48c2e9e..dfd4575 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -8,6 +8,7 @@ import ./components/dht import ./components/crawler import ./components/timetracker import ./components/nodestore +import ./components/dhtmetrics proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = var components: seq[Component] = newSeq[Component]() @@ -20,8 +21,12 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = let metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort) + without dhtMetrics =? createDhtMetrics(state, metrics), err: + return failure(err) + components.add(nodeStore) components.add(dht) components.add(Crawler.new(dht, state.config)) - components.add(TimeTracker.new(state.config)) + components.add(TimeTracker.new(state, nodeStore)) + components.add(dhtMetrics) return success(components) diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index e32e856..5670ce9 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -10,7 +10,7 @@ logScope: topics = "state" type - OnStep = proc(): Future[?!void] {.async: (raises: []), gcsafe.} + OnStep* = proc(): Future[?!void] {.async: (raises: []), gcsafe.} DhtNodeCheckEventData* = object id*: Nid @@ -32,7 +32,7 @@ type config*: Config events*: Events -proc whileRunning*(s: State, step: OnStep, delay: Duration) {.async.} = +method whileRunning*(s: State, step: OnStep, delay: Duration) {.async, base.} = proc worker(): Future[void] {.async.} = while s.status == ApplicationStatus.Running: if err =? (await step()).errorOption: diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim index 52a0ea5..81946d7 100644 --- a/codexcrawler/utils/asyncdataevent.nim +++ b/codexcrawler/utils/asyncdataevent.nim @@ -55,18 +55,26 @@ proc subscribe*[T]( event.subscriptions.add(subscription) subscription -proc fire*[T](event: AsyncDataEvent[T], data: T): Future[?!void] {.async.} = +proc fire*[T]( + event: AsyncDataEvent[T], data: T +): Future[?!void] {.async: (raises: []).} = event.queue.emit(data.some) var toUnsubscribe = newSeq[AsyncDataEventSubscription]() for sub in event.subscriptions: - await sub.fireEvent.wait() + try: + await sub.fireEvent.wait() + except CancelledError: + discard if err =? sub.lastResult.errorOption: return failure(err) if sub.delayedUnsubscribe: toUnsubscribe.add(sub) for sub in toUnsubscribe: - await event.unsubscribe(sub) + try: + await event.unsubscribe(sub) + except CatchableError as exc: + return failure(exc.msg) success() diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim new file mode 100644 index 0000000..83f1fb2 --- /dev/null +++ b/tests/codexcrawler/components/testtimetracker.nim @@ -0,0 +1,72 @@ +import pkg/chronos +import pkg/questionable/results +import pkg/asynctest/chronos/unittest + +import ../../../codexcrawler/components/timetracker +import ../../../codexcrawler/components/nodestore +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../../../codexcrawler/state +import ../mockstate +import ../mocknodestore +import ../helpers + +suite "TimeTracker": + var + nid: Nid + state: MockState + store: MockNodeStore + time: TimeTracker + expiredNodesReceived: seq[Nid] + sub: AsyncDataEventSubscription + + setup: + nid = genNid() + state = createMockState() + store = createMockNodeStore() + + # Subscribe to nodesExpired event + expiredNodesReceived = newSeq[Nid]() + proc onExpired(nids: seq[Nid]): Future[?!void] {.async.} = + expiredNodesReceived = nids + return success() + + sub = state.events.nodesExpired.subscribe(onExpired) + + state.config.revisitDelayMins = 22 + + time = TimeTracker.new(state, store) + + (await time.start()).tryGet() + + teardown: + (await time.stop()).tryGet() + await state.events.nodesExpired.unsubscribe(sub) + state.checkAllUnsubscribed() + + proc createNodeInStore(lastVisit: uint64): Nid = + let entry = NodeEntry(id: genNid(), lastVisit: lastVisit) + store.nodesToIterate.add(entry) + return entry.id + + test "onStep fires nodesExpired event for expired nodes": + let + expiredTimestamp = + (Moment.now().epochSeconds - ((1 + state.config.revisitDelayMins) * 60)).uint64 + expiredNodeId = createNodeInStore(expiredTimestamp) + + (await state.stepper()).tryGet() + + check: + expiredNodeId in expiredNodesReceived + + test "onStep does not fire nodesExpired event for nodes that are recent": + let + recentTimestamp = + (Moment.now().epochSeconds - ((state.config.revisitDelayMins - 1) * 60)).uint64 + recentNodeId = createNodeInStore(recentTimestamp) + + (await state.stepper()).tryGet() + + check: + recentNodeId notin expiredNodesReceived diff --git a/tests/codexcrawler/mocknodestore.nim b/tests/codexcrawler/mocknodestore.nim new file mode 100644 index 0000000..eb3ec19 --- /dev/null +++ b/tests/codexcrawler/mocknodestore.nim @@ -0,0 +1,24 @@ +import std/sequtils +import pkg/questionable/results +import pkg/chronos + +import ../../codexcrawler/components/nodestore + +type MockNodeStore* = ref object of NodeStore + nodesToIterate*: seq[NodeEntry] + +method iterateAll*( + s: MockNodeStore, onNode: OnNodeEntry +): Future[?!void] {.async: (raises: []).} = + for node in s.nodesToIterate: + ?await onNode(node) + return success() + +method start*(s: MockNodeStore): Future[?!void] {.async.} = + return success() + +method stop*(s: MockNodeStore): Future[?!void] {.async.} = + return success() + +proc createMockNodeStore*(): MockNodeStore = + MockNodeStore(nodesToIterate: newSeq[NodeEntry]()) diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim index 7721b08..bfa7e6f 100644 --- a/tests/codexcrawler/mockstate.nim +++ b/tests/codexcrawler/mockstate.nim @@ -5,6 +5,7 @@ import ../../codexcrawler/types import ../../codexcrawler/config type MockState* = ref object of State + stepper*: OnStep proc createMockState*(): MockState = MockState( @@ -18,9 +19,12 @@ proc createMockState*(): MockState = ), ) -proc checkAllUnsubscribed*(this: MockState) = +proc checkAllUnsubscribed*(s: MockState) = check: - this.events.nodesFound.listeners == 0 - this.events.newNodesDiscovered.listeners == 0 - this.events.dhtNodeCheck.listeners == 0 - this.events.nodesExpired.listeners == 0 + s.events.nodesFound.listeners == 0 + s.events.newNodesDiscovered.listeners == 0 + s.events.dhtNodeCheck.listeners == 0 + s.events.nodesExpired.listeners == 0 + +method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} = + s.stepper = step diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim index 4226e95..1b9a6b7 100644 --- a/tests/codexcrawler/testcomponents.nim +++ b/tests/codexcrawler/testcomponents.nim @@ -1,5 +1,6 @@ import ./components/testnodestore import ./components/testdhtmetrics import ./components/testtodolist +import ./components/testtimetracker {.warning[UnusedImport]: off.} diff --git a/tests/codexcrawler/teststate.nim b/tests/codexcrawler/teststate.nim index 3a1493e..4ea6bb5 100644 --- a/tests/codexcrawler/teststate.nim +++ b/tests/codexcrawler/teststate.nim @@ -3,14 +3,24 @@ import pkg/questionable/results import pkg/asynctest/chronos/unittest import ../../codexcrawler/state -import ./mockstate +import ../../codexcrawler/config +import ../../codexcrawler/types +import ../../codexcrawler/utils/asyncdataevent suite "State": var state: State setup: - # The behavior we're testing is the same for the mock - state = createMockState() + state = State( + status: ApplicationStatus.Running, + config: Config(), + events: Events( + nodesFound: newAsyncDataEvent[seq[Nid]](), + newNodesDiscovered: newAsyncDataEvent[seq[Nid]](), + dhtNodeCheck: newAsyncDataEvent[DhtNodeCheckEventData](), + nodesExpired: newAsyncDataEvent[seq[Nid]](), + ), + ) test "whileRunning": var counter = 0 From 25291f7625e03fbca62bae0918f182e08068fd12 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 13:25:37 +0100 Subject: [PATCH 15/20] Implementing and testing crawler --- codexcrawler/components/crawler.nim | 100 +++++---------- codexcrawler/components/dhtmetrics.nim | 3 +- codexcrawler/components/todolist.nim | 11 +- codexcrawler/installer.nim | 12 +- codexcrawler/{components => services}/dht.nim | 77 ++++++------ codexcrawler/{ => services}/metrics.nim | 0 codexcrawler/state.nim | 1 + tests/codexcrawler/components/testcrawler.nim | 115 ++++++++++++++++++ .../components/testtimetracker.nim | 7 +- tests/codexcrawler/mockdht.nim | 26 ++++ tests/codexcrawler/mockmetrics.nim | 2 +- tests/codexcrawler/mockstate.nim | 20 +-- tests/codexcrawler/mocktodolist.nim | 20 +++ tests/codexcrawler/testcomponents.nim | 1 + 14 files changed, 265 insertions(+), 130 deletions(-) rename codexcrawler/{components => services}/dht.nim (74%) rename codexcrawler/{ => services}/metrics.nim (100%) create mode 100644 tests/codexcrawler/components/testcrawler.nim create mode 100644 tests/codexcrawler/mockdht.nim create mode 100644 tests/codexcrawler/mocktodolist.nim diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/crawler.nim index e86de85..0f984a0 100644 --- a/codexcrawler/components/crawler.nim +++ b/codexcrawler/components/crawler.nim @@ -3,86 +3,53 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results -import ./dht -import ../list +import ../services/dht +import ./todolist import ../config -import ../component import ../types +import ../component import ../state import ../utils/asyncdataevent -import std/sequtils - logScope: topics = "crawler" type Crawler* = ref object of Component + state: State dht: Dht - config: Config - todoNodes: List - okNodes: List - nokNodes: List + todo: TodoList -# This is not going to stay this way. -proc isNew(c: Crawler, node: Node): bool = - not c.todoNodes.contains(node.id) and not c.okNodes.contains(node.id) and - not c.nokNodes.contains(node.id) +proc raiseCheckEvent(c: Crawler, nid: Nid, success: bool): Future[?!void] {.async: (raises: []).} = + let event = DhtNodeCheckEventData( + id: nid, + isOk: success + ) + if err =? (await c.state.events.dhtNodeCheck.fire(event)).errorOption: + return failure(err) + return success() -# proc handleNodeNotOk(c: Crawler, target: NodeEntry) {.async.} = -# if err =? (await c.nokNodes.add(target)).errorOption: -# error "Failed to add not-OK-node to list", err = err.msg +proc step(c: Crawler): Future[?!void] {.async: (raises: []).} = + without nid =? (await c.todo.pop()), err: + return failure(err) -# proc handleNodeOk(c: Crawler, target: NodeEntry) {.async.} = -# if err =? (await c.okNodes.add(target)).errorOption: -# error "Failed to add OK-node to list", err = err.msg + without response =? await c.dht.getNeighbors(nid), err: + return failure(err) -# proc addNewTodoNode(c: Crawler, nodeId: NodeId): Future[?!void] {.async.} = -# let entry = NodeEntry(id: nodeId, lastVisit: 0) -# return await c.todoNodes.add(entry) + if err =? (await c.raiseCheckEvent(nid, response.isResponsive)).errorOption: + return failure(err) -# proc addNewTodoNodes(c: Crawler, newNodes: seq[Node]) {.async.} = -# for node in newNodes: -# if err =? (await c.addNewTodoNode(node.id)).errorOption: -# error "Failed to add todo-node to list", err = err.msg + if err =? (await c.state.events.nodesFound.fire(response.nodeIds)).errorOption: + return failure(err) -# proc step(c: Crawler) {.async.} = -# logScope: -# todo = $c.todoNodes.len -# ok = $c.okNodes.len -# nok = $c.nokNodes.len - -# without var target =? (await c.todoNodes.pop()), err: -# error "Failed to get todo node", err = err.msg - -# target.lastVisit = Moment.now().epochSeconds.uint64 - -# without receivedNodes =? (await c.dht.getNeighbors(target.id)), err: -# await c.handleNodeNotOk(target) -# return - -# let newNodes = receivedNodes.filterIt(isNew(c, it)) -# if newNodes.len > 0: -# trace "Discovered new nodes", newNodes = newNodes.len - -# await c.handleNodeOk(target) -# await c.addNewTodoNodes(newNodes) - -# # Don't log the status every loop: -# if (c.todoNodes.len mod 10) == 0: -# trace "Status" - -proc worker(c: Crawler) {.async.} = - try: - while true: - # await c.step() - await sleepAsync(c.config.stepDelayMs.millis) - except Exception as exc: - error "Exception in crawler worker", msg = exc.msg - quit QuitFailure + return success() method start*(c: Crawler): Future[?!void] {.async.} = - info "Starting crawler...", stepDelayMs = $c.config.stepDelayMs - asyncSpawn c.worker() + info "Starting crawler..." + + proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} = + await c.step() + await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds) + return success() method stop*(c: Crawler): Future[?!void] {.async.} = @@ -90,13 +57,12 @@ method stop*(c: Crawler): Future[?!void] {.async.} = proc new*( T: type Crawler, + state: State, dht: Dht, - # todoNodes: List, - # okNodes: List, - # nokNodes: List, - config: Config, + todo: TodoList ): Crawler = Crawler( + state: state, dht: dht, - config: config, # todoNodes: todoNodes, okNodes: okNodes, nokNodes: nokNodes, + todo: todo ) diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim index 0ba6ca8..092abde 100644 --- a/codexcrawler/components/dhtmetrics.nim +++ b/codexcrawler/components/dhtmetrics.nim @@ -3,10 +3,9 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results -import ./dht import ../list import ../state -import ../metrics +import ../services/metrics import ../component import ../utils/asyncdataevent diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim index 7eac856..0d55ccb 100644 --- a/codexcrawler/components/todolist.nim +++ b/codexcrawler/components/todolist.nim @@ -30,12 +30,15 @@ proc addNodes(t: TodoList, nids: seq[Nid]) = s.complete() t.emptySignal = Future[void].none -proc pop*(t: TodoList): Future[?!Nid] {.async.} = +method pop*(t: TodoList): Future[?!Nid] {.async: (raises: []), base.} = if t.nids.len < 1: trace "List is empty. Waiting for new items..." let signal = newFuture[void]("list.emptySignal") t.emptySignal = some(signal) - await signal.wait(1.hours) + try: + await signal.wait(1.hours) + except CatchableError as exc: + return failure(exc.msg) if t.nids.len < 1: return failure("TodoList is empty.") @@ -63,5 +66,5 @@ method stop*(t: TodoList): Future[?!void] {.async.} = proc new*(_: type TodoList, state: State): TodoList = TodoList(nids: newSeq[Nid](), state: state, emptySignal: Future[void].none) -proc createTodoList*(state: State): ?!TodoList = - success(TodoList.new(state)) +proc createTodoList*(state: State): TodoList = + TodoList.new(state) diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index dfd4575..e8a7fd5 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -2,9 +2,9 @@ import pkg/chronos import pkg/questionable/results import ./state -import ./metrics +import ./services/metrics +import ./services/dht import ./component -import ./components/dht import ./components/crawler import ./components/timetracker import ./components/nodestore @@ -19,14 +19,18 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = without nodeStore =? createNodeStore(state), err: return failure(err) - let metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort) + let + metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort) + todoList = createTodoList(state) without dhtMetrics =? createDhtMetrics(state, metrics), err: return failure(err) + components.add(todoList) components.add(nodeStore) components.add(dht) - components.add(Crawler.new(dht, state.config)) + components.add(Crawler.new(state, dht, todoList)) components.add(TimeTracker.new(state, nodeStore)) components.add(dhtMetrics) + return success(components) diff --git a/codexcrawler/components/dht.nim b/codexcrawler/services/dht.nim similarity index 74% rename from codexcrawler/components/dht.nim rename to codexcrawler/services/dht.nim index 565af1e..c1c777c 100644 --- a/codexcrawler/components/dht.nim +++ b/codexcrawler/services/dht.nim @@ -1,5 +1,6 @@ import std/os import std/net +import std/sequtils import pkg/chronicles import pkg/chronos import pkg/libp2p @@ -11,63 +12,59 @@ from pkg/nimcrypto import keccak256 import ../utils/keyutils import ../utils/datastoreutils import ../utils/rng -import ../utils/asyncdataevent import ../component import ../state +import ../types export discv5 logScope: topics = "dht" -type Dht* = ref object of Component - state: State - protocol*: discv5.Protocol - key: PrivateKey - peerId: PeerId - announceAddrs*: seq[MultiAddress] - providerRecord*: ?SignedPeerRecord - dhtRecord*: ?SignedPeerRecord +type + GetNeighborsResponse* = ref object + isResponsive*: bool + nodeIds*: seq[Nid] -# proc toNodeId*(cid: Cid): NodeId = -# ## Cid to discovery id -# ## - -# readUintBE[256](keccak256.digest(cid.data.buffer).data) - -# proc toNodeId*(host: ca.Address): NodeId = -# ## Eth address to discovery id -# ## - -# readUintBE[256](keccak256.digest(host.toArray).data) + Dht* = ref object of Component + state: State + protocol*: discv5.Protocol + key: PrivateKey + peerId: PeerId + announceAddrs*: seq[MultiAddress] + providerRecord*: ?SignedPeerRecord + dhtRecord*: ?SignedPeerRecord proc getNode*(d: Dht, nodeId: NodeId): ?!Node = let node = d.protocol.getNode(nodeId) if node.isSome(): return success(node.get()) - return failure("Node not found for id: " & $(NodeId(nodeId))) + return failure("Node not found for id: " & nodeId.toHex()) -proc getRoutingTableNodeIds(d: Dht): seq[NodeId] = - var ids = newSeq[NodeId]() +method getRoutingTableNodeIds*(d: Dht): seq[Nid] {.base.} = + var ids = newSeq[Nid]() for bucket in d.protocol.routingTable.buckets: for node in bucket.nodes: ids.add(node.id) return ids -proc getNeighbors*(d: Dht, target: NodeId): Future[?!seq[Node]] {.async.} = +method getNeighbors*(d: Dht, target: Nid): Future[?!GetNeighborsResponse] {.async: (raises: []), base.} = without node =? d.getNode(target), err: return failure(err) let distances = @[256.uint16] - let response = await d.protocol.findNode(node, distances) + try: + let response = await d.protocol.findNode(node, distances) - if response.isOk(): - let nodes = response.get() - if nodes.len > 0: - return success(nodes) - - # Both returning 0 nodes and a failure result are treated as failure of getNeighbors - return failure("No nodes returned") + if response.isOk(): + let nodes = response.get() + return success(GetNeighborsResponse( + isResponsive: true, + nodeIds: nodes.mapIt(it.id)) + ) + return failure($response.error()) + except CatchableError as exc: + return failure(exc.msg) proc findPeer*(d: Dht, peerId: PeerId): Future[?PeerRecord] {.async.} = trace "protocol.resolve..." @@ -103,19 +100,19 @@ proc updateDhtRecord(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") -proc findRoutingTableNodes(d: Dht) {.async.} = - await sleepAsync(5.seconds) - let nodes = d.getRoutingTableNodeIds() +# proc findRoutingTableNodes(d: Dht) {.async.} = +# await sleepAsync(5.seconds) +# let nodes = d.getRoutingTableNodeIds() - if err =? (await d.state.events.nodesFound.fire(nodes)).errorOption: - error "Failed to raise routing-table nodes as found nodes", err = err.msg - else: - trace "Routing table nodes raise as found nodes", num = nodes.len +# if err =? (await d.state.events.nodesFound.fire(nodes)).errorOption: +# error "Failed to raise routing-table nodes as found nodes", err = err.msg +# else: +# trace "Routing table nodes raised as found nodes", num = nodes.len method start*(d: Dht): Future[?!void] {.async.} = d.protocol.open() await d.protocol.start() - asyncSpawn d.findRoutingTableNodes() + # asyncSpawn d.findRoutingTableNodes() return success() method stop*(d: Dht): Future[?!void] {.async.} = diff --git a/codexcrawler/metrics.nim b/codexcrawler/services/metrics.nim similarity index 100% rename from codexcrawler/metrics.nim rename to codexcrawler/services/metrics.nim diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index 5670ce9..5394ef1 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -40,4 +40,5 @@ method whileRunning*(s: State, step: OnStep, delay: Duration) {.async, base.} = s.status = ApplicationStatus.Stopping await sleepAsync(delay) + # todo this needs a delay because starts are still being called. asyncSpawn worker() diff --git a/tests/codexcrawler/components/testcrawler.nim b/tests/codexcrawler/components/testcrawler.nim new file mode 100644 index 0000000..2ec2948 --- /dev/null +++ b/tests/codexcrawler/components/testcrawler.nim @@ -0,0 +1,115 @@ +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import pkg/asynctest/chronos/unittest + +import ../../../codexcrawler/components/crawler +import ../../../codexcrawler/services/dht +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../../../codexcrawler/state +import ../mockstate +import ../mockdht +import ../mocktodolist +import ../helpers + +suite "Crawler": + var + nid1: Nid + nid2: Nid + state: MockState + todo: MockTodoList + dht: MockDht + crawler: Crawler + + setup: + nid1 = genNid() + nid2 = genNid() + state = createMockState() + todo = createMockTodoList() + dht = createMockDht() + + crawler = Crawler.new(state, dht, todo) + + (await crawler.start()).tryGet() + + teardown: + (await crawler.stop()).tryGet() + state.checkAllUnsubscribed() + + proc onStep() {.async.} = + (await state.stepper()).tryGet() + + proc responsive(nid: Nid): GetNeighborsResponse = + GetNeighborsResponse( + isResponsive: true, + nodeIds: @[nid] + ) + + proc unresponsive(nid: Nid): GetNeighborsResponse = + GetNeighborsResponse( + isResponsive: false, + nodeIds: @[nid] + ) + + test "onStep should pop a node from the todoList and getNeighbors for it": + todo.popReturn = success(nid1) + dht.getNeighborsReturn = success(responsive(nid1)) + + await onStep() + + check: + !(dht.getNeighborsArg) == nid1 + + test "nodes returned by getNeighbors are raised as nodesFound": + var nodesFound = newSeq[Nid]() + proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = + nodesFound = nids + return success() + let sub = state.events.nodesFound.subscribe(onNodesFound) + + todo.popReturn = success(nid1) + dht.getNeighborsReturn = success(responsive(nid2)) + + await onStep() + + check: + nid2 in nodesFound + + await state.events.nodesFound.unsubscribe(sub) + + test "responsive result from getNeighbors raises the node as successful dhtNodeCheck": + var checkEvent = DhtNodeCheckEventData() + proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} = + checkEvent = event + return success() + let sub = state.events.dhtNodeCheck.subscribe(onCheck) + + todo.popReturn = success(nid1) + dht.getNeighborsReturn = success(responsive(nid2)) + + await onStep() + + check: + checkEvent.id == nid1 + checkEvent.isOk == true + + await state.events.dhtNodeCheck.unsubscribe(sub) + + test "unresponsive result from getNeighbors raises the node as unsuccessful dhtNodeCheck": + var checkEvent = DhtNodeCheckEventData() + proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} = + checkEvent = event + return success() + let sub = state.events.dhtNodeCheck.subscribe(onCheck) + + todo.popReturn = success(nid1) + dht.getNeighborsReturn = success(unresponsive(nid2)) + + await onStep() + + check: + checkEvent.id == nid1 + checkEvent.isOk == false + + await state.events.dhtNodeCheck.unsubscribe(sub) diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim index 83f1fb2..72666ff 100644 --- a/tests/codexcrawler/components/testtimetracker.nim +++ b/tests/codexcrawler/components/testtimetracker.nim @@ -44,6 +44,9 @@ suite "TimeTracker": await state.events.nodesExpired.unsubscribe(sub) state.checkAllUnsubscribed() + proc onStep() {.async.} = + (await state.stepper()).tryGet() + proc createNodeInStore(lastVisit: uint64): Nid = let entry = NodeEntry(id: genNid(), lastVisit: lastVisit) store.nodesToIterate.add(entry) @@ -55,7 +58,7 @@ suite "TimeTracker": (Moment.now().epochSeconds - ((1 + state.config.revisitDelayMins) * 60)).uint64 expiredNodeId = createNodeInStore(expiredTimestamp) - (await state.stepper()).tryGet() + await onStep() check: expiredNodeId in expiredNodesReceived @@ -66,7 +69,7 @@ suite "TimeTracker": (Moment.now().epochSeconds - ((state.config.revisitDelayMins - 1) * 60)).uint64 recentNodeId = createNodeInStore(recentTimestamp) - (await state.stepper()).tryGet() + await onStep() check: recentNodeId notin expiredNodesReceived diff --git a/tests/codexcrawler/mockdht.nim b/tests/codexcrawler/mockdht.nim new file mode 100644 index 0000000..98232ad --- /dev/null +++ b/tests/codexcrawler/mockdht.nim @@ -0,0 +1,26 @@ +import pkg/chronos +import pkg/questionable +import pkg/questionable/results +import ../../codexcrawler/services/dht +import ../../codexcrawler/types + +type MockDht* = ref object of Dht + routingTable*: seq[Nid] + getNeighborsArg*: ?Nid + getNeighborsReturn*: ?!GetNeighborsResponse + +method getRoutingTableNodeIds*(d: MockDht): seq[Nid] = + return d.routingTable + +method getNeighbors*(d: MockDht, target: Nid): Future[?!GetNeighborsResponse] {.async: (raises: []).} = + d.getNeighborsArg = some(target) + return d.getNeighborsReturn + +method start*(d: MockDht): Future[?!void] {.async.} = + return success() + +method stop*(d: MockDht): Future[?!void] {.async.} = + return success() + +proc createMockDht*(): MockDht = + MockDht() diff --git a/tests/codexcrawler/mockmetrics.nim b/tests/codexcrawler/mockmetrics.nim index 021a8b0..16837b3 100644 --- a/tests/codexcrawler/mockmetrics.nim +++ b/tests/codexcrawler/mockmetrics.nim @@ -1,4 +1,4 @@ -import ../../codexcrawler/metrics +import ../../codexcrawler/services/metrics type MockMetrics* = ref object of Metrics todo*: int diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim index bfa7e6f..abc9594 100644 --- a/tests/codexcrawler/mockstate.nim +++ b/tests/codexcrawler/mockstate.nim @@ -7,6 +7,16 @@ import ../../codexcrawler/config type MockState* = ref object of State stepper*: OnStep +proc checkAllUnsubscribed*(s: MockState) = + check: + s.events.nodesFound.listeners == 0 + s.events.newNodesDiscovered.listeners == 0 + s.events.dhtNodeCheck.listeners == 0 + s.events.nodesExpired.listeners == 0 + +method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} = + s.stepper = step + proc createMockState*(): MockState = MockState( status: ApplicationStatus.Running, @@ -18,13 +28,3 @@ proc createMockState*(): MockState = nodesExpired: newAsyncDataEvent[seq[Nid]](), ), ) - -proc checkAllUnsubscribed*(s: MockState) = - check: - s.events.nodesFound.listeners == 0 - s.events.newNodesDiscovered.listeners == 0 - s.events.dhtNodeCheck.listeners == 0 - s.events.nodesExpired.listeners == 0 - -method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} = - s.stepper = step diff --git a/tests/codexcrawler/mocktodolist.nim b/tests/codexcrawler/mocktodolist.nim new file mode 100644 index 0000000..3ff9988 --- /dev/null +++ b/tests/codexcrawler/mocktodolist.nim @@ -0,0 +1,20 @@ +import pkg/chronos +import pkg/questionable/results + +import ../../codexcrawler/components/todolist +import ../../codexcrawler/types + +type MockTodoList* = ref object of TodoList + popReturn*: ?!Nid + +method pop*(t: MockTodoList): Future[?!Nid] {.async: (raises: []).} = + return t.popReturn + +method start*(t: MockTodoList): Future[?!void] {.async.} = + return success() + +method stop*(t: MockTodoList): Future[?!void] {.async.} = + return success() + +proc createMockTodoList*(): MockTodoList = + MockTodoList() diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim index 1b9a6b7..0773ec0 100644 --- a/tests/codexcrawler/testcomponents.nim +++ b/tests/codexcrawler/testcomponents.nim @@ -2,5 +2,6 @@ import ./components/testnodestore import ./components/testdhtmetrics import ./components/testtodolist import ./components/testtimetracker +import ./components/testcrawler {.warning[UnusedImport]: off.} From 605b561e30362ae81c8d945312e0660b82d20f11 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 13:50:12 +0100 Subject: [PATCH 16/20] applies delay at application start --- codexcrawler/components/crawler.nim | 23 ++++++----------- codexcrawler/installer.nim | 3 ++- codexcrawler/services/dht.nim | 25 +++++++++++++------ codexcrawler/state.nim | 11 ++++++-- tests/codexcrawler/components/testcrawler.nim | 15 +++++------ .../components/testtimetracker.nim | 2 +- tests/codexcrawler/mockdht.nim | 4 ++- tests/codexcrawler/mocktodolist.nim | 2 +- 8 files changed, 46 insertions(+), 39 deletions(-) diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/crawler.nim index 0f984a0..b4db6f0 100644 --- a/codexcrawler/components/crawler.nim +++ b/codexcrawler/components/crawler.nim @@ -19,11 +19,10 @@ type Crawler* = ref object of Component dht: Dht todo: TodoList -proc raiseCheckEvent(c: Crawler, nid: Nid, success: bool): Future[?!void] {.async: (raises: []).} = - let event = DhtNodeCheckEventData( - id: nid, - isOk: success - ) +proc raiseCheckEvent( + c: Crawler, nid: Nid, success: bool +): Future[?!void] {.async: (raises: []).} = + let event = DhtNodeCheckEventData(id: nid, isOk: success) if err =? (await c.state.events.dhtNodeCheck.fire(event)).errorOption: return failure(err) return success() @@ -48,6 +47,7 @@ method start*(c: Crawler): Future[?!void] {.async.} = proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} = await c.step() + await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds) return success() @@ -55,14 +55,5 @@ method start*(c: Crawler): Future[?!void] {.async.} = method stop*(c: Crawler): Future[?!void] {.async.} = return success() -proc new*( - T: type Crawler, - state: State, - dht: Dht, - todo: TodoList -): Crawler = - Crawler( - state: state, - dht: dht, - todo: todo - ) +proc new*(T: type Crawler, state: State, dht: Dht, todo: TodoList): Crawler = + Crawler(state: state, dht: dht, todo: todo) diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index e8a7fd5..38cf0d6 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -9,6 +9,7 @@ import ./components/crawler import ./components/timetracker import ./components/nodestore import ./components/dhtmetrics +import ./components/todolist proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = var components: seq[Component] = newSeq[Component]() @@ -32,5 +33,5 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = components.add(Crawler.new(state, dht, todoList)) components.add(TimeTracker.new(state, nodeStore)) components.add(dhtMetrics) - + return success(components) diff --git a/codexcrawler/services/dht.nim b/codexcrawler/services/dht.nim index c1c777c..345dd31 100644 --- a/codexcrawler/services/dht.nim +++ b/codexcrawler/services/dht.nim @@ -21,7 +21,7 @@ export discv5 logScope: topics = "dht" -type +type GetNeighborsResponse* = ref object isResponsive*: bool nodeIds*: seq[Nid] @@ -35,6 +35,12 @@ type providerRecord*: ?SignedPeerRecord dhtRecord*: ?SignedPeerRecord +proc responsive(nodeIds: seq[Nid]): GetNeighborsResponse = + GetNeighborsResponse(isResponsive: true, nodeIds: nodeIds) + +proc unresponsive(): GetNeighborsResponse = + GetNeighborsResponse(isResponsive: false, nodeIds: newSeq[Nid]()) + proc getNode*(d: Dht, nodeId: NodeId): ?!Node = let node = d.protocol.getNode(nodeId) if node.isSome(): @@ -48,9 +54,11 @@ method getRoutingTableNodeIds*(d: Dht): seq[Nid] {.base.} = ids.add(node.id) return ids -method getNeighbors*(d: Dht, target: Nid): Future[?!GetNeighborsResponse] {.async: (raises: []), base.} = +method getNeighbors*( + d: Dht, target: Nid +): Future[?!GetNeighborsResponse] {.async: (raises: []), base.} = without node =? d.getNode(target), err: - return failure(err) + return success(unresponsive()) let distances = @[256.uint16] try: @@ -58,11 +66,12 @@ method getNeighbors*(d: Dht, target: Nid): Future[?!GetNeighborsResponse] {.asyn if response.isOk(): let nodes = response.get() - return success(GetNeighborsResponse( - isResponsive: true, - nodeIds: nodes.mapIt(it.id)) - ) - return failure($response.error()) + return success(responsive(nodes.mapIt(it.id))) + else: + let errmsg = $(response.error()) + if errmsg == "Nodes message not received in time": + return success(unresponsive()) + return failure(errmsg) except CatchableError as exc: return failure(exc.msg) diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index 5394ef1..f4ca2e0 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -32,7 +32,9 @@ type config*: Config events*: Events -method whileRunning*(s: State, step: OnStep, delay: Duration) {.async, base.} = +proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async.} = + await sleepAsync(3.seconds) + proc worker(): Future[void] {.async.} = while s.status == ApplicationStatus.Running: if err =? (await step()).errorOption: @@ -40,5 +42,10 @@ method whileRunning*(s: State, step: OnStep, delay: Duration) {.async, base.} = s.status = ApplicationStatus.Stopping await sleepAsync(delay) - # todo this needs a delay because starts are still being called. asyncSpawn worker() + +method whileRunning*(s: State, step: OnStep, delay: Duration) {.async, base.} = + # We use a small delay before starting the workers because 'whileRunning' is likely called from + # component 'start' methods, which are executed sequentially in arbitrary order (to prevent temporal coupling). + # Worker steps might start raising events that other components haven't had time to subscribe to yet. + asyncSpawn s.delayedWorkerStart(step, delay) diff --git a/tests/codexcrawler/components/testcrawler.nim b/tests/codexcrawler/components/testcrawler.nim index 2ec2948..c2bbe8f 100644 --- a/tests/codexcrawler/components/testcrawler.nim +++ b/tests/codexcrawler/components/testcrawler.nim @@ -37,20 +37,14 @@ suite "Crawler": (await crawler.stop()).tryGet() state.checkAllUnsubscribed() - proc onStep() {.async.} = + proc onStep() {.async.} = (await state.stepper()).tryGet() proc responsive(nid: Nid): GetNeighborsResponse = - GetNeighborsResponse( - isResponsive: true, - nodeIds: @[nid] - ) + GetNeighborsResponse(isResponsive: true, nodeIds: @[nid]) proc unresponsive(nid: Nid): GetNeighborsResponse = - GetNeighborsResponse( - isResponsive: false, - nodeIds: @[nid] - ) + GetNeighborsResponse(isResponsive: false, nodeIds: @[nid]) test "onStep should pop a node from the todoList and getNeighbors for it": todo.popReturn = success(nid1) @@ -66,6 +60,7 @@ suite "Crawler": proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = nodesFound = nids return success() + let sub = state.events.nodesFound.subscribe(onNodesFound) todo.popReturn = success(nid1) @@ -83,6 +78,7 @@ suite "Crawler": proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} = checkEvent = event return success() + let sub = state.events.dhtNodeCheck.subscribe(onCheck) todo.popReturn = success(nid1) @@ -101,6 +97,7 @@ suite "Crawler": proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} = checkEvent = event return success() + let sub = state.events.dhtNodeCheck.subscribe(onCheck) todo.popReturn = success(nid1) diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim index 72666ff..587d1cb 100644 --- a/tests/codexcrawler/components/testtimetracker.nim +++ b/tests/codexcrawler/components/testtimetracker.nim @@ -44,7 +44,7 @@ suite "TimeTracker": await state.events.nodesExpired.unsubscribe(sub) state.checkAllUnsubscribed() - proc onStep() {.async.} = + proc onStep() {.async.} = (await state.stepper()).tryGet() proc createNodeInStore(lastVisit: uint64): Nid = diff --git a/tests/codexcrawler/mockdht.nim b/tests/codexcrawler/mockdht.nim index 98232ad..5aa821c 100644 --- a/tests/codexcrawler/mockdht.nim +++ b/tests/codexcrawler/mockdht.nim @@ -12,7 +12,9 @@ type MockDht* = ref object of Dht method getRoutingTableNodeIds*(d: MockDht): seq[Nid] = return d.routingTable -method getNeighbors*(d: MockDht, target: Nid): Future[?!GetNeighborsResponse] {.async: (raises: []).} = +method getNeighbors*( + d: MockDht, target: Nid +): Future[?!GetNeighborsResponse] {.async: (raises: []).} = d.getNeighborsArg = some(target) return d.getNeighborsReturn diff --git a/tests/codexcrawler/mocktodolist.nim b/tests/codexcrawler/mocktodolist.nim index 3ff9988..7623c89 100644 --- a/tests/codexcrawler/mocktodolist.nim +++ b/tests/codexcrawler/mocktodolist.nim @@ -6,7 +6,7 @@ import ../../codexcrawler/types type MockTodoList* = ref object of TodoList popReturn*: ?!Nid - + method pop*(t: MockTodoList): Future[?!Nid] {.async: (raises: []).} = return t.popReturn From 6574d53d5f40e775e02a15c26d6c892e286ba5fa Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 14:12:21 +0100 Subject: [PATCH 17/20] fixes app initialization --- codexcrawler/application.nim | 17 +++++++++++++---- codexcrawler/components/dhtmetrics.nim | 3 ++- codexcrawler/components/nodestore.nim | 7 +++++-- codexcrawler/components/timetracker.nim | 3 ++- codexcrawler/components/todolist.nim | 3 ++- codexcrawler/list.nim | 4 ++-- 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim index 3cd17cf..bbc0734 100644 --- a/codexcrawler/application.nim +++ b/codexcrawler/application.nim @@ -16,9 +16,10 @@ import ./types type Application* = ref object state: State + components: seq[Component] proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} = - let state = State( + app.state = State( status: ApplicationStatus.Running, config: config, events: Events( @@ -29,9 +30,10 @@ proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} = ), ) - without components =? (await createComponents(state)), err: + without components =? (await createComponents(app.state)), err: error "Failed to create componenents", err = err.msg return failure(err) + app.components = components for c in components: if err =? (await c.start()).errorOption: @@ -39,6 +41,11 @@ proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} = return success() +proc stopComponents(app: Application) {.async.} = + for c in app.components: + if err =? (await c.stop()).errorOption: + error "Failed to stop component", err = err.msg + proc stop*(app: Application) = app.state.status = ApplicationStatus.Stopping @@ -56,7 +63,6 @@ proc run*(app: Application) = info "Metrics endpoint initialized" info "Starting application" - app.state.status = ApplicationStatus.Running if err =? (waitFor app.initializeApp(config)).errorOption: app.state.status = ApplicationStatus.Stopping error "Failed to start application", err = err.msg @@ -68,4 +74,7 @@ proc run*(app: Application) = except Exception as exc: error "Unhandled exception", msg = exc.msg quit QuitFailure - notice "Application closed" + + notice "Application stopping..." + waitFor app.stopComponents() + notice "Application stopped" diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim index 092abde..46ed115 100644 --- a/codexcrawler/components/dhtmetrics.nim +++ b/codexcrawler/components/dhtmetrics.nim @@ -32,10 +32,11 @@ proc handleCheckEvent( d.metrics.setOkNodes(d.ok.len) d.metrics.setNokNodes(d.nok.len) + trace "metrics updated", ok = d.ok.len, nok = d.nok.len return success() method start*(d: DhtMetrics): Future[?!void] {.async.} = - info "Starting DhtMetrics..." + info "Starting..." ?await d.ok.load() ?await d.nok.load() diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index 51ae946..e303ced 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -14,6 +14,9 @@ import ../utils/asyncdataevent const nodestoreName = "nodestore" +logScope: + topics = "nodestore" + type NodeEntry* = object id*: Nid @@ -75,7 +78,6 @@ proc fireNewNodesDiscovered(s: NodeStore, nids: seq[Nid]): Future[?!void] {.asyn proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = var newNodes = newSeq[Nid]() - for nid in nids: without isNew =? (await s.storeNodeIsNew(nid)), err: return failure(err) @@ -83,6 +85,7 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = if isNew: newNodes.add(nid) + trace "Processed found nodes", total = nids.len, numNew = newNodes.len if newNodes.len > 0: ?await s.fireNewNodesDiscovered(newNodes) return success() @@ -109,7 +112,7 @@ method iterateAll*( return success() method start*(s: NodeStore): Future[?!void] {.async.} = - info "Starting nodestore..." + info "Starting..." proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = return await s.processFoundNodes(nids) diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index a89821a..d61a585 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -16,6 +16,7 @@ type TimeTracker* = ref object of Component nodestore: NodeStore proc step(t: TimeTracker): Future[?!void] {.async: (raises: []).} = + trace "Checking for expired nodes..." let expiry = (Moment.now().epochSeconds - (t.state.config.revisitDelayMins * 60)).uint64 @@ -30,7 +31,7 @@ proc step(t: TimeTracker): Future[?!void] {.async: (raises: []).} = return success() method start*(t: TimeTracker): Future[?!void] {.async.} = - info "Starting timetracker..." + info "Starting..." proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} = await t.step() diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim index 0d55ccb..19211b0 100644 --- a/codexcrawler/components/todolist.nim +++ b/codexcrawler/components/todolist.nim @@ -26,6 +26,7 @@ proc addNodes(t: TodoList, nids: seq[Nid]) = for nid in nids: t.nids.add(nid) + trace "Nodes added", nodes = nids.len if s =? t.emptySignal: s.complete() t.emptySignal = Future[void].none @@ -36,7 +37,7 @@ method pop*(t: TodoList): Future[?!Nid] {.async: (raises: []), base.} = let signal = newFuture[void]("list.emptySignal") t.emptySignal = some(signal) try: - await signal.wait(1.hours) + await signal.wait(InfiniteDuration) except CatchableError as exc: return failure(exc.msg) if t.nids.len < 1: diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index dd68314..d1985a4 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -70,8 +70,8 @@ method add*(this: List, nid: Nid): Future[?!void] {.async, base.} = return success() method remove*(this: List, nid: Nid): Future[?!void] {.async, base.} = - if this.items.len < 1: - return failure(this.name & "List is empty.") + if not this.contains(nid): + return success() this.items.excl(nid) without itemKey =? Key.init(this.name / $nid), err: From 82a1cd07155b1c6de4229cab246081cc28154780 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 14:25:54 +0100 Subject: [PATCH 18/20] makes timetracker periodically raise routingtable nodes --- codexcrawler/components/timetracker.nim | 21 +++++++++++++++--- codexcrawler/services/dht.nim | 12 +--------- codexcrawler/state.nim | 2 +- .../components/testtimetracker.nim | 22 ++++++++++++++++++- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index d61a585..afbfca0 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -3,6 +3,7 @@ import pkg/chronos import pkg/questionable/results import ./nodestore +import ../services/dht import ../component import ../state import ../types @@ -14,8 +15,9 @@ logScope: type TimeTracker* = ref object of Component state: State nodestore: NodeStore + dht: Dht -proc step(t: TimeTracker): Future[?!void] {.async: (raises: []).} = +proc checkForExpiredNodes(t: TimeTracker): Future[?!void] {.async: (raises: []).} = trace "Checking for expired nodes..." let expiry = (Moment.now().epochSeconds - (t.state.config.revisitDelayMins * 60)).uint64 @@ -30,6 +32,17 @@ proc step(t: TimeTracker): Future[?!void] {.async: (raises: []).} = ?await t.state.events.nodesExpired.fire(expired) return success() +proc raiseRoutingTableNodes(t: TimeTracker): Future[?!void] {.async: (raises: []).} = + let nids = t.dht.getRoutingTableNodeIds() + if err =? (await t.state.events.nodesFound.fire(nids)).errorOption: + return failure(err) + return success() + +proc step(t: TimeTracker): Future[?!void] {.async: (raises: []).} = + ?await t.checkForExpiredNodes() + ?await t.raiseRoutingTableNodes() + return success() + method start*(t: TimeTracker): Future[?!void] {.async.} = info "Starting..." @@ -46,5 +59,7 @@ method start*(t: TimeTracker): Future[?!void] {.async.} = method stop*(t: TimeTracker): Future[?!void] {.async.} = return success() -proc new*(T: type TimeTracker, state: State, nodestore: NodeStore): TimeTracker = - TimeTracker(state: state, nodestore: nodestore) +proc new*( + T: type TimeTracker, state: State, nodestore: NodeStore, dht: Dht +): TimeTracker = + TimeTracker(state: state, nodestore: nodestore, dht: dht) diff --git a/codexcrawler/services/dht.nim b/codexcrawler/services/dht.nim index 345dd31..e4726fc 100644 --- a/codexcrawler/services/dht.nim +++ b/codexcrawler/services/dht.nim @@ -47,7 +47,7 @@ proc getNode*(d: Dht, nodeId: NodeId): ?!Node = return success(node.get()) return failure("Node not found for id: " & nodeId.toHex()) -method getRoutingTableNodeIds*(d: Dht): seq[Nid] {.base.} = +method getRoutingTableNodeIds*(d: Dht): seq[Nid] {.base, gcsafe, raises: [].} = var ids = newSeq[Nid]() for bucket in d.protocol.routingTable.buckets: for node in bucket.nodes: @@ -109,19 +109,9 @@ proc updateDhtRecord(d: Dht, addrs: openArray[MultiAddress]) = if not d.protocol.isNil: d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR") -# proc findRoutingTableNodes(d: Dht) {.async.} = -# await sleepAsync(5.seconds) -# let nodes = d.getRoutingTableNodeIds() - -# if err =? (await d.state.events.nodesFound.fire(nodes)).errorOption: -# error "Failed to raise routing-table nodes as found nodes", err = err.msg -# else: -# trace "Routing table nodes raised as found nodes", num = nodes.len - method start*(d: Dht): Future[?!void] {.async.} = d.protocol.open() await d.protocol.start() - # asyncSpawn d.findRoutingTableNodes() return success() method stop*(d: Dht): Future[?!void] {.async.} = diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index f4ca2e0..d70dfb5 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -33,7 +33,7 @@ type events*: Events proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async.} = - await sleepAsync(3.seconds) + await sleepAsync(1.seconds) proc worker(): Future[void] {.async.} = while s.status == ApplicationStatus.Running: diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim index 587d1cb..7e29282 100644 --- a/tests/codexcrawler/components/testtimetracker.nim +++ b/tests/codexcrawler/components/testtimetracker.nim @@ -9,6 +9,7 @@ import ../../../codexcrawler/types import ../../../codexcrawler/state import ../mockstate import ../mocknodestore +import ../mockdht import ../helpers suite "TimeTracker": @@ -16,6 +17,7 @@ suite "TimeTracker": nid: Nid state: MockState store: MockNodeStore + dht: MockDht time: TimeTracker expiredNodesReceived: seq[Nid] sub: AsyncDataEventSubscription @@ -24,6 +26,7 @@ suite "TimeTracker": nid = genNid() state = createMockState() store = createMockNodeStore() + dht = createMockDht() # Subscribe to nodesExpired event expiredNodesReceived = newSeq[Nid]() @@ -35,7 +38,7 @@ suite "TimeTracker": state.config.revisitDelayMins = 22 - time = TimeTracker.new(state, store) + time = TimeTracker.new(state, store, dht) (await time.start()).tryGet() @@ -73,3 +76,20 @@ suite "TimeTracker": check: recentNodeId notin expiredNodesReceived + + test "onStep raises routingTable nodes as nodesFound": + var nodesFound = newSeq[Nid]() + proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} = + nodesFound = nids + return success() + + let sub = state.events.nodesFound.subscribe(onNodesFound) + + dht.routingTable.add(nid) + + await onStep() + + check: + nid in nodesFound + + await state.events.nodesFound.unsubscribe(sub) From 1a05ecc88f5a24a1132c589e464a193ac04ff76d Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 14:43:12 +0100 Subject: [PATCH 19/20] finished rework --- codexcrawler/components/dhtmetrics.nim | 9 +++++++-- codexcrawler/components/nodestore.nim | 3 +-- codexcrawler/components/timetracker.nim | 1 + codexcrawler/components/todolist.nim | 2 +- codexcrawler/config.nim | 4 ++-- codexcrawler/installer.nim | 4 ++-- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim index 46ed115..cbf2a9c 100644 --- a/codexcrawler/components/dhtmetrics.nim +++ b/codexcrawler/components/dhtmetrics.nim @@ -31,8 +31,6 @@ proc handleCheckEvent( d.metrics.setOkNodes(d.ok.len) d.metrics.setNokNodes(d.nok.len) - - trace "metrics updated", ok = d.ok.len, nok = d.nok.len return success() method start*(d: DhtMetrics): Future[?!void] {.async.} = @@ -44,6 +42,13 @@ method start*(d: DhtMetrics): Future[?!void] {.async.} = await d.handleCheckEvent(event) d.sub = d.state.events.dhtNodeCheck.subscribe(onCheck) + + proc logDhtMetrics(): Future[?!void] {.async: (raises: []), gcsafe.} = + trace "Metrics", ok = d.ok.len, nok = d.nok.len + return success() + + await d.state.whileRunning(logDhtMetrics, 1.minutes) + return success() method stop*(d: DhtMetrics): Future[?!void] {.async.} = diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index e303ced..1a7509e 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -81,12 +81,11 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = for nid in nids: without isNew =? (await s.storeNodeIsNew(nid)), err: return failure(err) - if isNew: newNodes.add(nid) - trace "Processed found nodes", total = nids.len, numNew = newNodes.len if newNodes.len > 0: + trace "Discovered new nodes", newNodes = newNodes.len ?await s.fireNewNodesDiscovered(newNodes) return success() diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim index afbfca0..67eb48c 100644 --- a/codexcrawler/components/timetracker.nim +++ b/codexcrawler/components/timetracker.nim @@ -33,6 +33,7 @@ proc checkForExpiredNodes(t: TimeTracker): Future[?!void] {.async: (raises: []). return success() proc raiseRoutingTableNodes(t: TimeTracker): Future[?!void] {.async: (raises: []).} = + trace "Raising routing table nodes..." let nids = t.dht.getRoutingTableNodeIds() if err =? (await t.state.events.nodesFound.fire(nids)).errorOption: return failure(err) diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim index 19211b0..f84e2c1 100644 --- a/codexcrawler/components/todolist.nim +++ b/codexcrawler/components/todolist.nim @@ -26,8 +26,8 @@ proc addNodes(t: TodoList, nids: seq[Nid]) = for nid in nids: t.nids.add(nid) - trace "Nodes added", nodes = nids.len if s =? t.emptySignal: + trace "Nodes added, resuming...", nodes = nids.len s.complete() t.emptySignal = Future[void].none diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index 01d9c5e..71e183b 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -14,14 +14,14 @@ Usage: Options: --logLevel= Sets log level [default: INFO] - --publicIp= Public IP address where this instance is reachable. [default: 45.82.185.194] + --publicIp= Public IP address where this instance is reachable. --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] --metricsPort=

Listen HTTP port of the metrics server [default: 8008] --dataDir=

Directory for storing data [default: crawler_data] --discoveryPort=

Port used for DHT [default: 8090] --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] --stepDelay= Delay in milliseconds per crawl step [default: 1000] - --revisitDelay= Delay in minutes after which a node can be revisited [default: 1] (24h) + --revisitDelay= Delay in minutes after which a node can be revisited [default: 10] (24h) """ import strutils diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index 38cf0d6..0dbd1fb 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -27,11 +27,11 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = without dhtMetrics =? createDhtMetrics(state, metrics), err: return failure(err) + components.add(dht) components.add(todoList) components.add(nodeStore) - components.add(dht) components.add(Crawler.new(state, dht, todoList)) - components.add(TimeTracker.new(state, nodeStore)) + components.add(TimeTracker.new(state, nodeStore, dht)) components.add(dhtMetrics) return success(components) From e4fb76f0b1183ba03a3a02159a81a566e962f88e Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Feb 2025 14:48:57 +0100 Subject: [PATCH 20/20] Moves mocks into mocks folder --- tests/codexcrawler/components/testcrawler.nim | 6 +++--- tests/codexcrawler/components/testdhtmetrics.nim | 6 +++--- tests/codexcrawler/components/testnodestore.nim | 2 +- tests/codexcrawler/components/testtimetracker.nim | 6 +++--- tests/codexcrawler/components/testtodolist.nim | 2 +- tests/codexcrawler/{ => mocks}/mockdht.nim | 4 ++-- tests/codexcrawler/{ => mocks}/mocklist.nim | 4 ++-- tests/codexcrawler/{ => mocks}/mockmetrics.nim | 2 +- tests/codexcrawler/{ => mocks}/mocknodestore.nim | 2 +- tests/codexcrawler/{ => mocks}/mockstate.nim | 8 ++++---- tests/codexcrawler/{ => mocks}/mocktodolist.nim | 4 ++-- 11 files changed, 23 insertions(+), 23 deletions(-) rename tests/codexcrawler/{ => mocks}/mockdht.nim (89%) rename tests/codexcrawler/{ => mocks}/mocklist.nim (92%) rename tests/codexcrawler/{ => mocks}/mockmetrics.nim (88%) rename tests/codexcrawler/{ => mocks}/mocknodestore.nim (92%) rename tests/codexcrawler/{ => mocks}/mockstate.nim (83%) rename tests/codexcrawler/{ => mocks}/mocktodolist.nim (83%) diff --git a/tests/codexcrawler/components/testcrawler.nim b/tests/codexcrawler/components/testcrawler.nim index c2bbe8f..25dfd53 100644 --- a/tests/codexcrawler/components/testcrawler.nim +++ b/tests/codexcrawler/components/testcrawler.nim @@ -8,9 +8,9 @@ import ../../../codexcrawler/services/dht import ../../../codexcrawler/utils/asyncdataevent import ../../../codexcrawler/types import ../../../codexcrawler/state -import ../mockstate -import ../mockdht -import ../mocktodolist +import ../mocks/mockstate +import ../mocks/mockdht +import ../mocks/mocktodolist import ../helpers suite "Crawler": diff --git a/tests/codexcrawler/components/testdhtmetrics.nim b/tests/codexcrawler/components/testdhtmetrics.nim index f3c5f3c..1620c87 100644 --- a/tests/codexcrawler/components/testdhtmetrics.nim +++ b/tests/codexcrawler/components/testdhtmetrics.nim @@ -6,9 +6,9 @@ import ../../../codexcrawler/components/dhtmetrics import ../../../codexcrawler/utils/asyncdataevent import ../../../codexcrawler/types import ../../../codexcrawler/state -import ../mockstate -import ../mocklist -import ../mockmetrics +import ../mocks/mockstate +import ../mocks/mocklist +import ../mocks/mockmetrics import ../helpers suite "DhtMetrics": diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim index 47eed00..79f9a38 100644 --- a/tests/codexcrawler/components/testnodestore.nim +++ b/tests/codexcrawler/components/testnodestore.nim @@ -8,7 +8,7 @@ import ../../../codexcrawler/components/nodestore import ../../../codexcrawler/utils/datastoreutils import ../../../codexcrawler/utils/asyncdataevent import ../../../codexcrawler/types -import ../mockstate +import ../mocks/mockstate import ../helpers suite "Nodestore": diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim index 7e29282..34873c5 100644 --- a/tests/codexcrawler/components/testtimetracker.nim +++ b/tests/codexcrawler/components/testtimetracker.nim @@ -7,9 +7,9 @@ import ../../../codexcrawler/components/nodestore import ../../../codexcrawler/utils/asyncdataevent import ../../../codexcrawler/types import ../../../codexcrawler/state -import ../mockstate -import ../mocknodestore -import ../mockdht +import ../mocks/mockstate +import ../mocks/mocknodestore +import ../mocks/mockdht import ../helpers suite "TimeTracker": diff --git a/tests/codexcrawler/components/testtodolist.nim b/tests/codexcrawler/components/testtodolist.nim index 8960859..7aa30df 100644 --- a/tests/codexcrawler/components/testtodolist.nim +++ b/tests/codexcrawler/components/testtodolist.nim @@ -6,7 +6,7 @@ import ../../../codexcrawler/components/todolist import ../../../codexcrawler/utils/asyncdataevent import ../../../codexcrawler/types import ../../../codexcrawler/state -import ../mockstate +import ../mocks/mockstate import ../helpers suite "TodoList": diff --git a/tests/codexcrawler/mockdht.nim b/tests/codexcrawler/mocks/mockdht.nim similarity index 89% rename from tests/codexcrawler/mockdht.nim rename to tests/codexcrawler/mocks/mockdht.nim index 5aa821c..4c12f33 100644 --- a/tests/codexcrawler/mockdht.nim +++ b/tests/codexcrawler/mocks/mockdht.nim @@ -1,8 +1,8 @@ import pkg/chronos import pkg/questionable import pkg/questionable/results -import ../../codexcrawler/services/dht -import ../../codexcrawler/types +import ../../../codexcrawler/services/dht +import ../../../codexcrawler/types type MockDht* = ref object of Dht routingTable*: seq[Nid] diff --git a/tests/codexcrawler/mocklist.nim b/tests/codexcrawler/mocks/mocklist.nim similarity index 92% rename from tests/codexcrawler/mocklist.nim rename to tests/codexcrawler/mocks/mocklist.nim index e077a0c..dfbd414 100644 --- a/tests/codexcrawler/mocklist.nim +++ b/tests/codexcrawler/mocks/mocklist.nim @@ -1,8 +1,8 @@ import pkg/chronos import pkg/questionable/results -import ../../codexcrawler/types -import ../../codexcrawler/list +import ../../../codexcrawler/types +import ../../../codexcrawler/list type MockList* = ref object of List loadCalled*: bool diff --git a/tests/codexcrawler/mockmetrics.nim b/tests/codexcrawler/mocks/mockmetrics.nim similarity index 88% rename from tests/codexcrawler/mockmetrics.nim rename to tests/codexcrawler/mocks/mockmetrics.nim index 16837b3..8152e76 100644 --- a/tests/codexcrawler/mockmetrics.nim +++ b/tests/codexcrawler/mocks/mockmetrics.nim @@ -1,4 +1,4 @@ -import ../../codexcrawler/services/metrics +import ../../../codexcrawler/services/metrics type MockMetrics* = ref object of Metrics todo*: int diff --git a/tests/codexcrawler/mocknodestore.nim b/tests/codexcrawler/mocks/mocknodestore.nim similarity index 92% rename from tests/codexcrawler/mocknodestore.nim rename to tests/codexcrawler/mocks/mocknodestore.nim index eb3ec19..d640f38 100644 --- a/tests/codexcrawler/mocknodestore.nim +++ b/tests/codexcrawler/mocks/mocknodestore.nim @@ -2,7 +2,7 @@ import std/sequtils import pkg/questionable/results import pkg/chronos -import ../../codexcrawler/components/nodestore +import ../../../codexcrawler/components/nodestore type MockNodeStore* = ref object of NodeStore nodesToIterate*: seq[NodeEntry] diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim similarity index 83% rename from tests/codexcrawler/mockstate.nim rename to tests/codexcrawler/mocks/mockstate.nim index abc9594..f7d212b 100644 --- a/tests/codexcrawler/mockstate.nim +++ b/tests/codexcrawler/mocks/mockstate.nim @@ -1,8 +1,8 @@ import pkg/asynctest/chronos/unittest -import ../../codexcrawler/state -import ../../codexcrawler/utils/asyncdataevent -import ../../codexcrawler/types -import ../../codexcrawler/config +import ../../../codexcrawler/state +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../../../codexcrawler/config type MockState* = ref object of State stepper*: OnStep diff --git a/tests/codexcrawler/mocktodolist.nim b/tests/codexcrawler/mocks/mocktodolist.nim similarity index 83% rename from tests/codexcrawler/mocktodolist.nim rename to tests/codexcrawler/mocks/mocktodolist.nim index 7623c89..1417ae4 100644 --- a/tests/codexcrawler/mocktodolist.nim +++ b/tests/codexcrawler/mocks/mocktodolist.nim @@ -1,8 +1,8 @@ import pkg/chronos import pkg/questionable/results -import ../../codexcrawler/components/todolist -import ../../codexcrawler/types +import ../../../codexcrawler/components/todolist +import ../../../codexcrawler/types type MockTodoList* = ref object of TodoList popReturn*: ?!Nid