diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim index 8d0b042..1eeb0d8 100644 --- a/codexcrawler/components/nodestore.nim +++ b/codexcrawler/components/nodestore.nim @@ -12,9 +12,11 @@ import ../utils/datastoreutils import ../utils/asyncdataevent type - NodeEntry = object - id: Nid - lastVisit: uint64 + OnNodeId = proc(item: Nid): Future[?!void] {.async: (raises: []), gcsafe.} + + NodeEntry* = object + id*: Nid + lastVisit*: uint64 NodeStore* = ref object of Component state: State @@ -45,11 +47,26 @@ proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry = return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit)) +proc encode*(e: NodeEntry): seq[byte] = + e.toBytes() + +proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T = + if bytes.len < 1: + return success(NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64)) + return NodeEntry.fromBytes(bytes) + proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} = # put the nodes in the store. # track all new ones, if any, raise newNodes event. return success() +proc iterateAll*(s: NodeStore, onNodeId: OnNodeId) {.async.} = + discard + # query iterator, yield items to callback. + # for item in this.items: + # onItem(item) + # await sleepAsync(1.millis) + method start*(s: NodeStore): Future[?!void] {.async.} = info "Starting nodestore..." @@ -63,9 +80,19 @@ method stop*(s: NodeStore): Future[?!void] {.async.} = await s.state.events.nodesFound.unsubscribe(s.sub) return success() +proc new*( + T: type NodeStore, + state: State, + store: TypedDatastore +): NodeStore = + NodeStore( + state: state, + store: store + ) + proc createNodeStore*(state: State): ?!NodeStore = without ds =? createTypedDatastore(state.config.dataDir / "nodestore"), err: error "Failed to create typed datastore for node store", err = err.msg return failure(err) - return success(NodeStore(state: state, store: ds)) + return success(NodeStore.new(state, ds)) diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim index aebebe2..638042b 100644 --- a/codexcrawler/list.nim +++ b/codexcrawler/list.nim @@ -20,7 +20,6 @@ logScope: type OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].} - OnItem = proc(item: Nid): void {.gcsafe, raises: [].} List* = ref object name: string @@ -119,7 +118,3 @@ proc pop*(this: List): Future[?!Nid] {.async.} = proc len*(this: List): int = this.items.len -proc iterateAll*(this: List, onItem: OnItem) {.async.} = - for item in this.items: - onItem(item) - await sleepAsync(1.millis) diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim index ead6430..9d8d4be 100644 --- a/codexcrawler/state.nim +++ b/codexcrawler/state.nim @@ -18,7 +18,7 @@ type dhtNodeCheck*: AsyncDataEvent[DhtNodeCheckEventData] nodesExpired*: AsyncDataEvent[seq[Nid]] - State* = ref object + State* = ref object of RootObj config*: Config events*: Events diff --git a/codexcrawler/types.nim b/codexcrawler/types.nim index 1f1d2d9..9895ed1 100644 --- a/codexcrawler/types.nim +++ b/codexcrawler/types.nim @@ -1,5 +1,5 @@ import pkg/stew/byteutils -import pkg/stew/endians2 +import pkg/stint/io import pkg/questionable/results import pkg/codexdht import pkg/libp2p @@ -7,7 +7,7 @@ import pkg/libp2p type Nid* = NodeId proc `$`*(nid: Nid): string = - $(NodeId(nid)) + nid.toHex() proc fromStr*(T: type Nid, s: string): Nid = Nid(UInt256.fromHex(s)) diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim new file mode 100644 index 0000000..2034e9f --- /dev/null +++ b/tests/codexcrawler/components/testnodestore.nim @@ -0,0 +1,127 @@ +import std/os +import pkg/chronos +import pkg/questionable/results +import pkg/asynctest/chronos/unittest +import pkg/datastore/typedds + +import ../../../codexcrawler/components/nodestore +import ../../../codexcrawler/utils/datastoreutils +import ../../../codexcrawler/utils/asyncdataevent +import ../../../codexcrawler/types +import ../mockstate +import ../helpers + +suite "Nodestore": + let + dsPath = getTempDir() / "testds" + nodestoreName = "nodestore" + + var + ds: TypedDatastore + state: MockState + store: NodeStore + + setup: + ds = createTypedDatastore(dsPath).tryGet() + state = createMockState() + + store = NodeStore.new( + state, ds + ) + + teardown: + (await ds.close()).tryGet() + # state.cleanupMock() + removeDir(dsPath) + + test "nodeEntry encoding": + let entry = NodeEntry( + id: genNid(), + lastVisit: 123.uint64 + ) + + let + bytes = entry.encode() + decoded = NodeEntry.decode(bytes).tryGet() + + check: + entry.id == decoded.id + entry.lastVisit == decoded.lastVisit + + test "nodesFound event should store nodes": + let + nid = genNid() + expectedKey = Key.init(nodestoreName / $nid).tryGet() + + (await state.events.nodesFound.fire(@[nid])).tryGet() + + check: + (await ds.has(expectedKey)).tryGet() + + let entry = (await get[NodeEntry](ds, expectedKey)).tryGet() + check: + entry.id == nid + + test "nodesFound event should fire newNodesDiscovered": + var newNodes = newSeq[Nid]() + proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} = + newNodes = nids + return success() + + let + sub = state.events.newNodesDiscovered.subscribe(onNewNodes) + nid = genNid() + + (await state.events.nodesFound.fire(@[nid])).tryGet() + + check: + newNodes == @[nid] + + await state.events.newNodesDiscovered.unsubscribe(sub) + + test "nodesFound event should not fire newNodesDiscovered for previously seen nodes": + let + nid = genNid() + + # Make nid known first. Then subscribe. + (await state.events.nodesFound.fire(@[nid])).tryGet() + + var + newNodes = newSeq[Nid]() + count = 0 + proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} = + newNodes = nids + inc count + return success() + + let + sub = state.events.newNodesDiscovered.subscribe(onNewNodes) + + # Firing the event again should not trigger newNodesDiscovered for nid + (await state.events.nodesFound.fire(@[nid])).tryGet() + + check: + newNodes.len == 0 + count == 0 + + await state.events.newNodesDiscovered.unsubscribe(sub) + + test "iterateAll yields all known nids": + let + nid1 = genNid() + nid2 = genNid() + nid3 = genNid() + + (await state.events.nodesFound.fire(@[nid1, nid2, nid3])).tryGet() + + var iterNodes = newSeq[Nid]() + proc onNodeId(nid: Nid): Future[?!void] {.async: (raises: []), gcsafe.} = + iterNodes.add(nid) + return success() + + await store.iterateAll(onNodeId) + + check: + nid1 in iterNodes + nid2 in iterNodes + nid3 in iterNodes diff --git a/tests/codexcrawler/helpers.nim b/tests/codexcrawler/helpers.nim new file mode 100644 index 0000000..5feb345 --- /dev/null +++ b/tests/codexcrawler/helpers.nim @@ -0,0 +1,6 @@ +import std/random +import pkg/stint +import ../../codexcrawler/types + +proc genNid*(): Nid = + Nid(rand(uint64).u256) diff --git a/tests/codexcrawler/mockstate.nim b/tests/codexcrawler/mockstate.nim new file mode 100644 index 0000000..af302b7 --- /dev/null +++ b/tests/codexcrawler/mockstate.nim @@ -0,0 +1,24 @@ +import ../../codexcrawler/state +import ../../codexcrawler/utils/asyncdataevent +import ../../codexcrawler/types +import ../../codexcrawler/config + +type + MockState* = ref object of State + # config*: Config + # events*: Events + + +proc createMockState*(): MockState = + MockState( + config: Config(), + events: Events( + nodesFound: newAsyncDataEvent[seq[Nid]](), + newNodesDiscovered: newAsyncDataEvent[seq[Nid]](), + dhtNodeCheck: newAsyncDataEvent[DhtNodeCheckEventData](), + nodesExpired: newAsyncDataEvent[seq[Nid]](), + ), + ) + +proc cleanupMock*(this: MockState) = + discard diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim new file mode 100644 index 0000000..1363c5a --- /dev/null +++ b/tests/codexcrawler/testcomponents.nim @@ -0,0 +1,3 @@ +import ./components/testnodestore + +{.warning[UnusedImport]: off.} diff --git a/tests/codexcrawler/testtypes.nim b/tests/codexcrawler/testtypes.nim new file mode 100644 index 0000000..7332651 --- /dev/null +++ b/tests/codexcrawler/testtypes.nim @@ -0,0 +1,25 @@ +import pkg/chronos +import pkg/asynctest/chronos/unittest +import pkg/questionable/results + +import ../../codexcrawler/types +import ./helpers + +suite "Types": + test "nid string encoding": + let + nid = genNid() + str = $nid + + check: + nid == Nid.fromStr(str) + + test "nid byte encoding": + let + nid = genNid() + bytes = nid.toBytes() + + check: + nid == Nid.fromBytes(bytes).tryGet() + + \ No newline at end of file diff --git a/tests/test.nim b/tests/test.nim index ca1588f..a310fc6 100644 --- a/tests/test.nim +++ b/tests/test.nim @@ -1,3 +1,5 @@ import ./codexcrawler/testutils +import ./codexcrawler/testcomponents +import ./codexcrawler/testtypes {.warning[UnusedImport]: off.}