Tests for nodestore

codexcrawler/components/nodestore.nim

@@ -12,9 +12,11 @@ import ../utils/datastoreutils
 import ../utils/asyncdataevent
 
 type
-  NodeEntry = object
-    id: Nid
-    lastVisit: uint64
+  OnNodeId = proc(item: Nid): Future[?!void] {.async: (raises: []), gcsafe.}
+
+  NodeEntry* = object
+    id*: Nid
+    lastVisit*: uint64
 
   NodeStore* = ref object of Component
     state: State
@@ -45,11 +47,26 @@ proc fromBytes*(_: type NodeEntry, data: openArray[byte]): ?!NodeEntry =
 
   return success(NodeEntry(id: Nid.fromStr(idStr), lastVisit: lastVisit))
 
+proc encode*(e: NodeEntry): seq[byte] =
+  e.toBytes()
+
+proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T =
+  if bytes.len < 1:
+    return success(NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64))
+  return NodeEntry.fromBytes(bytes)
+
 proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} =
   # put the nodes in the store.
   # track all new ones, if any, raise newNodes event.
   return success()
 
+proc iterateAll*(s: NodeStore, onNodeId: OnNodeId) {.async.} =
+  discard
+  # query iterator, yield items to callback.
+  # for item in this.items:
+  #   onItem(item)
+  #   await sleepAsync(1.millis)
+
 method start*(s: NodeStore): Future[?!void] {.async.} =
   info "Starting nodestore..."
 
@@ -63,9 +80,19 @@ method stop*(s: NodeStore): Future[?!void] {.async.} =
   await s.state.events.nodesFound.unsubscribe(s.sub)
   return success()
 
+proc new*(
+    T: type NodeStore,
+    state: State,
+    store: TypedDatastore
+): NodeStore =
+  NodeStore(
+    state: state,
+    store: store
+  )
+
 proc createNodeStore*(state: State): ?!NodeStore =
   without ds =? createTypedDatastore(state.config.dataDir / "nodestore"), err:
     error "Failed to create typed datastore for node store", err = err.msg
     return failure(err)
 
-  return success(NodeStore(state: state, store: ds))
+  return success(NodeStore.new(state, ds))

codexcrawler/list.nim

@@ -20,7 +20,6 @@ logScope:
 
 type
   OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].}
-  OnItem = proc(item: Nid): void {.gcsafe, raises: [].}
 
   List* = ref object
     name: string
@@ -119,7 +118,3 @@ proc pop*(this: List): Future[?!Nid] {.async.} =
 proc len*(this: List): int =
   this.items.len
 
-proc iterateAll*(this: List, onItem: OnItem) {.async.} =
-  for item in this.items:
-    onItem(item)
-    await sleepAsync(1.millis)

codexcrawler/state.nim

@@ -18,7 +18,7 @@ type
     dhtNodeCheck*: AsyncDataEvent[DhtNodeCheckEventData]
     nodesExpired*: AsyncDataEvent[seq[Nid]]
 
-  State* = ref object
+  State* = ref object of RootObj
     config*: Config
     events*: Events
 

codexcrawler/types.nim

@@ -1,5 +1,5 @@
 import pkg/stew/byteutils
-import pkg/stew/endians2
+import pkg/stint/io
 import pkg/questionable/results
 import pkg/codexdht
 import pkg/libp2p
@@ -7,7 +7,7 @@ import pkg/libp2p
 type Nid* = NodeId
 
 proc `$`*(nid: Nid): string =
-  $(NodeId(nid))
+  nid.toHex()
 
 proc fromStr*(T: type Nid, s: string): Nid =
   Nid(UInt256.fromHex(s))

tests/codexcrawler/components/testnodestore.nim

@@ -0,0 +1,127 @@
+import std/os
+import pkg/chronos
+import pkg/questionable/results
+import pkg/asynctest/chronos/unittest
+import pkg/datastore/typedds
+
+import ../../../codexcrawler/components/nodestore
+import ../../../codexcrawler/utils/datastoreutils
+import ../../../codexcrawler/utils/asyncdataevent
+import ../../../codexcrawler/types
+import ../mockstate
+import ../helpers
+
+suite "Nodestore":
+  let
+    dsPath = getTempDir() / "testds"
+    nodestoreName = "nodestore"
+
+  var 
+    ds: TypedDatastore
+    state: MockState
+    store: NodeStore
+
+  setup:
+    ds = createTypedDatastore(dsPath).tryGet()
+    state = createMockState()
+
+    store = NodeStore.new(
+      state, ds
+    )
+
+  teardown:
+    (await ds.close()).tryGet()
+    # state.cleanupMock()
+    removeDir(dsPath)
+
+  test "nodeEntry encoding":
+    let entry = NodeEntry(
+      id: genNid(),
+      lastVisit: 123.uint64
+    )
+
+    let
+      bytes = entry.encode()
+      decoded = NodeEntry.decode(bytes).tryGet()
+
+    check:
+      entry.id == decoded.id
+      entry.lastVisit == decoded.lastVisit
+
+  test "nodesFound event should store nodes":
+    let 
+      nid = genNid()
+      expectedKey = Key.init(nodestoreName / $nid).tryGet()
+
+    (await state.events.nodesFound.fire(@[nid])).tryGet()
+
+    check:
+      (await ds.has(expectedKey)).tryGet()
+    
+    let entry = (await get[NodeEntry](ds, expectedKey)).tryGet()
+    check:
+      entry.id == nid
+
+  test "nodesFound event should fire newNodesDiscovered":
+    var newNodes = newSeq[Nid]()
+    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} =
+      newNodes = nids
+      return success()
+
+    let 
+      sub = state.events.newNodesDiscovered.subscribe(onNewNodes)
+      nid = genNid()
+
+    (await state.events.nodesFound.fire(@[nid])).tryGet()
+
+    check:
+      newNodes == @[nid]
+
+    await state.events.newNodesDiscovered.unsubscribe(sub)
+    
+  test "nodesFound event should not fire newNodesDiscovered for previously seen nodes":
+    let 
+      nid = genNid()
+
+    # Make nid known first. Then subscribe.
+    (await state.events.nodesFound.fire(@[nid])).tryGet()
+
+    var
+      newNodes = newSeq[Nid]()
+      count = 0
+    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} =
+      newNodes = nids
+      inc count
+      return success()
+
+    let 
+      sub = state.events.newNodesDiscovered.subscribe(onNewNodes)
+      
+    # Firing the event again should not trigger newNodesDiscovered for nid
+    (await state.events.nodesFound.fire(@[nid])).tryGet()
+
+    check:
+      newNodes.len == 0
+      count == 0
+
+    await state.events.newNodesDiscovered.unsubscribe(sub)
+
+  test "iterateAll yields all known nids":
+    let 
+      nid1 = genNid()
+      nid2 = genNid()
+      nid3 = genNid()
+      
+    (await state.events.nodesFound.fire(@[nid1, nid2, nid3])).tryGet()
+
+    var iterNodes = newSeq[Nid]()
+    proc onNodeId(nid: Nid): Future[?!void] {.async: (raises: []), gcsafe.} =
+      iterNodes.add(nid)
+      return success()
+
+    await store.iterateAll(onNodeId)
+
+    check:
+      nid1 in iterNodes
+      nid2 in iterNodes
+      nid3 in iterNodes

tests/codexcrawler/helpers.nim

@@ -0,0 +1,6 @@
+import std/random
+import pkg/stint
+import ../../codexcrawler/types
+
+proc genNid*(): Nid =
+  Nid(rand(uint64).u256)

tests/codexcrawler/mockstate.nim

@@ -0,0 +1,24 @@
+import ../../codexcrawler/state
+import ../../codexcrawler/utils/asyncdataevent
+import ../../codexcrawler/types
+import ../../codexcrawler/config
+
+type
+  MockState* = ref object of State
+    # config*: Config
+    # events*: Events
+
+
+proc createMockState*(): MockState =
+  MockState(
+    config: Config(),
+    events: Events(
+      nodesFound: newAsyncDataEvent[seq[Nid]](),
+      newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),
+      dhtNodeCheck: newAsyncDataEvent[DhtNodeCheckEventData](),
+      nodesExpired: newAsyncDataEvent[seq[Nid]](),
+    ),
+  )
+
+proc cleanupMock*(this: MockState) =
+  discard

tests/codexcrawler/testcomponents.nim

@@ -0,0 +1,3 @@
+import ./components/testnodestore
+
+{.warning[UnusedImport]: off.}

tests/codexcrawler/testtypes.nim

@@ -0,0 +1,25 @@
+import pkg/chronos
+import pkg/asynctest/chronos/unittest
+import pkg/questionable/results
+
+import ../../codexcrawler/types
+import ./helpers
+
+suite "Types":
+  test "nid string encoding":
+    let
+      nid = genNid()
+      str = $nid
+
+    check:
+      nid == Nid.fromStr(str)
+
+  test "nid byte encoding":
+    let
+      nid = genNid()
+      bytes = nid.toBytes()
+
+    check:
+      nid == Nid.fromBytes(bytes).tryGet()
+      
+    

tests/test.nim

@@ -1,3 +1,5 @@
 import ./codexcrawler/testutils
+import ./codexcrawler/testcomponents
+import ./codexcrawler/testtypes
 
 {.warning[UnusedImport]: off.}