adds old-check to chain metrics entry handling

codexcrawler/components/chainmetrics.nim

@@ -6,6 +6,7 @@ import pkg/questionable/results
 import ../state
 import ../services/metrics
 import ../services/marketplace
+import ../services/clock
 import ../components/requeststore
 import ../component
 import ../types
@@ -19,12 +20,17 @@ type
     metrics: Metrics
     store: RequestStore
     marketplace: MarketplaceService
+    clock: Clock
 
   Update = ref object
     numRequests: int
     numSlots: int
     totalSize: int64
 
+proc isOld(c: ChainMetrics, entry: RequestEntry): bool =
+  let oneDay = 60 * 60 * 24
+  return entry.lastSeen < (c.clock.now - oneDay.uint64)
+
 proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
   var update = Update(numRequests: 0, numSlots: 0, totalSize: 0)
 
@@ -35,7 +41,8 @@ proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
       update.numSlots += info.slots.int
       update.totalSize += (info.slots * info.slotSize).int64
     else:
-      ?await c.store.remove(entry.id)
+      if c.isOld(entry):
+        ?await c.store.remove(entry.id)
     return success()
 
   ?await c.store.iterateAll(onRequest)
@@ -73,5 +80,6 @@ proc new*(
     metrics: Metrics,
     store: RequestStore,
     marketplace: MarketplaceService,
+    clock: Clock
 ): ChainMetrics =
-  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace)
+  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace, clock: clock)

codexcrawler/installer.nim

@@ -15,6 +15,8 @@ import ./components/nodestore
 import ./components/dhtmetrics
 import ./components/todolist
 import ./components/chainmetrics
+import ./components/chaincrawler
+import ./components/requeststore
 
 proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   var components: seq[Component] = newSeq[Component]()
@@ -26,11 +28,14 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   without nodeStore =? createNodeStore(state, clock), err:
     return failure(err)
 
+  without requestStore =? createRequestStore(state, clock), err:
+    return failure(err)
+
   let
     metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort)
     todoList = createTodoList(state, metrics)
     marketplace = createMarketplace(state, clock)
-    chainMetrics = ChainMetrics.new(state, metrics, marketplace)
+    chainMetrics = ChainMetrics.new(state, metrics, requestStore, marketplace, clock)
 
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
     return failure(err)
@@ -43,5 +48,6 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   components.add(dhtMetrics)
   components.add(marketplace)
   components.add(chainMetrics)
+  components.add(ChainCrawler.new(state, requestStore, marketplace))
 
   return success(components)

tests/codexcrawler/components/testchainmetrics.nim

@@ -11,6 +11,7 @@ import ../mocks/mockstate
 import ../mocks/mockmetrics
 import ../mocks/mockrequeststore
 import ../mocks/mockmarketplace
+import ../mocks/mockclock
 import ../helpers
 
 suite "ChainMetrics":
@@ -19,6 +20,7 @@ suite "ChainMetrics":
     metrics: MockMetrics
     store: MockRequestStore
     marketplace: MockMarketplaceService
+    clock: MockClock
     chain: ChainMetrics
 
   setup:
@@ -26,8 +28,9 @@ suite "ChainMetrics":
     metrics = createMockMetrics()
     store = createMockRequestStore()
     marketplace = createMockMarketplaceService()
+    clock = createMockClock()
 
-    chain = ChainMetrics.new(state, metrics, store, marketplace)
+    chain = ChainMetrics.new(state, metrics, store, marketplace, clock)
 
     (await chain.start()).tryGet()
 
@@ -43,10 +46,12 @@ suite "ChainMetrics":
       state.delays.len == 1
       state.delays[0] == 10.minutes
 
-  test "onStep should remove non-running requests from request store":
+  test "onStep should remove old non-running requests from request store":
     let rid = genRid()
-    store.iterateEntries.add(RequestEntry(id: rid))
+    let oneDay = (60 * 60 * 24).uint64
+    store.iterateEntries.add(RequestEntry(id: rid, lastSeen: 100.uint64))
 
+    clock.setNow = 100 + oneDay + 1
     marketplace.requestInfoReturns = none(RequestInfo)
 
     await onStep()
@@ -55,6 +60,20 @@ suite "ChainMetrics":
       marketplace.requestInfoRid == rid
       store.removeRid == rid
 
+  test "onStep should not remove recent non-running requests from request store":
+    let rid = genRid()
+    let now = 123456789.uint64
+    store.iterateEntries.add(RequestEntry(id: rid, lastSeen: now - 1))
+
+    clock.setNow = now
+    marketplace.requestInfoReturns = none(RequestInfo)
+
+    await onStep()
+
+    check:
+      marketplace.requestInfoRid == rid
+      not (store.removeRid == rid)
+
   test "onStep should count the number of active requests":
     let rid1 = genRid()
     let rid2 = genRid()