diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim index 8f8f899..80c6d3b 100644 --- a/codexcrawler/components/chainmetrics.nim +++ b/codexcrawler/components/chainmetrics.nim @@ -6,6 +6,7 @@ import pkg/questionable/results import ../state import ../services/metrics import ../services/marketplace +import ../services/clock import ../components/requeststore import ../component import ../types @@ -19,12 +20,17 @@ type metrics: Metrics store: RequestStore marketplace: MarketplaceService + clock: Clock Update = ref object numRequests: int numSlots: int totalSize: int64 +proc isOld(c: ChainMetrics, entry: RequestEntry): bool = + let oneDay = 60 * 60 * 24 + return entry.lastSeen < (c.clock.now - oneDay.uint64) + proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} = var update = Update(numRequests: 0, numSlots: 0, totalSize: 0) @@ -35,7 +41,8 @@ proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} = update.numSlots += info.slots.int update.totalSize += (info.slots * info.slotSize).int64 else: - ?await c.store.remove(entry.id) + if c.isOld(entry): + ?await c.store.remove(entry.id) return success() ?await c.store.iterateAll(onRequest) @@ -73,5 +80,6 @@ proc new*( metrics: Metrics, store: RequestStore, marketplace: MarketplaceService, + clock: Clock ): ChainMetrics = - ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace) + ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace, clock: clock) diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim index fbaf172..bdff348 100644 --- a/codexcrawler/installer.nim +++ b/codexcrawler/installer.nim @@ -15,6 +15,8 @@ import ./components/nodestore import ./components/dhtmetrics import ./components/todolist import ./components/chainmetrics +import ./components/chaincrawler +import ./components/requeststore proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = var components: seq[Component] = newSeq[Component]() @@ -26,11 +28,14 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = without nodeStore =? createNodeStore(state, clock), err: return failure(err) + without requestStore =? createRequestStore(state, clock), err: + return failure(err) + let metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort) todoList = createTodoList(state, metrics) marketplace = createMarketplace(state, clock) - chainMetrics = ChainMetrics.new(state, metrics, marketplace) + chainMetrics = ChainMetrics.new(state, metrics, requestStore, marketplace, clock) without dhtMetrics =? createDhtMetrics(state, metrics), err: return failure(err) @@ -43,5 +48,6 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} = components.add(dhtMetrics) components.add(marketplace) components.add(chainMetrics) + components.add(ChainCrawler.new(state, requestStore, marketplace)) return success(components) diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim index 81c062b..eaae1d9 100644 --- a/tests/codexcrawler/components/testchainmetrics.nim +++ b/tests/codexcrawler/components/testchainmetrics.nim @@ -11,6 +11,7 @@ import ../mocks/mockstate import ../mocks/mockmetrics import ../mocks/mockrequeststore import ../mocks/mockmarketplace +import ../mocks/mockclock import ../helpers suite "ChainMetrics": @@ -19,6 +20,7 @@ suite "ChainMetrics": metrics: MockMetrics store: MockRequestStore marketplace: MockMarketplaceService + clock: MockClock chain: ChainMetrics setup: @@ -26,8 +28,9 @@ suite "ChainMetrics": metrics = createMockMetrics() store = createMockRequestStore() marketplace = createMockMarketplaceService() + clock = createMockClock() - chain = ChainMetrics.new(state, metrics, store, marketplace) + chain = ChainMetrics.new(state, metrics, store, marketplace, clock) (await chain.start()).tryGet() @@ -43,10 +46,12 @@ suite "ChainMetrics": state.delays.len == 1 state.delays[0] == 10.minutes - test "onStep should remove non-running requests from request store": + test "onStep should remove old non-running requests from request store": let rid = genRid() - store.iterateEntries.add(RequestEntry(id: rid)) + let oneDay = (60 * 60 * 24).uint64 + store.iterateEntries.add(RequestEntry(id: rid, lastSeen: 100.uint64)) + clock.setNow = 100 + oneDay + 1 marketplace.requestInfoReturns = none(RequestInfo) await onStep() @@ -55,6 +60,20 @@ suite "ChainMetrics": marketplace.requestInfoRid == rid store.removeRid == rid + test "onStep should not remove recent non-running requests from request store": + let rid = genRid() + let now = 123456789.uint64 + store.iterateEntries.add(RequestEntry(id: rid, lastSeen: now - 1)) + + clock.setNow = now + marketplace.requestInfoReturns = none(RequestInfo) + + await onStep() + + check: + marketplace.requestInfoRid == rid + not (store.removeRid == rid) + test "onStep should count the number of active requests": let rid1 = genRid() let rid2 = genRid()