From 1704d3dc3004a3804bf75adee88a06d762082d33 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 10 Mar 2025 13:51:51 +0100
Subject: [PATCH 01/25] Sets up new config flags

---
 codexcrawler/config.nim     | 30 +++++++++++++++++++++++++-----
 docker/docker-entrypoint.sh |  8 +++++++-
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index 4411e3b..f81dd06 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -10,7 +10,7 @@ let doc =
 Codex Network Crawler. Generates network metrics.
 
 Usage:
-  codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>]  [--expiryDelay=<m>]
+  codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>]  [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>]
 
 Options:
   --logLevel=<l>                    Sets log level [default: INFO]
@@ -20,10 +20,14 @@ Options:
   --dataDir=<dir>                   Directory for storing data [default: crawler_data]
   --discoveryPort=<p>               Port used for DHT [default: 8090]
   --bootNodes=<n>                   Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
+  --dhtEnable=<e>                         Set to "1" to enable DHT crawler [default: 1]
   --stepDelay=<ms>                  Delay in milliseconds per node visit [default: 1000]
   --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
   --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
   --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
+  --marketplaceEnable=<e>                 Set to "1" to enable marketplace metrics [default: 1]
+  --ethProvider=<a>                Address including http(s) or ws of the eth provider
+  --marketplaceAddress=<a>         Eth address of Codex contracts deployment
 """
 
 import strutils
@@ -37,18 +41,27 @@ type Config* = ref object
   dataDir*: string
   discPort*: Port
   bootNodes*: seq[SignedPeerRecord]
+
+  dhtEnable*: bool
   stepDelayMs*: int
   revisitDelayMins*: int
   checkDelayMins*: int
   expiryDelayMins*: int
 
+  marketplaceEnable*: bool
+  ethProvider*: string
+  marketplaceAddress*: string
+
 proc `$`*(config: Config): string =
   "Crawler:" & " logLevel=" & config.logLevel & " publicIp=" & config.publicIp &
     " metricsAddress=" & $config.metricsAddress & " metricsPort=" & $config.metricsPort &
-    " dataDir=" & config.dataDir & " discPort=" & $config.discPort & " bootNodes=" &
-    config.bootNodes.mapIt($it).join(";") & " stepDelay=" & $config.stepDelayMs &
-    " revisitDelayMins=" & $config.revisitDelayMins & " expiryDelayMins=" &
-    $config.expiryDelayMins & " checkDelayMins=" & $config.checkDelayMins
+    " dataDir=" & config.dataDir & " discPort=" & $config.discPort & " dhtEnable=" &
+    $config.dhtEnable & " bootNodes=" & config.bootNodes.mapIt($it).join(";") &
+    " stepDelay=" & $config.stepDelayMs & " revisitDelayMins=" & $config.revisitDelayMins &
+    " expiryDelayMins=" & $config.expiryDelayMins & " checkDelayMins=" &
+    $config.checkDelayMins & " marketplaceEnable=" & $config.marketplaceEnable &
+    " ethProvider=" & config.ethProvider & " marketplaceAddress=" &
+    config.marketplaceAddress
 
 proc getDefaultTestnetBootNodes(): seq[string] =
   @[
@@ -86,6 +99,9 @@ proc stringToSpr(uri: string): SignedPeerRecord =
 proc getBootNodes(input: string): seq[SignedPeerRecord] =
   getBootNodeStrings(input).mapIt(stringToSpr(it))
 
+proc getEnable(input: string): bool =
+  input == "1"
+
 proc parseConfig*(): Config =
   let args = docopt(doc, version = crawlerFullVersion)
 
@@ -100,8 +116,12 @@ proc parseConfig*(): Config =
     dataDir: get("--dataDir"),
     discPort: Port(parseInt(get("--discoveryPort"))),
     bootNodes: getBootNodes(get("--bootNodes")),
+    dhtEnable: getEnable(get("--dhtEnable")),
     stepDelayMs: parseInt(get("--stepDelay")),
     revisitDelayMins: parseInt(get("--revisitDelay")),
     checkDelayMins: parseInt(get("--checkDelay")),
     expiryDelayMins: parseInt(get("--expiryDelay")),
+    marketplaceEnable: getEnable(get("--marketplaceEnable")),
+    ethProvider: get("--ethProvider"),
+    marketplaceAddress: get("--marketplaceAddress"),
   )
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index 9cad0bd..a9846a4 100644
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -9,13 +9,19 @@ METRICSPORT=${CRAWLER_METRICSPORT:-8008}
 DATADIR=${CRAWLER_DATADIR:-crawler_data}
 DISCPORT=${CRAWLER_DISCPORT:-8090}
 BOOTNODES=${CRAWLER_BOOTNODES:-testnet_sprs}
+
+DHTENABLE=${CRAWLER_DHTENABLE:-1}
 STEPDELAY=${CRAWLER_STEPDELAY:-1000}
 REVISITDELAY=${CRAWLER_REVISITDELAY:-60}
 CHECKDELAY=${CRAWLER_CHECKDELAY:-10}
 EXPIRYDELAY=${CRAWLER_EXPIRYDELAY:-1440}
 
+MARKETPLACEENABLE=${CRAWLER_MARKETPLACEENABLE:-1}
+ETHPROVIDER=${CRAWLER_ETHPROVIDER:-NULL}
+MARKETPLACEADDRESS=${CRAWLER_MARKETPLACEADDRESS:-NULL}
+
 # Update CLI arguments
-set -- "$@" --logLevel="${LOGLEVEL}" --publicIp="${PUBLICIP}" --metricsAddress="${METRICSADDRESS}" --metricsPort="${METRICSPORT}" --dataDir="${DATADIR}" --discoveryPort="${DISCPORT}" --bootNodes="${BOOTNODES}" --stepDelay="${STEPDELAY}" --revisitDelay="${REVISITDELAY}" --expiryDelay="${EXPIRYDELAY}" --checkDelay="${CHECKDELAY}"
+set -- "$@" --logLevel="${LOGLEVEL}" --publicIp="${PUBLICIP}" --metricsAddress="${METRICSADDRESS}" --metricsPort="${METRICSPORT}" --dataDir="${DATADIR}" --discoveryPort="${DISCPORT}" --bootNodes="${BOOTNODES}" --dhtEnable="${DHTENABLE}" --stepDelay="${STEPDELAY}" --revisitDelay="${REVISITDELAY}" --expiryDelay="${EXPIRYDELAY}" --checkDelay="${CHECKDELAY}" --marketplaceEnable="${MARKETPLACEENABLE}" --ethProvider="${ETHPROVIDER}" --marketplaceAddress="${MARKETPLACEADDRESS}"
 
 # Run
 echo "Run Codex Crawler"

From 5bd35720b56e2bed6103c2b2e1bf74d68557f24d Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 10 Mar 2025 14:09:41 +0100
Subject: [PATCH 02/25] implements dhtEnable flag

---
 codexcrawler/components/crawler.nim           |  3 ++-
 tests/codexcrawler/components/testcrawler.nim | 14 ++++++++++++++
 tests/codexcrawler/mocks/mockstate.nim        |  4 +++-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/crawler.nim
index c32f5ce..57602db 100644
--- a/codexcrawler/components/crawler.nim
+++ b/codexcrawler/components/crawler.nim
@@ -52,7 +52,8 @@ method start*(c: Crawler): Future[?!void] {.async.} =
   proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
     await c.step()
 
-  await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds)
+  if c.state.config.dhtEnable:
+    await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds)
 
   return success()
 
diff --git a/tests/codexcrawler/components/testcrawler.nim b/tests/codexcrawler/components/testcrawler.nim
index cbe9d75..6a51721 100644
--- a/tests/codexcrawler/components/testcrawler.nim
+++ b/tests/codexcrawler/components/testcrawler.nim
@@ -55,6 +55,20 @@ suite "Crawler":
     check:
       !(dht.getNeighborsArg) == nid1
 
+  test "onStep is not activated when config.dhtEnable is false":
+    # Recreate crawler, reset mockstate:
+    (await crawler.stop()).tryGet()
+    state.steppers = @[]
+    # disable DHT:
+    state.config.dhtEnable = false
+    (await crawler.start()).tryGet()
+
+    todo.popReturn = success(nid1)
+    dht.getNeighborsReturn = success(responsive(nid1))
+
+    check:
+      state.steppers.len == 0
+
   test "nodes returned by getNeighbors are raised as nodesFound":
     var nodesFound = newSeq[Nid]()
     proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} =
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index 00e1198..67b45a3 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -22,7 +22,9 @@ method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} =
 proc createMockState*(): MockState =
   MockState(
     status: ApplicationStatus.Running,
-    config: Config(),
+    config: Config(
+      dhtEnable: true
+    ),
     events: Events(
       nodesFound: newAsyncDataEvent[seq[Nid]](),
       newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),

From 56df7adfd82dc7ff63f0df06892f1deda52a6eea Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 11 Mar 2025 09:34:33 +0100
Subject: [PATCH 03/25] Imports nim-ethers and required glue-code from
 nim-codex.

---
 codexcrawler.nimble                           |   1 +
 codexcrawler/services/marketplace.nim         |   6 +
 codexcrawler/services/marketplace/config.nim  | 103 +++++++
 .../services/marketplace/logutils.nim         | 285 ++++++++++++++++++
 .../services/marketplace/marketplace.nim      | 184 +++++++++++
 codexcrawler/services/marketplace/proofs.nim  |  46 +++
 codexcrawler/services/marketplace/readme.md   |   2 +
 .../services/marketplace/requests.nim         | 220 ++++++++++++++
 8 files changed, 847 insertions(+)
 create mode 100644 codexcrawler/services/marketplace.nim
 create mode 100644 codexcrawler/services/marketplace/config.nim
 create mode 100644 codexcrawler/services/marketplace/logutils.nim
 create mode 100644 codexcrawler/services/marketplace/marketplace.nim
 create mode 100644 codexcrawler/services/marketplace/proofs.nim
 create mode 100644 codexcrawler/services/marketplace/readme.md
 create mode 100644 codexcrawler/services/marketplace/requests.nim

diff --git a/codexcrawler.nimble b/codexcrawler.nimble
index a38d166..3af92dc 100644
--- a/codexcrawler.nimble
+++ b/codexcrawler.nimble
@@ -25,6 +25,7 @@ requires "questionable >= 0.10.15 & < 0.11.0"
 requires "https://github.com/codex-storage/nim-codex-dht#f6eef1ac95c70053b2518f1e3909c909ed8701a6"
 requires "docopt >= 0.7.1 & < 1.0.0"
 requires "nph >= 0.6.1 & < 1.0.0"
+requires "ethers >= 1.0.0 & < 2.0.0"
 
 task format, "Formatting...":
   exec "nph ./"
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
new file mode 100644
index 0000000..80d6a36
--- /dev/null
+++ b/codexcrawler/services/marketplace.nim
@@ -0,0 +1,6 @@
+import ./marketplace/marketplace
+
+# todo
+
+proc aaa*() =
+  echo "aaa"
diff --git a/codexcrawler/services/marketplace/config.nim b/codexcrawler/services/marketplace/config.nim
new file mode 100644
index 0000000..3c31c8b
--- /dev/null
+++ b/codexcrawler/services/marketplace/config.nim
@@ -0,0 +1,103 @@
+import pkg/contractabi
+import pkg/ethers/fields
+import pkg/questionable/results
+
+export contractabi
+
+const DefaultRequestCacheSize* = 128.uint16
+
+type
+  MarketplaceConfig* = object
+    collateral*: CollateralConfig
+    proofs*: ProofConfig
+    reservations*: SlotReservationsConfig
+    requestDurationLimit*: uint64
+
+  CollateralConfig* = object
+    repairRewardPercentage*: uint8
+      # percentage of remaining collateral slot has after it has been freed
+    maxNumberOfSlashes*: uint8 # frees slot when the number of slashes reaches this value
+    slashPercentage*: uint8 # percentage of the collateral that is slashed
+    validatorRewardPercentage*: uint8
+      # percentage of the slashed amount going to the validators
+
+  ProofConfig* = object
+    period*: uint64 # proofs requirements are calculated per period (in seconds)
+    timeout*: uint64 # mark proofs as missing before the timeout (in seconds)
+    downtime*: uint8 # ignore this much recent blocks for proof requirements
+    downtimeProduct*: uint8
+    zkeyHash*: string # hash of the zkey file which is linked to the verifier
+    # Ensures the pointer does not remain in downtime for many consecutive
+    # periods. For each period increase, move the pointer `pointerProduct`
+    # blocks. Should be a prime number to ensure there are no cycles.
+
+  SlotReservationsConfig* = object
+    maxReservations*: uint8
+
+func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig =
+  ProofConfig(
+    period: tupl[0],
+    timeout: tupl[1],
+    downtime: tupl[2],
+    downtimeProduct: tupl[3],
+    zkeyHash: tupl[4],
+  )
+
+func fromTuple(_: type SlotReservationsConfig, tupl: tuple): SlotReservationsConfig =
+  SlotReservationsConfig(maxReservations: tupl[0])
+
+func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig =
+  CollateralConfig(
+    repairRewardPercentage: tupl[0],
+    maxNumberOfSlashes: tupl[1],
+    slashPercentage: tupl[2],
+    validatorRewardPercentage: tupl[3],
+  )
+
+func fromTuple(_: type MarketplaceConfig, tupl: tuple): MarketplaceConfig =
+  MarketplaceConfig(
+    collateral: tupl[0],
+    proofs: tupl[1],
+    reservations: tupl[2],
+    requestDurationLimit: tupl[3],
+  )
+
+func solidityType*(_: type SlotReservationsConfig): string =
+  solidityType(SlotReservationsConfig.fieldTypes)
+
+func solidityType*(_: type ProofConfig): string =
+  solidityType(ProofConfig.fieldTypes)
+
+func solidityType*(_: type CollateralConfig): string =
+  solidityType(CollateralConfig.fieldTypes)
+
+func solidityType*(_: type MarketplaceConfig): string =
+  solidityType(MarketplaceConfig.fieldTypes)
+
+func encode*(encoder: var AbiEncoder, slot: SlotReservationsConfig) =
+  encoder.write(slot.fieldValues)
+
+func encode*(encoder: var AbiEncoder, slot: ProofConfig) =
+  encoder.write(slot.fieldValues)
+
+func encode*(encoder: var AbiEncoder, slot: CollateralConfig) =
+  encoder.write(slot.fieldValues)
+
+func encode*(encoder: var AbiEncoder, slot: MarketplaceConfig) =
+  encoder.write(slot.fieldValues)
+
+func decode*(decoder: var AbiDecoder, T: type ProofConfig): ?!T =
+  let tupl = ?decoder.read(ProofConfig.fieldTypes)
+  success ProofConfig.fromTuple(tupl)
+
+func decode*(decoder: var AbiDecoder, T: type SlotReservationsConfig): ?!T =
+  let tupl = ?decoder.read(SlotReservationsConfig.fieldTypes)
+  success SlotReservationsConfig.fromTuple(tupl)
+
+func decode*(decoder: var AbiDecoder, T: type CollateralConfig): ?!T =
+  let tupl = ?decoder.read(CollateralConfig.fieldTypes)
+  success CollateralConfig.fromTuple(tupl)
+
+func decode*(decoder: var AbiDecoder, T: type MarketplaceConfig): ?!T =
+  let tupl = ?decoder.read(MarketplaceConfig.fieldTypes)
+  success MarketplaceConfig.fromTuple(tupl)
diff --git a/codexcrawler/services/marketplace/logutils.nim b/codexcrawler/services/marketplace/logutils.nim
new file mode 100644
index 0000000..39f99fb
--- /dev/null
+++ b/codexcrawler/services/marketplace/logutils.nim
@@ -0,0 +1,285 @@
+## logutils is a module that has several goals:
+## 1. Fix json logging output (run with `--log-format=json`) which was
+##    effectively broken for many types using default Chronicles json
+##    serialization.
+## 2. Ability to specify log output for textlines and json sinks together or
+##    separately
+##     - This is useful if consuming json in some kind of log parser and need
+##       valid json with real values
+##     - eg a shortened Cid is nice to see in a text log in stdout, but won't
+##       provide a real Cid when parsed in json
+## 4. Remove usages of `nim-json-serialization` from the codebase
+## 5. Remove need to declare `writeValue` for new types
+## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent
+##    conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467)
+##
+## When declaring a new type, one should consider importing the `codex/logutils`
+## module, and specifying `formatIt`. If textlines log output and json log output
+## need to be different, overload `formatIt` and specify a `LogFormat`. If json
+## serialization is needed, it can be declared with a `%` proc. `logutils`
+## imports and exports `nim-serde` which handles the de/serialization, examples
+## below. **Only `codex/logutils` needs to be imported.**
+##
+## Using `logutils` in the Codex codebase:
+## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils`
+##     - most of `chronicles` is exported by `logutils`
+## - Instead of importing `std/json`, import `pkg/serde/json`
+##     - `std/json` is exported by `serde` which is exported by `logutils`
+## - Instead of importing `pkg/nim-json-serialization`, import
+##   `pkg/serde/json` or use codex-specific overloads by importing `utils/json`
+##     - one of the goals is to remove the use of `nim-json-serialization`
+##
+## ```nim
+## import pkg/codex/logutils
+##
+## type
+##   BlockAddress* = object
+##     case leaf*: bool
+##     of true:
+##       treeCid* {.serialize.}: Cid
+##       index* {.serialize.}: Natural
+##     else:
+##       cid* {.serialize.}: Cid
+##
+## logutils.formatIt(LogFormat.textLines, BlockAddress):
+##   if it.leaf:
+##     "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index
+##   else:
+##     "cid: " & shortLog($it.cid)
+##
+## logutils.formatIt(LogFormat.json, BlockAddress): %it
+##
+## # chronicles textlines output
+## TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0"
+## # chronicles json output
+## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
+## ```
+## In this case, `BlockAddress` is just an object, so `nim-serde` can handle
+## serializing it without issue (only fields annotated with `{.serialize.}` will
+## serialize (aka opt-in serialization)).
+##
+## If one so wished, another option for the textlines log output, would be to
+## simply `toString` the serialised json:
+## ```nim
+## logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it
+## # or, more succinctly:
+## logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson
+## ```
+## In that case, both the textlines and json sinks would have the same output,
+## so we could reduce this even further by not specifying a `LogFormat`:
+## ```nim
+## type
+##   BlockAddress* = object
+##     case leaf*: bool
+##     of true:
+##       treeCid* {.serialize.}: Cid
+##       index* {.serialize.}: Natural
+##     else:
+##       cid* {.serialize.}: Cid
+##
+## logutils.formatIt(BlockAddress): %it
+##
+## # chronicles textlines output
+## TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}"
+## # chronicles json output
+## {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
+## ```
+
+import std/options
+import std/sequtils
+import std/strutils
+import std/sugar
+import std/typetraits
+
+import pkg/chronicles except toJson, `%`
+from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
+from pkg/ethers import Address
+import pkg/questionable
+import pkg/questionable/results
+import pkg/stew/byteutils
+import pkg/stint
+import pkg/serde/json
+import pkg/codexdht/discv5/node as dn
+import pkg/contractabi
+
+export byteutils
+export chronicles except toJson, formatIt, `%`
+export questionable
+export sequtils
+export json except formatIt
+export strutils
+export sugar
+export results
+
+proc fromJson*(_: type Cid, json: JsonNode): ?!Cid =
+  expectJsonKind(Cid, JString, json)
+  Cid.init(json.str).mapFailure
+
+func `%`*(cid: Cid): JsonNode =
+  % $cid
+
+func `%`*(obj: PeerId): JsonNode =
+  % $obj
+
+func `%`*(obj: SignedPeerRecord): JsonNode =
+  % $obj
+
+func `%`*(obj: dn.Address): JsonNode =
+  % $obj
+
+func `%`*(obj: AddressInfo): JsonNode =
+  % $obj.address
+
+func `%`*(obj: MultiAddress): JsonNode =
+  % $obj
+
+func `%`*(address: ethers.Address): JsonNode =
+  % $address
+
+func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string =
+  ## Returns compact string representation of ``long``.
+  var short = long
+  let minLen = start + ellipses.len + stop
+  if len(short) > minLen:
+    short.insert(ellipses, start)
+
+    when (NimMajor, NimMinor) > (1, 4):
+      short.delete(start + ellipses.len .. short.high - stop)
+    else:
+      short.delete(start + ellipses.len, short.high - stop)
+
+  short
+
+func shortHexLog*(long: string): string =
+  if long[0 .. 1] == "0x":
+    result &= "0x"
+  result &= long[2 .. long.high].shortLog("..", 4, 4)
+
+func short0xHexLog*[N: static[int], T: array[N, byte]](v: T): string =
+  v.to0xHex.shortHexLog
+
+func short0xHexLog*[T: distinct](v: T): string =
+  type BaseType = T.distinctBase
+  BaseType(v).short0xHexLog
+
+func short0xHexLog*[U: distinct, T: seq[U]](v: T): string =
+  type BaseType = U.distinctBase
+  "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]"
+
+func to0xHexLog*[T: distinct](v: T): string =
+  type BaseType = T.distinctBase
+  BaseType(v).to0xHex
+
+func to0xHexLog*[U: distinct, T: seq[U]](v: T): string =
+  type BaseType = U.distinctBase
+  "@[" & v.map(x => BaseType(x).to0xHex).join(",") & "]"
+
+proc formatTextLineSeq*(val: seq[string]): string =
+  "@[" & val.join(", ") & "]"
+
+template formatIt*(format: LogFormat, T: typedesc, body: untyped) =
+  # Provides formatters for logging with Chronicles for the given type and
+  # `LogFormat`.
+  # NOTE: `seq[T]`, `Option[T]`, and `seq[Option[T]]` are overridden
+  # since the base `setProperty` is generic using `auto` and conflicts with
+  # providing a generic `seq` and `Option` override.
+  when format == LogFormat.json:
+    proc formatJsonOption(val: ?T): JsonNode =
+      if it =? val:
+        json.`%`(body)
+      else:
+        newJNull()
+
+    proc formatJsonResult*(val: ?!T): JsonNode =
+      without it =? val, error:
+        let jObj = newJObject()
+        jObj["error"] = newJString(error.msg)
+        return jObj
+      json.`%`(body)
+
+    proc setProperty*(r: var JsonRecord, key: string, res: ?!T) =
+      var it {.inject, used.}: T
+      setProperty(r, key, res.formatJsonResult)
+
+    proc setProperty*(r: var JsonRecord, key: string, opt: ?T) =
+      var it {.inject, used.}: T
+      let v = opt.formatJsonOption
+      setProperty(r, key, v)
+
+    proc setProperty*(r: var JsonRecord, key: string, opts: seq[?T]) =
+      var it {.inject, used.}: T
+      let v = opts.map(opt => opt.formatJsonOption)
+      setProperty(r, key, json.`%`(v))
+
+    proc setProperty*(
+        r: var JsonRecord, key: string, val: seq[T]
+    ) {.raises: [ValueError, IOError].} =
+      var it {.inject, used.}: T
+      let v = val.map(it => body)
+      setProperty(r, key, json.`%`(v))
+
+    proc setProperty*(
+        r: var JsonRecord, key: string, val: T
+    ) {.raises: [ValueError, IOError].} =
+      var it {.inject, used.}: T = val
+      let v = body
+      setProperty(r, key, json.`%`(v))
+
+  elif format == LogFormat.textLines:
+    proc formatTextLineOption*(val: ?T): string =
+      var v = "none(" & $T & ")"
+      if it =? val:
+        v = "some(" & $(body) & ")" # that I used to know :)
+      v
+
+    proc formatTextLineResult*(val: ?!T): string =
+      without it =? val, error:
+        return "Error: " & error.msg
+      $(body)
+
+    proc setProperty*(r: var TextLineRecord, key: string, res: ?!T) =
+      var it {.inject, used.}: T
+      setProperty(r, key, res.formatTextLineResult)
+
+    proc setProperty*(r: var TextLineRecord, key: string, opt: ?T) =
+      var it {.inject, used.}: T
+      let v = opt.formatTextLineOption
+      setProperty(r, key, v)
+
+    proc setProperty*(r: var TextLineRecord, key: string, opts: seq[?T]) =
+      var it {.inject, used.}: T
+      let v = opts.map(opt => opt.formatTextLineOption)
+      setProperty(r, key, v.formatTextLineSeq)
+
+    proc setProperty*(
+        r: var TextLineRecord, key: string, val: seq[T]
+    ) {.raises: [ValueError, IOError].} =
+      var it {.inject, used.}: T
+      let v = val.map(it => body)
+      setProperty(r, key, v.formatTextLineSeq)
+
+    proc setProperty*(
+        r: var TextLineRecord, key: string, val: T
+    ) {.raises: [ValueError, IOError].} =
+      var it {.inject, used.}: T = val
+      let v = body
+      setProperty(r, key, v)
+
+template formatIt*(T: type, body: untyped) {.dirty.} =
+  formatIt(LogFormat.textLines, T):
+    body
+  formatIt(LogFormat.json, T):
+    body
+
+formatIt(LogFormat.textLines, Cid):
+  shortLog($it)
+formatIt(LogFormat.json, Cid):
+  $it
+formatIt(UInt256):
+  $it
+formatIt(MultiAddress):
+  $it
+formatIt(LogFormat.textLines, array[32, byte]):
+  it.short0xHexLog
+formatIt(LogFormat.json, array[32, byte]):
+  it.to0xHex
diff --git a/codexcrawler/services/marketplace/marketplace.nim b/codexcrawler/services/marketplace/marketplace.nim
new file mode 100644
index 0000000..1f0f90d
--- /dev/null
+++ b/codexcrawler/services/marketplace/marketplace.nim
@@ -0,0 +1,184 @@
+import pkg/ethers
+import pkg/ethers/erc20
+import pkg/json_rpc/rpcclient
+import pkg/stint
+import pkg/chronos
+import ./requests
+import ./proofs
+import ./config
+
+export stint
+export ethers except `%`, `%*`, toJson
+export erc20 except `%`, `%*`, toJson
+export config
+export requests
+
+type
+  Marketplace* = ref object of Contract
+
+  Marketplace_RepairRewardPercentageTooHigh* = object of SolidityError
+  Marketplace_SlashPercentageTooHigh* = object of SolidityError
+  Marketplace_MaximumSlashingTooHigh* = object of SolidityError
+  Marketplace_InvalidExpiry* = object of SolidityError
+  Marketplace_InvalidMaxSlotLoss* = object of SolidityError
+  Marketplace_InsufficientSlots* = object of SolidityError
+  Marketplace_InvalidClientAddress* = object of SolidityError
+  Marketplace_RequestAlreadyExists* = object of SolidityError
+  Marketplace_InvalidSlot* = object of SolidityError
+  Marketplace_SlotNotFree* = object of SolidityError
+  Marketplace_InvalidSlotHost* = object of SolidityError
+  Marketplace_AlreadyPaid* = object of SolidityError
+  Marketplace_TransferFailed* = object of SolidityError
+  Marketplace_UnknownRequest* = object of SolidityError
+  Marketplace_InvalidState* = object of SolidityError
+  Marketplace_StartNotBeforeExpiry* = object of SolidityError
+  Marketplace_SlotNotAcceptingProofs* = object of SolidityError
+  Marketplace_SlotIsFree* = object of SolidityError
+  Marketplace_ReservationRequired* = object of SolidityError
+  Marketplace_NothingToWithdraw* = object of SolidityError
+  Marketplace_InsufficientDuration* = object of SolidityError
+  Marketplace_InsufficientProofProbability* = object of SolidityError
+  Marketplace_InsufficientCollateral* = object of SolidityError
+  Marketplace_InsufficientReward* = object of SolidityError
+  Marketplace_InvalidCid* = object of SolidityError
+  Marketplace_DurationExceedsLimit* = object of SolidityError
+  Proofs_InsufficientBlockHeight* = object of SolidityError
+  Proofs_InvalidProof* = object of SolidityError
+  Proofs_ProofAlreadySubmitted* = object of SolidityError
+  Proofs_PeriodNotEnded* = object of SolidityError
+  Proofs_ValidationTimedOut* = object of SolidityError
+  Proofs_ProofNotMissing* = object of SolidityError
+  Proofs_ProofNotRequired* = object of SolidityError
+  Proofs_ProofAlreadyMarkedMissing* = object of SolidityError
+  Proofs_InvalidProbability* = object of SolidityError
+  Periods_InvalidSecondsPerPeriod* = object of SolidityError
+
+proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.}
+proc token*(marketplace: Marketplace): Address {.contract, view.}
+proc currentCollateral*(
+  marketplace: Marketplace, id: SlotId
+): UInt256 {.contract, view.}
+
+proc requestStorage*(
+  marketplace: Marketplace, request: StorageRequest
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_InvalidClientAddress, Marketplace_RequestAlreadyExists,
+    Marketplace_InvalidExpiry, Marketplace_InsufficientSlots,
+    Marketplace_InvalidMaxSlotLoss,
+  ]
+.}
+
+proc fillSlot*(
+  marketplace: Marketplace, requestId: RequestId, slotIndex: uint64, proof: Groth16Proof
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_InvalidSlot, Marketplace_ReservationRequired, Marketplace_SlotNotFree,
+    Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest,
+  ]
+.}
+
+proc withdrawFunds*(
+  marketplace: Marketplace, requestId: RequestId
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_InvalidClientAddress, Marketplace_InvalidState,
+    Marketplace_NothingToWithdraw, Marketplace_UnknownRequest,
+  ]
+.}
+
+proc withdrawFunds*(
+  marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_InvalidClientAddress, Marketplace_InvalidState,
+    Marketplace_NothingToWithdraw, Marketplace_UnknownRequest,
+  ]
+.}
+
+proc freeSlot*(
+  marketplace: Marketplace, id: SlotId
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid,
+    Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree,
+  ]
+.}
+
+proc freeSlot*(
+  marketplace: Marketplace,
+  id: SlotId,
+  rewardRecipient: Address,
+  collateralRecipient: Address,
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid,
+    Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree,
+  ]
+.}
+
+proc getRequest*(
+  marketplace: Marketplace, id: RequestId
+): StorageRequest {.contract, view, errors: [Marketplace_UnknownRequest].}
+
+proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.}
+proc getActiveSlot*(
+  marketplace: Marketplace, id: SlotId
+): Slot {.contract, view, errors: [Marketplace_SlotIsFree].}
+
+proc myRequests*(marketplace: Marketplace): seq[RequestId] {.contract, view.}
+proc mySlots*(marketplace: Marketplace): seq[SlotId] {.contract, view.}
+proc requestState*(
+  marketplace: Marketplace, requestId: RequestId
+): RequestState {.contract, view, errors: [Marketplace_UnknownRequest].}
+
+proc slotState*(marketplace: Marketplace, slotId: SlotId): SlotState {.contract, view.}
+proc requestEnd*(
+  marketplace: Marketplace, requestId: RequestId
+): int64 {.contract, view.}
+
+proc requestExpiry*(
+  marketplace: Marketplace, requestId: RequestId
+): int64 {.contract, view.}
+
+proc missingProofs*(marketplace: Marketplace, id: SlotId): UInt256 {.contract, view.}
+proc isProofRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.}
+proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.}
+proc getChallenge*(
+  marketplace: Marketplace, id: SlotId
+): array[32, byte] {.contract, view.}
+
+proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.}
+
+proc submitProof*(
+  marketplace: Marketplace, id: SlotId, proof: Groth16Proof
+): Confirmable {.
+  contract,
+  errors:
+    [Proofs_ProofAlreadySubmitted, Proofs_InvalidProof, Marketplace_UnknownRequest]
+.}
+
+proc markProofAsMissing*(
+  marketplace: Marketplace, id: SlotId, period: uint64
+): Confirmable {.
+  contract,
+  errors: [
+    Marketplace_SlotNotAcceptingProofs, Marketplace_StartNotBeforeExpiry,
+    Proofs_PeriodNotEnded, Proofs_ValidationTimedOut, Proofs_ProofNotMissing,
+    Proofs_ProofNotRequired, Proofs_ProofAlreadyMarkedMissing,
+  ]
+.}
+
+proc reserveSlot*(
+  marketplace: Marketplace, requestId: RequestId, slotIndex: uint64
+): Confirmable {.contract.}
+
+proc canReserveSlot*(
+  marketplace: Marketplace, requestId: RequestId, slotIndex: uint64
+): bool {.contract, view.}
diff --git a/codexcrawler/services/marketplace/proofs.nim b/codexcrawler/services/marketplace/proofs.nim
new file mode 100644
index 0000000..771d685
--- /dev/null
+++ b/codexcrawler/services/marketplace/proofs.nim
@@ -0,0 +1,46 @@
+import pkg/stint
+import pkg/contractabi
+import pkg/ethers/fields
+
+type
+  Groth16Proof* = object
+    a*: G1Point
+    b*: G2Point
+    c*: G1Point
+
+  G1Point* = object
+    x*: UInt256
+    y*: UInt256
+
+  # A field element F_{p^2} encoded as `real + i * imag`
+  Fp2Element* = object
+    real*: UInt256
+    imag*: UInt256
+
+  G2Point* = object
+    x*: Fp2Element
+    y*: Fp2Element
+
+func solidityType*(_: type G1Point): string =
+  solidityType(G1Point.fieldTypes)
+
+func solidityType*(_: type Fp2Element): string =
+  solidityType(Fp2Element.fieldTypes)
+
+func solidityType*(_: type G2Point): string =
+  solidityType(G2Point.fieldTypes)
+
+func solidityType*(_: type Groth16Proof): string =
+  solidityType(Groth16Proof.fieldTypes)
+
+func encode*(encoder: var AbiEncoder, point: G1Point) =
+  encoder.write(point.fieldValues)
+
+func encode*(encoder: var AbiEncoder, element: Fp2Element) =
+  encoder.write(element.fieldValues)
+
+func encode*(encoder: var AbiEncoder, point: G2Point) =
+  encoder.write(point.fieldValues)
+
+func encode*(encoder: var AbiEncoder, proof: Groth16Proof) =
+  encoder.write(proof.fieldValues)
diff --git a/codexcrawler/services/marketplace/readme.md b/codexcrawler/services/marketplace/readme.md
new file mode 100644
index 0000000..fd2b28a
--- /dev/null
+++ b/codexcrawler/services/marketplace/readme.md
@@ -0,0 +1,2 @@
+These are copied from nim-codex v0.2.0.
+There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
diff --git a/codexcrawler/services/marketplace/requests.nim b/codexcrawler/services/marketplace/requests.nim
new file mode 100644
index 0000000..a6e4281
--- /dev/null
+++ b/codexcrawler/services/marketplace/requests.nim
@@ -0,0 +1,220 @@
+import std/hashes
+import std/sequtils
+import std/typetraits
+import pkg/contractabi
+import pkg/nimcrypto
+import pkg/ethers/fields
+import pkg/results
+import pkg/questionable/results
+import pkg/stew/byteutils
+import pkg/libp2p/[cid, multicodec]
+import pkg/serde/json
+import ./logutils
+
+export contractabi
+
+type
+  StorageRequest* = object
+    client* {.serialize.}: Address
+    ask* {.serialize.}: StorageAsk
+    content* {.serialize.}: StorageContent
+    expiry* {.serialize.}: uint64
+    nonce*: Nonce
+
+  StorageAsk* = object
+    proofProbability* {.serialize.}: UInt256
+    pricePerBytePerSecond* {.serialize.}: UInt256
+    collateralPerByte* {.serialize.}: UInt256
+    slots* {.serialize.}: uint64
+    slotSize* {.serialize.}: uint64
+    duration* {.serialize.}: uint64
+    maxSlotLoss* {.serialize.}: uint64
+
+  StorageContent* = object
+    cid* {.serialize.}: Cid
+    merkleRoot*: array[32, byte]
+
+  Slot* = object
+    request* {.serialize.}: StorageRequest
+    slotIndex* {.serialize.}: uint64
+
+  SlotId* = distinct array[32, byte]
+  RequestId* = distinct array[32, byte]
+  Nonce* = distinct array[32, byte]
+  RequestState* {.pure.} = enum
+    New
+    Started
+    Cancelled
+    Finished
+    Failed
+
+  SlotState* {.pure.} = enum
+    Free
+    Filled
+    Finished
+    Failed
+    Paid
+    Cancelled
+    Repair
+
+template mapFailure*[T, V, E](
+    exp: Result[T, V], exc: typedesc[E]
+): Result[T, ref CatchableError] =
+  ## Convert `Result[T, E]` to `Result[E, ref CatchableError]`
+  ##
+
+  exp.mapErr(
+    proc(e: V): ref CatchableError =
+      (ref exc)(msg: $e)
+  )
+
+template mapFailure*[T, V](exp: Result[T, V]): Result[T, ref CatchableError] =
+  mapFailure(exp, CatchableError)
+
+proc `==`*(x, y: Nonce): bool {.borrow.}
+proc `==`*(x, y: RequestId): bool {.borrow.}
+proc `==`*(x, y: SlotId): bool {.borrow.}
+proc hash*(x: SlotId): Hash {.borrow.}
+proc hash*(x: Nonce): Hash {.borrow.}
+proc hash*(x: Address): Hash {.borrow.}
+
+func toArray*(id: RequestId | SlotId | Nonce): array[32, byte] =
+  array[32, byte](id)
+
+proc `$`*(id: RequestId | SlotId | Nonce): string =
+  id.toArray.toHex
+
+proc fromHex*(T: type RequestId, hex: string): T =
+  T array[32, byte].fromHex(hex)
+
+proc fromHex*(T: type SlotId, hex: string): T =
+  T array[32, byte].fromHex(hex)
+
+proc fromHex*(T: type Nonce, hex: string): T =
+  T array[32, byte].fromHex(hex)
+
+proc fromHex*[T: distinct](_: type T, hex: string): T =
+  type baseType = T.distinctBase
+  T baseType.fromHex(hex)
+
+proc toHex*[T: distinct](id: T): string =
+  type baseType = T.distinctBase
+  baseType(id).toHex
+
+logutils.formatIt(LogFormat.textLines, Nonce):
+  it.short0xHexLog
+logutils.formatIt(LogFormat.textLines, RequestId):
+  it.short0xHexLog
+logutils.formatIt(LogFormat.textLines, SlotId):
+  it.short0xHexLog
+logutils.formatIt(LogFormat.json, Nonce):
+  it.to0xHexLog
+logutils.formatIt(LogFormat.json, RequestId):
+  it.to0xHexLog
+logutils.formatIt(LogFormat.json, SlotId):
+  it.to0xHexLog
+
+func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest =
+  StorageRequest(
+    client: tupl[0], ask: tupl[1], content: tupl[2], expiry: tupl[3], nonce: tupl[4]
+  )
+
+func fromTuple(_: type Slot, tupl: tuple): Slot =
+  Slot(request: tupl[0], slotIndex: tupl[1])
+
+func fromTuple(_: type StorageAsk, tupl: tuple): StorageAsk =
+  StorageAsk(
+    proofProbability: tupl[0],
+    pricePerBytePerSecond: tupl[1],
+    collateralPerByte: tupl[2],
+    slots: tupl[3],
+    slotSize: tupl[4],
+    duration: tupl[5],
+    maxSlotLoss: tupl[6],
+  )
+
+func fromTuple(_: type StorageContent, tupl: tuple): StorageContent =
+  StorageContent(cid: tupl[0], merkleRoot: tupl[1])
+
+func solidityType*(_: type Cid): string =
+  solidityType(seq[byte])
+
+func solidityType*(_: type StorageContent): string =
+  solidityType(StorageContent.fieldTypes)
+
+func solidityType*(_: type StorageAsk): string =
+  solidityType(StorageAsk.fieldTypes)
+
+func solidityType*(_: type StorageRequest): string =
+  solidityType(StorageRequest.fieldTypes)
+
+# Note: it seems to be ok to ignore the vbuffer offset for now
+func encode*(encoder: var AbiEncoder, cid: Cid) =
+  encoder.write(cid.data.buffer)
+
+func encode*(encoder: var AbiEncoder, content: StorageContent) =
+  encoder.write(content.fieldValues)
+
+func encode*(encoder: var AbiEncoder, ask: StorageAsk) =
+  encoder.write(ask.fieldValues)
+
+func encode*(encoder: var AbiEncoder, id: RequestId | SlotId | Nonce) =
+  encoder.write(id.toArray)
+
+func encode*(encoder: var AbiEncoder, request: StorageRequest) =
+  encoder.write(request.fieldValues)
+
+func encode*(encoder: var AbiEncoder, slot: Slot) =
+  encoder.write(slot.fieldValues)
+
+func decode*(decoder: var AbiDecoder, T: type Cid): ?!T =
+  let data = ?decoder.read(seq[byte])
+  Cid.init(data).mapFailure
+
+func decode*(decoder: var AbiDecoder, T: type StorageContent): ?!T =
+  let tupl = ?decoder.read(StorageContent.fieldTypes)
+  success StorageContent.fromTuple(tupl)
+
+func decode*(decoder: var AbiDecoder, T: type StorageAsk): ?!T =
+  let tupl = ?decoder.read(StorageAsk.fieldTypes)
+  success StorageAsk.fromTuple(tupl)
+
+func decode*(decoder: var AbiDecoder, T: type StorageRequest): ?!T =
+  let tupl = ?decoder.read(StorageRequest.fieldTypes)
+  success StorageRequest.fromTuple(tupl)
+
+func decode*(decoder: var AbiDecoder, T: type Slot): ?!T =
+  let tupl = ?decoder.read(Slot.fieldTypes)
+  success Slot.fromTuple(tupl)
+
+func id*(request: StorageRequest): RequestId =
+  let encoding = AbiEncoder.encode((request,))
+  RequestId(keccak256.digest(encoding).data)
+
+func slotId*(requestId: RequestId, slotIndex: uint64): SlotId =
+  let encoding = AbiEncoder.encode((requestId, slotIndex))
+  SlotId(keccak256.digest(encoding).data)
+
+func slotId*(request: StorageRequest, slotIndex: uint64): SlotId =
+  slotId(request.id, slotIndex)
+
+func id*(slot: Slot): SlotId =
+  slotId(slot.request, slot.slotIndex)
+
+func pricePerSlotPerSecond*(ask: StorageAsk): UInt256 =
+  ask.pricePerBytePerSecond * ask.slotSize.u256
+
+func pricePerSlot*(ask: StorageAsk): UInt256 =
+  ask.duration.u256 * ask.pricePerSlotPerSecond
+
+func totalPrice*(ask: StorageAsk): UInt256 =
+  ask.slots.u256 * ask.pricePerSlot
+
+func totalPrice*(request: StorageRequest): UInt256 =
+  request.ask.totalPrice
+
+func collateralPerSlot*(ask: StorageAsk): UInt256 =
+  ask.collateralPerByte * ask.slotSize.u256
+
+func size*(ask: StorageAsk): uint64 =
+  ask.slots * ask.slotSize

From 483f319e2cce3b59c417e547519957fae81a0614 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 11 Mar 2025 09:36:09 +0100
Subject: [PATCH 04/25] formatting

---
 codexcrawler/services/marketplace/logutils.nim | 3 ++-
 tests/codexcrawler/mocks/mockstate.nim         | 4 +---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/codexcrawler/services/marketplace/logutils.nim b/codexcrawler/services/marketplace/logutils.nim
index 39f99fb..90415ea 100644
--- a/codexcrawler/services/marketplace/logutils.nim
+++ b/codexcrawler/services/marketplace/logutils.nim
@@ -92,7 +92,8 @@ import std/sugar
 import std/typetraits
 
 import pkg/chronicles except toJson, `%`
-from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
+from pkg/libp2p import
+  Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
 from pkg/ethers import Address
 import pkg/questionable
 import pkg/questionable/results
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index 67b45a3..329ca99 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -22,9 +22,7 @@ method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} =
 proc createMockState*(): MockState =
   MockState(
     status: ApplicationStatus.Running,
-    config: Config(
-      dhtEnable: true
-    ),
+    config: Config(dhtEnable: true),
     events: Events(
       nodesFound: newAsyncDataEvent[seq[Nid]](),
       newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),

From ed100ef27e02d88698294bd5d16447f08efb2994 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 13:42:22 +0100
Subject: [PATCH 05/25] fixes compilation of nim-codex marketplace wrapper code

---
 codexcrawler/config.nim                       |   4 +-
 codexcrawler/installer.nim                    |   5 +
 codexcrawler/services/marketplace.nim         |   4 +-
 codexcrawler/services/marketplace/market.nim  | 599 ++++++++++++++++++
 codexcrawler/services/marketplace/periods.nim |  17 +
 .../services/marketplace/provider.nim         | 120 ++++
 6 files changed, 744 insertions(+), 5 deletions(-)
 create mode 100644 codexcrawler/services/marketplace/market.nim
 create mode 100644 codexcrawler/services/marketplace/periods.nim
 create mode 100644 codexcrawler/services/marketplace/provider.nim

diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index f81dd06..f599a71 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -20,14 +20,14 @@ Options:
   --dataDir=<dir>                   Directory for storing data [default: crawler_data]
   --discoveryPort=<p>               Port used for DHT [default: 8090]
   --bootNodes=<n>                   Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
-  --dhtEnable=<e>                         Set to "1" to enable DHT crawler [default: 1]
+  --dhtEnable=<e>                   Set to "1" to enable DHT crawler [default: 0]
   --stepDelay=<ms>                  Delay in milliseconds per node visit [default: 1000]
   --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
   --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
   --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
-  --marketplaceEnable=<e>                 Set to "1" to enable marketplace metrics [default: 1]
   --ethProvider=<a>                Address including http(s) or ws of the eth provider
   --marketplaceAddress=<a>         Eth address of Codex contracts deployment
+  --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
 """
 
 import strutils
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index 8b952aa..5e3738e 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -5,6 +5,9 @@ import ./state
 import ./services/clock
 import ./services/metrics
 import ./services/dht
+
+import ./services/marketplace
+
 import ./component
 import ./components/crawler
 import ./components/timetracker
@@ -15,6 +18,8 @@ import ./components/todolist
 proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   var components: seq[Component] = newSeq[Component]()
 
+  aaa()
+
   let clock = createClock()
 
   without dht =? (await createDht(state)), err:
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 80d6a36..44197a7 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -1,6 +1,4 @@
-import ./marketplace/marketplace
-
-# todo
+import ./marketplace/market
 
 proc aaa*() =
   echo "aaa"
diff --git a/codexcrawler/services/marketplace/market.nim b/codexcrawler/services/marketplace/market.nim
new file mode 100644
index 0000000..294bccc
--- /dev/null
+++ b/codexcrawler/services/marketplace/market.nim
@@ -0,0 +1,599 @@
+import std/strutils
+import std/strformat
+import pkg/ethers
+import pkg/upraises
+import pkg/questionable
+import ./logutils
+import ./marketplace
+import ./proofs
+import ./provider
+import ./config
+import ./periods
+
+# Copy of nim-codex market.nim
+# Edited to remove signing, reward address, etc
+
+logScope:
+  topics = "marketplace onchain market"
+
+type
+  OnChainMarket* = ref object of RootObj
+    contract: Marketplace
+    configuration: ?MarketplaceConfig
+
+  Subscription = ref object of RootObj
+  MarketError* = object of CatchableError
+  MarketSubscription = market.Subscription
+  EventSubscription = ethers.Subscription
+  OnChainMarketSubscription = ref object of MarketSubscription
+    eventSubscription: EventSubscription
+
+  ProofChallenge* = array[32, byte]
+  # Event callback signatures:
+  OnRequest* =
+    proc(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, upraises: [].}
+  OnFulfillment* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
+  OnSlotFilled* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].}
+  OnSlotFreed* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].}
+  OnSlotReservationsFull* =
+    proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, upraises: [].}
+  OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
+  OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, upraises: [].}
+  OnProofSubmitted* = proc(id: SlotId) {.gcsafe, upraises: [].}
+
+  # Marketplace events
+  MarketplaceEvent* = Event
+  StorageRequested* = object of MarketplaceEvent
+    requestId*: RequestId
+    ask*: StorageAsk
+    expiry*: uint64
+
+  SlotFilled* = object of MarketplaceEvent
+    requestId* {.indexed.}: RequestId
+    slotIndex*: uint64
+
+  SlotFreed* = object of MarketplaceEvent
+    requestId* {.indexed.}: RequestId
+    slotIndex*: uint64
+
+  SlotReservationsFull* = object of MarketplaceEvent
+    requestId* {.indexed.}: RequestId
+    slotIndex*: uint64
+
+  RequestFulfilled* = object of MarketplaceEvent
+    requestId* {.indexed.}: RequestId
+
+  RequestCancelled* = object of MarketplaceEvent
+    requestId* {.indexed.}: RequestId
+
+  RequestFailed* = object of MarketplaceEvent
+    requestId* {.indexed.}: RequestId
+
+  ProofSubmitted* = object of MarketplaceEvent
+    id*: SlotId
+
+func new*(_: type OnChainMarket, contract: Marketplace): OnChainMarket =
+  OnChainMarket(contract: contract)
+
+proc raiseMarketError(message: string) {.raises: [MarketError].} =
+  raise newException(MarketError, message)
+
+proc msgDetail*(e: ref CatchableError): string =
+  var msg = e.msg
+  if e.parent != nil:
+    msg = fmt"{msg} Inner exception: {e.parent.msg}"
+  return msg
+
+template convertEthersError(body) =
+  try:
+    body
+  except EthersError as error:
+    raiseMarketError(error.msgDetail)
+
+proc loadConfig(
+    market: OnChainMarket
+): Future[?!void] {.async: (raises: [CancelledError, MarketError]).} =
+  try:
+    without config =? market.configuration:
+      let fetchedConfig = await market.contract.configuration()
+
+      market.configuration = some fetchedConfig
+
+    return success()
+  except AsyncLockError, EthersError, CatchableError:
+    let err = getCurrentException()
+    return failure newException(
+      MarketError,
+      "Failed to fetch the config from the Marketplace contract: " & err.msg,
+    )
+
+proc config(
+    market: OnChainMarket
+): Future[MarketplaceConfig] {.async: (raises: [CancelledError, MarketError]).} =
+  without resolvedConfig =? market.configuration:
+    if err =? (await market.loadConfig()).errorOption:
+      raiseMarketError(err.msg)
+
+    without config =? market.configuration:
+      raiseMarketError("Failed to access to config from the Marketplace contract")
+
+    return config
+
+  return resolvedConfig
+
+proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
+  raiseAssert("Not available: approveFunds")
+
+proc getZkeyHash*(
+    market: OnChainMarket
+): Future[?string] {.async: (raises: [CancelledError, MarketError]).} =
+  let config = await market.config()
+  return some config.proofs.zkeyHash
+
+proc periodicity*(
+    market: OnChainMarket
+): Future[Periodicity] {.async: (raises: [CancelledError, MarketError]).} =
+  convertEthersError:
+    let config = await market.config()
+    let period = config.proofs.period
+    return Periodicity(seconds: period)
+
+proc proofTimeout*(
+    market: OnChainMarket
+): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
+  convertEthersError:
+    let config = await market.config()
+    return config.proofs.timeout
+
+proc repairRewardPercentage*(
+    market: OnChainMarket
+): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
+  convertEthersError:
+    let config = await market.config()
+    return config.collateral.repairRewardPercentage
+
+proc requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async.} =
+  convertEthersError:
+    let config = await market.config()
+    return config.requestDurationLimit
+
+proc proofDowntime*(
+    market: OnChainMarket
+): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
+  convertEthersError:
+    let config = await market.config()
+    return config.proofs.downtime
+
+proc getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} =
+  convertEthersError:
+    let overrides = CallOverrides(blockTag: some BlockTag.pending)
+    return await market.contract.getPointer(slotId, overrides)
+
+proc myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} =
+  convertEthersError:
+    return await market.contract.myRequests
+
+proc mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} =
+  convertEthersError:
+    let slots = await market.contract.mySlots()
+    debug "Fetched my slots", numSlots = len(slots)
+
+    return slots
+
+proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async.} =
+  convertEthersError:
+    debug "Requesting storage"
+    await market.approveFunds(request.totalPrice())
+    discard await market.contract.requestStorage(request).confirm(1)
+
+proc getRequest*(
+    market: OnChainMarket, id: RequestId
+): Future[?StorageRequest] {.async: (raises: [CancelledError]).} =
+  try:
+    let key = $id
+
+    # if key in market.requestCache:
+    #   return some market.requestCache[key]
+
+    let request = await market.contract.getRequest(id)
+    # market.requestCache[key] = request
+    return some request
+  except Marketplace_UnknownRequest, KeyError:
+    warn "Cannot retrieve the request", error = getCurrentExceptionMsg()
+    return none StorageRequest
+  except EthersError, AsyncLockError:
+    error "Cannot retrieve the request", error = getCurrentExceptionMsg()
+    return none StorageRequest
+  except CatchableError as err:
+    error "Unknown error", error = err.msg
+    return none StorageRequest
+
+proc requestState*(
+    market: OnChainMarket, requestId: RequestId
+): Future[?RequestState] {.async.} =
+  convertEthersError:
+    try:
+      let overrides = CallOverrides(blockTag: some BlockTag.pending)
+      return some await market.contract.requestState(requestId, overrides)
+    except Marketplace_UnknownRequest:
+      return none RequestState
+
+proc slotState*(
+    market: OnChainMarket, slotId: SlotId
+): Future[SlotState] {.async: (raises: [CancelledError, MarketError]).} =
+  convertEthersError:
+    try:
+      let overrides = CallOverrides(blockTag: some BlockTag.pending)
+      return await market.contract.slotState(slotId, overrides)
+    except AsyncLockError as err:
+      raiseMarketError(
+        "Failed to fetch the slot state from the Marketplace contract: " & err.msg
+      )
+    except CatchableError as err:
+      raiseMarketError("Unknown error: " & err.msg)
+
+proc getRequestEnd*(market: OnChainMarket, id: RequestId): Future[int64] {.async.} =
+  convertEthersError:
+    return await market.contract.requestEnd(id)
+
+proc requestExpiresAt*(market: OnChainMarket, id: RequestId): Future[int64] {.async.} =
+  convertEthersError:
+    return await market.contract.requestExpiry(id)
+
+proc getHost(
+    market: OnChainMarket, requestId: RequestId, slotIndex: uint64
+): Future[?Address] {.async.} =
+  convertEthersError:
+    let slotId = slotId(requestId, slotIndex)
+    let address = await market.contract.getHost(slotId)
+    if address != Address.default:
+      return some address
+    else:
+      return none Address
+
+proc currentCollateral*(
+    market: OnChainMarket, slotId: SlotId
+): Future[UInt256] {.async.} =
+  convertEthersError:
+    return await market.contract.currentCollateral(slotId)
+
+proc getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async.} =
+  convertEthersError:
+    try:
+      return some await market.contract.getActiveSlot(slotId)
+    except Marketplace_SlotIsFree:
+      return none Slot
+
+proc fillSlot(
+    market: OnChainMarket,
+    requestId: RequestId,
+    slotIndex: uint64,
+    proof: Groth16Proof,
+    collateral: UInt256,
+) {.async.} =
+  convertEthersError:
+    logScope:
+      requestId
+      slotIndex
+
+    await market.approveFunds(collateral)
+    trace "calling fillSlot on contract"
+    discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(1)
+    trace "fillSlot transaction completed"
+
+proc freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
+  raiseAssert("Not available: freeSlot")
+
+proc withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async.} =
+  convertEthersError:
+    discard await market.contract.withdrawFunds(requestId).confirm(1)
+
+proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} =
+  convertEthersError:
+    try:
+      let overrides = CallOverrides(blockTag: some BlockTag.pending)
+      return await market.contract.isProofRequired(id, overrides)
+    except Marketplace_SlotIsFree:
+      return false
+
+proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} =
+  convertEthersError:
+    try:
+      let overrides = CallOverrides(blockTag: some BlockTag.pending)
+      return await market.contract.willProofBeRequired(id, overrides)
+    except Marketplace_SlotIsFree:
+      return false
+
+proc getChallenge*(
+    market: OnChainMarket, id: SlotId
+): Future[ProofChallenge] {.async.} =
+  convertEthersError:
+    let overrides = CallOverrides(blockTag: some BlockTag.pending)
+    return await market.contract.getChallenge(id, overrides)
+
+proc submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async.} =
+  convertEthersError:
+    discard await market.contract.submitProof(id, proof).confirm(1)
+
+proc markProofAsMissing*(market: OnChainMarket, id: SlotId, period: Period) {.async.} =
+  convertEthersError:
+    discard await market.contract.markProofAsMissing(id, period).confirm(1)
+
+proc canProofBeMarkedAsMissing*(
+    market: OnChainMarket, id: SlotId, period: Period
+): Future[bool] {.async.} =
+  let provider = market.contract.provider
+  let contractWithoutSigner = market.contract.connect(provider)
+  let overrides = CallOverrides(blockTag: some BlockTag.pending)
+  try:
+    discard await contractWithoutSigner.markProofAsMissing(id, period, overrides)
+    return true
+  except EthersError as e:
+    trace "Proof cannot be marked as missing", msg = e.msg
+    return false
+
+proc reserveSlot*(
+    market: OnChainMarket, requestId: RequestId, slotIndex: uint64
+) {.async.} =
+  convertEthersError:
+    discard await market.contract
+    .reserveSlot(
+      requestId,
+      slotIndex,
+      # reserveSlot runs out of gas for unknown reason, but 100k gas covers it
+      TransactionOverrides(gasLimit: some 100000.u256),
+    )
+    .confirm(1)
+
+proc canReserveSlot*(
+    market: OnChainMarket, requestId: RequestId, slotIndex: uint64
+): Future[bool] {.async.} =
+  convertEthersError:
+    return await market.contract.canReserveSlot(requestId, slotIndex)
+
+proc subscribeRequests*(
+    market: OnChainMarket, callback: OnRequest
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!StorageRequested) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in Request subscription", msg = eventErr.msg
+      return
+
+    callback(event.requestId, event.ask, event.expiry)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(StorageRequested, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeSlotFilled*(
+    market: OnChainMarket, callback: OnSlotFilled
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!SlotFilled) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in SlotFilled subscription", msg = eventErr.msg
+      return
+
+    callback(event.requestId, event.slotIndex)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(SlotFilled, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeSlotFilled*(
+    market: OnChainMarket,
+    requestId: RequestId,
+    slotIndex: uint64,
+    callback: OnSlotFilled,
+): Future[MarketSubscription] {.async.} =
+  proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: uint64) =
+    if eventRequestId == requestId and eventSlotIndex == slotIndex:
+      callback(requestId, slotIndex)
+
+  convertEthersError:
+    return await market.subscribeSlotFilled(onSlotFilled)
+
+proc subscribeSlotFreed*(
+    market: OnChainMarket, callback: OnSlotFreed
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!SlotFreed) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in SlotFreed subscription", msg = eventErr.msg
+      return
+
+    callback(event.requestId, event.slotIndex)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(SlotFreed, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeSlotReservationsFull*(
+    market: OnChainMarket, callback: OnSlotReservationsFull
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!SlotReservationsFull) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in SlotReservationsFull subscription",
+        msg = eventErr.msg
+      return
+
+    callback(event.requestId, event.slotIndex)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(SlotReservationsFull, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeFulfillment(
+    market: OnChainMarket, callback: OnFulfillment
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
+      return
+
+    callback(event.requestId)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeFulfillment(
+    market: OnChainMarket, requestId: RequestId, callback: OnFulfillment
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
+      return
+
+    if event.requestId == requestId:
+      callback(event.requestId)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(RequestFulfilled, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeRequestCancelled*(
+    market: OnChainMarket, callback: OnRequestCancelled
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in RequestCancelled subscription", msg = eventErr.msg
+      return
+
+    callback(event.requestId)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeRequestCancelled*(
+    market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in RequestCancelled subscription", msg = eventErr.msg
+      return
+
+    if event.requestId == requestId:
+      callback(event.requestId)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(RequestCancelled, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeRequestFailed*(
+    market: OnChainMarket, callback: OnRequestFailed
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in RequestFailed subscription", msg = eventErr.msg
+      return
+
+    callback(event.requestId)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(RequestFailed, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeRequestFailed*(
+    market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in RequestFailed subscription", msg = eventErr.msg
+      return
+
+    if event.requestId == requestId:
+      callback(event.requestId)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(RequestFailed, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc subscribeProofSubmission*(
+    market: OnChainMarket, callback: OnProofSubmitted
+): Future[MarketSubscription] {.async.} =
+  proc onEvent(eventResult: ?!ProofSubmitted) {.upraises: [].} =
+    without event =? eventResult, eventErr:
+      error "There was an error in ProofSubmitted subscription", msg = eventErr.msg
+      return
+
+    callback(event.id)
+
+  convertEthersError:
+    let subscription = await market.contract.subscribe(ProofSubmitted, onEvent)
+    return OnChainMarketSubscription(eventSubscription: subscription)
+
+proc unsubscribe*(subscription: OnChainMarketSubscription) {.async.} =
+  await subscription.eventSubscription.unsubscribe()
+
+proc queryPastSlotFilledEvents*(
+    market: OnChainMarket, fromBlock: BlockTag
+): Future[seq[SlotFilled]] {.async.} =
+  convertEthersError:
+    return await market.contract.queryFilter(SlotFilled, fromBlock, BlockTag.latest)
+
+proc queryPastSlotFilledEvents*(
+    market: OnChainMarket, blocksAgo: int
+): Future[seq[SlotFilled]] {.async.} =
+  convertEthersError:
+    let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
+
+    return await market.queryPastSlotFilledEvents(fromBlock)
+
+proc queryPastSlotFilledEvents*(
+    market: OnChainMarket, fromTime: int64
+): Future[seq[SlotFilled]] {.async.} =
+  convertEthersError:
+    let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
+    return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock))
+
+proc queryPastStorageRequestedEvents*(
+    market: OnChainMarket, fromBlock: BlockTag
+): Future[seq[StorageRequested]] {.async.} =
+  convertEthersError:
+    return
+      await market.contract.queryFilter(StorageRequested, fromBlock, BlockTag.latest)
+
+proc queryPastStorageRequestedEvents*(
+    market: OnChainMarket, blocksAgo: int
+): Future[seq[StorageRequested]] {.async.} =
+  convertEthersError:
+    let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
+
+    return await market.queryPastStorageRequestedEvents(fromBlock)
+
+proc slotCollateral*(
+    market: OnChainMarket, collateralPerSlot: UInt256, slotState: SlotState
+): ?!UInt256 {.raises: [].} =
+  if slotState == SlotState.Repair:
+    without repairRewardPercentage =?
+      market.configuration .? collateral .? repairRewardPercentage:
+      return failure newException(
+        MarketError,
+        "Failure calculating the slotCollateral, cannot get the reward percentage",
+      )
+
+    return success (
+      collateralPerSlot - (collateralPerSlot * repairRewardPercentage.u256).div(
+        100.u256
+      )
+    )
+
+  return success(collateralPerSlot)
+
+proc slotCollateral*(
+    market: OnChainMarket, requestId: RequestId, slotIndex: uint64
+): Future[?!UInt256] {.async: (raises: [CancelledError]).} =
+  let slotid = slotId(requestId, slotIndex)
+
+  try:
+    let slotState = await market.slotState(slotid)
+
+    without request =? await market.getRequest(requestId):
+      return failure newException(
+        MarketError, "Failure calculating the slotCollateral, cannot get the request"
+      )
+
+    return market.slotCollateral(request.ask.collateralPerSlot, slotState)
+  except MarketError as error:
+    error "Error when trying to calculate the slotCollateral", error = error.msg
+    return failure error
diff --git a/codexcrawler/services/marketplace/periods.nim b/codexcrawler/services/marketplace/periods.nim
new file mode 100644
index 0000000..cbb860e
--- /dev/null
+++ b/codexcrawler/services/marketplace/periods.nim
@@ -0,0 +1,17 @@
+import pkg/stint
+
+type
+  Periodicity* = object
+    seconds*: uint64
+
+  Period* = uint64
+  Timestamp* = uint64
+
+func periodOf*(periodicity: Periodicity, timestamp: Timestamp): Period =
+  timestamp div periodicity.seconds
+
+func periodStart*(periodicity: Periodicity, period: Period): Timestamp =
+  period * periodicity.seconds
+
+func periodEnd*(periodicity: Periodicity, period: Period): Timestamp =
+  periodicity.periodStart(period + 1)
diff --git a/codexcrawler/services/marketplace/provider.nim b/codexcrawler/services/marketplace/provider.nim
new file mode 100644
index 0000000..ac5a6c1
--- /dev/null
+++ b/codexcrawler/services/marketplace/provider.nim
@@ -0,0 +1,120 @@
+import pkg/ethers/provider
+import pkg/chronos
+import pkg/questionable
+import ./logutils
+
+logScope:
+  topics = "marketplace onchain provider"
+
+proc raiseProviderError(message: string) {.raises: [ProviderError].} =
+  raise newException(ProviderError, message)
+
+proc blockNumberAndTimestamp*(
+    provider: Provider, blockTag: BlockTag
+): Future[(UInt256, UInt256)] {.async: (raises: [ProviderError, CancelledError]).} =
+  without latestBlock =? await provider.getBlock(blockTag):
+    raiseProviderError("Could not get latest block")
+
+  without latestBlockNumber =? latestBlock.number:
+    raiseProviderError("Could not get latest block number")
+
+  return (latestBlockNumber, latestBlock.timestamp)
+
+proc binarySearchFindClosestBlock(
+    provider: Provider, epochTime: int, low: UInt256, high: UInt256
+): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} =
+  let (_, lowTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(low))
+  let (_, highTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(high))
+  if abs(lowTimestamp.truncate(int) - epochTime) <
+      abs(highTimestamp.truncate(int) - epochTime):
+    return low
+  else:
+    return high
+
+proc binarySearchBlockNumberForEpoch(
+    provider: Provider,
+    epochTime: UInt256,
+    latestBlockNumber: UInt256,
+    earliestBlockNumber: UInt256,
+): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} =
+  var low = earliestBlockNumber
+  var high = latestBlockNumber
+
+  while low <= high:
+    if low == 0 and high == 0:
+      return low
+    let mid = (low + high) div 2
+    let (midBlockNumber, midBlockTimestamp) =
+      await provider.blockNumberAndTimestamp(BlockTag.init(mid))
+
+    if midBlockTimestamp < epochTime:
+      low = mid + 1
+    elif midBlockTimestamp > epochTime:
+      high = mid - 1
+    else:
+      return midBlockNumber
+  # NOTICE that by how the binary search is implemented, when it finishes
+  # low is always greater than high - this is why we use high, where
+  # intuitively we would use low:
+  await provider.binarySearchFindClosestBlock(
+    epochTime.truncate(int), low = high, high = low
+  )
+
+proc blockNumberForEpoch*(
+    provider: Provider, epochTime: int64
+): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} =
+  let epochTimeUInt256 = epochTime.u256
+  let (latestBlockNumber, latestBlockTimestamp) =
+    await provider.blockNumberAndTimestamp(BlockTag.latest)
+  let (earliestBlockNumber, earliestBlockTimestamp) =
+    await provider.blockNumberAndTimestamp(BlockTag.earliest)
+
+  # Initially we used the average block time to predict
+  # the number of blocks we need to look back in order to find
+  # the block number corresponding to the given epoch time. 
+  # This estimation can be highly inaccurate if block time
+  # was changing in the past or is fluctuating and therefore
+  # we used that information initially only to find out
+  # if the available history is long enough to perform effective search.
+  # It turns out we do not have to do that. There is an easier way.
+  #
+  # First we check if the given epoch time equals the timestamp of either
+  # the earliest or the latest block. If it does, we just return the
+  # block number of that block.
+  #
+  # Otherwise, if the earliest available block is not the genesis block, 
+  # we should check the timestamp of that earliest block and if it is greater
+  # than the epoch time, we should issue a warning and return
+  # that earliest block number.
+  # In all other cases, thus when the earliest block is not the genesis
+  # block but its timestamp is not greater than the requested epoch time, or
+  # if the earliest available block is the genesis block, 
+  # (which means we have the whole history available), we should proceed with
+  # the binary search.
+  #
+  # Additional benefit of this method is that we do not have to rely
+  # on the average block time, which not only makes the whole thing
+  # more reliable, but also easier to test.
+
+  # Are lucky today?
+  if earliestBlockTimestamp == epochTimeUInt256:
+    return earliestBlockNumber
+  if latestBlockTimestamp == epochTimeUInt256:
+    return latestBlockNumber
+
+  if earliestBlockNumber > 0 and earliestBlockTimestamp > epochTimeUInt256:
+    let availableHistoryInDays =
+      (latestBlockTimestamp - earliestBlockTimestamp) div 1.days.secs.u256
+    warn "Short block history detected.",
+      earliestBlockTimestamp = earliestBlockTimestamp, days = availableHistoryInDays
+    return earliestBlockNumber
+
+  return await provider.binarySearchBlockNumberForEpoch(
+    epochTimeUInt256, latestBlockNumber, earliestBlockNumber
+  )
+
+proc pastBlockTag*(
+    provider: Provider, blocksAgo: int
+): Future[BlockTag] {.async: (raises: [ProviderError, CancelledError]).} =
+  let head = await provider.getBlockNumber()
+  return BlockTag.init(head - blocksAgo.abs.u256)

From b3e147ace12da4d4155c06cac49ce3683462c9b6 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 14:31:10 +0100
Subject: [PATCH 06/25] can successfully fetch zkeyhash from chain

---
 codexcrawler/installer.nim            |  2 +-
 codexcrawler/services/marketplace.nim | 30 +++++++++++++++++++++++++--
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index 5e3738e..42153bd 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -18,7 +18,7 @@ import ./components/todolist
 proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   var components: seq[Component] = newSeq[Component]()
 
-  aaa()
+  await aaa(state.config)
 
   let clock = createClock()
 
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 44197a7..4872ffe 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -1,4 +1,30 @@
-import ./marketplace/market
+import pkg/ethers
+import pkg/questionable
 
-proc aaa*() =
+import ./marketplace/market
+import ./marketplace/marketplace
+import ../config
+
+proc aaa*(config: Config) {.async.} = 
   echo "aaa"
+
+  let provider = JsonRpcProvider.new(config.ethProvider)
+  without marketplaceAddress =? Address.init(config.marketplaceAddress):
+    raiseAssert("A!")
+
+  let marketplace = Marketplace.new(marketplaceAddress, provider)
+  let market = OnChainMarket.new(marketplace)
+
+  echo "bbb"
+  echo "running with marketplace address: " & $marketplaceAddress
+
+  try:
+    without zkeyhash =? await market.getZkeyHash():
+      echo "couldn't get zkeyhash"
+      return
+    echo "zkeyhash=" & $zkeyhash
+
+  except CatchableError as err:
+    echo "catchable error! " & err.msg
+
+  echo "ccc"

From ce3e53949bb6ab01e8443e35dcc06b6653498942 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 14:55:52 +0100
Subject: [PATCH 07/25] converts to application component

---
 codexcrawler/installer.nim                   |  5 +-
 codexcrawler/services/marketplace.nim        | 55 ++++++++++++++------
 tests/codexcrawler/mocks/mockmarketplace.nim | 21 ++++++++
 3 files changed, 62 insertions(+), 19 deletions(-)
 create mode 100644 tests/codexcrawler/mocks/mockmarketplace.nim

diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index 42153bd..5101897 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -17,9 +17,6 @@ import ./components/todolist
 
 proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   var components: seq[Component] = newSeq[Component]()
-
-  await aaa(state.config)
-
   let clock = createClock()
 
   without dht =? (await createDht(state)), err:
@@ -31,6 +28,7 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   let
     metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort)
     todoList = createTodoList(state, metrics)
+    marketplace = createMarketplace(state)
 
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
     return failure(err)
@@ -41,5 +39,6 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   components.add(Crawler.new(state, dht, todoList))
   components.add(TimeTracker.new(state, nodeStore, dht, clock))
   components.add(dhtMetrics)
+  components.add(marketplace)
 
   return success(components)
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 4872ffe..c1f037b 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -4,27 +4,50 @@ import pkg/questionable
 import ./marketplace/market
 import ./marketplace/marketplace
 import ../config
+import ../component
+import ../state
 
-proc aaa*(config: Config) {.async.} = 
-  echo "aaa"
+logScope:
+  topics = "marketplace"
 
-  let provider = JsonRpcProvider.new(config.ethProvider)
-  without marketplaceAddress =? Address.init(config.marketplaceAddress):
-    raiseAssert("A!")
+type
+  MarketplaceService* = ref object of Component
+    state: State
+    market: ?OnChainMarket
+
+method getZkeyhash*(m: MarketplaceService): Future[?!string] {.async: (raises: []), base.} =
+  try:
+    if market =? m.market:
+      without zkeyhash =? await market.getZkeyHash():
+        return failure("Failed to get zkeyHash")
+      return success(zkeyhash)
+    return failure("MarketplaceService is not started")
+  except CatchableError as err:
+    return failure("Error while getting zkeyHash: " & err.msg)
+
+method start*(m: MarketplaceService): Future[?!void] {.async.} =
+  let provider = JsonRpcProvider.new(m.state.config.ethProvider)
+  without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
+    return failure("Invalid MarketplaceAddress provided")
 
   let marketplace = Marketplace.new(marketplaceAddress, provider)
-  let market = OnChainMarket.new(marketplace)
+  m.market = some(OnChainMarket.new(marketplace))
 
-  echo "bbb"
-  echo "running with marketplace address: " & $marketplaceAddress
+  return success()
 
-  try:
-    without zkeyhash =? await market.getZkeyHash():
-      echo "couldn't get zkeyhash"
-      return
-    echo "zkeyhash=" & $zkeyhash
+method stop*(m: MarketplaceService): Future[?!void] {.async.} =
+  return success()
 
-  except CatchableError as err:
-    echo "catchable error! " & err.msg
+proc new(
+    T: type MarketplaceService,
+    state: State
+): MarketplaceService =
+  return MarketplaceService(
+    state: state,
+    market: none(OnChainMarket)
+  )
 
-  echo "ccc"
+proc createMarketplace*(state: State): MarketplaceService =
+  return MarketplaceService.new(
+    state
+  )
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
new file mode 100644
index 0000000..c38d865
--- /dev/null
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -0,0 +1,21 @@
+import pkg/ethers
+import pkg/questionable
+
+import ./marketplace/market
+import ./marketplace/marketplace
+import ../config
+import ../component
+import ../state
+
+logScope:
+  topics = "marketplace"
+
+type
+  MockMarketplaceService* = ref object of MarketplaceService
+    zkeyHashReturn*: ?!string
+
+method getZkeyhash*(m: MockMarketplaceService): Future[?!string] {.async: (raises: []).} =
+  return m.zkeyHashReturn
+
+proc createMockMarketplaceService*(): MockMarketplaceService =
+  MockMarketplaceService()

From b801e68f1dcad3c98ed398bee5d76b189d28a568 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 15:05:35 +0100
Subject: [PATCH 08/25] replaces zkeyhash with recent slotfill events

---
 codexcrawler/services/marketplace.nim        | 21 ++++++++++----------
 tests/codexcrawler/mocks/mockmarketplace.nim |  6 +++---
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index c1f037b..318032e 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -1,6 +1,5 @@
 import pkg/ethers
 import pkg/questionable
-
 import ./marketplace/market
 import ./marketplace/marketplace
 import ../config
@@ -15,15 +14,17 @@ type
     state: State
     market: ?OnChainMarket
 
-method getZkeyhash*(m: MarketplaceService): Future[?!string] {.async: (raises: []), base.} =
-  try:
-    if market =? m.market:
-      without zkeyhash =? await market.getZkeyHash():
-        return failure("Failed to get zkeyHash")
-      return success(zkeyhash)
-    return failure("MarketplaceService is not started")
-  except CatchableError as err:
-    return failure("Error while getting zkeyHash: " & err.msg)
+method getRecentSlotFillEvents*(m: MarketplaceService): Future[?!seq[SlotFilled]] {.async: (raises: []), base.} =
+  # There is (aprox.) 1 block every 10 seconds.
+  # 10 seconds * 6 * 60 = 3600 = 1 hour.
+  let blocksAgo = 6 * 60;
+
+  if market =? m.market:
+    try:
+      return success(await market.queryPastSlotFilledEvents(blocksAgo))
+    except CatchableError as err:
+      return failure(err.msg)
+  return failure("MarketplaceService is not started")
 
 method start*(m: MarketplaceService): Future[?!void] {.async.} =
   let provider = JsonRpcProvider.new(m.state.config.ethProvider)
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
index c38d865..2f90e6b 100644
--- a/tests/codexcrawler/mocks/mockmarketplace.nim
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -12,10 +12,10 @@ logScope:
 
 type
   MockMarketplaceService* = ref object of MarketplaceService
-    zkeyHashReturn*: ?!string
+    recentSlotFillEventsReturn*: ?!seq[SlotFilled]
 
-method getZkeyhash*(m: MockMarketplaceService): Future[?!string] {.async: (raises: []).} =
-  return m.zkeyHashReturn
+method getRecentSlotFillEvents*(m: MarketplaceService): Future[?!seq[SlotFilled]] {.async: (raises: []).} =
+  return m.recentSlotFillEventsReturn
 
 proc createMockMarketplaceService*(): MockMarketplaceService =
   MockMarketplaceService()

From 002bdc73145db03491750adbd43e5b7360f58cbd Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 15:54:55 +0100
Subject: [PATCH 09/25] Implements chainmetrics that publishes slotFill events
 count

---
 codexcrawler/components/chainmetrics.nim      | 45 ++++++++++
 codexcrawler/services/metrics.nim             | 11 ++-
 .../components/testchainmetrics.nim           | 90 +++++++++++++++++++
 tests/codexcrawler/mocks/mockmarketplace.nim  |  9 +-
 tests/codexcrawler/mocks/mockmetrics.nim      |  4 +
 tests/codexcrawler/mocks/mockstate.nim        |  2 +-
 tests/codexcrawler/testcomponents.nim         |  9 +-
 7 files changed, 158 insertions(+), 12 deletions(-)
 create mode 100644 codexcrawler/components/chainmetrics.nim
 create mode 100644 tests/codexcrawler/components/testchainmetrics.nim

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
new file mode 100644
index 0000000..3aaf68c
--- /dev/null
+++ b/codexcrawler/components/chainmetrics.nim
@@ -0,0 +1,45 @@
+import pkg/chronicles
+import pkg/chronos
+import pkg/questionable
+import pkg/questionable/results
+
+import ../state
+import ../services/metrics
+import ../services/marketplace
+import ../component
+
+logScope:
+  topics = "ChainMetrics"
+
+type ChainMetrics* = ref object of Component
+  state: State
+  metrics: Metrics
+  marketplace: MarketplaceService
+
+proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
+  without slotFills =? (await c.marketplace.getRecentSlotFillEvents()), err:
+    trace "Unable to get recent slotFill events from chain", err = err.msg
+    return success() # We don't propagate this error.
+    # The call is allowed to fail and the app should continue as normal.
+
+  c.metrics.setSlotFill(slotFills.len)
+  return success()
+
+method start*(c: ChainMetrics): Future[?!void] {.async.} =
+  info "Starting..."  
+
+  proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
+    return await c.step()
+
+  if c.state.config.marketplaceEnable:
+    await c.state.whileRunning(onStep, 10.minutes)
+
+  return success()
+
+method stop*(c: ChainMetrics): Future[?!void] {.async.} =
+  return success()
+
+proc new*(
+    T: type ChainMetrics, state: State, metrics: Metrics, marketplace: MarketplaceService
+): ChainMetrics =
+  ChainMetrics(state: state, metrics: metrics, marketplace: marketplace)
diff --git a/codexcrawler/services/metrics.nim b/codexcrawler/services/metrics.nim
index 39d0b63..322b80c 100644
--- a/codexcrawler/services/metrics.nim
+++ b/codexcrawler/services/metrics.nim
@@ -6,6 +6,8 @@ declareGauge(todoNodesGauge, "DHT nodes to be visited")
 declareGauge(okNodesGauge, "DHT nodes successfully contacted")
 declareGauge(nokNodesGauge, "DHT nodes failed to contact")
 
+declareGauge(slotFillGauge, "Marketplace recent slots filled")
+
 type
   OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].}
 
@@ -13,6 +15,7 @@ type
     todoNodes: OnUpdateMetric
     okNodes: OnUpdateMetric
     nokNodes: OnUpdateMetric
+    onSlotFill: OnUpdateMetric
 
 proc startServer(metricsAddress: IpAddress, metricsPort: Port) =
   let metricsAddress = metricsAddress
@@ -34,6 +37,9 @@ method setOkNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
 method setNokNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
   m.nokNodes(value.int64)
 
+method setSlotFill*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
+  m.onSlotFill(value.int64)
+
 proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
   startServer(metricsAddress, metricsPort)
 
@@ -47,5 +53,8 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
 
   proc onNok(value: int64) =
     nokNodesGauge.set(value)
+    
+  proc onSlotFill(value: int64) =
+    slotFillGauge.set(value)
 
-  return Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok)
+  return Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok, onSlotFill: onSlotFill)
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
new file mode 100644
index 0000000..4668e41
--- /dev/null
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -0,0 +1,90 @@
+import pkg/chronos
+import pkg/questionable
+import pkg/questionable/results
+import pkg/asynctest/chronos/unittest
+import std/sequtils
+
+import ../../../codexcrawler/components/chainmetrics
+import ../../../codexcrawler/services/marketplace/market
+import ../../../codexcrawler/types
+import ../../../codexcrawler/state
+import ../mocks/mockstate
+import ../mocks/mockmetrics
+import ../mocks/mockmarketplace
+import ../helpers
+
+suite "ChainMetrics":
+  var
+    state: MockState
+    metrics: MockMetrics
+    marketplace: MockMarketplaceService
+    chain: ChainMetrics
+
+  setup:
+    state = createMockState()
+    metrics = createMockMetrics()
+    marketplace = createMockMarketplaceService()
+    
+    metrics.slotFill = -1
+    chain = ChainMetrics.new(state, metrics, marketplace)
+
+    (await chain.start()).tryGet()
+
+  teardown:
+    (await chain.stop()).tryGet()
+    state.checkAllUnsubscribed()
+
+  proc onStep() {.async.} =
+    (await state.steppers[0]()).tryGet()
+
+  test "start should start stepper for 10 minutes":
+    check:
+      state.delays.len == 1
+      state.delays[0] == 10.minutes
+
+  test "onStep is not activated when config.marketplaceEnable is false":
+    # Recreate chainMetrics, reset mockstate:
+    (await chain.stop()).tryGet()
+    state.steppers = @[]
+    # disable marketplace:
+    state.config.marketplaceEnable = false
+    (await chain.start()).tryGet()
+
+    check:
+      state.steppers.len == 0
+
+  test "step should not call setSlotFill when getRecentSlotFillEvents fails":
+    let testValue = -123
+    metrics.slotFill = testValue
+
+    marketplace.recentSlotFillEventsReturn = seq[SlotFilled].failure("testfailure")
+
+    await onStep()
+
+    check:
+      metrics.slotFill == testValue
+
+  test "step should setSlotFill to zero when getRecentSlotFillEvents returns empty seq":
+    metrics.slotFill = -123
+
+    marketplace.recentSlotFillEventsReturn = success(newSeq[SlotFilled]())
+
+    await onStep()
+
+    check:
+      metrics.slotFill == 0
+
+  test "step should setSlotFill to the length of seq returned from getRecentSlotFillEvents":
+    let fills = @[
+      SlotFilled(),
+      SlotFilled(),
+      SlotFilled(),
+      SlotFilled()
+    ]
+
+    marketplace.recentSlotFillEventsReturn = success(fills)    
+
+    await onStep()
+
+    check:
+      metrics.slotFill == fills.len
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
index 2f90e6b..bf4a717 100644
--- a/tests/codexcrawler/mocks/mockmarketplace.nim
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -1,11 +1,8 @@
 import pkg/ethers
 import pkg/questionable
 
-import ./marketplace/market
-import ./marketplace/marketplace
-import ../config
-import ../component
-import ../state
+import ../../../codexcrawler/services/marketplace
+import ../../../codexcrawler/services/marketplace/market
 
 logScope:
   topics = "marketplace"
@@ -14,7 +11,7 @@ type
   MockMarketplaceService* = ref object of MarketplaceService
     recentSlotFillEventsReturn*: ?!seq[SlotFilled]
 
-method getRecentSlotFillEvents*(m: MarketplaceService): Future[?!seq[SlotFilled]] {.async: (raises: []).} =
+method getRecentSlotFillEvents*(m: MockMarketplaceService): Future[?!seq[SlotFilled]] {.async: (raises: []).} =
   return m.recentSlotFillEventsReturn
 
 proc createMockMarketplaceService*(): MockMarketplaceService =
diff --git a/tests/codexcrawler/mocks/mockmetrics.nim b/tests/codexcrawler/mocks/mockmetrics.nim
index 8152e76..3b9a24a 100644
--- a/tests/codexcrawler/mocks/mockmetrics.nim
+++ b/tests/codexcrawler/mocks/mockmetrics.nim
@@ -4,6 +4,7 @@ type MockMetrics* = ref object of Metrics
   todo*: int
   ok*: int
   nok*: int
+  slotFill*: int
 
 method setTodoNodes*(m: MockMetrics, value: int) =
   m.todo = value
@@ -14,5 +15,8 @@ method setOkNodes*(m: MockMetrics, value: int) =
 method setNokNodes*(m: MockMetrics, value: int) =
   m.nok = value
 
+method setSlotFill*(m: MockMetrics, value: int) =
+  m.slotFill = value
+
 proc createMockMetrics*(): MockMetrics =
   MockMetrics()
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index 329ca99..e61ae92 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -22,7 +22,7 @@ method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} =
 proc createMockState*(): MockState =
   MockState(
     status: ApplicationStatus.Running,
-    config: Config(dhtEnable: true),
+    config: Config(dhtEnable: true, marketplaceEnable: true),
     events: Events(
       nodesFound: newAsyncDataEvent[seq[Nid]](),
       newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),
diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim
index 0773ec0..a81a08f 100644
--- a/tests/codexcrawler/testcomponents.nim
+++ b/tests/codexcrawler/testcomponents.nim
@@ -1,7 +1,8 @@
-import ./components/testnodestore
-import ./components/testdhtmetrics
-import ./components/testtodolist
-import ./components/testtimetracker
+import ./components/testchainmetrics
 import ./components/testcrawler
+import ./components/testdhtmetrics
+import ./components/testnodestore
+import ./components/testtimetracker
+import ./components/testtodolist
 
 {.warning[UnusedImport]: off.}

From 6a8a6c252f32be82ef42ff8329be6010655e515a Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 15:55:58 +0100
Subject: [PATCH 10/25] formatting

---
 codexcrawler/components/chainmetrics.nim      |  7 +++--
 codexcrawler/config.nim                       |  4 +--
 codexcrawler/services/marketplace.nim         | 27 +++++++------------
 codexcrawler/services/metrics.nim             |  5 ++--
 .../components/testchainmetrics.nim           | 11 +++-----
 tests/codexcrawler/mocks/mockmarketplace.nim  |  9 ++++---
 6 files changed, 28 insertions(+), 35 deletions(-)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 3aaf68c..9691192 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -26,7 +26,7 @@ proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
   return success()
 
 method start*(c: ChainMetrics): Future[?!void] {.async.} =
-  info "Starting..."  
+  info "Starting..."
 
   proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
     return await c.step()
@@ -40,6 +40,9 @@ method stop*(c: ChainMetrics): Future[?!void] {.async.} =
   return success()
 
 proc new*(
-    T: type ChainMetrics, state: State, metrics: Metrics, marketplace: MarketplaceService
+    T: type ChainMetrics,
+    state: State,
+    metrics: Metrics,
+    marketplace: MarketplaceService,
 ): ChainMetrics =
   ChainMetrics(state: state, metrics: metrics, marketplace: marketplace)
diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index f599a71..bc56e40 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -25,8 +25,8 @@ Options:
   --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
   --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
   --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
-  --ethProvider=<a>                Address including http(s) or ws of the eth provider
-  --marketplaceAddress=<a>         Eth address of Codex contracts deployment
+  --ethProvider=<a>                 Address including http(s) or ws of the eth provider
+  --marketplaceAddress=<a>          Eth address of Codex contracts deployment
   --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
 """
 
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 318032e..c959a22 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -9,15 +9,16 @@ import ../state
 logScope:
   topics = "marketplace"
 
-type
-  MarketplaceService* = ref object of Component
-    state: State
-    market: ?OnChainMarket
+type MarketplaceService* = ref object of Component
+  state: State
+  market: ?OnChainMarket
 
-method getRecentSlotFillEvents*(m: MarketplaceService): Future[?!seq[SlotFilled]] {.async: (raises: []), base.} =
+method getRecentSlotFillEvents*(
+    m: MarketplaceService
+): Future[?!seq[SlotFilled]] {.async: (raises: []), base.} =
   # There is (aprox.) 1 block every 10 seconds.
   # 10 seconds * 6 * 60 = 3600 = 1 hour.
-  let blocksAgo = 6 * 60;
+  let blocksAgo = 6 * 60
 
   if market =? m.market:
     try:
@@ -39,16 +40,8 @@ method start*(m: MarketplaceService): Future[?!void] {.async.} =
 method stop*(m: MarketplaceService): Future[?!void] {.async.} =
   return success()
 
-proc new(
-    T: type MarketplaceService,
-    state: State
-): MarketplaceService =
-  return MarketplaceService(
-    state: state,
-    market: none(OnChainMarket)
-  )
+proc new(T: type MarketplaceService, state: State): MarketplaceService =
+  return MarketplaceService(state: state, market: none(OnChainMarket))
 
 proc createMarketplace*(state: State): MarketplaceService =
-  return MarketplaceService.new(
-    state
-  )
+  return MarketplaceService.new(state)
diff --git a/codexcrawler/services/metrics.nim b/codexcrawler/services/metrics.nim
index 322b80c..13115ed 100644
--- a/codexcrawler/services/metrics.nim
+++ b/codexcrawler/services/metrics.nim
@@ -53,8 +53,9 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
 
   proc onNok(value: int64) =
     nokNodesGauge.set(value)
-    
+
   proc onSlotFill(value: int64) =
     slotFillGauge.set(value)
 
-  return Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok, onSlotFill: onSlotFill)
+  return
+    Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok, onSlotFill: onSlotFill)
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 4668e41..99c2a89 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -24,7 +24,7 @@ suite "ChainMetrics":
     state = createMockState()
     metrics = createMockMetrics()
     marketplace = createMockMarketplaceService()
-    
+
     metrics.slotFill = -1
     chain = ChainMetrics.new(state, metrics, marketplace)
 
@@ -75,14 +75,9 @@ suite "ChainMetrics":
       metrics.slotFill == 0
 
   test "step should setSlotFill to the length of seq returned from getRecentSlotFillEvents":
-    let fills = @[
-      SlotFilled(),
-      SlotFilled(),
-      SlotFilled(),
-      SlotFilled()
-    ]
+    let fills = @[SlotFilled(), SlotFilled(), SlotFilled(), SlotFilled()]
 
-    marketplace.recentSlotFillEventsReturn = success(fills)    
+    marketplace.recentSlotFillEventsReturn = success(fills)
 
     await onStep()
 
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
index bf4a717..2956e8d 100644
--- a/tests/codexcrawler/mocks/mockmarketplace.nim
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -7,11 +7,12 @@ import ../../../codexcrawler/services/marketplace/market
 logScope:
   topics = "marketplace"
 
-type
-  MockMarketplaceService* = ref object of MarketplaceService
-    recentSlotFillEventsReturn*: ?!seq[SlotFilled]
+type MockMarketplaceService* = ref object of MarketplaceService
+  recentSlotFillEventsReturn*: ?!seq[SlotFilled]
 
-method getRecentSlotFillEvents*(m: MockMarketplaceService): Future[?!seq[SlotFilled]] {.async: (raises: []).} =
+method getRecentSlotFillEvents*(
+    m: MockMarketplaceService
+): Future[?!seq[SlotFilled]] {.async: (raises: []).} =
   return m.recentSlotFillEventsReturn
 
 proc createMockMarketplaceService*(): MockMarketplaceService =

From eaeddb5a2819263604709609f31f2286872c2430 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Tue, 18 Mar 2025 18:22:31 +0100
Subject: [PATCH 11/25] Turns on chainmetrics components. Tested with local
 node. it works!

---
 codexcrawler/components/chainmetrics.nim | 2 +-
 codexcrawler/installer.nim               | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 9691192..b035b14 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -9,7 +9,7 @@ import ../services/marketplace
 import ../component
 
 logScope:
-  topics = "ChainMetrics"
+  topics = "chainmetrics"
 
 type ChainMetrics* = ref object of Component
   state: State
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index 5101897..aadb842 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -14,6 +14,7 @@ import ./components/timetracker
 import ./components/nodestore
 import ./components/dhtmetrics
 import ./components/todolist
+import ./components/chainmetrics
 
 proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   var components: seq[Component] = newSeq[Component]()
@@ -29,6 +30,7 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
     metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort)
     todoList = createTodoList(state, metrics)
     marketplace = createMarketplace(state)
+    chainMetrics = ChainMetrics.new(state, metrics, marketplace)
 
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
     return failure(err)
@@ -40,5 +42,6 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   components.add(TimeTracker.new(state, nodeStore, dht, clock))
   components.add(dhtMetrics)
   components.add(marketplace)
+  components.add(chainMetrics)
 
   return success(components)

From de22684ff3144551e62e6fd315d1816c4feb5435 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Wed, 19 Mar 2025 15:25:17 +0100
Subject: [PATCH 12/25] Implementing requeststore

---
 codexcrawler/components/requeststore.nim      | 157 ++++++++++++++++++
 codexcrawler/services/marketplace.nim         |  10 ++
 codexcrawler/services/marketplace/market.nim  |   8 +-
 codexcrawler/types.nim                        |  11 +-
 .../components/testrequeststore.nim           |  89 ++++++++++
 tests/codexcrawler/helpers.nim                |  20 +++
 tests/codexcrawler/mocks/mockclock.nim        |   2 +-
 tests/codexcrawler/testcomponents.nim         |   1 +
 8 files changed, 295 insertions(+), 3 deletions(-)
 create mode 100644 codexcrawler/components/requeststore.nim
 create mode 100644 tests/codexcrawler/components/testrequeststore.nim

diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
new file mode 100644
index 0000000..b101fce
--- /dev/null
+++ b/codexcrawler/components/requeststore.nim
@@ -0,0 +1,157 @@
+import std/os
+import pkg/datastore
+import pkg/datastore/typedds
+import pkg/questionable/results
+import pkg/chronicles
+import pkg/chronos
+import pkg/libp2p
+
+import ../types
+import ../component
+import ../state
+import ../utils/datastoreutils
+import ../utils/asyncdataevent
+import ../services/clock
+
+const requeststoreName = "requeststore"
+
+logScope:
+  topics = "requeststore"
+
+type
+  RequestEntry* = object
+    id*: Rid
+    lastSeen*: uint64
+
+  OnRequestEntry* =
+    proc(entry: RequestEntry): Future[?!void] {.async: (raises: []), gcsafe.}
+
+  RequestStore* = ref object of Component
+    state: State
+    store: TypedDatastore
+    clock: Clock
+
+proc `$`*(entry: RequestEntry): string =
+  $entry.id
+
+proc toBytes*(entry: RequestEntry): seq[byte] =
+  var buffer = initProtoBuffer()
+  buffer.write(1, $entry.id)
+  buffer.write(2, entry.lastSeen)
+  buffer.finish()
+  return buffer.buffer
+
+proc fromBytes*(_: type RequestEntry, data: openArray[byte]): ?!RequestEntry =
+  var
+    buffer = initProtoBuffer(data)
+    idStr: string
+    lastSeen: uint64
+
+  if buffer.getField(1, idStr).isErr:
+    return failure("Unable to decode `idStr`")
+  if buffer.getField(2, lastSeen).isErr:
+    return failure("Unable to decode `lastSeen`")
+
+  return success(RequestEntry(id: Rid.fromStr(idStr), lastSeen: lastSeen))
+
+proc encode*(e: RequestEntry): seq[byte] =
+  e.toBytes()
+
+proc decode*(T: type RequestEntry, bytes: seq[byte]): ?!T =
+  if bytes.len < 1:
+    return success(RequestEntry(lastSeen: 0))
+  return RequestEntry.fromBytes(bytes)
+
+proc update*(s: RequestStore, rid: Rid): Future[?!void] {.async.} =
+  without key =? Key.init(requeststoreName / $rid), err:
+    error "failed to format key", err = err.msg
+    return failure(err)
+
+  let entry = RequestEntry(id: rid, lastSeen: s.clock.now)
+  ?await s.store.put(key, entry)
+  trace "Request entry updated", id = $rid
+  return success()
+
+proc remove*(s: RequestStore, rid: Rid): Future[?!void] {.async.} =
+  without key =? Key.init(requeststoreName / $rid), err:
+    error "failed to format key", err = err.msg
+    return failure(err)
+
+  ?await s.store.delete(key)
+  trace "Request entry removed", id = $rid
+  return success()
+
+# proc storeNodeIsNew(s: RequestStore, nid: Nid): Future[?!bool] {.async.} =
+#   without key =? Key.init(requeststoreName / $nid), err:
+#     error "failed to format key", err = err.msg
+#     return failure(err)
+#   without exists =? (await s.store.has(key)), err:
+#     error "failed to check store for key", err = err.msg
+#     return failure(err)
+
+#   if not exists:
+#     let entry = RequestEntry(id: nid, lastVisit: 0, firstInactive: 0)
+#     ?await s.store.put(key, entry)
+#     info "New node", nodeId = $nid
+
+#   return success(not exists)
+
+# proc deleteEntry(s: RequestStore, nid: Nid): Future[?!bool] {.async.} =
+#   without key =? Key.init(requeststoreName / $nid), err:
+#     error "failed to format key", err = err.msg
+#     return failure(err)
+#   without exists =? (await s.store.has(key)), err:
+#     error "failed to check store for key", err = err.msg, key = $key
+#     return failure(err)
+
+#   if exists:
+#     ?await s.store.delete(key)
+
+#   return success(exists)
+
+# method iterateAll*(
+#     s: RequestStore, onNode: OnRequestEntry
+# ): Future[?!void] {.async: (raises: []), base.} =
+#   without queryKey =? Key.init(requeststoreName), err:
+#     error "failed to format key", err = err.msg
+#     return failure(err)
+#   try:
+#     without iter =? (await query[RequestEntry](s.store, Query.init(queryKey))), err:
+#       error "failed to create query", err = err.msg
+#       return failure(err)
+
+#     while not iter.finished:
+#       without item =? (await iter.next()), err:
+#         error "failure during query iteration", err = err.msg
+#         return failure(err)
+#       without value =? item.value, err:
+#         error "failed to get value from iterator", err = err.msg
+#         return failure(err)
+
+#       if value.lastSeen > 0:
+#         ?await onNode(value)
+
+#       await sleepAsync(1.millis)
+#   except CatchableError as exc:
+#     return failure(exc.msg)
+
+#   return success()
+
+method start*(s: RequestStore): Future[?!void] {.async.} =
+  info "Starting..."
+  return success()
+
+method stop*(s: RequestStore): Future[?!void] {.async.} =
+  return success()
+
+proc new*(
+    T: type RequestStore, state: State, store: TypedDatastore, clock: Clock
+): RequestStore =
+  RequestStore(state: state, store: store, clock: clock)
+
+proc createRequestStore*(state: State, clock: Clock): ?!RequestStore =
+  without ds =? createTypedDatastore(state.config.dataDir / "requeststore"), err:
+    error "Failed to create typed datastore for request store", err = err.msg
+    return failure(err)
+
+  return success(RequestStore.new(state, ds, clock))
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index c959a22..f652ad6 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -28,6 +28,16 @@ method getRecentSlotFillEvents*(
   return failure("MarketplaceService is not started")
 
 method start*(m: MarketplaceService): Future[?!void] {.async.} =
+  # Todo:
+  # - subscribe to requestSubmitted -> add id to list
+  # - queryPastStorageRequestedEvents from 3 months ago (max duration) -> add ids to list
+  # for list:
+  #   - get status of request
+  #   if running:
+  #     - sum total bytes
+  #   else:
+  #     - remove from list
+
   let provider = JsonRpcProvider.new(m.state.config.ethProvider)
   without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
     return failure("Invalid MarketplaceAddress provided")
diff --git a/codexcrawler/services/marketplace/market.nim b/codexcrawler/services/marketplace/market.nim
index 294bccc..c0b88fa 100644
--- a/codexcrawler/services/marketplace/market.nim
+++ b/codexcrawler/services/marketplace/market.nim
@@ -558,9 +558,15 @@ proc queryPastStorageRequestedEvents*(
 ): Future[seq[StorageRequested]] {.async.} =
   convertEthersError:
     let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
-
     return await market.queryPastStorageRequestedEvents(fromBlock)
 
+proc queryPastStorageRequestedEvents*(
+    market: OnChainMarket, fromTime: int64
+): Future[seq[StorageRequested]] {.async.} =
+  convertEthersError:
+    let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
+    return await market.queryPastStorageRequestedEvents(BlockTag.init(fromBlock))
+
 proc slotCollateral*(
     market: OnChainMarket, collateralPerSlot: UInt256, slotState: SlotState
 ): ?!UInt256 {.raises: [].} =
diff --git a/codexcrawler/types.nim b/codexcrawler/types.nim
index 9895ed1..5948eb2 100644
--- a/codexcrawler/types.nim
+++ b/codexcrawler/types.nim
@@ -4,7 +4,13 @@ import pkg/questionable/results
 import pkg/codexdht
 import pkg/libp2p
 
-type Nid* = NodeId
+import ./services/marketplace/requests
+
+export requests
+
+type
+  Nid* = NodeId
+  Rid* = requests.RequestId
 
 proc `$`*(nid: Nid): string =
   nid.toHex()
@@ -12,6 +18,9 @@ proc `$`*(nid: Nid): string =
 proc fromStr*(T: type Nid, s: string): Nid =
   Nid(UInt256.fromHex(s))
 
+proc fromStr*(T: type Rid, s: string): Rid =
+  Rid(requests.RequestId.fromHex(s))
+
 proc toBytes*(nid: Nid): seq[byte] =
   var buffer = initProtoBuffer()
   buffer.write(1, $nid)
diff --git a/tests/codexcrawler/components/testrequeststore.nim b/tests/codexcrawler/components/testrequeststore.nim
new file mode 100644
index 0000000..6072fab
--- /dev/null
+++ b/tests/codexcrawler/components/testrequeststore.nim
@@ -0,0 +1,89 @@
+import std/os
+import pkg/chronos
+import pkg/questionable/results
+import pkg/asynctest/chronos/unittest
+import pkg/datastore/typedds
+
+import ../../../codexcrawler/components/requeststore
+import ../../../codexcrawler/utils/datastoreutils
+import ../../../codexcrawler/types
+import ../../../codexcrawler/state
+import ../mocks/mockstate
+import ../mocks/mockclock
+import ../helpers
+
+suite "Requeststore":
+  let
+    dsPath = getTempDir() / "testds"
+    requeststoreName = "requeststore"
+
+  var
+    ds: TypedDatastore
+    state: MockState
+    clock: MockClock
+    store: RequestStore
+
+  setup:
+    ds = createTypedDatastore(dsPath).tryGet()
+    state = createMockState()
+    clock = createMockClock()
+
+    store = RequestStore.new(state, ds, clock)
+
+    (await store.start()).tryGet()
+
+  teardown:
+    (await store.stop()).tryGet()
+    (await ds.close()).tryGet()
+    state.checkAllUnsubscribed()
+    removeDir(dsPath)
+
+  test "requestEntry encoding":
+    let entry = RequestEntry(id: genRid(), lastSeen: 123.uint64)
+
+    let
+      bytes = entry.encode()
+      decoded = RequestEntry.decode(bytes).tryGet()
+
+    check:
+      entry.id == decoded.id
+      entry.lastSeen == decoded.lastSeen
+
+  test "update stores a new requestId with current time":
+    let rid = genRid()
+    (await store.update(rid)).tryGet()
+
+    let
+      key = Key.init(requeststoreName / $rid).tryGet()
+      stored = (await get[RequestEntry](ds, key)).tryGet()
+
+    check:
+      stored.id == rid
+      stored.lastSeen == clock.setNow
+
+  test "update updates the current time of an existing requestId with current time":
+    let rid = genRid()
+    (await store.update(rid)).tryGet()
+
+    clock.setNow = 1234
+    (await store.update(rid)).tryGet()
+
+    let
+      key = Key.init(requeststoreName / $rid).tryGet()
+      stored = (await get[RequestEntry](ds, key)).tryGet()
+
+    check:
+      stored.id == rid
+      stored.lastSeen == clock.setNow
+
+  test "remove will remove an entry":
+    let rid = genRid()
+    (await store.update(rid)).tryGet()
+    (await store.remove(rid)).tryGet()
+
+    let
+      key = Key.init(requeststoreName / $rid).tryGet()
+      isStored = (await ds.has(key)).tryGet()
+
+    check:
+      isStored == false
diff --git a/tests/codexcrawler/helpers.nim b/tests/codexcrawler/helpers.nim
index 5feb345..b7a5b9e 100644
--- a/tests/codexcrawler/helpers.nim
+++ b/tests/codexcrawler/helpers.nim
@@ -1,6 +1,26 @@
 import std/random
+import std/sequtils
+import std/typetraits
 import pkg/stint
+import pkg/stew/byteutils
 import ../../codexcrawler/types
 
+proc example*[T: SomeInteger](_: type T): T =
+  rand(T)
+
+proc example*[T, N](_: type array[N, T]): array[N, T] =
+  for item in result.mitems:
+    item = T.example
+
+proc example*(_: type UInt256): UInt256 =
+  UInt256.fromBytes(array[32, byte].example)
+
+proc example*[T: distinct](_: type T): T =
+  type baseType = T.distinctBase
+  T(baseType.example)
+
 proc genNid*(): Nid =
   Nid(rand(uint64).u256)
+
+proc genRid*(): Rid =
+  Rid(array[32, byte].example)
diff --git a/tests/codexcrawler/mocks/mockclock.nim b/tests/codexcrawler/mocks/mockclock.nim
index 5c5cf44..d4621bc 100644
--- a/tests/codexcrawler/mocks/mockclock.nim
+++ b/tests/codexcrawler/mocks/mockclock.nim
@@ -7,4 +7,4 @@ method now*(clock: MockClock): uint64 {.raises: [].} =
   clock.setNow
 
 proc createMockClock*(): MockClock =
-  MockClock()
+  MockClock(setNow: 12)
diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim
index a81a08f..99b5be9 100644
--- a/tests/codexcrawler/testcomponents.nim
+++ b/tests/codexcrawler/testcomponents.nim
@@ -2,6 +2,7 @@ import ./components/testchainmetrics
 import ./components/testcrawler
 import ./components/testdhtmetrics
 import ./components/testnodestore
+import ./components/testrequeststore
 import ./components/testtimetracker
 import ./components/testtodolist
 

From 9f8ba85d350ab9428e28118bd9e80861c64cfedf Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Wed, 19 Mar 2025 15:34:04 +0100
Subject: [PATCH 13/25] finishes implement of requeststore

---
 codexcrawler/components/requeststore.nim      | 93 ++++++++-----------
 .../components/testrequeststore.nim           | 26 ++++++
 2 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index b101fce..c421816 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -62,80 +62,61 @@ proc decode*(T: type RequestEntry, bytes: seq[byte]): ?!T =
     return success(RequestEntry(lastSeen: 0))
   return RequestEntry.fromBytes(bytes)
 
-proc update*(s: RequestStore, rid: Rid): Future[?!void] {.async.} =
+method update*(
+    s: RequestStore, rid: Rid
+): Future[?!void] {.async: (raises: []), base.} =
   without key =? Key.init(requeststoreName / $rid), err:
     error "failed to format key", err = err.msg
     return failure(err)
 
   let entry = RequestEntry(id: rid, lastSeen: s.clock.now)
-  ?await s.store.put(key, entry)
+  try:
+    ?await s.store.put(key, entry)
+  except CatchableError as exc:
+    return failure(exc.msg)
   trace "Request entry updated", id = $rid
   return success()
 
-proc remove*(s: RequestStore, rid: Rid): Future[?!void] {.async.} =
+method remove*(
+    s: RequestStore, rid: Rid
+): Future[?!void] {.async: (raises: []), base.} =
   without key =? Key.init(requeststoreName / $rid), err:
     error "failed to format key", err = err.msg
     return failure(err)
 
-  ?await s.store.delete(key)
+  try:
+    ?await s.store.delete(key)
+  except CatchableError as exc:
+    return failure(exc.msg)
   trace "Request entry removed", id = $rid
   return success()
 
-# proc storeNodeIsNew(s: RequestStore, nid: Nid): Future[?!bool] {.async.} =
-#   without key =? Key.init(requeststoreName / $nid), err:
-#     error "failed to format key", err = err.msg
-#     return failure(err)
-#   without exists =? (await s.store.has(key)), err:
-#     error "failed to check store for key", err = err.msg
-#     return failure(err)
+method iterateAll*(
+    s: RequestStore, onNode: OnRequestEntry
+): Future[?!void] {.async: (raises: []), base.} =
+  without queryKey =? Key.init(requeststoreName), err:
+    error "failed to format key", err = err.msg
+    return failure(err)
+  try:
+    without iter =? (await query[RequestEntry](s.store, Query.init(queryKey))), err:
+      error "failed to create query", err = err.msg
+      return failure(err)
 
-#   if not exists:
-#     let entry = RequestEntry(id: nid, lastVisit: 0, firstInactive: 0)
-#     ?await s.store.put(key, entry)
-#     info "New node", nodeId = $nid
+    while not iter.finished:
+      without item =? (await iter.next()), err:
+        error "failure during query iteration", err = err.msg
+        return failure(err)
+      without value =? item.value, err:
+        error "failed to get value from iterator", err = err.msg
+        return failure(err)
 
-#   return success(not exists)
+      if value.lastSeen > 0:
+        ?await onNode(value)
 
-# proc deleteEntry(s: RequestStore, nid: Nid): Future[?!bool] {.async.} =
-#   without key =? Key.init(requeststoreName / $nid), err:
-#     error "failed to format key", err = err.msg
-#     return failure(err)
-#   without exists =? (await s.store.has(key)), err:
-#     error "failed to check store for key", err = err.msg, key = $key
-#     return failure(err)
-
-#   if exists:
-#     ?await s.store.delete(key)
-
-#   return success(exists)
-
-# method iterateAll*(
-#     s: RequestStore, onNode: OnRequestEntry
-# ): Future[?!void] {.async: (raises: []), base.} =
-#   without queryKey =? Key.init(requeststoreName), err:
-#     error "failed to format key", err = err.msg
-#     return failure(err)
-#   try:
-#     without iter =? (await query[RequestEntry](s.store, Query.init(queryKey))), err:
-#       error "failed to create query", err = err.msg
-#       return failure(err)
-
-#     while not iter.finished:
-#       without item =? (await iter.next()), err:
-#         error "failure during query iteration", err = err.msg
-#         return failure(err)
-#       without value =? item.value, err:
-#         error "failed to get value from iterator", err = err.msg
-#         return failure(err)
-
-#       if value.lastSeen > 0:
-#         ?await onNode(value)
-
-#       await sleepAsync(1.millis)
-#   except CatchableError as exc:
-#     return failure(exc.msg)
-
-#   return success()
+      await sleepAsync(1.millis)
+  except CatchableError as exc:
+    return failure(exc.msg)
+  return success()
 
 method start*(s: RequestStore): Future[?!void] {.async.} =
   info "Starting..."
diff --git a/tests/codexcrawler/components/testrequeststore.nim b/tests/codexcrawler/components/testrequeststore.nim
index 6072fab..687c124 100644
--- a/tests/codexcrawler/components/testrequeststore.nim
+++ b/tests/codexcrawler/components/testrequeststore.nim
@@ -87,3 +87,29 @@ suite "Requeststore":
 
     check:
       isStored == false
+
+  test "iterateAll yields all entries":
+    let
+      rid1 = genRid()
+      rid2 = genRid()
+      rid3 = genRid()
+
+    (await store.update(rid1)).tryGet()
+    (await store.update(rid2)).tryGet()
+    (await store.update(rid3)).tryGet()
+
+    var entries = newSeq[RequestEntry]()
+    proc onEntry(entry: RequestEntry): Future[?!void] {.async: (raises: []), gcsafe.} =
+      entries.add(entry)
+      return success()
+
+    (await store.iterateAll(onEntry)).tryGet()
+
+    check:
+      entries.len == 3
+      entries[0].id == rid1
+      entries[0].lastSeen == clock.setNow
+      entries[1].id == rid2
+      entries[1].lastSeen == clock.setNow
+      entries[2].id == rid3
+      entries[2].lastSeen == clock.setNow

From fe9c29760cc5b8e92fa23dd0b5ecfcf02df4e8e0 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Wed, 19 Mar 2025 15:52:50 +0100
Subject: [PATCH 14/25] Renames crawler to dhtcrawler

---
 codexcrawler/components/chainmetrics.nim       |  2 +-
 .../components/{crawler.nim => dhtcrawler.nim} | 18 +++++++++---------
 codexcrawler/components/dhtmetrics.nim         |  2 +-
 codexcrawler/components/nodestore.nim          |  2 +-
 codexcrawler/components/requeststore.nim       |  2 +-
 codexcrawler/components/timetracker.nim        |  2 +-
 codexcrawler/installer.nim                     |  4 ++--
 .../{testcrawler.nim => testdhtcrawler.nim}    |  8 ++++----
 .../components/testrequeststore.nim            | 17 ++++++++++++++---
 tests/codexcrawler/testcomponents.nim          |  2 +-
 10 files changed, 35 insertions(+), 24 deletions(-)
 rename codexcrawler/components/{crawler.nim => dhtcrawler.nim} (75%)
 rename tests/codexcrawler/components/{testcrawler.nim => testdhtcrawler.nim} (95%)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index b035b14..6b053c9 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -26,7 +26,7 @@ proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
   return success()
 
 method start*(c: ChainMetrics): Future[?!void] {.async.} =
-  info "Starting..."
+  info "starting..."
 
   proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
     return await c.step()
diff --git a/codexcrawler/components/crawler.nim b/codexcrawler/components/dhtcrawler.nim
similarity index 75%
rename from codexcrawler/components/crawler.nim
rename to codexcrawler/components/dhtcrawler.nim
index 57602db..5fde23e 100644
--- a/codexcrawler/components/crawler.nim
+++ b/codexcrawler/components/dhtcrawler.nim
@@ -12,15 +12,15 @@ import ../state
 import ../utils/asyncdataevent
 
 logScope:
-  topics = "crawler"
+  topics = "dhtcrawler"
 
-type Crawler* = ref object of Component
+type DhtCrawler* = ref object of Component
   state: State
   dht: Dht
   todo: TodoList
 
 proc raiseCheckEvent(
-    c: Crawler, nid: Nid, success: bool
+    c: DhtCrawler, nid: Nid, success: bool
 ): Future[?!void] {.async: (raises: []).} =
   let event = DhtNodeCheckEventData(id: nid, isOk: success)
   if err =? (await c.state.events.dhtNodeCheck.fire(event)).errorOption:
@@ -28,7 +28,7 @@ proc raiseCheckEvent(
     return failure(err)
   return success()
 
-proc step(c: Crawler): Future[?!void] {.async: (raises: []).} =
+proc step(c: DhtCrawler): Future[?!void] {.async: (raises: []).} =
   without nid =? (await c.todo.pop()), err:
     error "failed to pop todolist", err = err.msg
     return failure(err)
@@ -46,8 +46,8 @@ proc step(c: Crawler): Future[?!void] {.async: (raises: []).} =
 
   return success()
 
-method start*(c: Crawler): Future[?!void] {.async.} =
-  info "Starting crawler..."
+method start*(c: DhtCrawler): Future[?!void] {.async.} =
+  info "starting..."
 
   proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
     await c.step()
@@ -57,8 +57,8 @@ method start*(c: Crawler): Future[?!void] {.async.} =
 
   return success()
 
-method stop*(c: Crawler): Future[?!void] {.async.} =
+method stop*(c: DhtCrawler): Future[?!void] {.async.} =
   return success()
 
-proc new*(T: type Crawler, state: State, dht: Dht, todo: TodoList): Crawler =
-  Crawler(state: state, dht: dht, todo: todo)
+proc new*(T: type DhtCrawler, state: State, dht: Dht, todo: TodoList): DhtCrawler =
+  DhtCrawler(state: state, dht: dht, todo: todo)
diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim
index af9c834..a8d0c74 100644
--- a/codexcrawler/components/dhtmetrics.nim
+++ b/codexcrawler/components/dhtmetrics.nim
@@ -46,7 +46,7 @@ proc handleDeleteEvent(d: DhtMetrics, nids: seq[Nid]): Future[?!void] {.async.}
   return success()
 
 method start*(d: DhtMetrics): Future[?!void] {.async.} =
-  info "Starting..."
+  info "starting..."
   ?await d.ok.load()
   ?await d.nok.load()
 
diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim
index 2bd42bc..ef22ae0 100644
--- a/codexcrawler/components/nodestore.nim
+++ b/codexcrawler/components/nodestore.nim
@@ -203,7 +203,7 @@ method deleteEntries*(
   return success()
 
 method start*(s: NodeStore): Future[?!void] {.async.} =
-  info "Starting..."
+  info "starting..."
 
   proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} =
     return await s.processFoundNodes(nids)
diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index c421816..566c308 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -119,7 +119,7 @@ method iterateAll*(
   return success()
 
 method start*(s: RequestStore): Future[?!void] {.async.} =
-  info "Starting..."
+  info "starting..."
   return success()
 
 method stop*(s: RequestStore): Future[?!void] {.async.} =
diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim
index b1af510..a4217cb 100644
--- a/codexcrawler/components/timetracker.nim
+++ b/codexcrawler/components/timetracker.nim
@@ -57,7 +57,7 @@ proc raiseRoutingTableNodes(t: TimeTracker): Future[?!void] {.async: (raises: []
   return success()
 
 method start*(t: TimeTracker): Future[?!void] {.async.} =
-  info "Starting..."
+  info "starting..."
 
   proc onCheckRevisitAndExpiry(): Future[?!void] {.async: (raises: []), gcsafe.} =
     await t.checkRevisitsAndExpiry()
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index aadb842..500bc46 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -9,7 +9,7 @@ import ./services/dht
 import ./services/marketplace
 
 import ./component
-import ./components/crawler
+import ./components/dhtcrawler
 import ./components/timetracker
 import ./components/nodestore
 import ./components/dhtmetrics
@@ -38,7 +38,7 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   components.add(dht)
   components.add(todoList)
   components.add(nodeStore)
-  components.add(Crawler.new(state, dht, todoList))
+  components.add(DhtCrawler.new(state, dht, todoList))
   components.add(TimeTracker.new(state, nodeStore, dht, clock))
   components.add(dhtMetrics)
   components.add(marketplace)
diff --git a/tests/codexcrawler/components/testcrawler.nim b/tests/codexcrawler/components/testdhtcrawler.nim
similarity index 95%
rename from tests/codexcrawler/components/testcrawler.nim
rename to tests/codexcrawler/components/testdhtcrawler.nim
index 6a51721..def5ddc 100644
--- a/tests/codexcrawler/components/testcrawler.nim
+++ b/tests/codexcrawler/components/testdhtcrawler.nim
@@ -3,7 +3,7 @@ import pkg/questionable
 import pkg/questionable/results
 import pkg/asynctest/chronos/unittest
 
-import ../../../codexcrawler/components/crawler
+import ../../../codexcrawler/components/dhtcrawler
 import ../../../codexcrawler/services/dht
 import ../../../codexcrawler/utils/asyncdataevent
 import ../../../codexcrawler/types
@@ -13,14 +13,14 @@ import ../mocks/mockdht
 import ../mocks/mocktodolist
 import ../helpers
 
-suite "Crawler":
+suite "DhtCrawler":
   var
     nid1: Nid
     nid2: Nid
     state: MockState
     todo: MockTodoList
     dht: MockDht
-    crawler: Crawler
+    crawler: DhtCrawler
 
   setup:
     nid1 = genNid()
@@ -29,7 +29,7 @@ suite "Crawler":
     todo = createMockTodoList()
     dht = createMockDht()
 
-    crawler = Crawler.new(state, dht, todo)
+    crawler = DhtCrawler.new(state, dht, todo)
 
     (await crawler.start()).tryGet()
 
diff --git a/tests/codexcrawler/components/testrequeststore.nim b/tests/codexcrawler/components/testrequeststore.nim
index 687c124..93858e9 100644
--- a/tests/codexcrawler/components/testrequeststore.nim
+++ b/tests/codexcrawler/components/testrequeststore.nim
@@ -107,9 +107,20 @@ suite "Requeststore":
 
     check:
       entries.len == 3
-      entries[0].id == rid1
+
+    let
+      ids = @[entries[0].id, entries[1].id, entries[2].id]
+      all = @[rid1, rid2, rid3]
+
+    for id in ids:
+      check:
+        id in all
+
+    for id in all:
+      check:
+        id in ids
+
+    check:
       entries[0].lastSeen == clock.setNow
-      entries[1].id == rid2
       entries[1].lastSeen == clock.setNow
-      entries[2].id == rid3
       entries[2].lastSeen == clock.setNow
diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim
index 99b5be9..fa7de7d 100644
--- a/tests/codexcrawler/testcomponents.nim
+++ b/tests/codexcrawler/testcomponents.nim
@@ -1,5 +1,5 @@
 import ./components/testchainmetrics
-import ./components/testcrawler
+import ./components/testdhtcrawler
 import ./components/testdhtmetrics
 import ./components/testnodestore
 import ./components/testrequeststore

From 1e8621b639e40e051b95486093ebf7a74dd063f7 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Wed, 19 Mar 2025 15:59:55 +0100
Subject: [PATCH 15/25] plans for chain crawler and updated chain metrics

---
 codexcrawler/components/chaincrawler.nim |  3 +++
 codexcrawler/components/chainmetrics.nim | 14 ++++++++++++++
 2 files changed, 17 insertions(+)
 create mode 100644 codexcrawler/components/chaincrawler.nim

diff --git a/codexcrawler/components/chaincrawler.nim b/codexcrawler/components/chaincrawler.nim
new file mode 100644
index 0000000..1eefa26
--- /dev/null
+++ b/codexcrawler/components/chaincrawler.nim
@@ -0,0 +1,3 @@
+# subscribe to newrequests
+# iterate past requests on start-up
+# push them into the request store
diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 6b053c9..44f8604 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -17,6 +17,20 @@ type ChainMetrics* = ref object of Component
   marketplace: MarketplaceService
 
 proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
+  # replace slotFills entirely:
+  # iterate all requests in requestStore:
+    # get state of request on chain
+    # if failed/canceled/error:
+      # if last-seen is old (1month?3months?)
+        # delete entry
+    # else (request is running):
+      # count:
+        # total running
+        # total num slots
+        # total size of request
+  # iter finished: update metrics!
+
+
   without slotFills =? (await c.marketplace.getRecentSlotFillEvents()), err:
     trace "Unable to get recent slotFill events from chain", err = err.msg
     return success() # We don't propagate this error.

From 86643e76472ae0bddc581cc7b33a6b6c36c2bae4 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Thu, 20 Mar 2025 15:27:17 +0100
Subject: [PATCH 16/25] Implements chaincrawler module

---
 codexcrawler/components/chaincrawler.nim      |  45 ++++++-
 codexcrawler/components/chainmetrics.nim      |  10 +-
 codexcrawler/config.nim                       |   2 +-
 codexcrawler/installer.nim                    |   2 +-
 codexcrawler/services/marketplace.nim         |  74 ++++++----
 .../components/testchaincrawler.nim           |  60 +++++++++
 .../components/testchainmetrics.nim           | 126 +++++++++---------
 tests/codexcrawler/mocks/mockmarketplace.nim  |  19 ++-
 tests/codexcrawler/mocks/mockrequeststore.nim |  33 +++++
 tests/codexcrawler/testcomponents.nim         |   1 +
 10 files changed, 266 insertions(+), 106 deletions(-)
 create mode 100644 tests/codexcrawler/components/testchaincrawler.nim
 create mode 100644 tests/codexcrawler/mocks/mockrequeststore.nim

diff --git a/codexcrawler/components/chaincrawler.nim b/codexcrawler/components/chaincrawler.nim
index 1eefa26..8295618 100644
--- a/codexcrawler/components/chaincrawler.nim
+++ b/codexcrawler/components/chaincrawler.nim
@@ -1,3 +1,42 @@
-# subscribe to newrequests
-# iterate past requests on start-up
-# push them into the request store
+
+import pkg/chronicles
+import pkg/chronos
+import pkg/questionable/results
+
+import ../state
+import ../components/requeststore
+import ../services/marketplace
+import ../component
+import ../types
+
+logScope:
+  topics = "chaincrawler"
+
+type ChainCrawler* = ref object of Component
+  state: State
+  store: RequestStore
+  marketplace: MarketplaceService
+
+proc onNewRequest(c: ChainCrawler, rid: Rid): Future[?!void] {.async: (raises: []).} =
+  return await c.store.update(rid)
+
+method start*(c: ChainCrawler): Future[?!void] {.async.} =
+  info "starting..."
+
+  proc onRequest(rid: Rid): Future[?!void] {.async: (raises: []).} =
+    return await c.onNewRequest(rid)
+
+  ?await c.marketplace.subscribeToNewRequests(onRequest)
+  ?await c.marketplace.iteratePastNewRequestEvents(onRequest)
+  return success()
+
+method stop*(c: ChainCrawler): Future[?!void] {.async.} =
+  return success()
+
+proc new*(
+    T: type ChainCrawler,
+    state: State,
+    store: RequestStore,
+    marketplace: MarketplaceService,
+): ChainCrawler =
+  ChainCrawler(state: state, store: store, marketplace: marketplace)
diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 44f8604..e08334b 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -31,12 +31,12 @@ proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
   # iter finished: update metrics!
 
 
-  without slotFills =? (await c.marketplace.getRecentSlotFillEvents()), err:
-    trace "Unable to get recent slotFill events from chain", err = err.msg
-    return success() # We don't propagate this error.
-    # The call is allowed to fail and the app should continue as normal.
+  # without slotFills =? (await c.marketplace.getRecentSlotFillEvents()), err:
+  #   trace "Unable to get recent slotFill events from chain", err = err.msg
+  #   return success() # We don't propagate this error.
+  #   # The call is allowed to fail and the app should continue as normal.
 
-  c.metrics.setSlotFill(slotFills.len)
+  # c.metrics.setSlotFill(slotFills.len)
   return success()
 
 method start*(c: ChainMetrics): Future[?!void] {.async.} =
diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index bc56e40..10f434b 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -25,9 +25,9 @@ Options:
   --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
   --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
   --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
+  --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
   --ethProvider=<a>                 Address including http(s) or ws of the eth provider
   --marketplaceAddress=<a>          Eth address of Codex contracts deployment
-  --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
 """
 
 import strutils
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index 500bc46..fbaf172 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -29,7 +29,7 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   let
     metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort)
     todoList = createTodoList(state, metrics)
-    marketplace = createMarketplace(state)
+    marketplace = createMarketplace(state, clock)
     chainMetrics = ChainMetrics.new(state, metrics, marketplace)
 
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index f652ad6..88a1d9f 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -1,43 +1,63 @@
 import pkg/ethers
 import pkg/questionable
+import pkg/upraises
 import ./marketplace/market
 import ./marketplace/marketplace
 import ../config
 import ../component
 import ../state
+import ../types
+import ./clock
 
 logScope:
   topics = "marketplace"
 
-type MarketplaceService* = ref object of Component
-  state: State
-  market: ?OnChainMarket
+type
+  MarketplaceService* = ref object of Component
+    state: State
+    market: ?OnChainMarket
+    clock: Clock
+  OnNewRequest* = proc(id: Rid): Future[?!void] {.async: (raises: []), gcsafe.}
 
-method getRecentSlotFillEvents*(
-    m: MarketplaceService
-): Future[?!seq[SlotFilled]] {.async: (raises: []), base.} =
-  # There is (aprox.) 1 block every 10 seconds.
-  # 10 seconds * 6 * 60 = 3600 = 1 hour.
-  let blocksAgo = 6 * 60
+proc notStarted() =
+  raiseAssert("MarketplaceService was called before it was started.")
+
+method subscribeToNewRequests*(m: MarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []), base.} =
+  proc resultWrapper(rid: Rid): Future[void] {.async.} =
+    let response = await onNewRequest(rid)
+    if error =? response.errorOption:
+      raiseAssert("Error result in handling of onNewRequest callback: " & error.msg)
+
+  proc onRequest(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, upraises: [].} =
+    asyncSpawn resultWrapper(Rid(id))
 
   if market =? m.market:
     try:
-      return success(await market.queryPastSlotFilledEvents(blocksAgo))
-    except CatchableError as err:
-      return failure(err.msg)
-  return failure("MarketplaceService is not started")
+      discard await market.subscribeRequests(onRequest)
+    except CatchableError as exc:
+      return failure(exc.msg)
+  else:
+    notStarted()
+  return success()
+  
+method iteratePastNewRequestEvents*(m: MarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []), base.} =
+  let
+    oneDay = 60 * 60 * 24
+    timespan = oneDay * 30
+    startTime = m.clock.now() - timespan.uint64
+
+  if market =? m.market:
+    try:
+      let requests = await market.queryPastStorageRequestedEvents(startTime.int64)
+      for request in requests:
+        if error =? (await onNewRequest(Rid(request.requestId))).errorOption:
+          return failure(error.msg)
+    except CatchableError as exc:
+      return failure(exc.msg)
+  else:
+    notStarted()
 
 method start*(m: MarketplaceService): Future[?!void] {.async.} =
-  # Todo:
-  # - subscribe to requestSubmitted -> add id to list
-  # - queryPastStorageRequestedEvents from 3 months ago (max duration) -> add ids to list
-  # for list:
-  #   - get status of request
-  #   if running:
-  #     - sum total bytes
-  #   else:
-  #     - remove from list
-
   let provider = JsonRpcProvider.new(m.state.config.ethProvider)
   without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
     return failure("Invalid MarketplaceAddress provided")
@@ -50,8 +70,8 @@ method start*(m: MarketplaceService): Future[?!void] {.async.} =
 method stop*(m: MarketplaceService): Future[?!void] {.async.} =
   return success()
 
-proc new(T: type MarketplaceService, state: State): MarketplaceService =
-  return MarketplaceService(state: state, market: none(OnChainMarket))
+proc new(T: type MarketplaceService, state: State, clock: Clock): MarketplaceService =
+  return MarketplaceService(state: state, market: none(OnChainMarket), clock: clock)
 
-proc createMarketplace*(state: State): MarketplaceService =
-  return MarketplaceService.new(state)
+proc createMarketplace*(state: State, clock: Clock): MarketplaceService =
+  return MarketplaceService.new(state, clock)
diff --git a/tests/codexcrawler/components/testchaincrawler.nim b/tests/codexcrawler/components/testchaincrawler.nim
new file mode 100644
index 0000000..8a50a78
--- /dev/null
+++ b/tests/codexcrawler/components/testchaincrawler.nim
@@ -0,0 +1,60 @@
+import pkg/chronos
+import pkg/questionable
+import pkg/questionable/results
+import pkg/asynctest/chronos/unittest
+import std/sequtils
+import std/options
+
+import ../../../codexcrawler/components/chaincrawler
+import ../../../codexcrawler/services/marketplace/market
+import ../../../codexcrawler/types
+import ../../../codexcrawler/state
+import ../mocks/mockstate
+import ../mocks/mockrequeststore
+import ../mocks/mockmarketplace
+import ../helpers
+
+suite "ChainCrawler":
+  var
+    state: MockState
+    store: MockRequestStore
+    marketplace: MockMarketplaceService
+    crawler: ChainCrawler
+
+  setup:
+    state = createMockState()
+    store = createMockRequestStore()
+    marketplace = createMockMarketplaceService()
+    
+    crawler = ChainCrawler.new(state, store, marketplace)
+
+    (await crawler.start()).tryGet()
+
+  teardown:
+    (await crawler.stop()).tryGet()
+    state.checkAllUnsubscribed()
+
+  # subscribe to newrequests
+  # iterate past requests on start-up
+  # push them into the request store
+
+  test "start should subscribe to new requests":
+    check:
+      marketplace.subNewRequestsCallback.isSome()
+
+  test "new-request subscription should add requestId to store":
+    let rid = genRid()
+    (await (marketplace.subNewRequestsCallback.get())(rid)).tryGet()
+
+    check:
+      store.updateRid == rid
+
+  test "start should iterate past requests and add then to store":
+    check:
+      marketplace.iterRequestsCallback.isSome()
+
+    let rid = genRid()
+    (await marketplace.iterRequestsCallback.get()(rid)).tryGet()
+
+    check:
+      store.updateRid == rid
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 99c2a89..2f0092f 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -1,85 +1,85 @@
-import pkg/chronos
-import pkg/questionable
-import pkg/questionable/results
-import pkg/asynctest/chronos/unittest
-import std/sequtils
+# import pkg/chronos
+# import pkg/questionable
+# import pkg/questionable/results
+# import pkg/asynctest/chronos/unittest
+# import std/sequtils
 
-import ../../../codexcrawler/components/chainmetrics
-import ../../../codexcrawler/services/marketplace/market
-import ../../../codexcrawler/types
-import ../../../codexcrawler/state
-import ../mocks/mockstate
-import ../mocks/mockmetrics
-import ../mocks/mockmarketplace
-import ../helpers
+# import ../../../codexcrawler/components/chainmetrics
+# import ../../../codexcrawler/services/marketplace/market
+# import ../../../codexcrawler/types
+# import ../../../codexcrawler/state
+# import ../mocks/mockstate
+# import ../mocks/mockmetrics
+# import ../mocks/mockmarketplace
+# import ../helpers
 
-suite "ChainMetrics":
-  var
-    state: MockState
-    metrics: MockMetrics
-    marketplace: MockMarketplaceService
-    chain: ChainMetrics
+# suite "ChainMetrics":
+#   var
+#     state: MockState
+#     metrics: MockMetrics
+#     marketplace: MockMarketplaceService
+#     chain: ChainMetrics
 
-  setup:
-    state = createMockState()
-    metrics = createMockMetrics()
-    marketplace = createMockMarketplaceService()
+#   setup:
+#     state = createMockState()
+#     metrics = createMockMetrics()
+#     marketplace = createMockMarketplaceService()
 
-    metrics.slotFill = -1
-    chain = ChainMetrics.new(state, metrics, marketplace)
+#     metrics.slotFill = -1
+#     chain = ChainMetrics.new(state, metrics, marketplace)
 
-    (await chain.start()).tryGet()
+#     (await chain.start()).tryGet()
 
-  teardown:
-    (await chain.stop()).tryGet()
-    state.checkAllUnsubscribed()
+#   teardown:
+#     (await chain.stop()).tryGet()
+#     state.checkAllUnsubscribed()
 
-  proc onStep() {.async.} =
-    (await state.steppers[0]()).tryGet()
+#   proc onStep() {.async.} =
+#     (await state.steppers[0]()).tryGet()
 
-  test "start should start stepper for 10 minutes":
-    check:
-      state.delays.len == 1
-      state.delays[0] == 10.minutes
+#   test "start should start stepper for 10 minutes":
+#     check:
+#       state.delays.len == 1
+#       state.delays[0] == 10.minutes
 
-  test "onStep is not activated when config.marketplaceEnable is false":
-    # Recreate chainMetrics, reset mockstate:
-    (await chain.stop()).tryGet()
-    state.steppers = @[]
-    # disable marketplace:
-    state.config.marketplaceEnable = false
-    (await chain.start()).tryGet()
+#   test "onStep is not activated when config.marketplaceEnable is false":
+#     # Recreate chainMetrics, reset mockstate:
+#     (await chain.stop()).tryGet()
+#     state.steppers = @[]
+#     # disable marketplace:
+#     state.config.marketplaceEnable = false
+#     (await chain.start()).tryGet()
 
-    check:
-      state.steppers.len == 0
+#     check:
+#       state.steppers.len == 0
 
-  test "step should not call setSlotFill when getRecentSlotFillEvents fails":
-    let testValue = -123
-    metrics.slotFill = testValue
+#   test "step should not call setSlotFill when getRecentSlotFillEvents fails":
+#     let testValue = -123
+#     metrics.slotFill = testValue
 
-    marketplace.recentSlotFillEventsReturn = seq[SlotFilled].failure("testfailure")
+#     marketplace.recentSlotFillEventsReturn = seq[SlotFilled].failure("testfailure")
 
-    await onStep()
+#     await onStep()
 
-    check:
-      metrics.slotFill == testValue
+#     check:
+#       metrics.slotFill == testValue
 
-  test "step should setSlotFill to zero when getRecentSlotFillEvents returns empty seq":
-    metrics.slotFill = -123
+#   test "step should setSlotFill to zero when getRecentSlotFillEvents returns empty seq":
+#     metrics.slotFill = -123
 
-    marketplace.recentSlotFillEventsReturn = success(newSeq[SlotFilled]())
+#     marketplace.recentSlotFillEventsReturn = success(newSeq[SlotFilled]())
 
-    await onStep()
+#     await onStep()
 
-    check:
-      metrics.slotFill == 0
+#     check:
+#       metrics.slotFill == 0
 
-  test "step should setSlotFill to the length of seq returned from getRecentSlotFillEvents":
-    let fills = @[SlotFilled(), SlotFilled(), SlotFilled(), SlotFilled()]
+#   test "step should setSlotFill to the length of seq returned from getRecentSlotFillEvents":
+#     let fills = @[SlotFilled(), SlotFilled(), SlotFilled(), SlotFilled()]
 
-    marketplace.recentSlotFillEventsReturn = success(fills)
+#     marketplace.recentSlotFillEventsReturn = success(fills)
 
-    await onStep()
+#     await onStep()
 
-    check:
-      metrics.slotFill == fills.len
+#     check:
+#       metrics.slotFill == fills.len
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
index 2956e8d..e93931c 100644
--- a/tests/codexcrawler/mocks/mockmarketplace.nim
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -8,12 +8,19 @@ logScope:
   topics = "marketplace"
 
 type MockMarketplaceService* = ref object of MarketplaceService
-  recentSlotFillEventsReturn*: ?!seq[SlotFilled]
+  subNewRequestsCallback*: ?OnNewRequest
+  iterRequestsCallback*: ?OnNewRequest
 
-method getRecentSlotFillEvents*(
-    m: MockMarketplaceService
-): Future[?!seq[SlotFilled]] {.async: (raises: []).} =
-  return m.recentSlotFillEventsReturn
+method subscribeToNewRequests*(m: MockMarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []).} =
+  m.subNewRequestsCallback = some(onNewRequest)
+  return success()
+  
+method iteratePastNewRequestEvents*(m: MockMarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []).} =
+  m.iterRequestsCallback = some(onNewRequest)
+  return success()
 
 proc createMockMarketplaceService*(): MockMarketplaceService =
-  MockMarketplaceService()
+  MockMarketplaceService(
+    subNewRequestsCallback: none(OnNewRequest),
+    iterRequestsCallback: none(OnNewRequest)
+  )
diff --git a/tests/codexcrawler/mocks/mockrequeststore.nim b/tests/codexcrawler/mocks/mockrequeststore.nim
new file mode 100644
index 0000000..7005594
--- /dev/null
+++ b/tests/codexcrawler/mocks/mockrequeststore.nim
@@ -0,0 +1,33 @@
+import std/sequtils
+import pkg/questionable/results
+import pkg/chronos
+
+import ../../../codexcrawler/components/requeststore
+import ../../../codexcrawler/types
+
+type MockRequestStore* = ref object of RequestStore
+  updateRid*: Rid
+  removeRid*: Rid
+  iterateEntries*: seq[RequestEntry]
+
+method update*(
+    s: MockRequestStore, rid: Rid
+): Future[?!void] {.async: (raises: []).} =
+  s.updateRid = rid
+  return success()
+
+method remove*(
+    s: MockRequestStore, rid: Rid
+): Future[?!void] {.async: (raises: []).} =
+  s.removeRid = rid
+  return success()
+
+method iterateAll*(
+    s: MockRequestStore, onNode: OnRequestEntry
+): Future[?!void] {.async: (raises: []).} =
+  for entry in s.iterateEntries:
+    ?await onNode(entry)
+  return success()
+
+proc createMockRequestStore*(): MockRequestStore =
+  MockRequestStore(iterateEntries: newSeq[RequestEntry]())
diff --git a/tests/codexcrawler/testcomponents.nim b/tests/codexcrawler/testcomponents.nim
index fa7de7d..bf3b66b 100644
--- a/tests/codexcrawler/testcomponents.nim
+++ b/tests/codexcrawler/testcomponents.nim
@@ -1,3 +1,4 @@
+import ./components/testchaincrawler
 import ./components/testchainmetrics
 import ./components/testdhtcrawler
 import ./components/testdhtmetrics

From fe4f5c2991319ae4da7c2445d45c9bcc7306f4e2 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Thu, 20 Mar 2025 16:20:37 +0100
Subject: [PATCH 17/25] sets up tests for new chainmetrics

---
 codexcrawler/components/chainmetrics.nim      |  16 +-
 codexcrawler/services/marketplace.nim         |  30 ++++
 codexcrawler/services/metrics.nim             |  31 +++-
 .../components/testchainmetrics.nim           | 151 ++++++++++--------
 tests/codexcrawler/helpers.nim                |   1 -
 tests/codexcrawler/mocks/mockmarketplace.nim  |   8 +-
 tests/codexcrawler/mocks/mockmetrics.nim      |  14 +-
 7 files changed, 165 insertions(+), 86 deletions(-)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index e08334b..62db5e6 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -6,6 +6,7 @@ import pkg/questionable/results
 import ../state
 import ../services/metrics
 import ../services/marketplace
+import ../components/requeststore
 import ../component
 
 logScope:
@@ -14,10 +15,10 @@ logScope:
 type ChainMetrics* = ref object of Component
   state: State
   metrics: Metrics
+  store: RequestStore
   marketplace: MarketplaceService
 
 proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
-  # replace slotFills entirely:
   # iterate all requests in requestStore:
     # get state of request on chain
     # if failed/canceled/error:
@@ -29,14 +30,6 @@ proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
         # total num slots
         # total size of request
   # iter finished: update metrics!
-
-
-  # without slotFills =? (await c.marketplace.getRecentSlotFillEvents()), err:
-  #   trace "Unable to get recent slotFill events from chain", err = err.msg
-  #   return success() # We don't propagate this error.
-  #   # The call is allowed to fail and the app should continue as normal.
-
-  # c.metrics.setSlotFill(slotFills.len)
   return success()
 
 method start*(c: ChainMetrics): Future[?!void] {.async.} =
@@ -57,6 +50,7 @@ proc new*(
     T: type ChainMetrics,
     state: State,
     metrics: Metrics,
-    marketplace: MarketplaceService,
+    store: RequestStore,
+    marketplace: MarketplaceService
 ): ChainMetrics =
-  ChainMetrics(state: state, metrics: metrics, marketplace: marketplace)
+  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace)
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 88a1d9f..eb4c73d 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -18,10 +18,25 @@ type
     market: ?OnChainMarket
     clock: Clock
   OnNewRequest* = proc(id: Rid): Future[?!void] {.async: (raises: []), gcsafe.}
+  RequestInfo* = ref object
+    slots*: uint64
+    slotSize*: uint64
 
 proc notStarted() =
   raiseAssert("MarketplaceService was called before it was started.")
 
+proc fetchRequestInfo(market: OnChainMarket, rid: Rid): Future[?RequestInfo] {.async: (raises: []).} =
+  try:
+    let request = await market.getRequest(rid)
+    if r =? request:
+      return some(RequestInfo(
+        slots: r.ask.slots,
+        slotSize: r.ask.slotSize
+      ))
+  except CatchableError as exc:
+    trace "Failed to get request info", err = exc.msg
+  return none(RequestInfo)
+
 method subscribeToNewRequests*(m: MarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []), base.} =
   proc resultWrapper(rid: Rid): Future[void] {.async.} =
     let response = await onNewRequest(rid)
@@ -57,6 +72,21 @@ method iteratePastNewRequestEvents*(m: MarketplaceService, onNewRequest: OnNewRe
   else:
     notStarted()
 
+method getRequestInfo*(m: MarketplaceService, rid: Rid): Future[?RequestInfo] {.async: (raises: []), base.} =
+  # If the request id exists and is running, fetch the request object and return the info object.
+  # otherwise, return none.
+  if market =? m.market:
+    try:
+      let state = await market.requestState(rid)
+      if s =? state:
+        if s == RequestState.Started:
+          return await market.fetchRequestInfo(rid)
+    except CatchableError as exc:
+      trace "Failed to get request state", err = exc.msg
+    return none(RequestInfo)
+  else:
+    notStarted()
+
 method start*(m: MarketplaceService): Future[?!void] {.async.} =
   let provider = JsonRpcProvider.new(m.state.config.ethProvider)
   without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
diff --git a/codexcrawler/services/metrics.nim b/codexcrawler/services/metrics.nim
index 13115ed..d4dc1e1 100644
--- a/codexcrawler/services/metrics.nim
+++ b/codexcrawler/services/metrics.nim
@@ -6,7 +6,9 @@ declareGauge(todoNodesGauge, "DHT nodes to be visited")
 declareGauge(okNodesGauge, "DHT nodes successfully contacted")
 declareGauge(nokNodesGauge, "DHT nodes failed to contact")
 
-declareGauge(slotFillGauge, "Marketplace recent slots filled")
+declareGauge(requestsGauge, "Marketplace active storage requests")
+declareGauge(requestSlotsGauge, "Marketplace active storage request slots")
+declareGauge(totalStorageSizeGauge, "Marketplace total bytes stored in active storage requests")
 
 type
   OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].}
@@ -15,7 +17,10 @@ type
     todoNodes: OnUpdateMetric
     okNodes: OnUpdateMetric
     nokNodes: OnUpdateMetric
-    onSlotFill: OnUpdateMetric
+
+    onRequests: OnUpdateMetric
+    onRequestSlots: OnUpdateMetric
+    onTotalSize: OnUpdateMetric
 
 proc startServer(metricsAddress: IpAddress, metricsPort: Port) =
   let metricsAddress = metricsAddress
@@ -37,8 +42,14 @@ method setOkNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
 method setNokNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
   m.nokNodes(value.int64)
 
-method setSlotFill*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
-  m.onSlotFill(value.int64)
+method setRequests*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
+  m.onRequests(value.int64)
+
+method setRequestSlots*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
+  m.onRequestSlots(value.int64)
+
+method setTotalSize*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
+  m.onTotalSize(value.int64)
 
 proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
   startServer(metricsAddress, metricsPort)
@@ -54,8 +65,14 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
   proc onNok(value: int64) =
     nokNodesGauge.set(value)
 
-  proc onSlotFill(value: int64) =
-    slotFillGauge.set(value)
+  proc onRequests(value: int64) =
+    requestsGauge.set(value)
+
+  proc onRequestSlots(value: int64) =
+    requestSlotsGauge.set(value)
+
+  proc onTotalSize(value: int64) =
+    totalStorageSizeGauge.set(value)
 
   return
-    Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok, onSlotFill: onSlotFill)
+    Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok, onRequests: onRequests, onRequestSlots: onRequestSlots, onTotalSize: onTotalSize)
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 2f0092f..84a079e 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -1,85 +1,110 @@
-# import pkg/chronos
-# import pkg/questionable
-# import pkg/questionable/results
-# import pkg/asynctest/chronos/unittest
-# import std/sequtils
+import pkg/chronos
+import pkg/questionable
+import pkg/questionable/results
+import pkg/asynctest/chronos/unittest
 
-# import ../../../codexcrawler/components/chainmetrics
-# import ../../../codexcrawler/services/marketplace/market
-# import ../../../codexcrawler/types
-# import ../../../codexcrawler/state
-# import ../mocks/mockstate
-# import ../mocks/mockmetrics
-# import ../mocks/mockmarketplace
-# import ../helpers
+import ../../../codexcrawler/components/chainmetrics
+import ../../../codexcrawler/components/requeststore
+import ../../../codexcrawler/services/marketplace
+import ../../../codexcrawler/types
+import ../mocks/mockstate
+import ../mocks/mockmetrics
+import ../mocks/mockrequeststore
+import ../mocks/mockmarketplace
+import ../helpers
 
-# suite "ChainMetrics":
-#   var
-#     state: MockState
-#     metrics: MockMetrics
-#     marketplace: MockMarketplaceService
-#     chain: ChainMetrics
+suite "ChainMetrics":
+  var
+    state: MockState
+    metrics: MockMetrics
+    store: MockRequestStore
+    marketplace: MockMarketplaceService
+    chain: ChainMetrics
 
-#   setup:
-#     state = createMockState()
-#     metrics = createMockMetrics()
-#     marketplace = createMockMarketplaceService()
+  setup:
+    state = createMockState()
+    metrics = createMockMetrics()
+    store = createMockRequestStore()
+    marketplace = createMockMarketplaceService()
 
-#     metrics.slotFill = -1
-#     chain = ChainMetrics.new(state, metrics, marketplace)
+    chain = ChainMetrics.new(state, metrics, store, marketplace)
 
-#     (await chain.start()).tryGet()
+    (await chain.start()).tryGet()
 
-#   teardown:
-#     (await chain.stop()).tryGet()
-#     state.checkAllUnsubscribed()
+  teardown:
+    (await chain.stop()).tryGet()
+    state.checkAllUnsubscribed()
 
-#   proc onStep() {.async.} =
-#     (await state.steppers[0]()).tryGet()
+  proc onStep() {.async.} =
+    (await state.steppers[0]()).tryGet()
 
-#   test "start should start stepper for 10 minutes":
-#     check:
-#       state.delays.len == 1
-#       state.delays[0] == 10.minutes
+  test "start should start stepper for 10 minutes":
+    check:
+      state.delays.len == 1
+      state.delays[0] == 10.minutes
 
-#   test "onStep is not activated when config.marketplaceEnable is false":
-#     # Recreate chainMetrics, reset mockstate:
-#     (await chain.stop()).tryGet()
-#     state.steppers = @[]
-#     # disable marketplace:
-#     state.config.marketplaceEnable = false
-#     (await chain.start()).tryGet()
+  # iterate all requests in requestStore:
+    # get state of request on chain
+    # if failed/canceled/error:
+      # if last-seen is old (1month?3months?)
+        # delete entry
+    # else (request is running):
+      # count:
+        # total running
+        # total num slots
+        # total size of request
+  # iter finished: update metrics!
 
-#     check:
-#       state.steppers.len == 0
+  test "onStep should remove non-running requests from request store":
+    let rid = genRid()
+    store.iterateEntries.add(RequestEntry(id: rid))
 
-#   test "step should not call setSlotFill when getRecentSlotFillEvents fails":
-#     let testValue = -123
-#     metrics.slotFill = testValue
+    marketplace.requestInfoReturns = none(RequestInfo)
 
-#     marketplace.recentSlotFillEventsReturn = seq[SlotFilled].failure("testfailure")
+    await onStep()
 
-#     await onStep()
+    check:
+      marketplace.requestInfoRid == rid
+      store.removeRid == rid
+    
+  test "onStep should count the number of active requests":
+    let rid1 = genRid()
+    let rid2 = genRid()
+    store.iterateEntries.add(RequestEntry(id: rid1))
+    store.iterateEntries.add(RequestEntry(id: rid2))
 
-#     check:
-#       metrics.slotFill == testValue
+    marketplace.requestInfoReturns = some(RequestInfo())
 
-#   test "step should setSlotFill to zero when getRecentSlotFillEvents returns empty seq":
-#     metrics.slotFill = -123
+    await onStep()
 
-#     marketplace.recentSlotFillEventsReturn = success(newSeq[SlotFilled]())
+    check:
+      metrics.requests == 2
 
-#     await onStep()
+  test "onStep should count the number of active slots":
+    let rid = genRid()
+    store.iterateEntries.add(RequestEntry(id: rid))
 
-#     check:
-#       metrics.slotFill == 0
+    let info = RequestInfo(
+      slots: 123
+    )
+    marketplace.requestInfoReturns = some(info)
 
-#   test "step should setSlotFill to the length of seq returned from getRecentSlotFillEvents":
-#     let fills = @[SlotFilled(), SlotFilled(), SlotFilled(), SlotFilled()]
+    await onStep()
 
-#     marketplace.recentSlotFillEventsReturn = success(fills)
+    check:
+      metrics.slots == info.slots.int
 
-#     await onStep()
+  test "onStep should count the total size of active slots":
+    let rid = genRid()
+    store.iterateEntries.add(RequestEntry(id: rid))
 
-#     check:
-#       metrics.slotFill == fills.len
+    let info = RequestInfo(
+      slots: 12,
+      slotSize: 23
+    )
+    marketplace.requestInfoReturns = some(info)
+
+    await onStep()
+
+    check:
+      metrics.totalSize == (info.slots * info.slotSize).int
diff --git a/tests/codexcrawler/helpers.nim b/tests/codexcrawler/helpers.nim
index b7a5b9e..1afb6a8 100644
--- a/tests/codexcrawler/helpers.nim
+++ b/tests/codexcrawler/helpers.nim
@@ -1,5 +1,4 @@
 import std/random
-import std/sequtils
 import std/typetraits
 import pkg/stint
 import pkg/stew/byteutils
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
index e93931c..a48f24f 100644
--- a/tests/codexcrawler/mocks/mockmarketplace.nim
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -2,7 +2,7 @@ import pkg/ethers
 import pkg/questionable
 
 import ../../../codexcrawler/services/marketplace
-import ../../../codexcrawler/services/marketplace/market
+import ../../../codexcrawler/types
 
 logScope:
   topics = "marketplace"
@@ -10,6 +10,8 @@ logScope:
 type MockMarketplaceService* = ref object of MarketplaceService
   subNewRequestsCallback*: ?OnNewRequest
   iterRequestsCallback*: ?OnNewRequest
+  requestInfoReturns*: ?RequestInfo
+  requestInfoRid*: Rid
 
 method subscribeToNewRequests*(m: MockMarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []).} =
   m.subNewRequestsCallback = some(onNewRequest)
@@ -19,6 +21,10 @@ method iteratePastNewRequestEvents*(m: MockMarketplaceService, onNewRequest: OnN
   m.iterRequestsCallback = some(onNewRequest)
   return success()
 
+method getRequestInfo*(m: MockMarketplaceService, rid: Rid): Future[?RequestInfo] {.async: (raises: []).} =
+  m.requestInfoRid = rid
+  return m.requestInfoReturns
+
 proc createMockMarketplaceService*(): MockMarketplaceService =
   MockMarketplaceService(
     subNewRequestsCallback: none(OnNewRequest),
diff --git a/tests/codexcrawler/mocks/mockmetrics.nim b/tests/codexcrawler/mocks/mockmetrics.nim
index 3b9a24a..cfe1b01 100644
--- a/tests/codexcrawler/mocks/mockmetrics.nim
+++ b/tests/codexcrawler/mocks/mockmetrics.nim
@@ -4,7 +4,9 @@ type MockMetrics* = ref object of Metrics
   todo*: int
   ok*: int
   nok*: int
-  slotFill*: int
+  requests*: int
+  slots*: int
+  totalSize*: int
 
 method setTodoNodes*(m: MockMetrics, value: int) =
   m.todo = value
@@ -15,8 +17,14 @@ method setOkNodes*(m: MockMetrics, value: int) =
 method setNokNodes*(m: MockMetrics, value: int) =
   m.nok = value
 
-method setSlotFill*(m: MockMetrics, value: int) =
-  m.slotFill = value
+method setRequests*(m: MockMetrics, value: int) =
+  m.requests = value
+
+method setRequestSlots*(m: MockMetrics, value: int) =
+  m.slots = value
+
+method setTotalSize*(m: MockMetrics, value: int) =
+  m.totalSize = value
 
 proc createMockMetrics*(): MockMetrics =
   MockMetrics()

From ec3c7c0be379744761436cab295ca4a37c9127f6 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Fri, 21 Mar 2025 09:41:05 +0100
Subject: [PATCH 18/25] implements chainmetrics component

---
 codexcrawler/components/chaincrawler.nim      |  1 -
 codexcrawler/components/chainmetrics.nim      | 55 +++++++++++++------
 codexcrawler/components/requeststore.nim      |  1 -
 codexcrawler/services/marketplace.nim         | 28 ++++++----
 codexcrawler/services/metrics.nim             | 16 ++++--
 .../components/testchaincrawler.nim           |  2 +-
 .../components/testchainmetrics.nim           | 23 +-------
 tests/codexcrawler/mocks/mockmarketplace.nim  | 17 ++++--
 tests/codexcrawler/mocks/mockmetrics.nim      |  4 +-
 tests/codexcrawler/mocks/mockrequeststore.nim |  8 +--
 10 files changed, 87 insertions(+), 68 deletions(-)

diff --git a/codexcrawler/components/chaincrawler.nim b/codexcrawler/components/chaincrawler.nim
index 8295618..70941fd 100644
--- a/codexcrawler/components/chaincrawler.nim
+++ b/codexcrawler/components/chaincrawler.nim
@@ -1,4 +1,3 @@
-
 import pkg/chronicles
 import pkg/chronos
 import pkg/questionable/results
diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 62db5e6..8f8f899 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -8,28 +8,49 @@ import ../services/metrics
 import ../services/marketplace
 import ../components/requeststore
 import ../component
+import ../types
 
 logScope:
   topics = "chainmetrics"
 
-type ChainMetrics* = ref object of Component
-  state: State
-  metrics: Metrics
-  store: RequestStore
-  marketplace: MarketplaceService
+type
+  ChainMetrics* = ref object of Component
+    state: State
+    metrics: Metrics
+    store: RequestStore
+    marketplace: MarketplaceService
+
+  Update = ref object
+    numRequests: int
+    numSlots: int
+    totalSize: int64
+
+proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
+  var update = Update(numRequests: 0, numSlots: 0, totalSize: 0)
+
+  proc onRequest(entry: RequestEntry): Future[?!void] {.async: (raises: []).} =
+    let response = await c.marketplace.getRequestInfo(entry.id)
+    if info =? response:
+      inc update.numRequests
+      update.numSlots += info.slots.int
+      update.totalSize += (info.slots * info.slotSize).int64
+    else:
+      ?await c.store.remove(entry.id)
+    return success()
+
+  ?await c.store.iterateAll(onRequest)
+  return success(update)
+
+proc updateMetrics(c: ChainMetrics, update: Update) =
+  c.metrics.setRequests(update.numRequests)
+  c.metrics.setRequestSlots(update.numSlots)
+  c.metrics.setTotalSize(update.totalSize)
 
 proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
-  # iterate all requests in requestStore:
-    # get state of request on chain
-    # if failed/canceled/error:
-      # if last-seen is old (1month?3months?)
-        # delete entry
-    # else (request is running):
-      # count:
-        # total running
-        # total num slots
-        # total size of request
-  # iter finished: update metrics!
+  without update =? (await c.collectUpdate()), err:
+    return failure(err)
+
+  c.updateMetrics(update)
   return success()
 
 method start*(c: ChainMetrics): Future[?!void] {.async.} =
@@ -51,6 +72,6 @@ proc new*(
     state: State,
     metrics: Metrics,
     store: RequestStore,
-    marketplace: MarketplaceService
+    marketplace: MarketplaceService,
 ): ChainMetrics =
   ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace)
diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index 566c308..e4f18b6 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -10,7 +10,6 @@ import ../types
 import ../component
 import ../state
 import ../utils/datastoreutils
-import ../utils/asyncdataevent
 import ../services/clock
 
 const requeststoreName = "requeststore"
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index eb4c73d..4a84d1a 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -17,6 +17,7 @@ type
     state: State
     market: ?OnChainMarket
     clock: Clock
+
   OnNewRequest* = proc(id: Rid): Future[?!void] {.async: (raises: []), gcsafe.}
   RequestInfo* = ref object
     slots*: uint64
@@ -25,25 +26,28 @@ type
 proc notStarted() =
   raiseAssert("MarketplaceService was called before it was started.")
 
-proc fetchRequestInfo(market: OnChainMarket, rid: Rid): Future[?RequestInfo] {.async: (raises: []).} =
+proc fetchRequestInfo(
+    market: OnChainMarket, rid: Rid
+): Future[?RequestInfo] {.async: (raises: []).} =
   try:
     let request = await market.getRequest(rid)
     if r =? request:
-      return some(RequestInfo(
-        slots: r.ask.slots,
-        slotSize: r.ask.slotSize
-      ))
+      return some(RequestInfo(slots: r.ask.slots, slotSize: r.ask.slotSize))
   except CatchableError as exc:
     trace "Failed to get request info", err = exc.msg
   return none(RequestInfo)
 
-method subscribeToNewRequests*(m: MarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []), base.} =
+method subscribeToNewRequests*(
+    m: MarketplaceService, onNewRequest: OnNewRequest
+): Future[?!void] {.async: (raises: []), base.} =
   proc resultWrapper(rid: Rid): Future[void] {.async.} =
     let response = await onNewRequest(rid)
     if error =? response.errorOption:
       raiseAssert("Error result in handling of onNewRequest callback: " & error.msg)
 
-  proc onRequest(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, upraises: [].} =
+  proc onRequest(
+      id: RequestId, ask: StorageAsk, expiry: uint64
+  ) {.gcsafe, upraises: [].} =
     asyncSpawn resultWrapper(Rid(id))
 
   if market =? m.market:
@@ -54,8 +58,10 @@ method subscribeToNewRequests*(m: MarketplaceService, onNewRequest: OnNewRequest
   else:
     notStarted()
   return success()
-  
-method iteratePastNewRequestEvents*(m: MarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []), base.} =
+
+method iteratePastNewRequestEvents*(
+    m: MarketplaceService, onNewRequest: OnNewRequest
+): Future[?!void] {.async: (raises: []), base.} =
   let
     oneDay = 60 * 60 * 24
     timespan = oneDay * 30
@@ -72,7 +78,9 @@ method iteratePastNewRequestEvents*(m: MarketplaceService, onNewRequest: OnNewRe
   else:
     notStarted()
 
-method getRequestInfo*(m: MarketplaceService, rid: Rid): Future[?RequestInfo] {.async: (raises: []), base.} =
+method getRequestInfo*(
+    m: MarketplaceService, rid: Rid
+): Future[?RequestInfo] {.async: (raises: []), base.} =
   # If the request id exists and is running, fetch the request object and return the info object.
   # otherwise, return none.
   if market =? m.market:
diff --git a/codexcrawler/services/metrics.nim b/codexcrawler/services/metrics.nim
index d4dc1e1..adc8852 100644
--- a/codexcrawler/services/metrics.nim
+++ b/codexcrawler/services/metrics.nim
@@ -8,7 +8,9 @@ declareGauge(nokNodesGauge, "DHT nodes failed to contact")
 
 declareGauge(requestsGauge, "Marketplace active storage requests")
 declareGauge(requestSlotsGauge, "Marketplace active storage request slots")
-declareGauge(totalStorageSizeGauge, "Marketplace total bytes stored in active storage requests")
+declareGauge(
+  totalStorageSizeGauge, "Marketplace total bytes stored in active storage requests"
+)
 
 type
   OnUpdateMetric = proc(value: int64): void {.gcsafe, raises: [].}
@@ -48,7 +50,7 @@ method setRequests*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
 method setRequestSlots*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
   m.onRequestSlots(value.int64)
 
-method setTotalSize*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
+method setTotalSize*(m: Metrics, value: int64) {.base, gcsafe, raises: [].} =
   m.onTotalSize(value.int64)
 
 proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
@@ -74,5 +76,11 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
   proc onTotalSize(value: int64) =
     totalStorageSizeGauge.set(value)
 
-  return
-    Metrics(todoNodes: onTodo, okNodes: onOk, nokNodes: onNok, onRequests: onRequests, onRequestSlots: onRequestSlots, onTotalSize: onTotalSize)
+  return Metrics(
+    todoNodes: onTodo,
+    okNodes: onOk,
+    nokNodes: onNok,
+    onRequests: onRequests,
+    onRequestSlots: onRequestSlots,
+    onTotalSize: onTotalSize,
+  )
diff --git a/tests/codexcrawler/components/testchaincrawler.nim b/tests/codexcrawler/components/testchaincrawler.nim
index 8a50a78..cdd74f3 100644
--- a/tests/codexcrawler/components/testchaincrawler.nim
+++ b/tests/codexcrawler/components/testchaincrawler.nim
@@ -25,7 +25,7 @@ suite "ChainCrawler":
     state = createMockState()
     store = createMockRequestStore()
     marketplace = createMockMarketplaceService()
-    
+
     crawler = ChainCrawler.new(state, store, marketplace)
 
     (await crawler.start()).tryGet()
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 84a079e..81c062b 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -43,18 +43,6 @@ suite "ChainMetrics":
       state.delays.len == 1
       state.delays[0] == 10.minutes
 
-  # iterate all requests in requestStore:
-    # get state of request on chain
-    # if failed/canceled/error:
-      # if last-seen is old (1month?3months?)
-        # delete entry
-    # else (request is running):
-      # count:
-        # total running
-        # total num slots
-        # total size of request
-  # iter finished: update metrics!
-
   test "onStep should remove non-running requests from request store":
     let rid = genRid()
     store.iterateEntries.add(RequestEntry(id: rid))
@@ -66,7 +54,7 @@ suite "ChainMetrics":
     check:
       marketplace.requestInfoRid == rid
       store.removeRid == rid
-    
+
   test "onStep should count the number of active requests":
     let rid1 = genRid()
     let rid2 = genRid()
@@ -84,9 +72,7 @@ suite "ChainMetrics":
     let rid = genRid()
     store.iterateEntries.add(RequestEntry(id: rid))
 
-    let info = RequestInfo(
-      slots: 123
-    )
+    let info = RequestInfo(slots: 123)
     marketplace.requestInfoReturns = some(info)
 
     await onStep()
@@ -98,10 +84,7 @@ suite "ChainMetrics":
     let rid = genRid()
     store.iterateEntries.add(RequestEntry(id: rid))
 
-    let info = RequestInfo(
-      slots: 12,
-      slotSize: 23
-    )
+    let info = RequestInfo(slots: 12, slotSize: 23)
     marketplace.requestInfoReturns = some(info)
 
     await onStep()
diff --git a/tests/codexcrawler/mocks/mockmarketplace.nim b/tests/codexcrawler/mocks/mockmarketplace.nim
index a48f24f..97d0ef1 100644
--- a/tests/codexcrawler/mocks/mockmarketplace.nim
+++ b/tests/codexcrawler/mocks/mockmarketplace.nim
@@ -13,20 +13,25 @@ type MockMarketplaceService* = ref object of MarketplaceService
   requestInfoReturns*: ?RequestInfo
   requestInfoRid*: Rid
 
-method subscribeToNewRequests*(m: MockMarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []).} =
+method subscribeToNewRequests*(
+    m: MockMarketplaceService, onNewRequest: OnNewRequest
+): Future[?!void] {.async: (raises: []).} =
   m.subNewRequestsCallback = some(onNewRequest)
   return success()
-  
-method iteratePastNewRequestEvents*(m: MockMarketplaceService, onNewRequest: OnNewRequest): Future[?!void] {.async: (raises: []).} =
+
+method iteratePastNewRequestEvents*(
+    m: MockMarketplaceService, onNewRequest: OnNewRequest
+): Future[?!void] {.async: (raises: []).} =
   m.iterRequestsCallback = some(onNewRequest)
   return success()
 
-method getRequestInfo*(m: MockMarketplaceService, rid: Rid): Future[?RequestInfo] {.async: (raises: []).} =
+method getRequestInfo*(
+    m: MockMarketplaceService, rid: Rid
+): Future[?RequestInfo] {.async: (raises: []).} =
   m.requestInfoRid = rid
   return m.requestInfoReturns
 
 proc createMockMarketplaceService*(): MockMarketplaceService =
   MockMarketplaceService(
-    subNewRequestsCallback: none(OnNewRequest),
-    iterRequestsCallback: none(OnNewRequest)
+    subNewRequestsCallback: none(OnNewRequest), iterRequestsCallback: none(OnNewRequest)
   )
diff --git a/tests/codexcrawler/mocks/mockmetrics.nim b/tests/codexcrawler/mocks/mockmetrics.nim
index cfe1b01..01787cc 100644
--- a/tests/codexcrawler/mocks/mockmetrics.nim
+++ b/tests/codexcrawler/mocks/mockmetrics.nim
@@ -6,7 +6,7 @@ type MockMetrics* = ref object of Metrics
   nok*: int
   requests*: int
   slots*: int
-  totalSize*: int
+  totalSize*: int64
 
 method setTodoNodes*(m: MockMetrics, value: int) =
   m.todo = value
@@ -23,7 +23,7 @@ method setRequests*(m: MockMetrics, value: int) =
 method setRequestSlots*(m: MockMetrics, value: int) =
   m.slots = value
 
-method setTotalSize*(m: MockMetrics, value: int) =
+method setTotalSize*(m: MockMetrics, value: int64) =
   m.totalSize = value
 
 proc createMockMetrics*(): MockMetrics =
diff --git a/tests/codexcrawler/mocks/mockrequeststore.nim b/tests/codexcrawler/mocks/mockrequeststore.nim
index 7005594..075fc42 100644
--- a/tests/codexcrawler/mocks/mockrequeststore.nim
+++ b/tests/codexcrawler/mocks/mockrequeststore.nim
@@ -10,15 +10,11 @@ type MockRequestStore* = ref object of RequestStore
   removeRid*: Rid
   iterateEntries*: seq[RequestEntry]
 
-method update*(
-    s: MockRequestStore, rid: Rid
-): Future[?!void] {.async: (raises: []).} =
+method update*(s: MockRequestStore, rid: Rid): Future[?!void] {.async: (raises: []).} =
   s.updateRid = rid
   return success()
 
-method remove*(
-    s: MockRequestStore, rid: Rid
-): Future[?!void] {.async: (raises: []).} =
+method remove*(s: MockRequestStore, rid: Rid): Future[?!void] {.async: (raises: []).} =
   s.removeRid = rid
   return success()
 

From a5618d0cd20d9e3fe218aaada9045f77082dca53 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Fri, 21 Mar 2025 10:39:46 +0100
Subject: [PATCH 19/25] adds old-check to chain metrics entry handling

---
 codexcrawler/components/chainmetrics.nim      | 12 +++++++--
 codexcrawler/installer.nim                    |  8 +++++-
 .../components/testchainmetrics.nim           | 25 ++++++++++++++++---
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 8f8f899..80c6d3b 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -6,6 +6,7 @@ import pkg/questionable/results
 import ../state
 import ../services/metrics
 import ../services/marketplace
+import ../services/clock
 import ../components/requeststore
 import ../component
 import ../types
@@ -19,12 +20,17 @@ type
     metrics: Metrics
     store: RequestStore
     marketplace: MarketplaceService
+    clock: Clock
 
   Update = ref object
     numRequests: int
     numSlots: int
     totalSize: int64
 
+proc isOld(c: ChainMetrics, entry: RequestEntry): bool =
+  let oneDay = 60 * 60 * 24
+  return entry.lastSeen < (c.clock.now - oneDay.uint64)
+
 proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
   var update = Update(numRequests: 0, numSlots: 0, totalSize: 0)
 
@@ -35,7 +41,8 @@ proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
       update.numSlots += info.slots.int
       update.totalSize += (info.slots * info.slotSize).int64
     else:
-      ?await c.store.remove(entry.id)
+      if c.isOld(entry):
+        ?await c.store.remove(entry.id)
     return success()
 
   ?await c.store.iterateAll(onRequest)
@@ -73,5 +80,6 @@ proc new*(
     metrics: Metrics,
     store: RequestStore,
     marketplace: MarketplaceService,
+    clock: Clock
 ): ChainMetrics =
-  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace)
+  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace, clock: clock)
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index fbaf172..bdff348 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -15,6 +15,8 @@ import ./components/nodestore
 import ./components/dhtmetrics
 import ./components/todolist
 import ./components/chainmetrics
+import ./components/chaincrawler
+import ./components/requeststore
 
 proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   var components: seq[Component] = newSeq[Component]()
@@ -26,11 +28,14 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   without nodeStore =? createNodeStore(state, clock), err:
     return failure(err)
 
+  without requestStore =? createRequestStore(state, clock), err:
+    return failure(err)
+
   let
     metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort)
     todoList = createTodoList(state, metrics)
     marketplace = createMarketplace(state, clock)
-    chainMetrics = ChainMetrics.new(state, metrics, marketplace)
+    chainMetrics = ChainMetrics.new(state, metrics, requestStore, marketplace, clock)
 
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
     return failure(err)
@@ -43,5 +48,6 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   components.add(dhtMetrics)
   components.add(marketplace)
   components.add(chainMetrics)
+  components.add(ChainCrawler.new(state, requestStore, marketplace))
 
   return success(components)
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 81c062b..eaae1d9 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -11,6 +11,7 @@ import ../mocks/mockstate
 import ../mocks/mockmetrics
 import ../mocks/mockrequeststore
 import ../mocks/mockmarketplace
+import ../mocks/mockclock
 import ../helpers
 
 suite "ChainMetrics":
@@ -19,6 +20,7 @@ suite "ChainMetrics":
     metrics: MockMetrics
     store: MockRequestStore
     marketplace: MockMarketplaceService
+    clock: MockClock
     chain: ChainMetrics
 
   setup:
@@ -26,8 +28,9 @@ suite "ChainMetrics":
     metrics = createMockMetrics()
     store = createMockRequestStore()
     marketplace = createMockMarketplaceService()
+    clock = createMockClock()
 
-    chain = ChainMetrics.new(state, metrics, store, marketplace)
+    chain = ChainMetrics.new(state, metrics, store, marketplace, clock)
 
     (await chain.start()).tryGet()
 
@@ -43,10 +46,12 @@ suite "ChainMetrics":
       state.delays.len == 1
       state.delays[0] == 10.minutes
 
-  test "onStep should remove non-running requests from request store":
+  test "onStep should remove old non-running requests from request store":
     let rid = genRid()
-    store.iterateEntries.add(RequestEntry(id: rid))
+    let oneDay = (60 * 60 * 24).uint64
+    store.iterateEntries.add(RequestEntry(id: rid, lastSeen: 100.uint64))
 
+    clock.setNow = 100 + oneDay + 1
     marketplace.requestInfoReturns = none(RequestInfo)
 
     await onStep()
@@ -55,6 +60,20 @@ suite "ChainMetrics":
       marketplace.requestInfoRid == rid
       store.removeRid == rid
 
+  test "onStep should not remove recent non-running requests from request store":
+    let rid = genRid()
+    let now = 123456789.uint64
+    store.iterateEntries.add(RequestEntry(id: rid, lastSeen: now - 1))
+
+    clock.setNow = now
+    marketplace.requestInfoReturns = none(RequestInfo)
+
+    await onStep()
+
+    check:
+      marketplace.requestInfoRid == rid
+      not (store.removeRid == rid)
+
   test "onStep should count the number of active requests":
     let rid1 = genRid()
     let rid2 = genRid()

From c1b0c30cc0a1adb68fb34f75991cb70577ecdfc2 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Fri, 21 Mar 2025 13:04:10 +0100
Subject: [PATCH 20/25] splits awake and start component methods

---
 codexcrawler/application.nim                      |  8 +++++++-
 codexcrawler/component.nim                        | 15 +++++++++++----
 codexcrawler/components/chaincrawler.nim          |  6 +++---
 codexcrawler/components/chainmetrics.nim          |  9 ++++-----
 codexcrawler/components/dhtcrawler.nim            |  3 ---
 codexcrawler/components/dhtmetrics.nim            | 11 ++++++-----
 codexcrawler/components/requeststore.nim          |  7 -------
 codexcrawler/components/timetracker.nim           |  3 ---
 codexcrawler/components/todolist.nim              |  8 ++++----
 codexcrawler/services/marketplace.nim             |  9 +++------
 .../codexcrawler/components/testchaincrawler.nim  |  6 ------
 .../codexcrawler/components/testchainmetrics.nim  |  2 --
 tests/codexcrawler/components/testdhtcrawler.nim  |  3 ---
 tests/codexcrawler/components/testdhtmetrics.nim  |  2 +-
 tests/codexcrawler/components/testnodestore.nim   |  1 -
 .../codexcrawler/components/testrequeststore.nim  |  3 ---
 tests/codexcrawler/components/testtimetracker.nim |  2 --
 tests/codexcrawler/components/testtodolist.nim    |  3 +--
 tests/codexcrawler/mocks/mockdht.nim              |  6 ------
 tests/codexcrawler/mocks/mocknodestore.nim        |  6 ------
 tests/codexcrawler/mocks/mocktodolist.nim         |  6 ------
 21 files changed, 40 insertions(+), 79 deletions(-)

diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim
index 0f7f73c..e5a0d07 100644
--- a/codexcrawler/application.nim
+++ b/codexcrawler/application.nim
@@ -36,9 +36,15 @@ proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} =
     return failure(err)
   app.components = components
 
+  for c in components:
+    if err =? (await c.awake()).errorOption:
+      error "Failed during component awake", err = err.msg
+      return failure(err)
+
   for c in components:
     if err =? (await c.start()).errorOption:
-      error "Failed to start component", err = err.msg
+      error "Failed during component start", err = err.msg
+      return failure(err)
 
   return success()
 
diff --git a/codexcrawler/component.nim b/codexcrawler/component.nim
index f36b0a2..ce1517b 100644
--- a/codexcrawler/component.nim
+++ b/codexcrawler/component.nim
@@ -1,12 +1,19 @@
 import pkg/chronos
 import pkg/questionable/results
 
-import ./state
-
 type Component* = ref object of RootObj
 
+method awake*(c: Component): Future[?!void] {.async, base.} =
+  # Awake is called on all components in an unspecified order.
+  # Use this method to subscribe/connect to other components.
+  return success()
+
 method start*(c: Component): Future[?!void] {.async, base.} =
-  raiseAssert("call to abstract method: component.start")
+  # Start is called on all components in an unspecified order.
+  # Is is guaranteed that all components have already successfulled handled 'awake'.
+  # Use this method to begin the work of this component.
+  return success()
 
 method stop*(c: Component): Future[?!void] {.async, base.} =
-  raiseAssert("call to abstract method: component.stop")
+  # Use this method to stop, unsubscribe, and clean up any resources.
+  return success()
diff --git a/codexcrawler/components/chaincrawler.nim b/codexcrawler/components/chaincrawler.nim
index 70941fd..b29e9f8 100644
--- a/codexcrawler/components/chaincrawler.nim
+++ b/codexcrawler/components/chaincrawler.nim
@@ -25,13 +25,13 @@ method start*(c: ChainCrawler): Future[?!void] {.async.} =
   proc onRequest(rid: Rid): Future[?!void] {.async: (raises: []).} =
     return await c.onNewRequest(rid)
 
+  # Normally subscriptions must be done in awake.
+  # Marketplace is a little different: It uses awake to set up its connections.
+  # And so it can't handle subscribes until we're in 'start'.
   ?await c.marketplace.subscribeToNewRequests(onRequest)
   ?await c.marketplace.iteratePastNewRequestEvents(onRequest)
   return success()
 
-method stop*(c: ChainCrawler): Future[?!void] {.async.} =
-  return success()
-
 proc new*(
     T: type ChainCrawler,
     state: State,
diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 80c6d3b..b23041c 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -71,15 +71,14 @@ method start*(c: ChainMetrics): Future[?!void] {.async.} =
 
   return success()
 
-method stop*(c: ChainMetrics): Future[?!void] {.async.} =
-  return success()
-
 proc new*(
     T: type ChainMetrics,
     state: State,
     metrics: Metrics,
     store: RequestStore,
     marketplace: MarketplaceService,
-    clock: Clock
+    clock: Clock,
 ): ChainMetrics =
-  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace, clock: clock)
+  ChainMetrics(
+    state: state, metrics: metrics, store: store, marketplace: marketplace, clock: clock
+  )
diff --git a/codexcrawler/components/dhtcrawler.nim b/codexcrawler/components/dhtcrawler.nim
index 5fde23e..0c594b2 100644
--- a/codexcrawler/components/dhtcrawler.nim
+++ b/codexcrawler/components/dhtcrawler.nim
@@ -57,8 +57,5 @@ method start*(c: DhtCrawler): Future[?!void] {.async.} =
 
   return success()
 
-method stop*(c: DhtCrawler): Future[?!void] {.async.} =
-  return success()
-
 proc new*(T: type DhtCrawler, state: State, dht: Dht, todo: TodoList): DhtCrawler =
   DhtCrawler(state: state, dht: dht, todo: todo)
diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim
index a8d0c74..374e169 100644
--- a/codexcrawler/components/dhtmetrics.nim
+++ b/codexcrawler/components/dhtmetrics.nim
@@ -45,11 +45,7 @@ proc handleDeleteEvent(d: DhtMetrics, nids: seq[Nid]): Future[?!void] {.async.}
   d.updateMetrics()
   return success()
 
-method start*(d: DhtMetrics): Future[?!void] {.async.} =
-  info "starting..."
-  ?await d.ok.load()
-  ?await d.nok.load()
-
+method awake*(d: DhtMetrics): Future[?!void] {.async.} =
   proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} =
     await d.handleCheckEvent(event)
 
@@ -58,7 +54,12 @@ method start*(d: DhtMetrics): Future[?!void] {.async.} =
 
   d.subCheck = d.state.events.dhtNodeCheck.subscribe(onCheck)
   d.subDel = d.state.events.nodesDeleted.subscribe(onDelete)
+  return success()
 
+method start*(d: DhtMetrics): Future[?!void] {.async.} =
+  info "starting..."
+  ?await d.ok.load()
+  ?await d.nok.load()
   return success()
 
 method stop*(d: DhtMetrics): Future[?!void] {.async.} =
diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index e4f18b6..130a6e5 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -117,13 +117,6 @@ method iterateAll*(
     return failure(exc.msg)
   return success()
 
-method start*(s: RequestStore): Future[?!void] {.async.} =
-  info "starting..."
-  return success()
-
-method stop*(s: RequestStore): Future[?!void] {.async.} =
-  return success()
-
 proc new*(
     T: type RequestStore, state: State, store: TypedDatastore, clock: Clock
 ): RequestStore =
diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim
index a4217cb..2683b0a 100644
--- a/codexcrawler/components/timetracker.nim
+++ b/codexcrawler/components/timetracker.nim
@@ -71,9 +71,6 @@ method start*(t: TimeTracker): Future[?!void] {.async.} =
   await t.state.whileRunning(onRoutingTable, 30.minutes)
   return success()
 
-method stop*(t: TimeTracker): Future[?!void] {.async.} =
-  return success()
-
 proc new*(
     T: type TimeTracker, state: State, nodestore: NodeStore, dht: Dht, clock: Clock
 ): TimeTracker =
diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim
index e271f1e..33144ae 100644
--- a/codexcrawler/components/todolist.nim
+++ b/codexcrawler/components/todolist.nim
@@ -32,13 +32,13 @@ proc addNodes(t: TodoList, nids: seq[Nid]) =
   t.metrics.setTodoNodes(t.nids.len)
 
   if s =? t.emptySignal:
-    trace "Nodes added, resuming...", nodes = nids.len
+    trace "nodes added, resuming...", nodes = nids.len
     s.complete()
     t.emptySignal = Future[void].none
 
 method pop*(t: TodoList): Future[?!Nid] {.async: (raises: []), base.} =
   if t.nids.len < 1:
-    trace "List is empty. Waiting for new items..."
+    trace "list is empty. Waiting for new items..."
     let signal = newFuture[void]("list.emptySignal")
     t.emptySignal = some(signal)
     try:
@@ -54,8 +54,8 @@ method pop*(t: TodoList): Future[?!Nid] {.async: (raises: []), base.} =
 
   return success(item)
 
-method start*(t: TodoList): Future[?!void] {.async.} =
-  info "Starting TodoList..."
+method awake*(t: TodoList): Future[?!void] {.async.} =
+  info "initializing..."
 
   proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} =
     t.addNodes(nids)
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 4a84d1a..494ff4c 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -53,11 +53,11 @@ method subscribeToNewRequests*(
   if market =? m.market:
     try:
       discard await market.subscribeRequests(onRequest)
+      return success()
     except CatchableError as exc:
       return failure(exc.msg)
   else:
     notStarted()
-  return success()
 
 method iteratePastNewRequestEvents*(
     m: MarketplaceService, onNewRequest: OnNewRequest
@@ -73,6 +73,7 @@ method iteratePastNewRequestEvents*(
       for request in requests:
         if error =? (await onNewRequest(Rid(request.requestId))).errorOption:
           return failure(error.msg)
+      return success()
     except CatchableError as exc:
       return failure(exc.msg)
   else:
@@ -95,17 +96,13 @@ method getRequestInfo*(
   else:
     notStarted()
 
-method start*(m: MarketplaceService): Future[?!void] {.async.} =
+method awake*(m: MarketplaceService): Future[?!void] {.async.} =
   let provider = JsonRpcProvider.new(m.state.config.ethProvider)
   without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
     return failure("Invalid MarketplaceAddress provided")
 
   let marketplace = Marketplace.new(marketplaceAddress, provider)
   m.market = some(OnChainMarket.new(marketplace))
-
-  return success()
-
-method stop*(m: MarketplaceService): Future[?!void] {.async.} =
   return success()
 
 proc new(T: type MarketplaceService, state: State, clock: Clock): MarketplaceService =
diff --git a/tests/codexcrawler/components/testchaincrawler.nim b/tests/codexcrawler/components/testchaincrawler.nim
index cdd74f3..172b2b3 100644
--- a/tests/codexcrawler/components/testchaincrawler.nim
+++ b/tests/codexcrawler/components/testchaincrawler.nim
@@ -27,17 +27,11 @@ suite "ChainCrawler":
     marketplace = createMockMarketplaceService()
 
     crawler = ChainCrawler.new(state, store, marketplace)
-
     (await crawler.start()).tryGet()
 
   teardown:
-    (await crawler.stop()).tryGet()
     state.checkAllUnsubscribed()
 
-  # subscribe to newrequests
-  # iterate past requests on start-up
-  # push them into the request store
-
   test "start should subscribe to new requests":
     check:
       marketplace.subNewRequestsCallback.isSome()
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index eaae1d9..1098f45 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -31,11 +31,9 @@ suite "ChainMetrics":
     clock = createMockClock()
 
     chain = ChainMetrics.new(state, metrics, store, marketplace, clock)
-
     (await chain.start()).tryGet()
 
   teardown:
-    (await chain.stop()).tryGet()
     state.checkAllUnsubscribed()
 
   proc onStep() {.async.} =
diff --git a/tests/codexcrawler/components/testdhtcrawler.nim b/tests/codexcrawler/components/testdhtcrawler.nim
index def5ddc..4ae4d21 100644
--- a/tests/codexcrawler/components/testdhtcrawler.nim
+++ b/tests/codexcrawler/components/testdhtcrawler.nim
@@ -30,11 +30,9 @@ suite "DhtCrawler":
     dht = createMockDht()
 
     crawler = DhtCrawler.new(state, dht, todo)
-
     (await crawler.start()).tryGet()
 
   teardown:
-    (await crawler.stop()).tryGet()
     state.checkAllUnsubscribed()
 
   proc onStep() {.async.} =
@@ -57,7 +55,6 @@ suite "DhtCrawler":
 
   test "onStep is not activated when config.dhtEnable is false":
     # Recreate crawler, reset mockstate:
-    (await crawler.stop()).tryGet()
     state.steppers = @[]
     # disable DHT:
     state.config.dhtEnable = false
diff --git a/tests/codexcrawler/components/testdhtmetrics.nim b/tests/codexcrawler/components/testdhtmetrics.nim
index 6475d05..98c48ab 100644
--- a/tests/codexcrawler/components/testdhtmetrics.nim
+++ b/tests/codexcrawler/components/testdhtmetrics.nim
@@ -28,7 +28,7 @@ suite "DhtMetrics":
     metrics = createMockMetrics()
 
     dhtmetrics = DhtMetrics.new(state, okList, nokList, metrics)
-
+    (await dhtmetrics.awake()).tryGet()
     (await dhtmetrics.start()).tryGet()
 
   teardown:
diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim
index f2a3a17..9ae8698 100644
--- a/tests/codexcrawler/components/testnodestore.nim
+++ b/tests/codexcrawler/components/testnodestore.nim
@@ -30,7 +30,6 @@ suite "Nodestore":
     clock = createMockClock()
 
     store = NodeStore.new(state, ds, clock)
-
     (await store.start()).tryGet()
 
   teardown:
diff --git a/tests/codexcrawler/components/testrequeststore.nim b/tests/codexcrawler/components/testrequeststore.nim
index 93858e9..c3202f6 100644
--- a/tests/codexcrawler/components/testrequeststore.nim
+++ b/tests/codexcrawler/components/testrequeststore.nim
@@ -30,10 +30,7 @@ suite "Requeststore":
 
     store = RequestStore.new(state, ds, clock)
 
-    (await store.start()).tryGet()
-
   teardown:
-    (await store.stop()).tryGet()
     (await ds.close()).tryGet()
     state.checkAllUnsubscribed()
     removeDir(dsPath)
diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim
index 0c9b4cd..1e405ad 100644
--- a/tests/codexcrawler/components/testtimetracker.nim
+++ b/tests/codexcrawler/components/testtimetracker.nim
@@ -47,11 +47,9 @@ suite "TimeTracker":
     state.config.expiryDelayMins = 22
 
     time = TimeTracker.new(state, store, dht, clock)
-
     (await time.start()).tryGet()
 
   teardown:
-    (await time.stop()).tryGet()
     await state.events.nodesToRevisit.unsubscribe(sub)
     state.checkAllUnsubscribed()
 
diff --git a/tests/codexcrawler/components/testtodolist.nim b/tests/codexcrawler/components/testtodolist.nim
index cca8512..b861f7b 100644
--- a/tests/codexcrawler/components/testtodolist.nim
+++ b/tests/codexcrawler/components/testtodolist.nim
@@ -23,8 +23,7 @@ suite "TodoList":
     metrics = createMockMetrics()
 
     todo = TodoList.new(state, metrics)
-
-    (await todo.start()).tryGet()
+    (await todo.awake()).tryGet()
 
   teardown:
     (await todo.stop()).tryGet()
diff --git a/tests/codexcrawler/mocks/mockdht.nim b/tests/codexcrawler/mocks/mockdht.nim
index 4c12f33..59c6d0b 100644
--- a/tests/codexcrawler/mocks/mockdht.nim
+++ b/tests/codexcrawler/mocks/mockdht.nim
@@ -18,11 +18,5 @@ method getNeighbors*(
   d.getNeighborsArg = some(target)
   return d.getNeighborsReturn
 
-method start*(d: MockDht): Future[?!void] {.async.} =
-  return success()
-
-method stop*(d: MockDht): Future[?!void] {.async.} =
-  return success()
-
 proc createMockDht*(): MockDht =
   MockDht()
diff --git a/tests/codexcrawler/mocks/mocknodestore.nim b/tests/codexcrawler/mocks/mocknodestore.nim
index ed78f31..d596565 100644
--- a/tests/codexcrawler/mocks/mocknodestore.nim
+++ b/tests/codexcrawler/mocks/mocknodestore.nim
@@ -22,11 +22,5 @@ method deleteEntries*(
   s.nodesToDelete = nids
   return success()
 
-method start*(s: MockNodeStore): Future[?!void] {.async.} =
-  return success()
-
-method stop*(s: MockNodeStore): Future[?!void] {.async.} =
-  return success()
-
 proc createMockNodeStore*(): MockNodeStore =
   MockNodeStore(nodesToIterate: newSeq[NodeEntry](), nodesToDelete: newSeq[Nid]())
diff --git a/tests/codexcrawler/mocks/mocktodolist.nim b/tests/codexcrawler/mocks/mocktodolist.nim
index 1417ae4..e7b11cd 100644
--- a/tests/codexcrawler/mocks/mocktodolist.nim
+++ b/tests/codexcrawler/mocks/mocktodolist.nim
@@ -10,11 +10,5 @@ type MockTodoList* = ref object of TodoList
 method pop*(t: MockTodoList): Future[?!Nid] {.async: (raises: []).} =
   return t.popReturn
 
-method start*(t: MockTodoList): Future[?!void] {.async.} =
-  return success()
-
-method stop*(t: MockTodoList): Future[?!void] {.async.} =
-  return success()
-
 proc createMockTodoList*(): MockTodoList =
   MockTodoList()

From 693b421a61995c8ad53d551b7153c830edcf44ae Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Mon, 24 Mar 2025 09:05:07 +0100
Subject: [PATCH 21/25] Adds requestCheckDelay config option

---
 codexcrawler/components/chainmetrics.nim           | 2 +-
 codexcrawler/config.nim                            | 9 +++++++--
 docker/docker-entrypoint.sh                        | 3 ++-
 tests/codexcrawler/components/testchainmetrics.nim | 4 ++--
 tests/codexcrawler/mocks/mockstate.nim             | 6 +++++-
 5 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index b23041c..4956ba0 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -67,7 +67,7 @@ method start*(c: ChainMetrics): Future[?!void] {.async.} =
     return await c.step()
 
   if c.state.config.marketplaceEnable:
-    await c.state.whileRunning(onStep, 10.minutes)
+    await c.state.whileRunning(onStep, c.state.config.requestCheckDelay.minutes)
 
   return success()
 
diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index 10f434b..47fca12 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -10,7 +10,7 @@ let doc =
 Codex Network Crawler. Generates network metrics.
 
 Usage:
-  codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>]  [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>]
+  codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>]  [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
 
 Options:
   --logLevel=<l>                    Sets log level [default: INFO]
@@ -20,14 +20,17 @@ Options:
   --dataDir=<dir>                   Directory for storing data [default: crawler_data]
   --discoveryPort=<p>               Port used for DHT [default: 8090]
   --bootNodes=<n>                   Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
+
   --dhtEnable=<e>                   Set to "1" to enable DHT crawler [default: 0]
   --stepDelay=<ms>                  Delay in milliseconds per node visit [default: 1000]
   --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
   --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
   --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
+
   --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
   --ethProvider=<a>                 Address including http(s) or ws of the eth provider
   --marketplaceAddress=<a>          Eth address of Codex contracts deployment
+  --requestCheckDelay=<m>           Delay in minutes after which storage contract status is (re)checked [default: 10]
 """
 
 import strutils
@@ -51,6 +54,7 @@ type Config* = ref object
   marketplaceEnable*: bool
   ethProvider*: string
   marketplaceAddress*: string
+  requestCheckDelay*: int
 
 proc `$`*(config: Config): string =
   "Crawler:" & " logLevel=" & config.logLevel & " publicIp=" & config.publicIp &
@@ -61,7 +65,7 @@ proc `$`*(config: Config): string =
     " expiryDelayMins=" & $config.expiryDelayMins & " checkDelayMins=" &
     $config.checkDelayMins & " marketplaceEnable=" & $config.marketplaceEnable &
     " ethProvider=" & config.ethProvider & " marketplaceAddress=" &
-    config.marketplaceAddress
+    config.marketplaceAddress & " requestCheckDelay=" & $config.requestCheckDelay
 
 proc getDefaultTestnetBootNodes(): seq[string] =
   @[
@@ -124,4 +128,5 @@ proc parseConfig*(): Config =
     marketplaceEnable: getEnable(get("--marketplaceEnable")),
     ethProvider: get("--ethProvider"),
     marketplaceAddress: get("--marketplaceAddress"),
+    requestCheckDelay: parseInt(get("--requestCheckDelay"))
   )
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index a9846a4..d7b1abc 100644
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -19,9 +19,10 @@ EXPIRYDELAY=${CRAWLER_EXPIRYDELAY:-1440}
 MARKETPLACEENABLE=${CRAWLER_MARKETPLACEENABLE:-1}
 ETHPROVIDER=${CRAWLER_ETHPROVIDER:-NULL}
 MARKETPLACEADDRESS=${CRAWLER_MARKETPLACEADDRESS:-NULL}
+REQUESTCHECKDELAY=${CRAWLER_REQUESTCHECKDELAY:-10}
 
 # Update CLI arguments
-set -- "$@" --logLevel="${LOGLEVEL}" --publicIp="${PUBLICIP}" --metricsAddress="${METRICSADDRESS}" --metricsPort="${METRICSPORT}" --dataDir="${DATADIR}" --discoveryPort="${DISCPORT}" --bootNodes="${BOOTNODES}" --dhtEnable="${DHTENABLE}" --stepDelay="${STEPDELAY}" --revisitDelay="${REVISITDELAY}" --expiryDelay="${EXPIRYDELAY}" --checkDelay="${CHECKDELAY}" --marketplaceEnable="${MARKETPLACEENABLE}" --ethProvider="${ETHPROVIDER}" --marketplaceAddress="${MARKETPLACEADDRESS}"
+set -- "$@" --logLevel="${LOGLEVEL}" --publicIp="${PUBLICIP}" --metricsAddress="${METRICSADDRESS}" --metricsPort="${METRICSPORT}" --dataDir="${DATADIR}" --discoveryPort="${DISCPORT}" --bootNodes="${BOOTNODES}" --dhtEnable="${DHTENABLE}" --stepDelay="${STEPDELAY}" --revisitDelay="${REVISITDELAY}" --expiryDelay="${EXPIRYDELAY}" --checkDelay="${CHECKDELAY}" --marketplaceEnable="${MARKETPLACEENABLE}" --ethProvider="${ETHPROVIDER}" --marketplaceAddress="${MARKETPLACEADDRESS}" --requestCheckDelay="${REQUESTCHECKDELAY}"
 
 # Run
 echo "Run Codex Crawler"
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 1098f45..4d279dd 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -39,10 +39,10 @@ suite "ChainMetrics":
   proc onStep() {.async.} =
     (await state.steppers[0]()).tryGet()
 
-  test "start should start stepper for 10 minutes":
+  test "start should start stepper for config.requestCheckDelay minutes":
     check:
       state.delays.len == 1
-      state.delays[0] == 10.minutes
+      state.delays[0] == state.config.requestCheckDelay.minutes
 
   test "onStep should remove old non-running requests from request store":
     let rid = genRid()
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index e61ae92..cefe98d 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -22,7 +22,11 @@ method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} =
 proc createMockState*(): MockState =
   MockState(
     status: ApplicationStatus.Running,
-    config: Config(dhtEnable: true, marketplaceEnable: true),
+    config: Config(
+      dhtEnable: true,
+      marketplaceEnable: true,
+      requestCheckDelay: 4
+    ),
     events: Events(
       nodesFound: newAsyncDataEvent[seq[Nid]](),
       newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),

From 46264b90b54b7acadddf5915cad32594e83e6249 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Mon, 24 Mar 2025 09:27:52 +0100
Subject: [PATCH 22/25] fixes formatting

---
 codexcrawler/config.nim                | 2 +-
 tests/codexcrawler/mocks/mockstate.nim | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index 47fca12..abb3ecd 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -128,5 +128,5 @@ proc parseConfig*(): Config =
     marketplaceEnable: getEnable(get("--marketplaceEnable")),
     ethProvider: get("--ethProvider"),
     marketplaceAddress: get("--marketplaceAddress"),
-    requestCheckDelay: parseInt(get("--requestCheckDelay"))
+    requestCheckDelay: parseInt(get("--requestCheckDelay")),
   )
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index cefe98d..d805f0e 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -22,11 +22,7 @@ method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} =
 proc createMockState*(): MockState =
   MockState(
     status: ApplicationStatus.Running,
-    config: Config(
-      dhtEnable: true,
-      marketplaceEnable: true,
-      requestCheckDelay: 4
-    ),
+    config: Config(dhtEnable: true, marketplaceEnable: true, requestCheckDelay: 4),
     events: Events(
       nodesFound: newAsyncDataEvent[seq[Nid]](),
       newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),

From a695b0ee318642ca7ea5ced9343f94dfe44da611 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Mon, 24 Mar 2025 14:11:32 +0100
Subject: [PATCH 23/25] replaces timestamp check with pending-status check

---
 codexcrawler/components/chaincrawler.nim      |  2 +-
 codexcrawler/components/chainmetrics.nim      | 27 ++++++-------
 codexcrawler/components/requeststore.nim      | 29 +++++---------
 codexcrawler/config.nim                       |  2 +-
 codexcrawler/installer.nim                    |  4 +-
 codexcrawler/services/marketplace.nim         |  5 ++-
 codexcrawler/services/metrics.nim             |  9 +++++
 .../components/testchaincrawler.nim           |  4 +-
 .../components/testchainmetrics.nim           | 37 ++++++++---------
 .../components/testrequeststore.nim           | 40 ++++---------------
 tests/codexcrawler/mocks/mockmetrics.nim      |  4 ++
 tests/codexcrawler/mocks/mockrequeststore.nim |  7 ++--
 12 files changed, 74 insertions(+), 96 deletions(-)

diff --git a/codexcrawler/components/chaincrawler.nim b/codexcrawler/components/chaincrawler.nim
index b29e9f8..f957630 100644
--- a/codexcrawler/components/chaincrawler.nim
+++ b/codexcrawler/components/chaincrawler.nim
@@ -17,7 +17,7 @@ type ChainCrawler* = ref object of Component
   marketplace: MarketplaceService
 
 proc onNewRequest(c: ChainCrawler, rid: Rid): Future[?!void] {.async: (raises: []).} =
-  return await c.store.update(rid)
+  return await c.store.add(rid)
 
 method start*(c: ChainCrawler): Future[?!void] {.async.} =
   info "starting..."
diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 4956ba0..85395ca 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -6,7 +6,6 @@ import pkg/questionable/results
 import ../state
 import ../services/metrics
 import ../services/marketplace
-import ../services/clock
 import ../components/requeststore
 import ../component
 import ../types
@@ -20,29 +19,29 @@ type
     metrics: Metrics
     store: RequestStore
     marketplace: MarketplaceService
-    clock: Clock
 
   Update = ref object
     numRequests: int
+    numPending: int
     numSlots: int
     totalSize: int64
 
-proc isOld(c: ChainMetrics, entry: RequestEntry): bool =
-  let oneDay = 60 * 60 * 24
-  return entry.lastSeen < (c.clock.now - oneDay.uint64)
-
 proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
-  var update = Update(numRequests: 0, numSlots: 0, totalSize: 0)
+  var update = Update(numRequests: 0, numPending: 0, numSlots: 0, totalSize: 0)
 
   proc onRequest(entry: RequestEntry): Future[?!void] {.async: (raises: []).} =
     let response = await c.marketplace.getRequestInfo(entry.id)
     if info =? response:
-      inc update.numRequests
-      update.numSlots += info.slots.int
-      update.totalSize += (info.slots * info.slotSize).int64
+      if info.pending:
+        trace "request is pending", id = $entry.id
+        inc update.numPending
+      else:
+        trace "request is running", id = $entry.id
+        inc update.numRequests
+        update.numSlots += info.slots.int
+        update.totalSize += (info.slots * info.slotSize).int64
     else:
-      if c.isOld(entry):
-        ?await c.store.remove(entry.id)
+      ?await c.store.remove(entry.id)
     return success()
 
   ?await c.store.iterateAll(onRequest)
@@ -50,6 +49,7 @@ proc collectUpdate(c: ChainMetrics): Future[?!Update] {.async: (raises: []).} =
 
 proc updateMetrics(c: ChainMetrics, update: Update) =
   c.metrics.setRequests(update.numRequests)
+  c.metrics.setPendingRequests(update.numPending)
   c.metrics.setRequestSlots(update.numSlots)
   c.metrics.setTotalSize(update.totalSize)
 
@@ -77,8 +77,7 @@ proc new*(
     metrics: Metrics,
     store: RequestStore,
     marketplace: MarketplaceService,
-    clock: Clock,
 ): ChainMetrics =
   ChainMetrics(
-    state: state, metrics: metrics, store: store, marketplace: marketplace, clock: clock
+    state: state, metrics: metrics, store: store, marketplace: marketplace
   )
diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index 130a6e5..8e0d38d 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -10,7 +10,6 @@ import ../types
 import ../component
 import ../state
 import ../utils/datastoreutils
-import ../services/clock
 
 const requeststoreName = "requeststore"
 
@@ -20,7 +19,7 @@ logScope:
 type
   RequestEntry* = object
     id*: Rid
-    lastSeen*: uint64
+    isValid: bool
 
   OnRequestEntry* =
     proc(entry: RequestEntry): Future[?!void] {.async: (raises: []), gcsafe.}
@@ -28,7 +27,6 @@ type
   RequestStore* = ref object of Component
     state: State
     store: TypedDatastore
-    clock: Clock
 
 proc `$`*(entry: RequestEntry): string =
   $entry.id
@@ -36,7 +34,6 @@ proc `$`*(entry: RequestEntry): string =
 proc toBytes*(entry: RequestEntry): seq[byte] =
   var buffer = initProtoBuffer()
   buffer.write(1, $entry.id)
-  buffer.write(2, entry.lastSeen)
   buffer.finish()
   return buffer.buffer
 
@@ -44,36 +41,32 @@ proc fromBytes*(_: type RequestEntry, data: openArray[byte]): ?!RequestEntry =
   var
     buffer = initProtoBuffer(data)
     idStr: string
-    lastSeen: uint64
 
   if buffer.getField(1, idStr).isErr:
     return failure("Unable to decode `idStr`")
-  if buffer.getField(2, lastSeen).isErr:
-    return failure("Unable to decode `lastSeen`")
 
-  return success(RequestEntry(id: Rid.fromStr(idStr), lastSeen: lastSeen))
+  return success(RequestEntry(id: Rid.fromStr(idStr), isValid: true))
 
 proc encode*(e: RequestEntry): seq[byte] =
   e.toBytes()
 
 proc decode*(T: type RequestEntry, bytes: seq[byte]): ?!T =
   if bytes.len < 1:
-    return success(RequestEntry(lastSeen: 0))
+    return success(RequestEntry(isValid: false))
   return RequestEntry.fromBytes(bytes)
 
-method update*(
+method add*(
     s: RequestStore, rid: Rid
 ): Future[?!void] {.async: (raises: []), base.} =
   without key =? Key.init(requeststoreName / $rid), err:
     error "failed to format key", err = err.msg
     return failure(err)
 
-  let entry = RequestEntry(id: rid, lastSeen: s.clock.now)
   try:
-    ?await s.store.put(key, entry)
+    ?await s.store.put(key, RequestEntry(id: rid))
   except CatchableError as exc:
     return failure(exc.msg)
-  trace "Request entry updated", id = $rid
+  trace "Request entry added", id = $rid
   return success()
 
 method remove*(
@@ -109,7 +102,7 @@ method iterateAll*(
         error "failed to get value from iterator", err = err.msg
         return failure(err)
 
-      if value.lastSeen > 0:
+      if value.isValid:
         ?await onNode(value)
 
       await sleepAsync(1.millis)
@@ -118,13 +111,13 @@ method iterateAll*(
   return success()
 
 proc new*(
-    T: type RequestStore, state: State, store: TypedDatastore, clock: Clock
+    T: type RequestStore, state: State, store: TypedDatastore
 ): RequestStore =
-  RequestStore(state: state, store: store, clock: clock)
+  RequestStore(state: state, store: store)
 
-proc createRequestStore*(state: State, clock: Clock): ?!RequestStore =
+proc createRequestStore*(state: State): ?!RequestStore =
   without ds =? createTypedDatastore(state.config.dataDir / "requeststore"), err:
     error "Failed to create typed datastore for request store", err = err.msg
     return failure(err)
 
-  return success(RequestStore.new(state, ds, clock))
+  return success(RequestStore.new(state, ds))
diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index abb3ecd..f4a39cc 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -21,7 +21,7 @@ Options:
   --discoveryPort=<p>               Port used for DHT [default: 8090]
   --bootNodes=<n>                   Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
 
-  --dhtEnable=<e>                   Set to "1" to enable DHT crawler [default: 0]
+  --dhtEnable=<e>                   Set to "1" to enable DHT crawler [default: 1]
   --stepDelay=<ms>                  Delay in milliseconds per node visit [default: 1000]
   --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
   --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index bdff348..80fa924 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -28,14 +28,14 @@ proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
   without nodeStore =? createNodeStore(state, clock), err:
     return failure(err)
 
-  without requestStore =? createRequestStore(state, clock), err:
+  without requestStore =? createRequestStore(state), err:
     return failure(err)
 
   let
     metrics = createMetrics(state.config.metricsAddress, state.config.metricsPort)
     todoList = createTodoList(state, metrics)
     marketplace = createMarketplace(state, clock)
-    chainMetrics = ChainMetrics.new(state, metrics, requestStore, marketplace, clock)
+    chainMetrics = ChainMetrics.new(state, metrics, requestStore, marketplace)
 
   without dhtMetrics =? createDhtMetrics(state, metrics), err:
     return failure(err)
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 494ff4c..d349158 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -20,6 +20,7 @@ type
 
   OnNewRequest* = proc(id: Rid): Future[?!void] {.async: (raises: []), gcsafe.}
   RequestInfo* = ref object
+    pending*: bool
     slots*: uint64
     slotSize*: uint64
 
@@ -32,7 +33,7 @@ proc fetchRequestInfo(
   try:
     let request = await market.getRequest(rid)
     if r =? request:
-      return some(RequestInfo(slots: r.ask.slots, slotSize: r.ask.slotSize))
+      return some(RequestInfo(pending: false, slots: r.ask.slots, slotSize: r.ask.slotSize))
   except CatchableError as exc:
     trace "Failed to get request info", err = exc.msg
   return none(RequestInfo)
@@ -88,6 +89,8 @@ method getRequestInfo*(
     try:
       let state = await market.requestState(rid)
       if s =? state:
+        if s == RequestState.New:
+          return some(RequestInfo(pending: true))
         if s == RequestState.Started:
           return await market.fetchRequestInfo(rid)
     except CatchableError as exc:
diff --git a/codexcrawler/services/metrics.nim b/codexcrawler/services/metrics.nim
index adc8852..020c5aa 100644
--- a/codexcrawler/services/metrics.nim
+++ b/codexcrawler/services/metrics.nim
@@ -7,6 +7,7 @@ declareGauge(okNodesGauge, "DHT nodes successfully contacted")
 declareGauge(nokNodesGauge, "DHT nodes failed to contact")
 
 declareGauge(requestsGauge, "Marketplace active storage requests")
+declareGauge(pendingGauge, "Marketplace pending storage requests")
 declareGauge(requestSlotsGauge, "Marketplace active storage request slots")
 declareGauge(
   totalStorageSizeGauge, "Marketplace total bytes stored in active storage requests"
@@ -21,6 +22,7 @@ type
     nokNodes: OnUpdateMetric
 
     onRequests: OnUpdateMetric
+    onPending: OnUpdateMetric
     onRequestSlots: OnUpdateMetric
     onTotalSize: OnUpdateMetric
 
@@ -47,6 +49,9 @@ method setNokNodes*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
 method setRequests*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
   m.onRequests(value.int64)
 
+method setPendingRequests*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
+  m.onPending(value.int64)
+
 method setRequestSlots*(m: Metrics, value: int) {.base, gcsafe, raises: [].} =
   m.onRequestSlots(value.int64)
 
@@ -70,6 +75,9 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
   proc onRequests(value: int64) =
     requestsGauge.set(value)
 
+  proc onPending(value: int64) =
+    pendingGauge.set(value)
+
   proc onRequestSlots(value: int64) =
     requestSlotsGauge.set(value)
 
@@ -81,6 +89,7 @@ proc createMetrics*(metricsAddress: IpAddress, metricsPort: Port): Metrics =
     okNodes: onOk,
     nokNodes: onNok,
     onRequests: onRequests,
+    onPending: onPending,
     onRequestSlots: onRequestSlots,
     onTotalSize: onTotalSize,
   )
diff --git a/tests/codexcrawler/components/testchaincrawler.nim b/tests/codexcrawler/components/testchaincrawler.nim
index 172b2b3..04f98b6 100644
--- a/tests/codexcrawler/components/testchaincrawler.nim
+++ b/tests/codexcrawler/components/testchaincrawler.nim
@@ -41,7 +41,7 @@ suite "ChainCrawler":
     (await (marketplace.subNewRequestsCallback.get())(rid)).tryGet()
 
     check:
-      store.updateRid == rid
+      store.addRid == rid
 
   test "start should iterate past requests and add then to store":
     check:
@@ -51,4 +51,4 @@ suite "ChainCrawler":
     (await marketplace.iterRequestsCallback.get()(rid)).tryGet()
 
     check:
-      store.updateRid == rid
+      store.addRid == rid
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index 4d279dd..be222d2 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -20,7 +20,6 @@ suite "ChainMetrics":
     metrics: MockMetrics
     store: MockRequestStore
     marketplace: MockMarketplaceService
-    clock: MockClock
     chain: ChainMetrics
 
   setup:
@@ -28,9 +27,8 @@ suite "ChainMetrics":
     metrics = createMockMetrics()
     store = createMockRequestStore()
     marketplace = createMockMarketplaceService()
-    clock = createMockClock()
 
-    chain = ChainMetrics.new(state, metrics, store, marketplace, clock)
+    chain = ChainMetrics.new(state, metrics, store, marketplace)
     (await chain.start()).tryGet()
 
   teardown:
@@ -44,12 +42,10 @@ suite "ChainMetrics":
       state.delays.len == 1
       state.delays[0] == state.config.requestCheckDelay.minutes
 
-  test "onStep should remove old non-running requests from request store":
+  test "onStep removes requests from request store when info can't be fetched":
     let rid = genRid()
-    let oneDay = (60 * 60 * 24).uint64
-    store.iterateEntries.add(RequestEntry(id: rid, lastSeen: 100.uint64))
+    store.iterateEntries.add(RequestEntry(id: rid))
 
-    clock.setNow = 100 + oneDay + 1
     marketplace.requestInfoReturns = none(RequestInfo)
 
     await onStep()
@@ -58,20 +54,6 @@ suite "ChainMetrics":
       marketplace.requestInfoRid == rid
       store.removeRid == rid
 
-  test "onStep should not remove recent non-running requests from request store":
-    let rid = genRid()
-    let now = 123456789.uint64
-    store.iterateEntries.add(RequestEntry(id: rid, lastSeen: now - 1))
-
-    clock.setNow = now
-    marketplace.requestInfoReturns = none(RequestInfo)
-
-    await onStep()
-
-    check:
-      marketplace.requestInfoRid == rid
-      not (store.removeRid == rid)
-
   test "onStep should count the number of active requests":
     let rid1 = genRid()
     let rid2 = genRid()
@@ -85,6 +67,19 @@ suite "ChainMetrics":
     check:
       metrics.requests == 2
 
+  test "onStep should count the number of pending requests":
+    let rid1 = genRid()
+    let rid2 = genRid()
+    store.iterateEntries.add(RequestEntry(id: rid1))
+    store.iterateEntries.add(RequestEntry(id: rid2))
+
+    marketplace.requestInfoReturns = some(RequestInfo(pending: true))
+
+    await onStep()
+
+    check:
+      metrics.pending == 2
+
   test "onStep should count the number of active slots":
     let rid = genRid()
     store.iterateEntries.add(RequestEntry(id: rid))
diff --git a/tests/codexcrawler/components/testrequeststore.nim b/tests/codexcrawler/components/testrequeststore.nim
index c3202f6..32c6c70 100644
--- a/tests/codexcrawler/components/testrequeststore.nim
+++ b/tests/codexcrawler/components/testrequeststore.nim
@@ -9,7 +9,6 @@ import ../../../codexcrawler/utils/datastoreutils
 import ../../../codexcrawler/types
 import ../../../codexcrawler/state
 import ../mocks/mockstate
-import ../mocks/mockclock
 import ../helpers
 
 suite "Requeststore":
@@ -20,15 +19,13 @@ suite "Requeststore":
   var
     ds: TypedDatastore
     state: MockState
-    clock: MockClock
     store: RequestStore
 
   setup:
     ds = createTypedDatastore(dsPath).tryGet()
     state = createMockState()
-    clock = createMockClock()
 
-    store = RequestStore.new(state, ds, clock)
+    store = RequestStore.new(state, ds)
 
   teardown:
     (await ds.close()).tryGet()
@@ -36,7 +33,7 @@ suite "Requeststore":
     removeDir(dsPath)
 
   test "requestEntry encoding":
-    let entry = RequestEntry(id: genRid(), lastSeen: 123.uint64)
+    let entry = RequestEntry(id: genRid())
 
     let
       bytes = entry.encode()
@@ -44,11 +41,10 @@ suite "Requeststore":
 
     check:
       entry.id == decoded.id
-      entry.lastSeen == decoded.lastSeen
 
-  test "update stores a new requestId with current time":
+  test "add stores a new requestId":
     let rid = genRid()
-    (await store.update(rid)).tryGet()
+    (await store.add(rid)).tryGet()
 
     let
       key = Key.init(requeststoreName / $rid).tryGet()
@@ -56,26 +52,10 @@ suite "Requeststore":
 
     check:
       stored.id == rid
-      stored.lastSeen == clock.setNow
-
-  test "update updates the current time of an existing requestId with current time":
-    let rid = genRid()
-    (await store.update(rid)).tryGet()
-
-    clock.setNow = 1234
-    (await store.update(rid)).tryGet()
-
-    let
-      key = Key.init(requeststoreName / $rid).tryGet()
-      stored = (await get[RequestEntry](ds, key)).tryGet()
-
-    check:
-      stored.id == rid
-      stored.lastSeen == clock.setNow
 
   test "remove will remove an entry":
     let rid = genRid()
-    (await store.update(rid)).tryGet()
+    (await store.add(rid)).tryGet()
     (await store.remove(rid)).tryGet()
 
     let
@@ -91,9 +71,9 @@ suite "Requeststore":
       rid2 = genRid()
       rid3 = genRid()
 
-    (await store.update(rid1)).tryGet()
-    (await store.update(rid2)).tryGet()
-    (await store.update(rid3)).tryGet()
+    (await store.add(rid1)).tryGet()
+    (await store.add(rid2)).tryGet()
+    (await store.add(rid3)).tryGet()
 
     var entries = newSeq[RequestEntry]()
     proc onEntry(entry: RequestEntry): Future[?!void] {.async: (raises: []), gcsafe.} =
@@ -117,7 +97,3 @@ suite "Requeststore":
       check:
         id in ids
 
-    check:
-      entries[0].lastSeen == clock.setNow
-      entries[1].lastSeen == clock.setNow
-      entries[2].lastSeen == clock.setNow
diff --git a/tests/codexcrawler/mocks/mockmetrics.nim b/tests/codexcrawler/mocks/mockmetrics.nim
index 01787cc..29c61e9 100644
--- a/tests/codexcrawler/mocks/mockmetrics.nim
+++ b/tests/codexcrawler/mocks/mockmetrics.nim
@@ -5,6 +5,7 @@ type MockMetrics* = ref object of Metrics
   ok*: int
   nok*: int
   requests*: int
+  pending*: int
   slots*: int
   totalSize*: int64
 
@@ -20,6 +21,9 @@ method setNokNodes*(m: MockMetrics, value: int) =
 method setRequests*(m: MockMetrics, value: int) =
   m.requests = value
 
+method setPendingRequests*(m: MockMetrics, value: int) =
+  m.pending = value
+
 method setRequestSlots*(m: MockMetrics, value: int) =
   m.slots = value
 
diff --git a/tests/codexcrawler/mocks/mockrequeststore.nim b/tests/codexcrawler/mocks/mockrequeststore.nim
index 075fc42..e17606e 100644
--- a/tests/codexcrawler/mocks/mockrequeststore.nim
+++ b/tests/codexcrawler/mocks/mockrequeststore.nim
@@ -1,4 +1,3 @@
-import std/sequtils
 import pkg/questionable/results
 import pkg/chronos
 
@@ -6,12 +5,12 @@ import ../../../codexcrawler/components/requeststore
 import ../../../codexcrawler/types
 
 type MockRequestStore* = ref object of RequestStore
-  updateRid*: Rid
+  addRid*: Rid
   removeRid*: Rid
   iterateEntries*: seq[RequestEntry]
 
-method update*(s: MockRequestStore, rid: Rid): Future[?!void] {.async: (raises: []).} =
-  s.updateRid = rid
+method add*(s: MockRequestStore, rid: Rid): Future[?!void] {.async: (raises: []).} =
+  s.addRid = rid
   return success()
 
 method remove*(s: MockRequestStore, rid: Rid): Future[?!void] {.async: (raises: []).} =

From 538be676c2adc8efeecd805c3c006bc48e5da791 Mon Sep 17 00:00:00 2001
From: ThatBen <thatbenbierens@gmail.com>
Date: Mon, 24 Mar 2025 14:16:12 +0100
Subject: [PATCH 24/25] fixes formatting

---
 codexcrawler/components/chainmetrics.nim           | 4 +---
 codexcrawler/components/requeststore.nim           | 8 ++------
 codexcrawler/services/marketplace.nim              | 3 ++-
 tests/codexcrawler/components/testrequeststore.nim | 1 -
 4 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index 85395ca..dac810d 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -78,6 +78,4 @@ proc new*(
     store: RequestStore,
     marketplace: MarketplaceService,
 ): ChainMetrics =
-  ChainMetrics(
-    state: state, metrics: metrics, store: store, marketplace: marketplace
-  )
+  ChainMetrics(state: state, metrics: metrics, store: store, marketplace: marketplace)
diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index 8e0d38d..836e1fb 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -55,9 +55,7 @@ proc decode*(T: type RequestEntry, bytes: seq[byte]): ?!T =
     return success(RequestEntry(isValid: false))
   return RequestEntry.fromBytes(bytes)
 
-method add*(
-    s: RequestStore, rid: Rid
-): Future[?!void] {.async: (raises: []), base.} =
+method add*(s: RequestStore, rid: Rid): Future[?!void] {.async: (raises: []), base.} =
   without key =? Key.init(requeststoreName / $rid), err:
     error "failed to format key", err = err.msg
     return failure(err)
@@ -110,9 +108,7 @@ method iterateAll*(
     return failure(exc.msg)
   return success()
 
-proc new*(
-    T: type RequestStore, state: State, store: TypedDatastore
-): RequestStore =
+proc new*(T: type RequestStore, state: State, store: TypedDatastore): RequestStore =
   RequestStore(state: state, store: store)
 
 proc createRequestStore*(state: State): ?!RequestStore =
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index d349158..12ebfa7 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -33,7 +33,8 @@ proc fetchRequestInfo(
   try:
     let request = await market.getRequest(rid)
     if r =? request:
-      return some(RequestInfo(pending: false, slots: r.ask.slots, slotSize: r.ask.slotSize))
+      return
+        some(RequestInfo(pending: false, slots: r.ask.slots, slotSize: r.ask.slotSize))
   except CatchableError as exc:
     trace "Failed to get request info", err = exc.msg
   return none(RequestInfo)
diff --git a/tests/codexcrawler/components/testrequeststore.nim b/tests/codexcrawler/components/testrequeststore.nim
index 32c6c70..340826f 100644
--- a/tests/codexcrawler/components/testrequeststore.nim
+++ b/tests/codexcrawler/components/testrequeststore.nim
@@ -96,4 +96,3 @@ suite "Requeststore":
     for id in all:
       check:
         id in ids
-

From 9e61b8b67d57e906ef7e3de77cb4009e4bd55e19 Mon Sep 17 00:00:00 2001
From: Slava <20563034+veaceslavdoina@users.noreply.github.com>
Date: Mon, 24 Mar 2025 18:14:11 +0200
Subject: [PATCH 25/25] fix: update nimble in ci and docker

---
 .github/workflows/main.yml | 13 +++++++++++++
 docker/crawler.Dockerfile  |  1 +
 2 files changed, 14 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3147d59..a5c4774 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -25,9 +25,22 @@ jobs:
     - uses: actions/checkout@v2
       with:
         submodules: true
+
     - uses: iffy/install-nim@v4
       with:
         version: ${{ matrix.nim }}
+
+    - name: Update nimble
+      run: |
+        nimble install nimble
+        nimble --version
+
+    - name: Use updated nimble version on Windows
+      if: contains(matrix.os, 'windows')
+      run: |
+        del $HOME\.nimble\bin\nimble.exe
+        nimble --version
+
     - name: Build
       run: nimble build -y
     - name: Test
diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile
index 3811fcf..455dfbc 100644
--- a/docker/crawler.Dockerfile
+++ b/docker/crawler.Dockerfile
@@ -19,6 +19,7 @@ RUN apt-get update && apt-get install -y cmake build-essential
 
 WORKDIR ${BUILD_HOME}
 COPY . .
+RUN nimble install nimble
 RUN nimble build
 
 # Create