From cc02dbcbfc2652b977a4b30801187152f62fc225 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 11:06:18 +0200
Subject: [PATCH 01/31] setting up

---
 .gitmodules                       |  99 ++++++++++++++++++++++++
 build.nims                        | 123 ++++++++++++++++++++++++++++++
 env.sh                            |   7 ++
 nim-stew                          |   1 +
 vendor/asynctest                  |   1 +
 vendor/codex-contracts-eth        |   1 +
 vendor/lrucache.nim               |   1 +
 vendor/nim-bearssl                |   1 +
 vendor/nim-blscurve               |   1 +
 vendor/nim-chronicles             |   1 +
 vendor/nim-chronos                |   1 +
 vendor/nim-codex-dht              |   1 +
 vendor/nim-confutils              |   1 +
 vendor/nim-contract-abi           |   1 +
 vendor/nim-eth                    |   1 +
 vendor/nim-ethers                 |   1 +
 vendor/nim-json-rpc               |   1 +
 vendor/nim-json-serialization     |   1 +
 vendor/nim-libbacktrace           |   1 +
 vendor/nim-libp2p                 |   1 +
 vendor/nim-metrics                |   1 +
 vendor/nim-nat-traversal          |   1 +
 vendor/nim-protobuf-serialization |   1 +
 vendor/nim-results                |   1 +
 vendor/nim-secp256k1              |   1 +
 vendor/nim-serde                  |   1 +
 vendor/nim-serialization          |   1 +
 vendor/nim-testutils              |   1 +
 vendor/nim-unittest2              |   1 +
 vendor/nim-websock                |   1 +
 vendor/nimbus-build-system        |   1 +
 vendor/nimcrypto                  |   1 +
 vendor/nph                        |   1 +
 vendor/questionable               |   1 +
 vendor/stint                      |   1 +
 vendor/upraises                   |   1 +
 36 files changed, 262 insertions(+)
 create mode 100644 .gitmodules
 create mode 100644 build.nims
 create mode 100644 env.sh
 create mode 160000 nim-stew
 create mode 160000 vendor/asynctest
 create mode 160000 vendor/codex-contracts-eth
 create mode 160000 vendor/lrucache.nim
 create mode 160000 vendor/nim-bearssl
 create mode 160000 vendor/nim-blscurve
 create mode 160000 vendor/nim-chronicles
 create mode 160000 vendor/nim-chronos
 create mode 160000 vendor/nim-codex-dht
 create mode 160000 vendor/nim-confutils
 create mode 160000 vendor/nim-contract-abi
 create mode 160000 vendor/nim-eth
 create mode 160000 vendor/nim-ethers
 create mode 160000 vendor/nim-json-rpc
 create mode 160000 vendor/nim-json-serialization
 create mode 160000 vendor/nim-libbacktrace
 create mode 160000 vendor/nim-libp2p
 create mode 160000 vendor/nim-metrics
 create mode 160000 vendor/nim-nat-traversal
 create mode 160000 vendor/nim-protobuf-serialization
 create mode 160000 vendor/nim-results
 create mode 160000 vendor/nim-secp256k1
 create mode 160000 vendor/nim-serde
 create mode 160000 vendor/nim-serialization
 create mode 160000 vendor/nim-testutils
 create mode 160000 vendor/nim-unittest2
 create mode 160000 vendor/nim-websock
 create mode 160000 vendor/nimbus-build-system
 create mode 160000 vendor/nimcrypto
 create mode 160000 vendor/nph
 create mode 160000 vendor/questionable
 create mode 160000 vendor/stint
 create mode 160000 vendor/upraises

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..6b5e018
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,99 @@
+[submodule "vendor/nimbus-build-system"]
+	path = vendor/nimbus-build-system
+	url = https://github.com/status-im/nimbus-build-system.git
+[submodule "vendor/nim-libp2p"]
+	path = vendor/nim-libp2p
+	url = https://github.com/vacp2p/nim-libp2p.git
+[submodule "vendor/nimcrypto"]
+	path = vendor/nimcrypto
+	url = https://github.com/cheatfate/nimcrypto.git
+[submodule "vendor/nim-chronicles"]
+	path = vendor/nim-chronicles
+	url = https://github.com/status-im/nim-chronicles.git
+[submodule "vendor/nim-metrics"]
+	path = vendor/nim-metrics
+	url = https://github.com/status-im/nim-metrics.git
+[submodule "vendor/nim-secp256k1"]
+	path = vendor/nim-secp256k1
+	url = https://github.com/status-im/nim-secp256k1.git
+[submodule "nim-stew"]
+	path = nim-stew
+	url = https://github.com/status-im/nim-stew.git
+[submodule "vendor/questionable"]
+	path = vendor/questionable
+	url = https://github.com/status-im/questionable.git
+[submodule "vendor/upraises"]
+	path = vendor/upraises
+	url = https://github.com/markspanbroek/upraises.git
+[submodule "vendor/asynctest"]
+	path = vendor/asynctest
+	url = https://github.com/status-im/asynctest.git
+[submodule "vendor/nim-confutils"]
+	path = vendor/nim-confutils
+	url = https://github.com/status-im/nim-confutils.git
+[submodule "vendor/nim-nat-traversal"]
+	path = vendor/nim-nat-traversal
+	url = https://github.com/status-im/nim-nat-traversal.git
+[submodule "vendor/nim-libbacktrace"]
+	path = vendor/nim-libbacktrace
+	url = https://github.com/status-im/nim-libbacktrace.git
+[submodule "vendor/nim-chronos"]
+	path = vendor/nim-chronos
+	url = https://github.com/status-im/nim-chronos.git
+[submodule "vendor/nim-json-serialization"]
+	path = vendor/nim-json-serialization
+	url = https://github.com/status-im/nim-json-serialization.git
+[submodule "vendor/nim-serialization"]
+	path = vendor/nim-serialization
+	url = https://github.com/status-im/nim-serialization.git
+[submodule "vendor/nim-bearssl"]
+	path = vendor/nim-bearssl
+	url = https://github.com/status-im/nim-bearssl.git
+[submodule "vendor/stint"]
+	path = vendor/stint
+	url = https://github.com/status-im/stint.git
+[submodule "vendor/nim-unittest2"]
+	path = vendor/nim-unittest2
+	url = https://github.com/status-im/nim-unittest2.git
+[submodule "vendor/nim-websock"]
+	path = vendor/nim-websock
+	url = https://github.com/status-im/nim-websock.git
+[submodule "vendor/nim-contract-abi"]
+	path = vendor/nim-contract-abi
+	url = https://github.com/status-im/nim-contract-abi
+[submodule "vendor/nim-json-rpc"]
+	path = vendor/nim-json-rpc
+	url = https://github.com/status-im/nim-json-rpc
+[submodule "vendor/nim-ethers"]
+	path = vendor/nim-ethers
+	url = https://github.com/status-im/nim-ethers
+[submodule "vendor/lrucache.nim"]
+	path = vendor/lrucache.nim
+	url = https://github.com/status-im/lrucache.nim
+[submodule "vendor/nim-blscurve"]
+	path = vendor/nim-blscurve
+	url = https://github.com/status-im/nim-blscurve.git
+[submodule "vendor/nim-codex-dht"]
+	path = vendor/nim-codex-dht
+	url = https://github.com/codex-storage/nim-codex-dht.git
+[submodule "vendor/nim-eth"]
+	path = vendor/nim-eth
+	url = https://github.com/status-im/nim-eth
+[submodule "vendor/codex-contracts-eth"]
+	path = vendor/codex-contracts-eth
+	url = https://github.com/status-im/codex-contracts-eth
+[submodule "vendor/nim-protobuf-serialization"]
+	path = vendor/nim-protobuf-serialization
+	url = https://github.com/status-im/nim-protobuf-serialization
+[submodule "vendor/nim-results"]
+	path = vendor/nim-results
+	url = https://github.com/arnetheduck/nim-results
+[submodule "vendor/nim-testutils"]
+	path = vendor/nim-testutils
+	url = https://github.com/status-im/nim-testutils
+[submodule "vendor/nim-serde"]
+	path = vendor/nim-serde
+	url = https://github.com/codex-storage/nim-serde.git
+[submodule "vendor/nph"]
+	path = vendor/nph
+	url = https://github.com/arnetheduck/nph.git
diff --git a/build.nims b/build.nims
new file mode 100644
index 0000000..8866032
--- /dev/null
+++ b/build.nims
@@ -0,0 +1,123 @@
+mode = ScriptMode.Verbose
+
+import std/os except commandLineParams
+
+### Helper functions
+proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
+  if not dirExists "build":
+    mkDir "build"
+
+  # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
+  var extra_params = params
+  when compiles(commandLineParams):
+    for param in commandLineParams():
+      extra_params &= " " & param
+  else:
+    for i in 2 ..< paramCount():
+      extra_params &= " " & paramStr(i)
+
+  let
+    # Place build output in 'build' folder, even if name includes a longer path.
+    outName = os.lastPathPart(name)
+    cmd =
+      "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir &
+      name & ".nim"
+
+  exec(cmd)
+
+proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
+  buildBinary name, srcDir, params
+  exec "build/" & name
+
+task codex, "build codex binary":
+  buildBinary "codex",
+    params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
+
+task toolsCirdl, "build tools/cirdl binary":
+  buildBinary "tools/cirdl/cirdl"
+
+task testCodex, "Build & run Codex tests":
+  test "testCodex", params = "-d:codex_enable_proof_failures=true"
+
+task testContracts, "Build & run Codex Contract tests":
+  test "testContracts"
+
+task testIntegration, "Run integration tests":
+  buildBinary "codex",
+    params =
+      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
+  test "testIntegration"
+  # use params to enable logging from the integration test executable
+  # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
+  #   "-d:chronicles_enabled_topics:integration:TRACE"  
+
+task build, "build codex binary":
+  codexTask()
+
+task test, "Run tests":
+  testCodexTask()
+
+task testTools, "Run Tools tests":
+  toolsCirdlTask()
+  test "testTools"
+
+task testAll, "Run all tests (except for Taiko L2 tests)":
+  testCodexTask()
+  testContractsTask()
+  testIntegrationTask()
+  testToolsTask()
+
+task testTaiko, "Run Taiko L2 tests":
+  codexTask()
+  test "testTaiko"
+
+import strutils
+import os
+
+task coverage, "generates code coverage report":
+  var (output, exitCode) = gorgeEx("which lcov")
+  if exitCode != 0:
+    echo "  ************************** ⛔️ ERROR ⛔️ **************************"
+    echo "  **   ERROR: lcov not found, it must be installed to run code   **"
+    echo "  **   coverage locally                                          **"
+    echo "  *****************************************************************"
+    quit 1
+
+  (output, exitCode) = gorgeEx("gcov --version")
+  if output.contains("Apple LLVM"):
+    echo "  ************************* ⚠️ WARNING ⚠️  *************************"
+    echo "  **   WARNING: Using Apple's llvm-cov in place of gcov, which   **"
+    echo "  **   emulates an old version of gcov (4.2.0) and therefore     **"
+    echo "  **   coverage results will differ than those on CI (which      **"
+    echo "  **   uses a much newer version of gcov).                       **"
+    echo "  *****************************************************************"
+
+  var nimSrcs = " "
+  for f in walkDirRec("codex", {pcFile}):
+    if f.endswith(".nim"):
+      nimSrcs.add " " & f.absolutePath.quoteShell()
+
+  echo "======== Running Tests ======== "
+  test "coverage",
+    srcDir = "tests/",
+    params =
+      " --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true"
+  exec("rm nimcache/coverage/*.c")
+  rmDir("coverage")
+  mkDir("coverage")
+  echo " ======== Running LCOV ======== "
+  exec(
+    "lcov --capture --keep-going --directory nimcache/coverage --output-file coverage/coverage.info"
+  )
+  exec(
+    "lcov --extract coverage/coverage.info --keep-going --output-file coverage/coverage.f.info " &
+      nimSrcs
+  )
+  echo " ======== Generating HTML coverage report ======== "
+  exec("genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report ")
+  echo " ======== Coverage report Done ======== "
+
+task showCoverage, "open coverage html":
+  echo " ======== Opening HTML coverage report in browser... ======== "
+  if findExe("open") != "":
+    exec("open coverage/report/index.html")
diff --git a/env.sh b/env.sh
new file mode 100644
index 0000000..697a426
--- /dev/null
+++ b/env.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
+# and we fall back to a Zsh-specific special var to also support Zsh.
+REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
+ABS_PATH="$(cd ${REL_PATH}; pwd)"
+source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
diff --git a/nim-stew b/nim-stew
new file mode 160000
index 0000000..58abb48
--- /dev/null
+++ b/nim-stew
@@ -0,0 +1 @@
+Subproject commit 58abb4891f97c6cdc07335e868414e0c7b736c68
diff --git a/vendor/asynctest b/vendor/asynctest
new file mode 160000
index 0000000..572c897
--- /dev/null
+++ b/vendor/asynctest
@@ -0,0 +1 @@
+Subproject commit 572c897a4e1177e905105a3bafe8c5573d9bae83
diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth
new file mode 160000
index 0000000..0bf1385
--- /dev/null
+++ b/vendor/codex-contracts-eth
@@ -0,0 +1 @@
+Subproject commit 0bf138512b7c1c3b8d77c48376e47f702e47106c
diff --git a/vendor/lrucache.nim b/vendor/lrucache.nim
new file mode 160000
index 0000000..8767ade
--- /dev/null
+++ b/vendor/lrucache.nim
@@ -0,0 +1 @@
+Subproject commit 8767ade0b76ea5b5d4ce24a52d0c58a6ebeb66cd
diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl
new file mode 160000
index 0000000..667b404
--- /dev/null
+++ b/vendor/nim-bearssl
@@ -0,0 +1 @@
+Subproject commit 667b40440a53a58e9f922e29e20818720c62d9ac
diff --git a/vendor/nim-blscurve b/vendor/nim-blscurve
new file mode 160000
index 0000000..de2d3c7
--- /dev/null
+++ b/vendor/nim-blscurve
@@ -0,0 +1 @@
+Subproject commit de2d3c79264bba18dbea469c8c5c4b3bb3c8bc55
diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles
new file mode 160000
index 0000000..81a4a7a
--- /dev/null
+++ b/vendor/nim-chronicles
@@ -0,0 +1 @@
+Subproject commit 81a4a7a360c78be9c80c8f735c76b6d4a1517304
diff --git a/vendor/nim-chronos b/vendor/nim-chronos
new file mode 160000
index 0000000..c04576d
--- /dev/null
+++ b/vendor/nim-chronos
@@ -0,0 +1 @@
+Subproject commit c04576d829b8a0a1b12baaa8bc92037501b3a4a0
diff --git a/vendor/nim-codex-dht b/vendor/nim-codex-dht
new file mode 160000
index 0000000..f6eef1a
--- /dev/null
+++ b/vendor/nim-codex-dht
@@ -0,0 +1 @@
+Subproject commit f6eef1ac95c70053b2518f1e3909c909ed8701a6
diff --git a/vendor/nim-confutils b/vendor/nim-confutils
new file mode 160000
index 0000000..cb858a2
--- /dev/null
+++ b/vendor/nim-confutils
@@ -0,0 +1 @@
+Subproject commit cb858a27f4347be949d10ed74b58713d687936d2
diff --git a/vendor/nim-contract-abi b/vendor/nim-contract-abi
new file mode 160000
index 0000000..842f489
--- /dev/null
+++ b/vendor/nim-contract-abi
@@ -0,0 +1 @@
+Subproject commit 842f48910be4f388bcbf8abf1f02aba1d5e2ee64
diff --git a/vendor/nim-eth b/vendor/nim-eth
new file mode 160000
index 0000000..dcfbc42
--- /dev/null
+++ b/vendor/nim-eth
@@ -0,0 +1 @@
+Subproject commit dcfbc4291d39b59563828c3e32be4d51a2f25931
diff --git a/vendor/nim-ethers b/vendor/nim-ethers
new file mode 160000
index 0000000..bbced46
--- /dev/null
+++ b/vendor/nim-ethers
@@ -0,0 +1 @@
+Subproject commit bbced4673316763c6ef931b4d0a08069cde2474c
diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc
new file mode 160000
index 0000000..2743721
--- /dev/null
+++ b/vendor/nim-json-rpc
@@ -0,0 +1 @@
+Subproject commit 274372132de497e6b7b793c9d5d5474b71bf80a2
diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization
new file mode 160000
index 0000000..6eadb6e
--- /dev/null
+++ b/vendor/nim-json-serialization
@@ -0,0 +1 @@
+Subproject commit 6eadb6e939ffa7882ff5437033c11a9464d3385c
diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace
new file mode 160000
index 0000000..6da0cda
--- /dev/null
+++ b/vendor/nim-libbacktrace
@@ -0,0 +1 @@
+Subproject commit 6da0cda88ab7780bd5fd342327adb91ab84692aa
diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
new file mode 160000
index 0000000..c08d807
--- /dev/null
+++ b/vendor/nim-libp2p
@@ -0,0 +1 @@
+Subproject commit c08d80734989b028b3d1705f2188d783a343aac0
diff --git a/vendor/nim-metrics b/vendor/nim-metrics
new file mode 160000
index 0000000..cacfdc1
--- /dev/null
+++ b/vendor/nim-metrics
@@ -0,0 +1 @@
+Subproject commit cacfdc12454a0804c65112b9f4f50d1375208dcd
diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal
new file mode 160000
index 0000000..6508ce7
--- /dev/null
+++ b/vendor/nim-nat-traversal
@@ -0,0 +1 @@
+Subproject commit 6508ce75060878dfcdfa21f94721672c69a1823b
diff --git a/vendor/nim-protobuf-serialization b/vendor/nim-protobuf-serialization
new file mode 160000
index 0000000..5a31137
--- /dev/null
+++ b/vendor/nim-protobuf-serialization
@@ -0,0 +1 @@
+Subproject commit 5a31137a82c2b6a989c9ed979bb636c7a49f570e
diff --git a/vendor/nim-results b/vendor/nim-results
new file mode 160000
index 0000000..df8113d
--- /dev/null
+++ b/vendor/nim-results
@@ -0,0 +1 @@
+Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27
diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1
new file mode 160000
index 0000000..2acbbdc
--- /dev/null
+++ b/vendor/nim-secp256k1
@@ -0,0 +1 @@
+Subproject commit 2acbbdcc0e63002a013fff49f015708522875832
diff --git a/vendor/nim-serde b/vendor/nim-serde
new file mode 160000
index 0000000..5ced7c8
--- /dev/null
+++ b/vendor/nim-serde
@@ -0,0 +1 @@
+Subproject commit 5ced7c88b97d99c582285ce796957fb71fd42434
diff --git a/vendor/nim-serialization b/vendor/nim-serialization
new file mode 160000
index 0000000..2086c99
--- /dev/null
+++ b/vendor/nim-serialization
@@ -0,0 +1 @@
+Subproject commit 2086c99608b4bf472e1ef5fe063710f280243396
diff --git a/vendor/nim-testutils b/vendor/nim-testutils
new file mode 160000
index 0000000..4d37244
--- /dev/null
+++ b/vendor/nim-testutils
@@ -0,0 +1 @@
+Subproject commit 4d37244f9f5e1acd8592a4ceb5c3fc47bc160181
diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2
new file mode 160000
index 0000000..845b6af
--- /dev/null
+++ b/vendor/nim-unittest2
@@ -0,0 +1 @@
+Subproject commit 845b6af28b9f68f02d320e03ad18eccccea7ddb9
diff --git a/vendor/nim-websock b/vendor/nim-websock
new file mode 160000
index 0000000..ebe308a
--- /dev/null
+++ b/vendor/nim-websock
@@ -0,0 +1 @@
+Subproject commit ebe308a79a7b440a11dfbe74f352be86a3883508
diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system
new file mode 160000
index 0000000..0be0663
--- /dev/null
+++ b/vendor/nimbus-build-system
@@ -0,0 +1 @@
+Subproject commit 0be0663e1af76e869837226a4ef3e586fcc737d3
diff --git a/vendor/nimcrypto b/vendor/nimcrypto
new file mode 160000
index 0000000..dc07e30
--- /dev/null
+++ b/vendor/nimcrypto
@@ -0,0 +1 @@
+Subproject commit dc07e3058c6904eef965394493b6ea99aa2adefc
diff --git a/vendor/nph b/vendor/nph
new file mode 160000
index 0000000..f1f0477
--- /dev/null
+++ b/vendor/nph
@@ -0,0 +1 @@
+Subproject commit f1f047760c6cb38d5c55d0ddb29b57a9c008a976
diff --git a/vendor/questionable b/vendor/questionable
new file mode 160000
index 0000000..47692e0
--- /dev/null
+++ b/vendor/questionable
@@ -0,0 +1 @@
+Subproject commit 47692e0d923ada8f7f731275b2a87614c0150987
diff --git a/vendor/stint b/vendor/stint
new file mode 160000
index 0000000..5c5e01c
--- /dev/null
+++ b/vendor/stint
@@ -0,0 +1 @@
+Subproject commit 5c5e01cef089a261474b7abfe246b37447aaa8ed
diff --git a/vendor/upraises b/vendor/upraises
new file mode 160000
index 0000000..bc26289
--- /dev/null
+++ b/vendor/upraises
@@ -0,0 +1 @@
+Subproject commit bc2628989b63854d980e92dadbd58f83e34b6f25

From 83aa7839d28defc891644b8bd31d62f7ad6352d6 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 11:22:26 +0200
Subject: [PATCH 02/31] wip

---
 .gitignore                  |  10 +-
 .gitmodules                 |   7 +-
 Makefile                    | 222 ++++++++++++++++++++++++++++++++++++
 build.nims                  |  92 +--------------
 vendor/nim-faststreams      |   1 +
 nim-stew => vendor/nim-stew |   0
 6 files changed, 235 insertions(+), 97 deletions(-)
 create mode 100644 Makefile
 create mode 160000 vendor/nim-faststreams
 rename nim-stew => vendor/nim-stew (100%)

diff --git a/.gitignore b/.gitignore
index ca02be0..fbe4999 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,17 +1,9 @@
 *
 !*.*
 !*/
-coverage
-nimcache
-tests/testAll
-nimble.develop
 nimble.paths
-nim.cfg
 nimbus-build-system.paths
 vendor/*
-NimBinaries
-.update.timestamp
-*.dSYM
-.vscode/*
 *.exe
 crawler_data
+.update.timestamp
diff --git a/.gitmodules b/.gitmodules
index 6b5e018..c058544 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -16,8 +16,8 @@
 [submodule "vendor/nim-secp256k1"]
 	path = vendor/nim-secp256k1
 	url = https://github.com/status-im/nim-secp256k1.git
-[submodule "nim-stew"]
-	path = nim-stew
+[submodule "vendor/nim-stew"]
+	path = vendor/nim-stew
 	url = https://github.com/status-im/nim-stew.git
 [submodule "vendor/questionable"]
 	path = vendor/questionable
@@ -97,3 +97,6 @@
 [submodule "vendor/nph"]
 	path = vendor/nph
 	url = https://github.com/arnetheduck/nph.git
+[submodule "vendor/nim-faststreams"]
+	path = vendor/nim-faststreams
+	url = https://github.com/status-im/nim-faststreams.git
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..d80a0d1
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,222 @@
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# This is the Nim version used locally and in regular CI builds.
+# Can be a specific version tag, a branch name, or a commit hash.
+# Can be overridden by setting the NIM_COMMIT environment variable
+# before calling make.
+#
+# For readability in CI, if NIM_COMMIT is set to "pinned",
+# this will also default to the version pinned here.
+#
+# If NIM_COMMIT is set to "nimbusbuild", this will use the
+# version pinned by nimbus-build-system.
+PINNED_NIM_VERSION := v2.2.4
+
+ifeq ($(NIM_COMMIT),)
+NIM_COMMIT := $(PINNED_NIM_VERSION)
+else ifeq ($(NIM_COMMIT),pinned)
+NIM_COMMIT := $(PINNED_NIM_VERSION)
+endif
+
+ifeq ($(NIM_COMMIT),nimbusbuild)
+undefine NIM_COMMIT
+else
+export NIM_COMMIT
+endif
+
+SHELL := bash # the shell used internally by Make
+
+# used inside the included makefiles
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+
+# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
+# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
+DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
+
+LINK_PCRE := 0
+
+ifeq ($(OS),Windows_NT)
+    ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
+        ARCH = x86_64
+    endif
+    ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
+        ARCH = arm64
+    endif
+else
+    UNAME_P := $(shell uname -m)
+    ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
+        ARCH = x86_64
+    endif
+    ifneq ($(filter $(UNAME_P), aarch64 arm),)
+        ARCH = arm64
+    endif
+endif
+
+ifeq ($(ARCH), x86_64)
+    CXXFLAGS ?= -std=c++17 -mssse3
+else
+    CXXFLAGS ?= -std=c++17
+endif
+export CXXFLAGS
+
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
+
+.PHONY: \
+	all \
+	clean \
+	coverage \
+	deps \
+	libbacktrace \
+	test \
+	update
+
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+# default target, because it's the first one that doesn't start with '.'
+
+# Builds the crawler binary
+all: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
+
+# must be included after the default target
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+
+# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
+ifeq ($(USE_LIBBACKTRACE), 0)
+NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
+else
+NIM_PARAMS := $(NIM_PARAMS) -d:release
+endif
+
+deps: | deps-common nat-libs
+ifneq ($(USE_LIBBACKTRACE), 0)
+deps: | libbacktrace
+endif
+
+update: | update-common
+
+# detecting the os
+ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
+ detected_OS := Windows
+else ifeq ($(strip $(shell uname)),Darwin)
+ detected_OS := macOS
+else
+ # e.g. Linux
+ detected_OS := $(strip $(shell uname))
+endif
+
+# Builds and run a part of the test suite
+test: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
+
+# Builds and runs the smart contract tests
+testContracts: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
+
+# Builds and runs the integration tests
+testIntegration: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
+
+# Builds and runs all tests (except for Taiko L2 tests)
+testAll: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims
+
+# Builds and runs Taiko L2 tests
+testTaiko: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims
+
+# Builds and runs tool tests
+testTools: | cirdl
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims
+
+# nim-libbacktrace
+LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
+libbacktrace:
+ifeq ($(detected_OS), Windows)
+# MSYS2 detection
+ifneq ($(MSYSTEM),)
+	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
+else
+	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
+endif
+else
+	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
+endif
+
+# usual cleaning
+clean: | clean-common
+	rm -rf build
+ifneq ($(USE_LIBBACKTRACE), 0)
+	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
+endif
+
+############
+## Format ##
+############
+.PHONY: build-nph install-nph-hook clean-nph print-nph-path
+
+# Default location for nph binary shall be next to nim binary to make it available on the path.
+NPH:=$(shell dirname $(NIM_BINARY))/nph
+
+build-nph:
+ifeq ("$(wildcard $(NPH))","")
+		$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
+		mv vendor/nph/src/nph $(shell dirname $(NPH))
+		echo "nph utility is available at " $(NPH)
+endif
+
+GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
+
+install-nph-hook: build-nph
+ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
+	cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
+else
+	echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
+	exit 1
+endif
+
+nph/%: build-nph
+	echo -e $(FORMAT_MSG) "nph/$*" && \
+		$(NPH) $*
+
+format:
+	$(NPH) *.nim
+	$(NPH) codexcrawler/
+	$(NPH) tests/
+
+clean-nph:
+	rm -f $(NPH)
+
+# To avoid hardcoding nph binary location in several places
+print-nph-path:
+	echo "$(NPH)"
+
+clean: | clean-nph
+
+endif # "variables.mk" was not included
diff --git a/build.nims b/build.nims
index 8866032..cfe8193 100644
--- a/build.nims
+++ b/build.nims
@@ -29,95 +29,15 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
   buildBinary name, srcDir, params
   exec "build/" & name
 
-task codex, "build codex binary":
-  buildBinary "codex",
+task codexcrawler, "build codexcrawler binary":
+  buildBinary "codexcrawler",
     params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
 
-task toolsCirdl, "build tools/cirdl binary":
-  buildBinary "tools/cirdl/cirdl"
-
-task testCodex, "Build & run Codex tests":
-  test "testCodex", params = "-d:codex_enable_proof_failures=true"
-
-task testContracts, "Build & run Codex Contract tests":
-  test "testContracts"
-
-task testIntegration, "Run integration tests":
-  buildBinary "codex",
-    params =
-      "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
-  test "testIntegration"
-  # use params to enable logging from the integration test executable
-  # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
-  #   "-d:chronicles_enabled_topics:integration:TRACE"  
+task testCodexcrawler, "Build & run Codex Crawler tests":
+  test "testCodexCrawler", params = "-d:codex_enable_proof_failures=true"
 
 task build, "build codex binary":
-  codexTask()
+  codexCrawlerTask()
 
 task test, "Run tests":
-  testCodexTask()
-
-task testTools, "Run Tools tests":
-  toolsCirdlTask()
-  test "testTools"
-
-task testAll, "Run all tests (except for Taiko L2 tests)":
-  testCodexTask()
-  testContractsTask()
-  testIntegrationTask()
-  testToolsTask()
-
-task testTaiko, "Run Taiko L2 tests":
-  codexTask()
-  test "testTaiko"
-
-import strutils
-import os
-
-task coverage, "generates code coverage report":
-  var (output, exitCode) = gorgeEx("which lcov")
-  if exitCode != 0:
-    echo "  ************************** ⛔️ ERROR ⛔️ **************************"
-    echo "  **   ERROR: lcov not found, it must be installed to run code   **"
-    echo "  **   coverage locally                                          **"
-    echo "  *****************************************************************"
-    quit 1
-
-  (output, exitCode) = gorgeEx("gcov --version")
-  if output.contains("Apple LLVM"):
-    echo "  ************************* ⚠️ WARNING ⚠️  *************************"
-    echo "  **   WARNING: Using Apple's llvm-cov in place of gcov, which   **"
-    echo "  **   emulates an old version of gcov (4.2.0) and therefore     **"
-    echo "  **   coverage results will differ than those on CI (which      **"
-    echo "  **   uses a much newer version of gcov).                       **"
-    echo "  *****************************************************************"
-
-  var nimSrcs = " "
-  for f in walkDirRec("codex", {pcFile}):
-    if f.endswith(".nim"):
-      nimSrcs.add " " & f.absolutePath.quoteShell()
-
-  echo "======== Running Tests ======== "
-  test "coverage",
-    srcDir = "tests/",
-    params =
-      " --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true"
-  exec("rm nimcache/coverage/*.c")
-  rmDir("coverage")
-  mkDir("coverage")
-  echo " ======== Running LCOV ======== "
-  exec(
-    "lcov --capture --keep-going --directory nimcache/coverage --output-file coverage/coverage.info"
-  )
-  exec(
-    "lcov --extract coverage/coverage.info --keep-going --output-file coverage/coverage.f.info " &
-      nimSrcs
-  )
-  echo " ======== Generating HTML coverage report ======== "
-  exec("genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report ")
-  echo " ======== Coverage report Done ======== "
-
-task showCoverage, "open coverage html":
-  echo " ======== Opening HTML coverage report in browser... ======== "
-  if findExe("open") != "":
-    exec("open coverage/report/index.html")
+  testCodexCrawlerTask()
diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams
new file mode 160000
index 0000000..c51315d
--- /dev/null
+++ b/vendor/nim-faststreams
@@ -0,0 +1 @@
+Subproject commit c51315d0ae5eb2594d0bf41181d0e1aca1b3c01d
diff --git a/nim-stew b/vendor/nim-stew
similarity index 100%
rename from nim-stew
rename to vendor/nim-stew

From e74f4722fe2f8aacb00aa08213c78f66c5fd8a66 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 11:28:13 +0200
Subject: [PATCH 03/31] wip

---
 .gitmodules            | 3 +++
 vendor/nim-faststreams | 2 +-
 vendor/nim-poseidon2   | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)
 create mode 160000 vendor/nim-poseidon2

diff --git a/.gitmodules b/.gitmodules
index c058544..66f1c6f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -100,3 +100,6 @@
 [submodule "vendor/nim-faststreams"]
 	path = vendor/nim-faststreams
 	url = https://github.com/status-im/nim-faststreams.git
+[submodule "vendor/nim-poseidon2"]
+	path = vendor/nim-poseidon2
+	url = https://github.com/codex-storage/nim-poseidon2.git
diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams
index c51315d..cf8d4d2 160000
--- a/vendor/nim-faststreams
+++ b/vendor/nim-faststreams
@@ -1 +1 @@
-Subproject commit c51315d0ae5eb2594d0bf41181d0e1aca1b3c01d
+Subproject commit cf8d4d22636b8e514caf17e49f9c786ac56b0e85
diff --git a/vendor/nim-poseidon2 b/vendor/nim-poseidon2
new file mode 160000
index 0000000..4e2c6e6
--- /dev/null
+++ b/vendor/nim-poseidon2
@@ -0,0 +1 @@
+Subproject commit 4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3

From 5651c289a30e14760e0c2f77aba1d645143d0e72 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 11:33:30 +0200
Subject: [PATCH 04/31] wip

---
 .gitmodules        | 3 +++
 vendor/constantine | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/constantine

diff --git a/.gitmodules b/.gitmodules
index 66f1c6f..eaef5e5 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -103,3 +103,6 @@
 [submodule "vendor/nim-poseidon2"]
 	path = vendor/nim-poseidon2
 	url = https://github.com/codex-storage/nim-poseidon2.git
+[submodule "vendor/constantine"]
+	path = vendor/constantine
+	url = https://github.com/mratsim/constantine.git
diff --git a/vendor/constantine b/vendor/constantine
new file mode 160000
index 0000000..8d6a6a3
--- /dev/null
+++ b/vendor/constantine
@@ -0,0 +1 @@
+Subproject commit 8d6a6a38b90fb8ee3ec2230839773e69aab36d80

From 2bbc2cd73a475b395460a9bdee81ec122291a585 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 11:38:09 +0200
Subject: [PATCH 05/31] wip

---
 .gitmodules          | 3 +++
 vendor/nim-taskpools | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-taskpools

diff --git a/.gitmodules b/.gitmodules
index eaef5e5..35b8ac8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -106,3 +106,6 @@
 [submodule "vendor/constantine"]
 	path = vendor/constantine
 	url = https://github.com/mratsim/constantine.git
+[submodule "vendor/nim-taskpools"]
+	path = vendor/nim-taskpools
+	url = https://github.com/status-im/nim-taskpools.git
diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools
new file mode 160000
index 0000000..66585e2
--- /dev/null
+++ b/vendor/nim-taskpools
@@ -0,0 +1 @@
+Subproject commit 66585e2e960b7695e48ea60377fb3aeac96406e8

From 5af9a21dc07f26f538ff8201b1e4ffe6f9947f50 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:09:59 +0200
Subject: [PATCH 06/31] wip

---
 .gitmodules     | 3 +++
 vendor/nim-quic | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-quic

diff --git a/.gitmodules b/.gitmodules
index 35b8ac8..ef44119 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -109,3 +109,6 @@
 [submodule "vendor/nim-taskpools"]
 	path = vendor/nim-taskpools
 	url = https://github.com/status-im/nim-taskpools.git
+[submodule "vendor/nim-quic"]
+	path = vendor/nim-quic
+	url = https://github.com/vacp2p/nim-quic.git
diff --git a/vendor/nim-quic b/vendor/nim-quic
new file mode 160000
index 0000000..ddcb31f
--- /dev/null
+++ b/vendor/nim-quic
@@ -0,0 +1 @@
+Subproject commit ddcb31ffb74b5460ab37fd13547eca90594248bc

From 4442d500f051eeaf28fcb712c904d034e0374e19 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:14:46 +0200
Subject: [PATCH 07/31] wip

---
 .gitmodules       | 3 +++
 vendor/nim-ngtcp2 | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-ngtcp2

diff --git a/.gitmodules b/.gitmodules
index ef44119..602e297 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -112,3 +112,6 @@
 [submodule "vendor/nim-quic"]
 	path = vendor/nim-quic
 	url = https://github.com/vacp2p/nim-quic.git
+[submodule "vendor/nim-ngtcp2"]
+	path = vendor/nim-ngtcp2
+	url = https://github.com/vacp2p/nim-ngtcp2.git
diff --git a/vendor/nim-ngtcp2 b/vendor/nim-ngtcp2
new file mode 160000
index 0000000..6834f47
--- /dev/null
+++ b/vendor/nim-ngtcp2
@@ -0,0 +1 @@
+Subproject commit 6834f4756b6af58356ac9c4fef3d71db3c3ae5fe

From 0ed8561cdb0286a47e02dbec87e5c11993bc61c6 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:20:47 +0200
Subject: [PATCH 08/31] wip

---
 vendor/nim-stew | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendor/nim-stew b/vendor/nim-stew
index 58abb48..a6e1981 160000
--- a/vendor/nim-stew
+++ b/vendor/nim-stew
@@ -1 +1 @@
-Subproject commit 58abb4891f97c6cdc07335e868414e0c7b736c68
+Subproject commit a6e198132097fb544d04959aeb3b839e1408f942

From 546e2688d1e0027ee875a9444b24209c25f63f7a Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:25:41 +0200
Subject: [PATCH 09/31] wip

---
 .gitmodules          | 3 +++
 vendor/nim-datastore | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-datastore

diff --git a/.gitmodules b/.gitmodules
index 602e297..7e5c40e 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -115,3 +115,6 @@
 [submodule "vendor/nim-ngtcp2"]
 	path = vendor/nim-ngtcp2
 	url = https://github.com/vacp2p/nim-ngtcp2.git
+[submodule "vendor/nim-datastore"]
+	path = vendor/nim-datastore
+	url = https://github.com/status-im/nim-datastore.git
diff --git a/vendor/nim-datastore b/vendor/nim-datastore
new file mode 160000
index 0000000..5778e37
--- /dev/null
+++ b/vendor/nim-datastore
@@ -0,0 +1 @@
+Subproject commit 5778e373fa97286f389e0aef61f1e8f30a934dab

From 44fcf51bd21d9ca32d7aa7e7c660d4d454e01054 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:30:42 +0200
Subject: [PATCH 10/31] wip

---
 .gitmodules            | 3 +++
 vendor/nim-sqlite3-abi | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-sqlite3-abi

diff --git a/.gitmodules b/.gitmodules
index 7e5c40e..ea58c06 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -118,3 +118,6 @@
 [submodule "vendor/nim-datastore"]
 	path = vendor/nim-datastore
 	url = https://github.com/status-im/nim-datastore.git
+[submodule "vendor/nim-sqlite3-abi"]
+	path = vendor/nim-sqlite3-abi
+	url = https://github.com/arnetheduck/nim-sqlite3-abi.git
diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi
new file mode 160000
index 0000000..05bbff1
--- /dev/null
+++ b/vendor/nim-sqlite3-abi
@@ -0,0 +1 @@
+Subproject commit 05bbff1af4e8fe2d972ba4b0667b89ca94d3ebba

From 077f7d9abb6fb6074836ee40a107159a8e3b87ad Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:36:56 +0200
Subject: [PATCH 11/31] a

---
 .gitmodules              | 3 +++
 vendor/nim-leveldbstatic | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-leveldbstatic

diff --git a/.gitmodules b/.gitmodules
index ea58c06..8c712d0 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -121,3 +121,6 @@
 [submodule "vendor/nim-sqlite3-abi"]
 	path = vendor/nim-sqlite3-abi
 	url = https://github.com/arnetheduck/nim-sqlite3-abi.git
+[submodule "vendor/nim-leveldbstatic"]
+	path = vendor/nim-leveldbstatic
+	url = https://github.com/codex-storage/nim-leveldb.git
diff --git a/vendor/nim-leveldbstatic b/vendor/nim-leveldbstatic
new file mode 160000
index 0000000..378ef63
--- /dev/null
+++ b/vendor/nim-leveldbstatic
@@ -0,0 +1 @@
+Subproject commit 378ef63e261e3b5834a3567404edc3ce838498b3

From ca6dabac41dbf1b7405e47014163b5902686a197 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:42:33 +0200
Subject: [PATCH 12/31] a

---
 .gitmodules       | 3 +++
 vendor/nim-docopt | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-docopt

diff --git a/.gitmodules b/.gitmodules
index 8c712d0..dddce6f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -124,3 +124,6 @@
 [submodule "vendor/nim-leveldbstatic"]
 	path = vendor/nim-leveldbstatic
 	url = https://github.com/codex-storage/nim-leveldb.git
+[submodule "vendor/nim-docopt"]
+	path = vendor/nim-docopt
+	url = https://github.com/docopt/docopt.nim.git
diff --git a/vendor/nim-docopt b/vendor/nim-docopt
new file mode 160000
index 0000000..efaa112
--- /dev/null
+++ b/vendor/nim-docopt
@@ -0,0 +1 @@
+Subproject commit efaa112b6df172a9168c4eb581ab8dda1fbcfe2a

From 7564ba1f6f20d39f684e241dc4d43afb26edcd70 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 13:53:10 +0200
Subject: [PATCH 13/31] a

---
 .gitmodules          | 6 ++++++
 vendor/nim-regex     | 1 +
 vendor/nim-unicodedb | 1 +
 3 files changed, 8 insertions(+)
 create mode 160000 vendor/nim-regex
 create mode 160000 vendor/nim-unicodedb

diff --git a/.gitmodules b/.gitmodules
index dddce6f..be40c2a 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -127,3 +127,9 @@
 [submodule "vendor/nim-docopt"]
 	path = vendor/nim-docopt
 	url = https://github.com/docopt/docopt.nim.git
+[submodule "vendor/nim-regex"]
+	path = vendor/nim-regex
+	url = https://github.com/nitely/nim-regex.git
+[submodule "vendor/nim-unicodedb"]
+	path = vendor/nim-unicodedb
+	url = https://github.com/nitely/nim-unicodedb.git
diff --git a/vendor/nim-regex b/vendor/nim-regex
new file mode 160000
index 0000000..88f634b
--- /dev/null
+++ b/vendor/nim-regex
@@ -0,0 +1 @@
+Subproject commit 88f634b95651ce0d53961668be1c414341429b7f
diff --git a/vendor/nim-unicodedb b/vendor/nim-unicodedb
new file mode 160000
index 0000000..66f2458
--- /dev/null
+++ b/vendor/nim-unicodedb
@@ -0,0 +1 @@
+Subproject commit 66f2458710dc641dd4640368f9483c8a0ec70561

From fcc4da86a604f9accfffb1f24c09e42caaeca13c Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 14:05:51 +0200
Subject: [PATCH 14/31] a

---
 .gitmodules                                    | 3 +++
 codexcrawler/services/marketplace/requests.nim | 2 +-
 vendor/nim-http-utils                          | 1 +
 3 files changed, 5 insertions(+), 1 deletion(-)
 create mode 160000 vendor/nim-http-utils

diff --git a/.gitmodules b/.gitmodules
index be40c2a..acbb4c6 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -133,3 +133,6 @@
 [submodule "vendor/nim-unicodedb"]
 	path = vendor/nim-unicodedb
 	url = https://github.com/nitely/nim-unicodedb.git
+[submodule "vendor/nim-http-utils"]
+	path = vendor/nim-http-utils
+	url = https://github.com/status-im/nim-http-utils.git
diff --git a/codexcrawler/services/marketplace/requests.nim b/codexcrawler/services/marketplace/requests.nim
index a6e4281..8abdaa6 100644
--- a/codexcrawler/services/marketplace/requests.nim
+++ b/codexcrawler/services/marketplace/requests.nim
@@ -3,7 +3,7 @@ import std/sequtils
 import std/typetraits
 import pkg/contractabi
 import pkg/nimcrypto
-import pkg/ethers/fields
+import pkg/ethers/contracts/fields
 import pkg/results
 import pkg/questionable/results
 import pkg/stew/byteutils
diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils
new file mode 160000
index 0000000..8bb1acb
--- /dev/null
+++ b/vendor/nim-http-utils
@@ -0,0 +1 @@
+Subproject commit 8bb1acbaa4b86eb866145b0d468eff64a57d1897

From bd756ca80e9b91fd76d669213a2058b7265240c2 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 14:11:25 +0200
Subject: [PATCH 15/31] a

---
 .gitmodules     | 3 +++
 vendor/nim-zlib | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 vendor/nim-zlib

diff --git a/.gitmodules b/.gitmodules
index acbb4c6..88eb8e6 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -136,3 +136,6 @@
 [submodule "vendor/nim-http-utils"]
 	path = vendor/nim-http-utils
 	url = https://github.com/status-im/nim-http-utils.git
+[submodule "vendor/nim-zlib"]
+	path = vendor/nim-zlib
+	url = https://github.com/status-im/nim-zlib
diff --git a/vendor/nim-zlib b/vendor/nim-zlib
new file mode 160000
index 0000000..91cf360
--- /dev/null
+++ b/vendor/nim-zlib
@@ -0,0 +1 @@
+Subproject commit 91cf360b1aeb2e0c753ff8bac6de22a41c5ed8cd

From e06d03e40cadaf5bd174d8aeef38044c99d4d16d Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 14:16:17 +0200
Subject: [PATCH 16/31] a

---
 codexcrawler/services/marketplace/proofs.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/codexcrawler/services/marketplace/proofs.nim b/codexcrawler/services/marketplace/proofs.nim
index 771d685..c0d80b7 100644
--- a/codexcrawler/services/marketplace/proofs.nim
+++ b/codexcrawler/services/marketplace/proofs.nim
@@ -1,6 +1,6 @@
 import pkg/stint
 import pkg/contractabi
-import pkg/ethers/fields
+import pkg/ethers/contracts/fields
 
 type
   Groth16Proof* = object

From 230b9b8fb29a11027c480db5614b6817c7141261 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 14:30:28 +0200
Subject: [PATCH 17/31] updating

---
 codexcrawler/application.nim                 |  4 +-
 codexcrawler/components/chaincrawler.nim     |  2 +-
 codexcrawler/components/chainmetrics.nim     |  2 +-
 codexcrawler/components/dhtcrawler.nim       |  2 +-
 codexcrawler/components/dhtmetrics.nim       | 14 +--
 codexcrawler/components/nodestore.nim        | 18 ++--
 codexcrawler/components/timetracker.nim      |  2 +-
 codexcrawler/components/todolist.nim         |  6 +-
 codexcrawler/installer.nim                   |  2 +-
 codexcrawler/list.nim                        |  2 +-
 codexcrawler/services/dht.nim                | 25 +++---
 codexcrawler/services/marketplace.nim        |  4 +-
 codexcrawler/services/marketplace/config.nim |  2 +-
 codexcrawler/services/marketplace/market.nim | 94 ++++++++++----------
 codexcrawler/state.nim                       |  4 +-
 codexcrawler/utils/asyncdataevent.nim        |  8 +-
 16 files changed, 97 insertions(+), 94 deletions(-)

diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim
index e5a0d07..d3b34dd 100644
--- a/codexcrawler/application.nim
+++ b/codexcrawler/application.nim
@@ -18,7 +18,7 @@ type Application* = ref object
   state: State
   components: seq[Component]
 
-proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} =
+proc initializeApp(app: Application, config: Config): Future[?!void] {.async: (raises: [CancelledError]).} =
   app.state = State(
     status: ApplicationStatus.Running,
     config: config,
@@ -48,7 +48,7 @@ proc initializeApp(app: Application, config: Config): Future[?!void] {.async.} =
 
   return success()
 
-proc stopComponents(app: Application) {.async.} =
+proc stopComponents(app: Application) {.async: (raises: [CancelledError]).} =
   for c in app.components:
     if err =? (await c.stop()).errorOption:
       error "Failed to stop component", err = err.msg
diff --git a/codexcrawler/components/chaincrawler.nim b/codexcrawler/components/chaincrawler.nim
index f957630..3f9cd8d 100644
--- a/codexcrawler/components/chaincrawler.nim
+++ b/codexcrawler/components/chaincrawler.nim
@@ -19,7 +19,7 @@ type ChainCrawler* = ref object of Component
 proc onNewRequest(c: ChainCrawler, rid: Rid): Future[?!void] {.async: (raises: []).} =
   return await c.store.add(rid)
 
-method start*(c: ChainCrawler): Future[?!void] {.async.} =
+method start*(c: ChainCrawler): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
 
   proc onRequest(rid: Rid): Future[?!void] {.async: (raises: []).} =
diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim
index bf7d8f6..6d6340d 100644
--- a/codexcrawler/components/chainmetrics.nim
+++ b/codexcrawler/components/chainmetrics.nim
@@ -65,7 +65,7 @@ proc step(c: ChainMetrics): Future[?!void] {.async: (raises: []).} =
   c.updateMetrics(update)
   return success()
 
-method start*(c: ChainMetrics): Future[?!void] {.async.} =
+method start*(c: ChainMetrics): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
 
   proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
diff --git a/codexcrawler/components/dhtcrawler.nim b/codexcrawler/components/dhtcrawler.nim
index 0c594b2..fa80170 100644
--- a/codexcrawler/components/dhtcrawler.nim
+++ b/codexcrawler/components/dhtcrawler.nim
@@ -46,7 +46,7 @@ proc step(c: DhtCrawler): Future[?!void] {.async: (raises: []).} =
 
   return success()
 
-method start*(c: DhtCrawler): Future[?!void] {.async.} =
+method start*(c: DhtCrawler): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
 
   proc onStep(): Future[?!void] {.async: (raises: []), gcsafe.} =
diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim
index 374e169..54d4d33 100644
--- a/codexcrawler/components/dhtmetrics.nim
+++ b/codexcrawler/components/dhtmetrics.nim
@@ -27,7 +27,7 @@ proc updateMetrics(d: DhtMetrics) =
 
 proc handleCheckEvent(
     d: DhtMetrics, event: DhtNodeCheckEventData
-): Future[?!void] {.async.} =
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   if event.isOk:
     ?await d.ok.add(event.id)
     ?await d.nok.remove(event.id)
@@ -38,31 +38,31 @@ proc handleCheckEvent(
   d.updateMetrics()
   return success()
 
-proc handleDeleteEvent(d: DhtMetrics, nids: seq[Nid]): Future[?!void] {.async.} =
+proc handleDeleteEvent(d: DhtMetrics, nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
   for nid in nids:
     ?await d.ok.remove(nid)
     ?await d.nok.remove(nid)
   d.updateMetrics()
   return success()
 
-method awake*(d: DhtMetrics): Future[?!void] {.async.} =
-  proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} =
+method awake*(d: DhtMetrics): Future[?!void] {.async: (raises: [CancelledError]).} =
+  proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
     await d.handleCheckEvent(event)
 
-  proc onDelete(nids: seq[Nid]): Future[?!void] {.async.} =
+  proc onDelete(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
     await d.handleDeleteEvent(nids)
 
   d.subCheck = d.state.events.dhtNodeCheck.subscribe(onCheck)
   d.subDel = d.state.events.nodesDeleted.subscribe(onDelete)
   return success()
 
-method start*(d: DhtMetrics): Future[?!void] {.async.} =
+method start*(d: DhtMetrics): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
   ?await d.ok.load()
   ?await d.nok.load()
   return success()
 
-method stop*(d: DhtMetrics): Future[?!void] {.async.} =
+method stop*(d: DhtMetrics): Future[?!void] {.async: (raises: [CancelledError]).} =
   await d.state.events.dhtNodeCheck.unsubscribe(d.subCheck)
   await d.state.events.nodesDeleted.unsubscribe(d.subDel)
   return success()
diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim
index ef22ae0..f10ff85 100644
--- a/codexcrawler/components/nodestore.nim
+++ b/codexcrawler/components/nodestore.nim
@@ -76,7 +76,7 @@ proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T =
     )
   return NodeEntry.fromBytes(bytes)
 
-proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async.} =
+proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async: (raises: [CancelledError]).} =
   without key =? Key.init(nodestoreName / $nid), err:
     error "failed to format key", err = err.msg
     return failure(err)
@@ -91,7 +91,7 @@ proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async.} =
 
   return success(not exists)
 
-proc fireNewNodesDiscovered(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} =
+proc fireNewNodesDiscovered(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
   await s.state.events.newNodesDiscovered.fire(nids)
 
 proc fireNodesDeleted(
@@ -99,7 +99,7 @@ proc fireNodesDeleted(
 ): Future[?!void] {.async: (raises: []).} =
   await s.state.events.nodesDeleted.fire(nids)
 
-proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} =
+proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
   var newNodes = newSeq[Nid]()
   for nid in nids:
     without isNew =? (await s.storeNodeIsNew(nid)), err:
@@ -114,7 +114,7 @@ proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async.} =
 
 proc processNodeCheck(
     s: NodeStore, event: DhtNodeCheckEventData
-): Future[?!void] {.async.} =
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   without key =? Key.init(nodestoreName / $(event.id)), err:
     error "failed to format key", err = err.msg
     return failure(err)
@@ -142,7 +142,7 @@ proc processNodeCheck(
   ?await s.store.put(key, entry)
   return success()
 
-proc deleteEntry(s: NodeStore, nid: Nid): Future[?!bool] {.async.} =
+proc deleteEntry(s: NodeStore, nid: Nid): Future[?!bool] {.async: (raises: [CancelledError]).} =
   without key =? Key.init(nodestoreName / $nid), err:
     error "failed to format key", err = err.msg
     return failure(err)
@@ -202,20 +202,20 @@ method deleteEntries*(
   ?await s.fireNodesDeleted(deleted)
   return success()
 
-method start*(s: NodeStore): Future[?!void] {.async.} =
+method start*(s: NodeStore): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
 
-  proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} =
+  proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
     return await s.processFoundNodes(nids)
 
-  proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} =
+  proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
     return await s.processNodeCheck(event)
 
   s.subFound = s.state.events.nodesFound.subscribe(onNodesFound)
   s.subCheck = s.state.events.dhtNodeCheck.subscribe(onCheck)
   return success()
 
-method stop*(s: NodeStore): Future[?!void] {.async.} =
+method stop*(s: NodeStore): Future[?!void] {.async: (raises: [CancelledError]).} =
   await s.state.events.nodesFound.unsubscribe(s.subFound)
   await s.state.events.dhtNodeCheck.unsubscribe(s.subCheck)
   return success()
diff --git a/codexcrawler/components/timetracker.nim b/codexcrawler/components/timetracker.nim
index 2683b0a..b38d2a0 100644
--- a/codexcrawler/components/timetracker.nim
+++ b/codexcrawler/components/timetracker.nim
@@ -56,7 +56,7 @@ proc raiseRoutingTableNodes(t: TimeTracker): Future[?!void] {.async: (raises: []
     return failure(err)
   return success()
 
-method start*(t: TimeTracker): Future[?!void] {.async.} =
+method start*(t: TimeTracker): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
 
   proc onCheckRevisitAndExpiry(): Future[?!void] {.async: (raises: []), gcsafe.} =
diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim
index 33144ae..2d4093e 100644
--- a/codexcrawler/components/todolist.nim
+++ b/codexcrawler/components/todolist.nim
@@ -54,10 +54,10 @@ method pop*(t: TodoList): Future[?!Nid] {.async: (raises: []), base.} =
 
   return success(item)
 
-method awake*(t: TodoList): Future[?!void] {.async.} =
+method awake*(t: TodoList): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "initializing..."
 
-  proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} =
+  proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
     t.addNodes(nids)
     return success()
 
@@ -65,7 +65,7 @@ method awake*(t: TodoList): Future[?!void] {.async.} =
   t.subRev = t.state.events.nodesToRevisit.subscribe(onNewNodes)
   return success()
 
-method stop*(t: TodoList): Future[?!void] {.async.} =
+method stop*(t: TodoList): Future[?!void] {.async: (raises: [CancelledError]).} =
   await t.state.events.newNodesDiscovered.unsubscribe(t.subNew)
   await t.state.events.nodesToRevisit.unsubscribe(t.subRev)
   return success()
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index 80fa924..e8a12f1 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -18,7 +18,7 @@ import ./components/chainmetrics
 import ./components/chaincrawler
 import ./components/requeststore
 
-proc createComponents*(state: State): Future[?!seq[Component]] {.async.} =
+proc createComponents*(state: State): Future[?!seq[Component]] {.async: (raises: [CancelledError]).} =
   var components: seq[Component] = newSeq[Component]()
   let clock = createClock()
 
diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim
index d1985a4..7ead86c 100644
--- a/codexcrawler/list.nim
+++ b/codexcrawler/list.nim
@@ -32,7 +32,7 @@ proc decode(T: type Nid, bytes: seq[byte]): ?!T =
     return success(Nid.fromStr("0"))
   return Nid.fromBytes(bytes)
 
-proc saveItem(this: List, item: Nid): Future[?!void] {.async.} =
+proc saveItem(this: List, item: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
   without itemKey =? Key.init(this.name / $item), err:
     return failure(err)
   ?await this.store.put(itemKey, item)
diff --git a/codexcrawler/services/dht.nim b/codexcrawler/services/dht.nim
index e4726fc..e5068db 100644
--- a/codexcrawler/services/dht.nim
+++ b/codexcrawler/services/dht.nim
@@ -75,15 +75,18 @@ method getNeighbors*(
   except CatchableError as exc:
     return failure(exc.msg)
 
-proc findPeer*(d: Dht, peerId: PeerId): Future[?PeerRecord] {.async.} =
+proc findPeer*(d: Dht, peerId: PeerId): Future[?PeerRecord] {.async: (raises: [CancelledError]).} =
   trace "protocol.resolve..."
-  let node = await d.protocol.resolve(toNodeId(peerId))
-
-  return
-    if node.isSome():
-      node.get().record.data.some
-    else:
-      PeerRecord.none
+  try:
+    let node = await d.protocol.resolve(toNodeId(peerId))
+    return
+      if node.isSome():
+        node.get().record.data.some
+      else:
+        PeerRecord.none
+  except CatchableError as exc:
+    error "CatchableError in protocol.resolve", err = exc.msg
+    return PeerRecord.none
 
 method removeProvider*(d: Dht, peerId: PeerId): Future[void] {.base, gcsafe.} =
   trace "Removing provider", peerId
@@ -109,12 +112,12 @@ proc updateDhtRecord(d: Dht, addrs: openArray[MultiAddress]) =
   if not d.protocol.isNil:
     d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR")
 
-method start*(d: Dht): Future[?!void] {.async.} =
+method start*(d: Dht): Future[?!void] {.async: (raises: [CancelledError]).} =
   d.protocol.open()
   await d.protocol.start()
   return success()
 
-method stop*(d: Dht): Future[?!void] {.async.} =
+method stop*(d: Dht): Future[?!void] {.async: (raises: [CancelledError]).} =
   await d.protocol.closeWait()
   return success()
 
@@ -154,7 +157,7 @@ proc new(
 
   self
 
-proc createDht*(state: State): Future[?!Dht] {.async.} =
+proc createDht*(state: State): Future[?!Dht] {.async: (raises: [CancelledError]).} =
   without dhtStore =? createDatastore(state.config.dataDir / "dht"), err:
     return failure(err)
   let keyPath = state.config.dataDir / "privatekey"
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 7513d02..4c69fda 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -50,7 +50,7 @@ proc fetchRequestInfo(
 method subscribeToNewRequests*(
     m: MarketplaceService, onNewRequest: OnNewRequest
 ): Future[?!void] {.async: (raises: []), base.} =
-  proc resultWrapper(rid: Rid): Future[void] {.async.} =
+  proc resultWrapper(rid: Rid): Future[void] {.async: (raises: [CancelledError]).} =
     let response = await onNewRequest(rid)
     if error =? response.errorOption:
       raiseAssert("Error result in handling of onNewRequest callback: " & error.msg)
@@ -109,7 +109,7 @@ method getRequestInfo*(
   else:
     notStarted()
 
-method awake*(m: MarketplaceService): Future[?!void] {.async.} =
+method awake*(m: MarketplaceService): Future[?!void] {.async: (raises: [CancelledError]).} =
   let provider = JsonRpcProvider.new(m.state.config.ethProvider)
   without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
     return failure("Invalid MarketplaceAddress provided")
diff --git a/codexcrawler/services/marketplace/config.nim b/codexcrawler/services/marketplace/config.nim
index 3c31c8b..83b39c0 100644
--- a/codexcrawler/services/marketplace/config.nim
+++ b/codexcrawler/services/marketplace/config.nim
@@ -1,5 +1,5 @@
 import pkg/contractabi
-import pkg/ethers/fields
+import pkg/ethers/contracts/fields
 import pkg/questionable/results
 
 export contractabi
diff --git a/codexcrawler/services/marketplace/market.nim b/codexcrawler/services/marketplace/market.nim
index 6b00376..64fa8c2 100644
--- a/codexcrawler/services/marketplace/market.nim
+++ b/codexcrawler/services/marketplace/market.nim
@@ -121,7 +121,7 @@ proc config(
 
   return resolvedConfig
 
-proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
+proc approveFunds(market: OnChainMarket, amount: UInt256) {.async: (raises: [CancelledError]).} =
   raiseAssert("Not available: approveFunds")
 
 proc getZkeyHash*(
@@ -138,43 +138,43 @@ proc periodicity*(
     let period = config.proofs.period
     return Periodicity(seconds: period)
 
-proc proofTimeout*(market: OnChainMarket): Future[uint64] {.async.} =
+proc proofTimeout*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let config = await market.config()
     return config.proofs.timeout
 
-proc repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async.} =
+proc repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let config = await market.config()
     return config.collateral.repairRewardPercentage
 
-proc requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async.} =
+proc requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let config = await market.config()
     return config.requestDurationLimit
 
-proc proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} =
+proc proofDowntime*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let config = await market.config()
     return config.proofs.downtime
 
-proc getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} =
+proc getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.getPointer(slotId, overrides)
 
-proc myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} =
+proc myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return await market.contract.myRequests
 
-proc mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} =
+proc mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let slots = await market.contract.mySlots()
     debug "Fetched my slots", numSlots = len(slots)
 
     return slots
 
-proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async.} =
+proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async: (raises: [CancelledError]).} =
   convertEthersError:
     debug "Requesting storage"
     await market.approveFunds(request.totalPrice())
@@ -182,7 +182,7 @@ proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async.} =
 
 proc getRequest*(
     market: OnChainMarket, id: RequestId
-): Future[?StorageRequest] {.async.} =
+): Future[?StorageRequest] {.async: (raises: [CancelledError]).} =
   let key = $id
 
   convertEthersError:
@@ -194,7 +194,7 @@ proc getRequest*(
 
 proc requestState*(
     market: OnChainMarket, requestId: RequestId
-): Future[?RequestState] {.async.} =
+): Future[?RequestState] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -202,22 +202,22 @@ proc requestState*(
     except Marketplace_UnknownRequest:
       return none RequestState
 
-proc slotState*(market: OnChainMarket, slotId: SlotId): Future[SlotState] {.async.} =
+proc slotState*(market: OnChainMarket, slotId: SlotId): Future[SlotState] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.slotState(slotId, overrides)
 
-proc getRequestEnd*(market: OnChainMarket, id: RequestId): Future[uint64] {.async.} =
+proc getRequestEnd*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return (await market.contract.requestEnd(id)).uint64
 
-proc requestExpiresAt*(market: OnChainMarket, id: RequestId): Future[uint64] {.async.} =
+proc requestExpiresAt*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return (await market.contract.requestExpiry(id)).uint64
 
 proc getHost(
     market: OnChainMarket, requestId: RequestId, slotIndex: uint64
-): Future[?Address] {.async.} =
+): Future[?Address] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let slotId = slotId(requestId, slotIndex)
     let address = await market.contract.getHost(slotId)
@@ -228,11 +228,11 @@ proc getHost(
 
 proc currentCollateral*(
     market: OnChainMarket, slotId: SlotId
-): Future[UInt256] {.async.} =
+): Future[UInt256] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return await market.contract.currentCollateral(slotId)
 
-proc getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async.} =
+proc getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     try:
       return some await market.contract.getActiveSlot(slotId)
@@ -245,7 +245,7 @@ proc fillSlot(
     slotIndex: uint64,
     proof: Groth16Proof,
     collateral: UInt256,
-) {.async.} =
+) {.async: (raises: [CancelledError]).} =
   convertEthersError:
     logScope:
       requestId
@@ -256,14 +256,14 @@ proc fillSlot(
     discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(1)
     trace "fillSlot transaction completed"
 
-proc freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} =
+proc freeSlot*(market: OnChainMarket, slotId: SlotId) {.async: (raises: [CancelledError]).} =
   raiseAssert("Not supported")
 
-proc withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async.} =
+proc withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async: (raises: [CancelledError]).} =
   convertEthersError:
     discard await market.contract.withdrawFunds(requestId).confirm(1)
 
-proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} =
+proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -271,7 +271,7 @@ proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.}
     except Marketplace_SlotIsFree:
       return false
 
-proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} =
+proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -281,22 +281,22 @@ proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.asy
 
 proc getChallenge*(
     market: OnChainMarket, id: SlotId
-): Future[ProofChallenge] {.async.} =
+): Future[ProofChallenge] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.getChallenge(id, overrides)
 
-proc submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async.} =
+proc submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async: (raises: [CancelledError]).} =
   convertEthersError:
     discard await market.contract.submitProof(id, proof).confirm(1)
 
-proc markProofAsMissing*(market: OnChainMarket, id: SlotId, period: Period) {.async.} =
+proc markProofAsMissing*(market: OnChainMarket, id: SlotId, period: Period) {.async: (raises: [CancelledError]).} =
   convertEthersError:
     discard await market.contract.markProofAsMissing(id, period).confirm(1)
 
 proc canProofBeMarkedAsMissing*(
     market: OnChainMarket, id: SlotId, period: Period
-): Future[bool] {.async.} =
+): Future[bool] {.async: (raises: [CancelledError]).} =
   let provider = market.contract.provider
   let contractWithoutSigner = market.contract.connect(provider)
   let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -309,7 +309,7 @@ proc canProofBeMarkedAsMissing*(
 
 proc reserveSlot*(
     market: OnChainMarket, requestId: RequestId, slotIndex: uint64
-) {.async.} =
+) {.async: (raises: [CancelledError]).} =
   convertEthersError:
     discard await market.contract
     .reserveSlot(
@@ -322,13 +322,13 @@ proc reserveSlot*(
 
 proc canReserveSlot*(
     market: OnChainMarket, requestId: RequestId, slotIndex: uint64
-): Future[bool] {.async.} =
+): Future[bool] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return await market.contract.canReserveSlot(requestId, slotIndex)
 
 proc subscribeRequests*(
     market: OnChainMarket, callback: OnRequest
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!StorageRequested) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in Request subscription", msg = eventErr.msg
@@ -342,7 +342,7 @@ proc subscribeRequests*(
 
 proc subscribeSlotFilled*(
     market: OnChainMarket, callback: OnSlotFilled
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!SlotFilled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in SlotFilled subscription", msg = eventErr.msg
@@ -359,7 +359,7 @@ proc subscribeSlotFilled*(
     requestId: RequestId,
     slotIndex: uint64,
     callback: OnSlotFilled,
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: uint64) =
     if eventRequestId == requestId and eventSlotIndex == slotIndex:
       callback(requestId, slotIndex)
@@ -369,7 +369,7 @@ proc subscribeSlotFilled*(
 
 proc subscribeSlotFreed*(
     market: OnChainMarket, callback: OnSlotFreed
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!SlotFreed) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in SlotFreed subscription", msg = eventErr.msg
@@ -383,7 +383,7 @@ proc subscribeSlotFreed*(
 
 proc subscribeSlotReservationsFull*(
     market: OnChainMarket, callback: OnSlotReservationsFull
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!SlotReservationsFull) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in SlotReservationsFull subscription",
@@ -398,7 +398,7 @@ proc subscribeSlotReservationsFull*(
 
 proc subscribeFulfillment(
     market: OnChainMarket, callback: OnFulfillment
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
@@ -412,7 +412,7 @@ proc subscribeFulfillment(
 
 proc subscribeFulfillment(
     market: OnChainMarket, requestId: RequestId, callback: OnFulfillment
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
@@ -427,7 +427,7 @@ proc subscribeFulfillment(
 
 proc subscribeRequestCancelled*(
     market: OnChainMarket, callback: OnRequestCancelled
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestCancelled subscription", msg = eventErr.msg
@@ -441,7 +441,7 @@ proc subscribeRequestCancelled*(
 
 proc subscribeRequestCancelled*(
     market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestCancelled subscription", msg = eventErr.msg
@@ -456,7 +456,7 @@ proc subscribeRequestCancelled*(
 
 proc subscribeRequestFailed*(
     market: OnChainMarket, callback: OnRequestFailed
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFailed subscription", msg = eventErr.msg
@@ -470,7 +470,7 @@ proc subscribeRequestFailed*(
 
 proc subscribeRequestFailed*(
     market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFailed subscription", msg = eventErr.msg
@@ -485,7 +485,7 @@ proc subscribeRequestFailed*(
 
 proc subscribeProofSubmission*(
     market: OnChainMarket, callback: OnProofSubmitted
-): Future[MarketSubscription] {.async.} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
   proc onEvent(eventResult: ?!ProofSubmitted) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in ProofSubmitted subscription", msg = eventErr.msg
@@ -497,18 +497,18 @@ proc subscribeProofSubmission*(
     let subscription = await market.contract.subscribe(ProofSubmitted, onEvent)
     return OnChainMarketSubscription(eventSubscription: subscription)
 
-proc unsubscribe*(subscription: OnChainMarketSubscription) {.async.} =
+proc unsubscribe*(subscription: OnChainMarketSubscription) {.async: (raises: [CancelledError]).} =
   await subscription.eventSubscription.unsubscribe()
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, fromBlock: BlockTag
-): Future[seq[SlotFilled]] {.async.} =
+): Future[seq[SlotFilled]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return await market.contract.queryFilter(SlotFilled, fromBlock, BlockTag.latest)
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, blocksAgo: int
-): Future[seq[SlotFilled]] {.async.} =
+): Future[seq[SlotFilled]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
 
@@ -516,21 +516,21 @@ proc queryPastSlotFilledEvents*(
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, fromTime: int64
-): Future[seq[SlotFilled]] {.async.} =
+): Future[seq[SlotFilled]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
     return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock))
 
 proc queryPastStorageRequestedEvents*(
     market: OnChainMarket, fromBlock: BlockTag
-): Future[seq[StorageRequested]] {.async.} =
+): Future[seq[StorageRequested]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     return
       await market.contract.queryFilter(StorageRequested, fromBlock, BlockTag.latest)
 
 proc queryPastStorageRequestedEvents*(
     market: OnChainMarket, blocksAgo: int
-): Future[seq[StorageRequested]] {.async.} =
+): Future[seq[StorageRequested]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
 
@@ -538,7 +538,7 @@ proc queryPastStorageRequestedEvents*(
 
 proc queryPastStorageRequestedEventsFromTime*(
     market: OnChainMarket, fromTime: int64
-): Future[seq[StorageRequested]] {.async.} =
+): Future[seq[StorageRequested]] {.async: (raises: [CancelledError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
 
diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim
index 76e293c..d98c337 100644
--- a/codexcrawler/state.nim
+++ b/codexcrawler/state.nim
@@ -33,10 +33,10 @@ type
     config*: Config
     events*: Events
 
-proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async.} =
+proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async: (raises: [CancelledError]).} =
   await sleepAsync(1.seconds)
 
-  proc worker(): Future[void] {.async.} =
+  proc worker(): Future[void] {.async: (raises: [CancelledError]).} =
     while s.status == ApplicationStatus.Running:
       if err =? (await step()).errorOption:
         error "Failure-result caught in main loop. Stopping...", err = err.msg
diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim
index e847c7d..aba1020 100644
--- a/codexcrawler/utils/asyncdataevent.nim
+++ b/codexcrawler/utils/asyncdataevent.nim
@@ -24,7 +24,7 @@ proc newAsyncDataEvent*[T](): AsyncDataEvent[T] =
 
 proc performUnsubscribe[T](
     event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription
-) {.async.} =
+) {.async: (raises: [CancelledError]).} =
   if subscription in event.subscriptions:
     await subscription.listenFuture.cancelAndWait()
     event.subscriptions.delete(event.subscriptions.find(subscription))
@@ -40,7 +40,7 @@ proc subscribe*[T](
     delayedUnsubscribe: false,
   )
 
-  proc listener() {.async.} =
+  proc listener() {.async: (raises: [CancelledError]).} =
     while true:
       let items = await event.queue.waitEvents(subscription.key)
       for item in items:
@@ -81,13 +81,13 @@ proc fire*[T](
 
 proc unsubscribe*[T](
     event: AsyncDataEvent[T], subscription: AsyncDataEventSubscription
-) {.async.} =
+) {.async: (raises: [CancelledError]).} =
   if subscription.inHandler:
     subscription.delayedUnsubscribe = true
   else:
     await event.performUnsubscribe(subscription)
 
-proc unsubscribeAll*[T](event: AsyncDataEvent[T]) {.async.} =
+proc unsubscribeAll*[T](event: AsyncDataEvent[T]) {.async: (raises: [CancelledError]).} =
   let all = event.subscriptions
   for subscription in all:
     await event.unsubscribe(subscription)

From c374dfc84425c6007d2be6d5df68b328114824a9 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 15:00:32 +0200
Subject: [PATCH 18/31] it builds

---
 codexcrawler/component.nim                   |  6 +-
 codexcrawler/components/dhtcrawler.nim       |  5 +-
 codexcrawler/components/nodestore.nim        | 13 +--
 codexcrawler/components/requeststore.nim     |  9 +-
 codexcrawler/list.nim                        | 15 ++--
 codexcrawler/services/dht.nim                | 12 ++-
 codexcrawler/services/marketplace.nim        | 15 ++--
 codexcrawler/services/marketplace/market.nim | 95 ++++++++++----------
 codexcrawler/state.nim                       |  2 +-
 codexcrawler/utils/asyncdataevent.nim        | 21 +++--
 10 files changed, 111 insertions(+), 82 deletions(-)

diff --git a/codexcrawler/component.nim b/codexcrawler/component.nim
index ce1517b..623e757 100644
--- a/codexcrawler/component.nim
+++ b/codexcrawler/component.nim
@@ -3,17 +3,17 @@ import pkg/questionable/results
 
 type Component* = ref object of RootObj
 
-method awake*(c: Component): Future[?!void] {.async, base.} =
+method awake*(c: Component): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   # Awake is called on all components in an unspecified order.
   # Use this method to subscribe/connect to other components.
   return success()
 
-method start*(c: Component): Future[?!void] {.async, base.} =
+method start*(c: Component): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   # Start is called on all components in an unspecified order.
   # Is is guaranteed that all components have already successfulled handled 'awake'.
   # Use this method to begin the work of this component.
   return success()
 
-method stop*(c: Component): Future[?!void] {.async, base.} =
+method stop*(c: Component): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   # Use this method to stop, unsubscribe, and clean up any resources.
   return success()
diff --git a/codexcrawler/components/dhtcrawler.nim b/codexcrawler/components/dhtcrawler.nim
index fa80170..7b8846e 100644
--- a/codexcrawler/components/dhtcrawler.nim
+++ b/codexcrawler/components/dhtcrawler.nim
@@ -53,7 +53,10 @@ method start*(c: DhtCrawler): Future[?!void] {.async: (raises: [CancelledError])
     await c.step()
 
   if c.state.config.dhtEnable:
-    await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds)
+    try:
+      await c.state.whileRunning(onStep, c.state.config.stepDelayMs.milliseconds)
+    except CatchableError as err:
+      return failure(err.msg)
 
   return success()
 
diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim
index f10ff85..7d68fbe 100644
--- a/codexcrawler/components/nodestore.nim
+++ b/codexcrawler/components/nodestore.nim
@@ -70,11 +70,14 @@ proc encode*(e: NodeEntry): seq[byte] =
   e.toBytes()
 
 proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T =
-  if bytes.len < 1:
-    return success(
-      NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64, firstInactive: 0.uint64)
-    )
-  return NodeEntry.fromBytes(bytes)
+  try:
+    if bytes.len < 1:
+      return success(
+        NodeEntry(id: Nid.fromStr("0"), lastVisit: 0.uint64, firstInactive: 0.uint64)
+      )
+    return NodeEntry.fromBytes(bytes)
+  except ValueError as err:
+    return failure(err.msg)
 
 proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async: (raises: [CancelledError]).} =
   without key =? Key.init(nodestoreName / $nid), err:
diff --git a/codexcrawler/components/requeststore.nim b/codexcrawler/components/requeststore.nim
index 836e1fb..6658087 100644
--- a/codexcrawler/components/requeststore.nim
+++ b/codexcrawler/components/requeststore.nim
@@ -51,9 +51,12 @@ proc encode*(e: RequestEntry): seq[byte] =
   e.toBytes()
 
 proc decode*(T: type RequestEntry, bytes: seq[byte]): ?!T =
-  if bytes.len < 1:
-    return success(RequestEntry(isValid: false))
-  return RequestEntry.fromBytes(bytes)
+  try:
+    if bytes.len < 1:
+      return success(RequestEntry(isValid: false))
+    return RequestEntry.fromBytes(bytes)
+  except ValueError as err:
+    return failure(err.msg)
 
 method add*(s: RequestStore, rid: Rid): Future[?!void] {.async: (raises: []), base.} =
   without key =? Key.init(requeststoreName / $rid), err:
diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim
index 7ead86c..d19ebdf 100644
--- a/codexcrawler/list.nim
+++ b/codexcrawler/list.nim
@@ -28,9 +28,12 @@ proc encode(s: Nid): seq[byte] =
   s.toBytes()
 
 proc decode(T: type Nid, bytes: seq[byte]): ?!T =
-  if bytes.len < 1:
-    return success(Nid.fromStr("0"))
-  return Nid.fromBytes(bytes)
+  try:
+    if bytes.len < 1:
+      return success(Nid.fromStr("0"))
+    return Nid.fromBytes(bytes)
+  except ValueError as err:
+    return failure(err.msg)
 
 proc saveItem(this: List, item: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
   without itemKey =? Key.init(this.name / $item), err:
@@ -38,7 +41,7 @@ proc saveItem(this: List, item: Nid): Future[?!void] {.async: (raises: [Cancelle
   ?await this.store.put(itemKey, item)
   return success()
 
-method load*(this: List): Future[?!void] {.async, base.} =
+method load*(this: List): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   without queryKey =? Key.init(this.name), err:
     return failure(err)
   without iter =? (await query[Nid](this.store, Query.init(queryKey))), err:
@@ -58,7 +61,7 @@ method load*(this: List): Future[?!void] {.async, base.} =
 proc contains*(this: List, nid: Nid): bool =
   this.items.anyIt(it == nid)
 
-method add*(this: List, nid: Nid): Future[?!void] {.async, base.} =
+method add*(this: List, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   if this.contains(nid):
     return success()
 
@@ -69,7 +72,7 @@ method add*(this: List, nid: Nid): Future[?!void] {.async, base.} =
 
   return success()
 
-method remove*(this: List, nid: Nid): Future[?!void] {.async, base.} =
+method remove*(this: List, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   if not this.contains(nid):
     return success()
 
diff --git a/codexcrawler/services/dht.nim b/codexcrawler/services/dht.nim
index e5068db..254f3a3 100644
--- a/codexcrawler/services/dht.nim
+++ b/codexcrawler/services/dht.nim
@@ -113,12 +113,18 @@ proc updateDhtRecord(d: Dht, addrs: openArray[MultiAddress]) =
     d.protocol.updateRecord(d.dhtRecord).expect("Should update SPR")
 
 method start*(d: Dht): Future[?!void] {.async: (raises: [CancelledError]).} =
-  d.protocol.open()
-  await d.protocol.start()
+  try:
+    d.protocol.open()
+    await d.protocol.start()
+  except CatchableError as exc:
+    return failure(exc.msg)
   return success()
 
 method stop*(d: Dht): Future[?!void] {.async: (raises: [CancelledError]).} =
-  await d.protocol.closeWait()
+  try:
+    await d.protocol.closeWait()
+  except CatchableError as exc:
+    return failure(exc.msg)
   return success()
 
 proc new(
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index 4c69fda..fd2c4f3 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -110,13 +110,16 @@ method getRequestInfo*(
     notStarted()
 
 method awake*(m: MarketplaceService): Future[?!void] {.async: (raises: [CancelledError]).} =
-  let provider = JsonRpcProvider.new(m.state.config.ethProvider)
-  without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
-    return failure("Invalid MarketplaceAddress provided")
+  try:
+    let provider = JsonRpcProvider.new(m.state.config.ethProvider)
+    without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
+      return failure("Invalid MarketplaceAddress provided")
 
-  let marketplace = Marketplace.new(marketplaceAddress, provider)
-  m.market = some(OnChainMarket.new(marketplace))
-  return success()
+    let marketplace = Marketplace.new(marketplaceAddress, provider)
+    m.market = some(OnChainMarket.new(marketplace))
+    return success()
+  except JsonRpcProviderError as err:
+    return failure(err.msg)
 
 proc new(T: type MarketplaceService, state: State, clock: Clock): MarketplaceService =
   return MarketplaceService(state: state, market: none(OnChainMarket), clock: clock)
diff --git a/codexcrawler/services/marketplace/market.nim b/codexcrawler/services/marketplace/market.nim
index 64fa8c2..4dfc501 100644
--- a/codexcrawler/services/marketplace/market.nim
+++ b/codexcrawler/services/marketplace/market.nim
@@ -89,6 +89,8 @@ template convertEthersError(body) =
     body
   except EthersError as error:
     raiseMarketError(error.msgDetail)
+  except CatchableError as error:
+    raiseMarketError(error.msg)
 
 proc loadConfig(
     market: OnChainMarket
@@ -138,43 +140,43 @@ proc periodicity*(
     let period = config.proofs.period
     return Periodicity(seconds: period)
 
-proc proofTimeout*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError]).} =
+proc proofTimeout*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.proofs.timeout
 
-proc repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError]).} =
+proc repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.collateral.repairRewardPercentage
 
-proc requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError]).} =
+proc requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.requestDurationLimit
 
-proc proofDowntime*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError]).} =
+proc proofDowntime*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.proofs.downtime
 
-proc getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async: (raises: [CancelledError]).} =
+proc getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.getPointer(slotId, overrides)
 
-proc myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async: (raises: [CancelledError]).} =
+proc myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return await market.contract.myRequests
 
-proc mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async: (raises: [CancelledError]).} =
+proc mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let slots = await market.contract.mySlots()
     debug "Fetched my slots", numSlots = len(slots)
 
     return slots
 
-proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async: (raises: [CancelledError]).} =
+proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     debug "Requesting storage"
     await market.approveFunds(request.totalPrice())
@@ -182,7 +184,7 @@ proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async: (ra
 
 proc getRequest*(
     market: OnChainMarket, id: RequestId
-): Future[?StorageRequest] {.async: (raises: [CancelledError]).} =
+): Future[?StorageRequest] {.async: (raises: [CancelledError, MarketError]).} =
   let key = $id
 
   convertEthersError:
@@ -194,7 +196,7 @@ proc getRequest*(
 
 proc requestState*(
     market: OnChainMarket, requestId: RequestId
-): Future[?RequestState] {.async: (raises: [CancelledError]).} =
+): Future[?RequestState] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -202,22 +204,22 @@ proc requestState*(
     except Marketplace_UnknownRequest:
       return none RequestState
 
-proc slotState*(market: OnChainMarket, slotId: SlotId): Future[SlotState] {.async: (raises: [CancelledError]).} =
+proc slotState*(market: OnChainMarket, slotId: SlotId): Future[SlotState] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.slotState(slotId, overrides)
 
-proc getRequestEnd*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError]).} =
+proc getRequestEnd*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return (await market.contract.requestEnd(id)).uint64
 
-proc requestExpiresAt*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError]).} =
+proc requestExpiresAt*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return (await market.contract.requestExpiry(id)).uint64
 
 proc getHost(
     market: OnChainMarket, requestId: RequestId, slotIndex: uint64
-): Future[?Address] {.async: (raises: [CancelledError]).} =
+): Future[?Address] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let slotId = slotId(requestId, slotIndex)
     let address = await market.contract.getHost(slotId)
@@ -228,11 +230,11 @@ proc getHost(
 
 proc currentCollateral*(
     market: OnChainMarket, slotId: SlotId
-): Future[UInt256] {.async: (raises: [CancelledError]).} =
+): Future[UInt256] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return await market.contract.currentCollateral(slotId)
 
-proc getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async: (raises: [CancelledError]).} =
+proc getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       return some await market.contract.getActiveSlot(slotId)
@@ -245,7 +247,7 @@ proc fillSlot(
     slotIndex: uint64,
     proof: Groth16Proof,
     collateral: UInt256,
-) {.async: (raises: [CancelledError]).} =
+) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     logScope:
       requestId
@@ -259,11 +261,11 @@ proc fillSlot(
 proc freeSlot*(market: OnChainMarket, slotId: SlotId) {.async: (raises: [CancelledError]).} =
   raiseAssert("Not supported")
 
-proc withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async: (raises: [CancelledError]).} =
+proc withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract.withdrawFunds(requestId).confirm(1)
 
-proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError]).} =
+proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -271,7 +273,7 @@ proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async:
     except Marketplace_SlotIsFree:
       return false
 
-proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError]).} =
+proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -281,16 +283,16 @@ proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.asy
 
 proc getChallenge*(
     market: OnChainMarket, id: SlotId
-): Future[ProofChallenge] {.async: (raises: [CancelledError]).} =
+): Future[ProofChallenge] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.getChallenge(id, overrides)
 
-proc submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async: (raises: [CancelledError]).} =
+proc submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract.submitProof(id, proof).confirm(1)
 
-proc markProofAsMissing*(market: OnChainMarket, id: SlotId, period: Period) {.async: (raises: [CancelledError]).} =
+proc markProofAsMissing*(market: OnChainMarket, id: SlotId, period: Period) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract.markProofAsMissing(id, period).confirm(1)
 
@@ -309,7 +311,7 @@ proc canProofBeMarkedAsMissing*(
 
 proc reserveSlot*(
     market: OnChainMarket, requestId: RequestId, slotIndex: uint64
-) {.async: (raises: [CancelledError]).} =
+) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract
     .reserveSlot(
@@ -322,13 +324,13 @@ proc reserveSlot*(
 
 proc canReserveSlot*(
     market: OnChainMarket, requestId: RequestId, slotIndex: uint64
-): Future[bool] {.async: (raises: [CancelledError]).} =
+): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return await market.contract.canReserveSlot(requestId, slotIndex)
 
 proc subscribeRequests*(
     market: OnChainMarket, callback: OnRequest
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!StorageRequested) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in Request subscription", msg = eventErr.msg
@@ -342,7 +344,7 @@ proc subscribeRequests*(
 
 proc subscribeSlotFilled*(
     market: OnChainMarket, callback: OnSlotFilled
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!SlotFilled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in SlotFilled subscription", msg = eventErr.msg
@@ -359,7 +361,7 @@ proc subscribeSlotFilled*(
     requestId: RequestId,
     slotIndex: uint64,
     callback: OnSlotFilled,
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: uint64) =
     if eventRequestId == requestId and eventSlotIndex == slotIndex:
       callback(requestId, slotIndex)
@@ -369,7 +371,7 @@ proc subscribeSlotFilled*(
 
 proc subscribeSlotFreed*(
     market: OnChainMarket, callback: OnSlotFreed
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!SlotFreed) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in SlotFreed subscription", msg = eventErr.msg
@@ -383,7 +385,7 @@ proc subscribeSlotFreed*(
 
 proc subscribeSlotReservationsFull*(
     market: OnChainMarket, callback: OnSlotReservationsFull
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!SlotReservationsFull) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in SlotReservationsFull subscription",
@@ -398,7 +400,7 @@ proc subscribeSlotReservationsFull*(
 
 proc subscribeFulfillment(
     market: OnChainMarket, callback: OnFulfillment
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
@@ -412,7 +414,7 @@ proc subscribeFulfillment(
 
 proc subscribeFulfillment(
     market: OnChainMarket, requestId: RequestId, callback: OnFulfillment
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!RequestFulfilled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFulfillment subscription", msg = eventErr.msg
@@ -427,7 +429,7 @@ proc subscribeFulfillment(
 
 proc subscribeRequestCancelled*(
     market: OnChainMarket, callback: OnRequestCancelled
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestCancelled subscription", msg = eventErr.msg
@@ -441,7 +443,7 @@ proc subscribeRequestCancelled*(
 
 proc subscribeRequestCancelled*(
     market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!RequestCancelled) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestCancelled subscription", msg = eventErr.msg
@@ -456,7 +458,7 @@ proc subscribeRequestCancelled*(
 
 proc subscribeRequestFailed*(
     market: OnChainMarket, callback: OnRequestFailed
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFailed subscription", msg = eventErr.msg
@@ -470,7 +472,7 @@ proc subscribeRequestFailed*(
 
 proc subscribeRequestFailed*(
     market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!RequestFailed) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in RequestFailed subscription", msg = eventErr.msg
@@ -485,7 +487,7 @@ proc subscribeRequestFailed*(
 
 proc subscribeProofSubmission*(
     market: OnChainMarket, callback: OnProofSubmitted
-): Future[MarketSubscription] {.async: (raises: [CancelledError]).} =
+): Future[MarketSubscription] {.async: (raises: [CancelledError, MarketError]).} =
   proc onEvent(eventResult: ?!ProofSubmitted) {.upraises: [].} =
     without event =? eventResult, eventErr:
       error "There was an error in ProofSubmitted subscription", msg = eventErr.msg
@@ -497,18 +499,21 @@ proc subscribeProofSubmission*(
     let subscription = await market.contract.subscribe(ProofSubmitted, onEvent)
     return OnChainMarketSubscription(eventSubscription: subscription)
 
-proc unsubscribe*(subscription: OnChainMarketSubscription) {.async: (raises: [CancelledError]).} =
-  await subscription.eventSubscription.unsubscribe()
+proc unsubscribe*(subscription: OnChainMarketSubscription) {.async: (raises: [CancelledError, MarketError]).} =
+  try:
+    await subscription.eventSubscription.unsubscribe()
+  except ProviderError as err:
+    raiseMarketError(err.msg)    
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, fromBlock: BlockTag
-): Future[seq[SlotFilled]] {.async: (raises: [CancelledError]).} =
+): Future[seq[SlotFilled]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return await market.contract.queryFilter(SlotFilled, fromBlock, BlockTag.latest)
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, blocksAgo: int
-): Future[seq[SlotFilled]] {.async: (raises: [CancelledError]).} =
+): Future[seq[SlotFilled]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
 
@@ -516,21 +521,21 @@ proc queryPastSlotFilledEvents*(
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, fromTime: int64
-): Future[seq[SlotFilled]] {.async: (raises: [CancelledError]).} =
+): Future[seq[SlotFilled]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
     return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock))
 
 proc queryPastStorageRequestedEvents*(
     market: OnChainMarket, fromBlock: BlockTag
-): Future[seq[StorageRequested]] {.async: (raises: [CancelledError]).} =
+): Future[seq[StorageRequested]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return
       await market.contract.queryFilter(StorageRequested, fromBlock, BlockTag.latest)
 
 proc queryPastStorageRequestedEvents*(
     market: OnChainMarket, blocksAgo: int
-): Future[seq[StorageRequested]] {.async: (raises: [CancelledError]).} =
+): Future[seq[StorageRequested]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo)
 
@@ -538,7 +543,7 @@ proc queryPastStorageRequestedEvents*(
 
 proc queryPastStorageRequestedEventsFromTime*(
     market: OnChainMarket, fromTime: int64
-): Future[seq[StorageRequested]] {.async: (raises: [CancelledError]).} =
+): Future[seq[StorageRequested]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime)
 
diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim
index d98c337..f29d6e1 100644
--- a/codexcrawler/state.nim
+++ b/codexcrawler/state.nim
@@ -45,7 +45,7 @@ proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async: (raise
 
   asyncSpawn worker()
 
-method whileRunning*(s: State, step: OnStep, delay: Duration) {.async, base.} =
+method whileRunning*(s: State, step: OnStep, delay: Duration) {.async: (raises: []), base.} =
   # We use a small delay before starting the workers because 'whileRunning' is likely called from
   # component 'start' methods, which are executed sequentially in arbitrary order (to prevent temporal coupling).
   # Worker steps might start raising events that other components haven't had time to subscribe to yet.
diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim
index aba1020..823871f 100644
--- a/codexcrawler/utils/asyncdataevent.nim
+++ b/codexcrawler/utils/asyncdataevent.nim
@@ -15,7 +15,7 @@ type
     queue: AsyncEventQueue[?T]
     subscriptions: seq[AsyncDataEventSubscription]
 
-  AsyncDataEventHandler*[T] = proc(data: T): Future[?!void]
+  AsyncDataEventHandler*[T] = proc(data: T): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).}
 
 proc newAsyncDataEvent*[T](): AsyncDataEvent[T] =
   AsyncDataEvent[T](
@@ -41,14 +41,17 @@ proc subscribe*[T](
   )
 
   proc listener() {.async: (raises: [CancelledError]).} =
-    while true:
-      let items = await event.queue.waitEvents(subscription.key)
-      for item in items:
-        if data =? item:
-          subscription.inHandler = true
-          subscription.lastResult = (await handler(data))
-          subscription.inHandler = false
-      subscription.fireEvent.fire()
+    try:
+      while true:
+        let items = await event.queue.waitEvents(subscription.key)
+        for item in items:
+          if data =? item:
+            subscription.inHandler = true
+            subscription.lastResult = (await handler(data))
+            subscription.inHandler = false
+        subscription.fireEvent.fire()
+    except AsyncEventQueueFullError as err:
+      raiseAssert("AsyncEventQueueFullError in asyncdataevent.listener()")
 
   subscription.listenFuture = listener()
 

From 6514e32919bfa86ca64b07ba56ed20e12bd497d9 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 15:30:12 +0200
Subject: [PATCH 19/31] tests are building

---
 Makefile                                      | 25 -------------------
 build.nims                                    |  4 +--
 .../components/testchainmetrics.nim           |  9 ++++---
 .../components/testdhtcrawler.nim             | 15 ++++++-----
 .../components/testdhtmetrics.nim             | 15 +++++++----
 .../codexcrawler/components/testnodestore.nim | 22 ++++++++++------
 .../components/testtimetracker.nim            | 18 ++++++++-----
 .../codexcrawler/components/testtodolist.nim  | 14 ++++++++---
 tests/codexcrawler/mocks/mocklist.nim         |  6 ++---
 tests/codexcrawler/mocks/mockstate.nim        |  2 +-
 .../codexcrawler/utils/testasyncdataevent.nim | 24 +++++++++---------
 tests/test.nimble                             |  2 +-
 tests/{test.nim => testCodexCrawler.nim}      |  0
 13 files changed, 80 insertions(+), 76 deletions(-)
 rename tests/{test.nim => testCodexCrawler.nim} (100%)

diff --git a/Makefile b/Makefile
index d80a0d1..4268fad 100644
--- a/Makefile
+++ b/Makefile
@@ -130,31 +130,6 @@ test: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
 
-# Builds and runs the smart contract tests
-testContracts: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
-
-# Builds and runs the integration tests
-testIntegration: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
-
-# Builds and runs all tests (except for Taiko L2 tests)
-testAll: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims
-
-# Builds and runs Taiko L2 tests
-testTaiko: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims
-
-# Builds and runs tool tests
-testTools: | cirdl
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims
-
 # nim-libbacktrace
 LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
 libbacktrace:
diff --git a/build.nims b/build.nims
index cfe8193..bf7ca43 100644
--- a/build.nims
+++ b/build.nims
@@ -34,9 +34,9 @@ task codexcrawler, "build codexcrawler binary":
     params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
 
 task testCodexcrawler, "Build & run Codex Crawler tests":
-  test "testCodexCrawler", params = "-d:codex_enable_proof_failures=true"
+  test "testCodexCrawler"
 
-task build, "build codex binary":
+task build, "build codex crawler binary":
   codexCrawlerTask()
 
 task test, "Run tests":
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index d3c1ea0..e1450b7 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -34,9 +34,12 @@ suite "ChainMetrics":
   teardown:
     state.checkAllUnsubscribed()
 
-  proc onStep() {.async.} =
-    (await state.steppers[0]()).tryGet()
-
+  proc onStep() {.async: (raises: []).} =
+    try:
+      (await state.steppers[0]()).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in onStep")
+    
   test "start should start stepper for config.requestCheckDelay minutes":
     check:
       state.delays.len == 1
diff --git a/tests/codexcrawler/components/testdhtcrawler.nim b/tests/codexcrawler/components/testdhtcrawler.nim
index 4ae4d21..3b2616e 100644
--- a/tests/codexcrawler/components/testdhtcrawler.nim
+++ b/tests/codexcrawler/components/testdhtcrawler.nim
@@ -35,9 +35,12 @@ suite "DhtCrawler":
   teardown:
     state.checkAllUnsubscribed()
 
-  proc onStep() {.async.} =
-    (await state.steppers[0]()).tryGet()
-
+  proc onStep() {.async: (raises: []).} =
+    try:
+      (await state.steppers[0]()).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in onStep")
+    
   proc responsive(nid: Nid): GetNeighborsResponse =
     GetNeighborsResponse(isResponsive: true, nodeIds: @[nid])
 
@@ -68,7 +71,7 @@ suite "DhtCrawler":
 
   test "nodes returned by getNeighbors are raised as nodesFound":
     var nodesFound = newSeq[Nid]()
-    proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       nodesFound = nids
       return success()
 
@@ -86,7 +89,7 @@ suite "DhtCrawler":
 
   test "responsive result from getNeighbors raises the node as successful dhtNodeCheck":
     var checkEvent = DhtNodeCheckEventData()
-    proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} =
+    proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
       checkEvent = event
       return success()
 
@@ -105,7 +108,7 @@ suite "DhtCrawler":
 
   test "unresponsive result from getNeighbors raises the node as unsuccessful dhtNodeCheck":
     var checkEvent = DhtNodeCheckEventData()
-    proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async.} =
+    proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
       checkEvent = event
       return success()
 
diff --git a/tests/codexcrawler/components/testdhtmetrics.nim b/tests/codexcrawler/components/testdhtmetrics.nim
index 98c48ab..3bbbb02 100644
--- a/tests/codexcrawler/components/testdhtmetrics.nim
+++ b/tests/codexcrawler/components/testdhtmetrics.nim
@@ -35,13 +35,18 @@ suite "DhtMetrics":
     (await dhtmetrics.stop()).tryGet()
     state.checkAllUnsubscribed()
 
-  proc fireDhtNodeCheckEvent(isOk: bool) {.async.} =
+  proc fireDhtNodeCheckEvent(isOk: bool) {.async: (raises: []).} =
     let event = DhtNodeCheckEventData(id: nid, isOk: isOk)
+    try:
+      (await state.events.dhtNodeCheck.fire(event)).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in fireDhtNodeCheckEvent")
 
-    (await state.events.dhtNodeCheck.fire(event)).tryGet()
-
-  proc fireNodesDeletedEvent(nids: seq[Nid]) {.async.} =
-    (await state.events.nodesDeleted.fire(nids)).tryGet()
+  proc fireNodesDeletedEvent(nids: seq[Nid]) {.async: (raises: []).} =
+    try:
+      (await state.events.nodesDeleted.fire(nids)).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in fireNodesDeletedEvent")
 
   test "dhtmetrics start should load both lists":
     check:
diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim
index 9ae8698..dc05e8a 100644
--- a/tests/codexcrawler/components/testnodestore.nim
+++ b/tests/codexcrawler/components/testnodestore.nim
@@ -38,11 +38,17 @@ suite "Nodestore":
     state.checkAllUnsubscribed()
     removeDir(dsPath)
 
-  proc fireNodeFoundEvent(nids: seq[Nid]) {.async.} =
-    (await state.events.nodesFound.fire(nids)).tryGet()
+  proc fireNodeFoundEvent(nids: seq[Nid]) {.async: (raises: []).} =
+    try:
+      (await state.events.nodesFound.fire(nids)).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in fireNodeFoundEvent")
 
-  proc fireCheckEvent(nid: Nid, isOk: bool) {.async.} =
-    (await state.events.dhtNodeCheck.fire(DhtNodeCheckEventData(id: nid, isOk: isOk))).tryGet()
+  proc fireCheckEvent(nid: Nid, isOk: bool) {.async: (raises: []).} =
+    try:
+      (await state.events.dhtNodeCheck.fire(DhtNodeCheckEventData(id: nid, isOk: isOk))).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in fireCheckEvent")
 
   test "nodeEntry encoding":
     let entry =
@@ -73,7 +79,7 @@ suite "Nodestore":
 
   test "nodesFound event should fire newNodesDiscovered":
     var newNodes = newSeq[Nid]()
-    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       newNodes = nids
       return success()
 
@@ -97,7 +103,7 @@ suite "Nodestore":
     var
       newNodes = newSeq[Nid]()
       count = 0
-    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       newNodes = nids
       inc count
       return success()
@@ -175,7 +181,7 @@ suite "Nodestore":
 
   test "deleteEntries fires nodesDeleted event":
     var deletedNodes = newSeq[Nid]()
-    proc onDeleted(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onDeleted(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       deletedNodes = nids
       return success()
 
@@ -245,7 +251,7 @@ suite "Nodestore":
 
   test "dhtNodeCheck event for non-existing node should fire nodesDeleted":
     var deletedNodes = newSeq[Nid]()
-    proc onDeleted(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onDeleted(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       deletedNodes = nids
       return success()
 
diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim
index 1e405ad..59c09eb 100644
--- a/tests/codexcrawler/components/testtimetracker.nim
+++ b/tests/codexcrawler/components/testtimetracker.nim
@@ -37,7 +37,7 @@ suite "TimeTracker":
 
     # Subscribe to nodesToRevisit event
     nodesToRevisitReceived = newSeq[Nid]()
-    proc onToRevisit(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onToRevisit(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       nodesToRevisitReceived = nids
       return success()
 
@@ -53,11 +53,17 @@ suite "TimeTracker":
     await state.events.nodesToRevisit.unsubscribe(sub)
     state.checkAllUnsubscribed()
 
-  proc onStepCheck() {.async.} =
-    (await state.steppers[0]()).tryGet()
+  proc onStepCheck() {.async: (raises: []).} =
+    try:
+      (await state.steppers[0]()).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in onStepCheck")
 
-  proc onStepRt() {.async.} =
-    (await state.steppers[1]()).tryGet()
+  proc onStepRt() {.async: (raises: []).} =
+    try:
+      (await state.steppers[1]()).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in onStepRt")
 
   proc createNodeInStore(lastVisit: uint64, firstInactive = 0.uint64): Nid =
     let entry =
@@ -120,7 +126,7 @@ suite "TimeTracker":
 
   test "onStep raises routingTable nodes as nodesFound":
     var nodesFound = newSeq[Nid]()
-    proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async.} =
+    proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
       nodesFound = nids
       return success()
 
diff --git a/tests/codexcrawler/components/testtodolist.nim b/tests/codexcrawler/components/testtodolist.nim
index b861f7b..da01b90 100644
--- a/tests/codexcrawler/components/testtodolist.nim
+++ b/tests/codexcrawler/components/testtodolist.nim
@@ -29,11 +29,17 @@ suite "TodoList":
     (await todo.stop()).tryGet()
     state.checkAllUnsubscribed()
 
-  proc fireNewNodesDiscoveredEvent(nids: seq[Nid]) {.async.} =
-    (await state.events.newNodesDiscovered.fire(nids)).tryGet()
+  proc fireNewNodesDiscoveredEvent(nids: seq[Nid]) {.async: (raises: []).} =
+    try:
+      (await state.events.newNodesDiscovered.fire(nids)).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in fireNewNodesDiscoveredEvent")
 
-  proc fireNodesToRevisitEvent(nids: seq[Nid]) {.async.} =
-    (await state.events.nodesToRevisit.fire(nids)).tryGet()
+  proc fireNodesToRevisitEvent(nids: seq[Nid]) {.async: (raises: []).} =
+    try:
+      (await state.events.nodesToRevisit.fire(nids)).tryGet()
+    except CatchableError:
+      raiseAssert("CatchableError in fireNodesToRevisitEvent")
 
   test "discovered nodes are added to todo list":
     await fireNewNodesDiscoveredEvent(@[nid])
diff --git a/tests/codexcrawler/mocks/mocklist.nim b/tests/codexcrawler/mocks/mocklist.nim
index dfbd414..5cee33e 100644
--- a/tests/codexcrawler/mocks/mocklist.nim
+++ b/tests/codexcrawler/mocks/mocklist.nim
@@ -12,17 +12,17 @@ type MockList* = ref object of List
   removeSuccess*: bool
   length*: int
 
-method load*(this: MockList): Future[?!void] {.async.} =
+method load*(this: MockList): Future[?!void] {.async: (raises: [CancelledError]).} =
   this.loadCalled = true
   return success()
 
-method add*(this: MockList, nid: Nid): Future[?!void] {.async.} =
+method add*(this: MockList, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
   this.added.add(nid)
   if this.addSuccess:
     return success()
   return failure("test failure")
 
-method remove*(this: MockList, nid: Nid): Future[?!void] {.async.} =
+method remove*(this: MockList, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
   this.removed.add(nid)
   if this.removeSuccess:
     return success()
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index d805f0e..27026e1 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -15,7 +15,7 @@ proc checkAllUnsubscribed*(s: MockState) =
     s.events.dhtNodeCheck.listeners == 0
     s.events.nodesToRevisit.listeners == 0
 
-method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} =
+method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async: (raises: [CancelledError]).} =
   s.steppers.add(step)
   s.delays.add(delay)
 
diff --git a/tests/codexcrawler/utils/testasyncdataevent.nim b/tests/codexcrawler/utils/testasyncdataevent.nim
index 34cf1b2..ef24b1b 100644
--- a/tests/codexcrawler/utils/testasyncdataevent.nim
+++ b/tests/codexcrawler/utils/testasyncdataevent.nim
@@ -20,7 +20,7 @@ suite "AsyncDataEvent":
 
   test "Successful event":
     var data = ""
-    proc eventHandler(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data = e.s
       success()
 
@@ -34,7 +34,7 @@ suite "AsyncDataEvent":
 
   test "Multiple events":
     var counter = 0
-    proc eventHandler(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       inc counter
       success()
 
@@ -54,15 +54,15 @@ suite "AsyncDataEvent":
       data1 = ""
       data2 = ""
       data3 = ""
-    proc eventHandler1(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler1(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data1 = e.s
       success()
 
-    proc eventHandler2(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler2(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data2 = e.s
       success()
 
-    proc eventHandler3(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler3(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data3 = e.s
       success()
 
@@ -82,7 +82,7 @@ suite "AsyncDataEvent":
     await event.unsubscribe(sub3)
 
   test "Failed event preserves error message":
-    proc eventHandler(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       failure(msg)
 
     let s = event.subscribe(eventHandler)
@@ -100,15 +100,15 @@ suite "AsyncDataEvent":
       data2 = ""
       data3 = ""
 
-    proc handler1(e: ExampleData): Future[?!void] {.async.} =
+    proc handler1(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data1 = e.s
       success()
 
-    proc handler2(e: ExampleData): Future[?!void] {.async.} =
+    proc handler2(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data2 = e.s
       success()
 
-    proc handler3(e: ExampleData): Future[?!void] {.async.} =
+    proc handler3(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       data3 = e.s
       success()
 
@@ -134,18 +134,18 @@ suite "AsyncDataEvent":
       isOK(await event.fire(ExampleData(s: msg)))
 
   test "Can unsubscribe in handler":
-    proc doNothing() {.async, closure.} =
+    proc doNothing() {.async: (raises: [CancelledError]), closure.} =
       await sleepAsync(1.millis)
 
     var callback = doNothing
 
-    proc eventHandler(e: ExampleData): Future[?!void] {.async.} =
+    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
       await callback()
       success()
 
     let s = event.subscribe(eventHandler)
 
-    proc doUnsubscribe() {.async.} =
+    proc doUnsubscribe() {.async: (raises: [CancelledError]).} =
       await event.unsubscribe(s)
 
     callback = doUnsubscribe
diff --git a/tests/test.nimble b/tests/test.nimble
index f7596d1..e128ee7 100644
--- a/tests/test.nimble
+++ b/tests/test.nimble
@@ -11,4 +11,4 @@ requires "asynctest >= 0.5.2 & < 0.6.0"
 requires "unittest2 <= 0.3.0"
 
 task test, "Run tests":
-  exec "nim c -r test.nim"
+  exec "nim c -r testCodexCrawler.nim"
diff --git a/tests/test.nim b/tests/testCodexCrawler.nim
similarity index 100%
rename from tests/test.nim
rename to tests/testCodexCrawler.nim

From 1b7137a75eadb8136fb5692726457d3bf874be75 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 15:40:26 +0200
Subject: [PATCH 20/31] fixes tests

---
 tests/codexcrawler/mocks/mockstate.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index 27026e1..cf2f151 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -15,7 +15,7 @@ proc checkAllUnsubscribed*(s: MockState) =
     s.events.dhtNodeCheck.listeners == 0
     s.events.nodesToRevisit.listeners == 0
 
-method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async: (raises: [CancelledError]).} =
+method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async: (raises: []).} =
   s.steppers.add(step)
   s.delays.add(delay)
 

From 5f03dc1f35ba3c910de44be960cddf757290eb25 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 15:41:19 +0200
Subject: [PATCH 21/31] attempt to fix docker build

---
 docker/crawler.Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile
index 455dfbc..c43416a 100644
--- a/docker/crawler.Dockerfile
+++ b/docker/crawler.Dockerfile
@@ -19,8 +19,8 @@ RUN apt-get update && apt-get install -y cmake build-essential
 
 WORKDIR ${BUILD_HOME}
 COPY . .
-RUN nimble install nimble
-RUN nimble build
+RUN make -j ${MAKE_PARALLEL} update
+RUN make -j ${MAKE_PARALLEL}
 
 # Create
 FROM ${IMAGE}

From 8b87f06ee2bf16749ec081a97cbc551e11460aa9 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 15:59:43 +0200
Subject: [PATCH 22/31] trying to fix docker build

---
 docker/crawler.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile
index c43416a..bdd8b4e 100644
--- a/docker/crawler.Dockerfile
+++ b/docker/crawler.Dockerfile
@@ -15,7 +15,7 @@ ARG NIMFLAGS
 ARG USE_LIBBACKTRACE
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y cmake build-essential
+RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential
 
 WORKDIR ${BUILD_HOME}
 COPY . .

From 0228e7614f11d4532c255d21c6db6957ba3d8b9e Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 16:16:41 +0200
Subject: [PATCH 23/31] formatting

---
 codexcrawler/application.nim                  |  4 +-
 codexcrawler/component.nim                    |  8 +-
 codexcrawler/components/dhtmetrics.nim        |  8 +-
 codexcrawler/components/nodestore.nim         | 24 ++++--
 codexcrawler/components/todolist.nim          |  4 +-
 codexcrawler/installer.nim                    |  4 +-
 codexcrawler/list.nim                         | 12 ++-
 codexcrawler/services/dht.nim                 |  4 +-
 codexcrawler/services/marketplace.nim         |  4 +-
 codexcrawler/services/marketplace/market.nim  | 82 ++++++++++++++-----
 codexcrawler/state.nim                        |  8 +-
 codexcrawler/utils/asyncdataevent.nim         |  7 +-
 .../components/testchainmetrics.nim           |  2 +-
 .../components/testdhtcrawler.nim             | 14 +++-
 .../codexcrawler/components/testnodestore.nim | 16 +++-
 .../components/testtimetracker.nim            |  8 +-
 tests/codexcrawler/mocks/mocklist.nim         |  8 +-
 tests/codexcrawler/mocks/mockstate.nim        |  4 +-
 .../codexcrawler/utils/testasyncdataevent.nim | 40 ++++++---
 19 files changed, 194 insertions(+), 67 deletions(-)

diff --git a/codexcrawler/application.nim b/codexcrawler/application.nim
index d3b34dd..889b054 100644
--- a/codexcrawler/application.nim
+++ b/codexcrawler/application.nim
@@ -18,7 +18,9 @@ type Application* = ref object
   state: State
   components: seq[Component]
 
-proc initializeApp(app: Application, config: Config): Future[?!void] {.async: (raises: [CancelledError]).} =
+proc initializeApp(
+    app: Application, config: Config
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   app.state = State(
     status: ApplicationStatus.Running,
     config: config,
diff --git a/codexcrawler/component.nim b/codexcrawler/component.nim
index 623e757..9be452c 100644
--- a/codexcrawler/component.nim
+++ b/codexcrawler/component.nim
@@ -3,12 +3,16 @@ import pkg/questionable/results
 
 type Component* = ref object of RootObj
 
-method awake*(c: Component): Future[?!void] {.async: (raises: [CancelledError]), base.} =
+method awake*(
+    c: Component
+): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   # Awake is called on all components in an unspecified order.
   # Use this method to subscribe/connect to other components.
   return success()
 
-method start*(c: Component): Future[?!void] {.async: (raises: [CancelledError]), base.} =
+method start*(
+    c: Component
+): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   # Start is called on all components in an unspecified order.
   # Is is guaranteed that all components have already successfulled handled 'awake'.
   # Use this method to begin the work of this component.
diff --git a/codexcrawler/components/dhtmetrics.nim b/codexcrawler/components/dhtmetrics.nim
index 54d4d33..847e1b2 100644
--- a/codexcrawler/components/dhtmetrics.nim
+++ b/codexcrawler/components/dhtmetrics.nim
@@ -38,7 +38,9 @@ proc handleCheckEvent(
   d.updateMetrics()
   return success()
 
-proc handleDeleteEvent(d: DhtMetrics, nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+proc handleDeleteEvent(
+    d: DhtMetrics, nids: seq[Nid]
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   for nid in nids:
     ?await d.ok.remove(nid)
     ?await d.nok.remove(nid)
@@ -46,7 +48,9 @@ proc handleDeleteEvent(d: DhtMetrics, nids: seq[Nid]): Future[?!void] {.async: (
   return success()
 
 method awake*(d: DhtMetrics): Future[?!void] {.async: (raises: [CancelledError]).} =
-  proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
+  proc onCheck(
+      event: DhtNodeCheckEventData
+  ): Future[?!void] {.async: (raises: [CancelledError]).} =
     await d.handleCheckEvent(event)
 
   proc onDelete(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
diff --git a/codexcrawler/components/nodestore.nim b/codexcrawler/components/nodestore.nim
index 7d68fbe..e502428 100644
--- a/codexcrawler/components/nodestore.nim
+++ b/codexcrawler/components/nodestore.nim
@@ -79,7 +79,9 @@ proc decode*(T: type NodeEntry, bytes: seq[byte]): ?!T =
   except ValueError as err:
     return failure(err.msg)
 
-proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async: (raises: [CancelledError]).} =
+proc storeNodeIsNew(
+    s: NodeStore, nid: Nid
+): Future[?!bool] {.async: (raises: [CancelledError]).} =
   without key =? Key.init(nodestoreName / $nid), err:
     error "failed to format key", err = err.msg
     return failure(err)
@@ -94,7 +96,9 @@ proc storeNodeIsNew(s: NodeStore, nid: Nid): Future[?!bool] {.async: (raises: [C
 
   return success(not exists)
 
-proc fireNewNodesDiscovered(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+proc fireNewNodesDiscovered(
+    s: NodeStore, nids: seq[Nid]
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   await s.state.events.newNodesDiscovered.fire(nids)
 
 proc fireNodesDeleted(
@@ -102,7 +106,9 @@ proc fireNodesDeleted(
 ): Future[?!void] {.async: (raises: []).} =
   await s.state.events.nodesDeleted.fire(nids)
 
-proc processFoundNodes(s: NodeStore, nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+proc processFoundNodes(
+    s: NodeStore, nids: seq[Nid]
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   var newNodes = newSeq[Nid]()
   for nid in nids:
     without isNew =? (await s.storeNodeIsNew(nid)), err:
@@ -145,7 +151,9 @@ proc processNodeCheck(
   ?await s.store.put(key, entry)
   return success()
 
-proc deleteEntry(s: NodeStore, nid: Nid): Future[?!bool] {.async: (raises: [CancelledError]).} =
+proc deleteEntry(
+    s: NodeStore, nid: Nid
+): Future[?!bool] {.async: (raises: [CancelledError]).} =
   without key =? Key.init(nodestoreName / $nid), err:
     error "failed to format key", err = err.msg
     return failure(err)
@@ -208,10 +216,14 @@ method deleteEntries*(
 method start*(s: NodeStore): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "starting..."
 
-  proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+  proc onNodesFound(
+      nids: seq[Nid]
+  ): Future[?!void] {.async: (raises: [CancelledError]).} =
     return await s.processFoundNodes(nids)
 
-  proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
+  proc onCheck(
+      event: DhtNodeCheckEventData
+  ): Future[?!void] {.async: (raises: [CancelledError]).} =
     return await s.processNodeCheck(event)
 
   s.subFound = s.state.events.nodesFound.subscribe(onNodesFound)
diff --git a/codexcrawler/components/todolist.nim b/codexcrawler/components/todolist.nim
index 2d4093e..8e9beca 100644
--- a/codexcrawler/components/todolist.nim
+++ b/codexcrawler/components/todolist.nim
@@ -57,7 +57,9 @@ method pop*(t: TodoList): Future[?!Nid] {.async: (raises: []), base.} =
 method awake*(t: TodoList): Future[?!void] {.async: (raises: [CancelledError]).} =
   info "initializing..."
 
-  proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+  proc onNewNodes(
+      nids: seq[Nid]
+  ): Future[?!void] {.async: (raises: [CancelledError]).} =
     t.addNodes(nids)
     return success()
 
diff --git a/codexcrawler/installer.nim b/codexcrawler/installer.nim
index e8a12f1..a29a836 100644
--- a/codexcrawler/installer.nim
+++ b/codexcrawler/installer.nim
@@ -18,7 +18,9 @@ import ./components/chainmetrics
 import ./components/chaincrawler
 import ./components/requeststore
 
-proc createComponents*(state: State): Future[?!seq[Component]] {.async: (raises: [CancelledError]).} =
+proc createComponents*(
+    state: State
+): Future[?!seq[Component]] {.async: (raises: [CancelledError]).} =
   var components: seq[Component] = newSeq[Component]()
   let clock = createClock()
 
diff --git a/codexcrawler/list.nim b/codexcrawler/list.nim
index d19ebdf..8587dc4 100644
--- a/codexcrawler/list.nim
+++ b/codexcrawler/list.nim
@@ -35,7 +35,9 @@ proc decode(T: type Nid, bytes: seq[byte]): ?!T =
   except ValueError as err:
     return failure(err.msg)
 
-proc saveItem(this: List, item: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
+proc saveItem(
+    this: List, item: Nid
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   without itemKey =? Key.init(this.name / $item), err:
     return failure(err)
   ?await this.store.put(itemKey, item)
@@ -61,7 +63,9 @@ method load*(this: List): Future[?!void] {.async: (raises: [CancelledError]), ba
 proc contains*(this: List, nid: Nid): bool =
   this.items.anyIt(it == nid)
 
-method add*(this: List, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]), base.} =
+method add*(
+    this: List, nid: Nid
+): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   if this.contains(nid):
     return success()
 
@@ -72,7 +76,9 @@ method add*(this: List, nid: Nid): Future[?!void] {.async: (raises: [CancelledEr
 
   return success()
 
-method remove*(this: List, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]), base.} =
+method remove*(
+    this: List, nid: Nid
+): Future[?!void] {.async: (raises: [CancelledError]), base.} =
   if not this.contains(nid):
     return success()
 
diff --git a/codexcrawler/services/dht.nim b/codexcrawler/services/dht.nim
index 254f3a3..19d6661 100644
--- a/codexcrawler/services/dht.nim
+++ b/codexcrawler/services/dht.nim
@@ -75,7 +75,9 @@ method getNeighbors*(
   except CatchableError as exc:
     return failure(exc.msg)
 
-proc findPeer*(d: Dht, peerId: PeerId): Future[?PeerRecord] {.async: (raises: [CancelledError]).} =
+proc findPeer*(
+    d: Dht, peerId: PeerId
+): Future[?PeerRecord] {.async: (raises: [CancelledError]).} =
   trace "protocol.resolve..."
   try:
     let node = await d.protocol.resolve(toNodeId(peerId))
diff --git a/codexcrawler/services/marketplace.nim b/codexcrawler/services/marketplace.nim
index fd2c4f3..08819f8 100644
--- a/codexcrawler/services/marketplace.nim
+++ b/codexcrawler/services/marketplace.nim
@@ -109,7 +109,9 @@ method getRequestInfo*(
   else:
     notStarted()
 
-method awake*(m: MarketplaceService): Future[?!void] {.async: (raises: [CancelledError]).} =
+method awake*(
+    m: MarketplaceService
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   try:
     let provider = JsonRpcProvider.new(m.state.config.ethProvider)
     without marketplaceAddress =? Address.init(m.state.config.marketplaceAddress):
diff --git a/codexcrawler/services/marketplace/market.nim b/codexcrawler/services/marketplace/market.nim
index 4dfc501..f0a2f0b 100644
--- a/codexcrawler/services/marketplace/market.nim
+++ b/codexcrawler/services/marketplace/market.nim
@@ -123,7 +123,9 @@ proc config(
 
   return resolvedConfig
 
-proc approveFunds(market: OnChainMarket, amount: UInt256) {.async: (raises: [CancelledError]).} =
+proc approveFunds(
+    market: OnChainMarket, amount: UInt256
+) {.async: (raises: [CancelledError]).} =
   raiseAssert("Not available: approveFunds")
 
 proc getZkeyHash*(
@@ -140,43 +142,59 @@ proc periodicity*(
     let period = config.proofs.period
     return Periodicity(seconds: period)
 
-proc proofTimeout*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
+proc proofTimeout*(
+    market: OnChainMarket
+): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.proofs.timeout
 
-proc repairRewardPercentage*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
+proc repairRewardPercentage*(
+    market: OnChainMarket
+): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.collateral.repairRewardPercentage
 
-proc requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
+proc requestDurationLimit*(
+    market: OnChainMarket
+): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.requestDurationLimit
 
-proc proofDowntime*(market: OnChainMarket): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
+proc proofDowntime*(
+    market: OnChainMarket
+): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let config = await market.config()
     return config.proofs.downtime
 
-proc getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
+proc getPointer*(
+    market: OnChainMarket, slotId: SlotId
+): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.getPointer(slotId, overrides)
 
-proc myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async: (raises: [CancelledError, MarketError]).} =
+proc myRequests*(
+    market: OnChainMarket
+): Future[seq[RequestId]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return await market.contract.myRequests
 
-proc mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async: (raises: [CancelledError, MarketError]).} =
+proc mySlots*(
+    market: OnChainMarket
+): Future[seq[SlotId]] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let slots = await market.contract.mySlots()
     debug "Fetched my slots", numSlots = len(slots)
 
     return slots
 
-proc requestStorage(market: OnChainMarket, request: StorageRequest) {.async: (raises: [CancelledError, MarketError]).} =
+proc requestStorage(
+    market: OnChainMarket, request: StorageRequest
+) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     debug "Requesting storage"
     await market.approveFunds(request.totalPrice())
@@ -204,16 +222,22 @@ proc requestState*(
     except Marketplace_UnknownRequest:
       return none RequestState
 
-proc slotState*(market: OnChainMarket, slotId: SlotId): Future[SlotState] {.async: (raises: [CancelledError, MarketError]).} =
+proc slotState*(
+    market: OnChainMarket, slotId: SlotId
+): Future[SlotState] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.slotState(slotId, overrides)
 
-proc getRequestEnd*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
+proc getRequestEnd*(
+    market: OnChainMarket, id: RequestId
+): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return (await market.contract.requestEnd(id)).uint64
 
-proc requestExpiresAt*(market: OnChainMarket, id: RequestId): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
+proc requestExpiresAt*(
+    market: OnChainMarket, id: RequestId
+): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     return (await market.contract.requestExpiry(id)).uint64
 
@@ -234,7 +258,9 @@ proc currentCollateral*(
   convertEthersError:
     return await market.contract.currentCollateral(slotId)
 
-proc getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async: (raises: [CancelledError, MarketError]).} =
+proc getActiveSlot*(
+    market: OnChainMarket, slotId: SlotId
+): Future[?Slot] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       return some await market.contract.getActiveSlot(slotId)
@@ -258,14 +284,20 @@ proc fillSlot(
     discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(1)
     trace "fillSlot transaction completed"
 
-proc freeSlot*(market: OnChainMarket, slotId: SlotId) {.async: (raises: [CancelledError]).} =
+proc freeSlot*(
+    market: OnChainMarket, slotId: SlotId
+) {.async: (raises: [CancelledError]).} =
   raiseAssert("Not supported")
 
-proc withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async: (raises: [CancelledError, MarketError]).} =
+proc withdrawFunds(
+    market: OnChainMarket, requestId: RequestId
+) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract.withdrawFunds(requestId).confirm(1)
 
-proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
+proc isProofRequired*(
+    market: OnChainMarket, id: SlotId
+): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -273,7 +305,9 @@ proc isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async:
     except Marketplace_SlotIsFree:
       return false
 
-proc willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
+proc willProofBeRequired*(
+    market: OnChainMarket, id: SlotId
+): Future[bool] {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     try:
       let overrides = CallOverrides(blockTag: some BlockTag.pending)
@@ -288,11 +322,15 @@ proc getChallenge*(
     let overrides = CallOverrides(blockTag: some BlockTag.pending)
     return await market.contract.getChallenge(id, overrides)
 
-proc submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async: (raises: [CancelledError, MarketError]).} =
+proc submitProof*(
+    market: OnChainMarket, id: SlotId, proof: Groth16Proof
+) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract.submitProof(id, proof).confirm(1)
 
-proc markProofAsMissing*(market: OnChainMarket, id: SlotId, period: Period) {.async: (raises: [CancelledError, MarketError]).} =
+proc markProofAsMissing*(
+    market: OnChainMarket, id: SlotId, period: Period
+) {.async: (raises: [CancelledError, MarketError]).} =
   convertEthersError:
     discard await market.contract.markProofAsMissing(id, period).confirm(1)
 
@@ -499,11 +537,13 @@ proc subscribeProofSubmission*(
     let subscription = await market.contract.subscribe(ProofSubmitted, onEvent)
     return OnChainMarketSubscription(eventSubscription: subscription)
 
-proc unsubscribe*(subscription: OnChainMarketSubscription) {.async: (raises: [CancelledError, MarketError]).} =
+proc unsubscribe*(
+    subscription: OnChainMarketSubscription
+) {.async: (raises: [CancelledError, MarketError]).} =
   try:
     await subscription.eventSubscription.unsubscribe()
   except ProviderError as err:
-    raiseMarketError(err.msg)    
+    raiseMarketError(err.msg)
 
 proc queryPastSlotFilledEvents*(
     market: OnChainMarket, fromBlock: BlockTag
diff --git a/codexcrawler/state.nim b/codexcrawler/state.nim
index f29d6e1..d0d8056 100644
--- a/codexcrawler/state.nim
+++ b/codexcrawler/state.nim
@@ -33,7 +33,9 @@ type
     config*: Config
     events*: Events
 
-proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async: (raises: [CancelledError]).} =
+proc delayedWorkerStart(
+    s: State, step: OnStep, delay: Duration
+) {.async: (raises: [CancelledError]).} =
   await sleepAsync(1.seconds)
 
   proc worker(): Future[void] {.async: (raises: [CancelledError]).} =
@@ -45,7 +47,9 @@ proc delayedWorkerStart(s: State, step: OnStep, delay: Duration) {.async: (raise
 
   asyncSpawn worker()
 
-method whileRunning*(s: State, step: OnStep, delay: Duration) {.async: (raises: []), base.} =
+method whileRunning*(
+    s: State, step: OnStep, delay: Duration
+) {.async: (raises: []), base.} =
   # We use a small delay before starting the workers because 'whileRunning' is likely called from
   # component 'start' methods, which are executed sequentially in arbitrary order (to prevent temporal coupling).
   # Worker steps might start raising events that other components haven't had time to subscribe to yet.
diff --git a/codexcrawler/utils/asyncdataevent.nim b/codexcrawler/utils/asyncdataevent.nim
index 823871f..8a47838 100644
--- a/codexcrawler/utils/asyncdataevent.nim
+++ b/codexcrawler/utils/asyncdataevent.nim
@@ -15,7 +15,8 @@ type
     queue: AsyncEventQueue[?T]
     subscriptions: seq[AsyncDataEventSubscription]
 
-  AsyncDataEventHandler*[T] = proc(data: T): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).}
+  AsyncDataEventHandler*[T] =
+    proc(data: T): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).}
 
 proc newAsyncDataEvent*[T](): AsyncDataEvent[T] =
   AsyncDataEvent[T](
@@ -90,7 +91,9 @@ proc unsubscribe*[T](
   else:
     await event.performUnsubscribe(subscription)
 
-proc unsubscribeAll*[T](event: AsyncDataEvent[T]) {.async: (raises: [CancelledError]).} =
+proc unsubscribeAll*[T](
+    event: AsyncDataEvent[T]
+) {.async: (raises: [CancelledError]).} =
   let all = event.subscriptions
   for subscription in all:
     await event.unsubscribe(subscription)
diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim
index e1450b7..cbeec8c 100644
--- a/tests/codexcrawler/components/testchainmetrics.nim
+++ b/tests/codexcrawler/components/testchainmetrics.nim
@@ -39,7 +39,7 @@ suite "ChainMetrics":
       (await state.steppers[0]()).tryGet()
     except CatchableError:
       raiseAssert("CatchableError in onStep")
-    
+
   test "start should start stepper for config.requestCheckDelay minutes":
     check:
       state.delays.len == 1
diff --git a/tests/codexcrawler/components/testdhtcrawler.nim b/tests/codexcrawler/components/testdhtcrawler.nim
index 3b2616e..1c38697 100644
--- a/tests/codexcrawler/components/testdhtcrawler.nim
+++ b/tests/codexcrawler/components/testdhtcrawler.nim
@@ -40,7 +40,7 @@ suite "DhtCrawler":
       (await state.steppers[0]()).tryGet()
     except CatchableError:
       raiseAssert("CatchableError in onStep")
-    
+
   proc responsive(nid: Nid): GetNeighborsResponse =
     GetNeighborsResponse(isResponsive: true, nodeIds: @[nid])
 
@@ -71,7 +71,9 @@ suite "DhtCrawler":
 
   test "nodes returned by getNeighbors are raised as nodesFound":
     var nodesFound = newSeq[Nid]()
-    proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onNodesFound(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       nodesFound = nids
       return success()
 
@@ -89,7 +91,9 @@ suite "DhtCrawler":
 
   test "responsive result from getNeighbors raises the node as successful dhtNodeCheck":
     var checkEvent = DhtNodeCheckEventData()
-    proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onCheck(
+        event: DhtNodeCheckEventData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       checkEvent = event
       return success()
 
@@ -108,7 +112,9 @@ suite "DhtCrawler":
 
   test "unresponsive result from getNeighbors raises the node as unsuccessful dhtNodeCheck":
     var checkEvent = DhtNodeCheckEventData()
-    proc onCheck(event: DhtNodeCheckEventData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onCheck(
+        event: DhtNodeCheckEventData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       checkEvent = event
       return success()
 
diff --git a/tests/codexcrawler/components/testnodestore.nim b/tests/codexcrawler/components/testnodestore.nim
index dc05e8a..29709fb 100644
--- a/tests/codexcrawler/components/testnodestore.nim
+++ b/tests/codexcrawler/components/testnodestore.nim
@@ -79,7 +79,9 @@ suite "Nodestore":
 
   test "nodesFound event should fire newNodesDiscovered":
     var newNodes = newSeq[Nid]()
-    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onNewNodes(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       newNodes = nids
       return success()
 
@@ -103,7 +105,9 @@ suite "Nodestore":
     var
       newNodes = newSeq[Nid]()
       count = 0
-    proc onNewNodes(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onNewNodes(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       newNodes = nids
       inc count
       return success()
@@ -181,7 +185,9 @@ suite "Nodestore":
 
   test "deleteEntries fires nodesDeleted event":
     var deletedNodes = newSeq[Nid]()
-    proc onDeleted(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onDeleted(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       deletedNodes = nids
       return success()
 
@@ -251,7 +257,9 @@ suite "Nodestore":
 
   test "dhtNodeCheck event for non-existing node should fire nodesDeleted":
     var deletedNodes = newSeq[Nid]()
-    proc onDeleted(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onDeleted(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       deletedNodes = nids
       return success()
 
diff --git a/tests/codexcrawler/components/testtimetracker.nim b/tests/codexcrawler/components/testtimetracker.nim
index 59c09eb..fa2421e 100644
--- a/tests/codexcrawler/components/testtimetracker.nim
+++ b/tests/codexcrawler/components/testtimetracker.nim
@@ -37,7 +37,9 @@ suite "TimeTracker":
 
     # Subscribe to nodesToRevisit event
     nodesToRevisitReceived = newSeq[Nid]()
-    proc onToRevisit(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onToRevisit(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       nodesToRevisitReceived = nids
       return success()
 
@@ -126,7 +128,9 @@ suite "TimeTracker":
 
   test "onStep raises routingTable nodes as nodesFound":
     var nodesFound = newSeq[Nid]()
-    proc onNodesFound(nids: seq[Nid]): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc onNodesFound(
+        nids: seq[Nid]
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       nodesFound = nids
       return success()
 
diff --git a/tests/codexcrawler/mocks/mocklist.nim b/tests/codexcrawler/mocks/mocklist.nim
index 5cee33e..721af59 100644
--- a/tests/codexcrawler/mocks/mocklist.nim
+++ b/tests/codexcrawler/mocks/mocklist.nim
@@ -16,13 +16,17 @@ method load*(this: MockList): Future[?!void] {.async: (raises: [CancelledError])
   this.loadCalled = true
   return success()
 
-method add*(this: MockList, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
+method add*(
+    this: MockList, nid: Nid
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   this.added.add(nid)
   if this.addSuccess:
     return success()
   return failure("test failure")
 
-method remove*(this: MockList, nid: Nid): Future[?!void] {.async: (raises: [CancelledError]).} =
+method remove*(
+    this: MockList, nid: Nid
+): Future[?!void] {.async: (raises: [CancelledError]).} =
   this.removed.add(nid)
   if this.removeSuccess:
     return success()
diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim
index cf2f151..ad92c7d 100644
--- a/tests/codexcrawler/mocks/mockstate.nim
+++ b/tests/codexcrawler/mocks/mockstate.nim
@@ -15,7 +15,9 @@ proc checkAllUnsubscribed*(s: MockState) =
     s.events.dhtNodeCheck.listeners == 0
     s.events.nodesToRevisit.listeners == 0
 
-method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async: (raises: []).} =
+method whileRunning*(
+    s: MockState, step: OnStep, delay: Duration
+) {.async: (raises: []).} =
   s.steppers.add(step)
   s.delays.add(delay)
 
diff --git a/tests/codexcrawler/utils/testasyncdataevent.nim b/tests/codexcrawler/utils/testasyncdataevent.nim
index ef24b1b..92ca859 100644
--- a/tests/codexcrawler/utils/testasyncdataevent.nim
+++ b/tests/codexcrawler/utils/testasyncdataevent.nim
@@ -20,7 +20,9 @@ suite "AsyncDataEvent":
 
   test "Successful event":
     var data = ""
-    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data = e.s
       success()
 
@@ -34,7 +36,9 @@ suite "AsyncDataEvent":
 
   test "Multiple events":
     var counter = 0
-    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       inc counter
       success()
 
@@ -54,15 +58,21 @@ suite "AsyncDataEvent":
       data1 = ""
       data2 = ""
       data3 = ""
-    proc eventHandler1(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler1(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data1 = e.s
       success()
 
-    proc eventHandler2(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler2(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data2 = e.s
       success()
 
-    proc eventHandler3(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler3(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data3 = e.s
       success()
 
@@ -82,7 +92,9 @@ suite "AsyncDataEvent":
     await event.unsubscribe(sub3)
 
   test "Failed event preserves error message":
-    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       failure(msg)
 
     let s = event.subscribe(eventHandler)
@@ -100,15 +112,21 @@ suite "AsyncDataEvent":
       data2 = ""
       data3 = ""
 
-    proc handler1(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc handler1(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data1 = e.s
       success()
 
-    proc handler2(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc handler2(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data2 = e.s
       success()
 
-    proc handler3(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc handler3(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       data3 = e.s
       success()
 
@@ -139,7 +157,9 @@ suite "AsyncDataEvent":
 
     var callback = doNothing
 
-    proc eventHandler(e: ExampleData): Future[?!void] {.async: (raises: [CancelledError]).} =
+    proc eventHandler(
+        e: ExampleData
+    ): Future[?!void] {.async: (raises: [CancelledError]).} =
       await callback()
       success()
 

From 27fcf142a6cb3bcd24430afa864ff38f6e183b6d Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 16:19:47 +0200
Subject: [PATCH 24/31] a

---
 .dockerignore | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..6427da1
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,6 @@
+.github
+build
+docs
+metrics
+nimcache
+tests

From ef48391e3a68d880f241316e822a154cacbc3fdd Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 16:30:35 +0200
Subject: [PATCH 25/31] attempt to fix docker build

---
 docker/crawler.Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile
index bdd8b4e..0d68a08 100644
--- a/docker/crawler.Dockerfile
+++ b/docker/crawler.Dockerfile
@@ -1,6 +1,6 @@
 # Variables
-ARG BUILDER=codexstorage/nim-lang:2.0.14
-ARG IMAGE=ubuntu:24.04
+ARG BUILDER=ubuntu:24.04
+ARG IMAGE=${BUILDER}
 ARG BUILD_HOME=/src
 ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
 ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
@@ -15,7 +15,7 @@ ARG NIMFLAGS
 ARG USE_LIBBACKTRACE
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential
+RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
 
 WORKDIR ${BUILD_HOME}
 COPY . .

From 9f50119863f757276b12edc2fabd787d61eaa725 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 16:34:51 +0200
Subject: [PATCH 26/31] attempt to set ci

---
 .github/workflows/main.yml | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c6671c4..b1016a4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
-        nim: [2.0.14]
+        nim: [2.2.4]
     steps:
     - uses: actions/checkout@v2
       with:
@@ -30,22 +30,18 @@ jobs:
       with:
         version: ${{ matrix.nim }}
 
-    - name: Enable git long paths
+    - name: Build Nim and dependencies
       run: |
-        git config --system core.longpaths true
+        which gcc
+        gcc --version
+        make -j${ncpu} update
+        ./env.sh nim --version
+        make -j${ncpu}
 
-    - name: Update nimble
+    - name: Run tests
       run: |
-        nimble install nimble
-        nimble --version
+        make -j${ncpu} test
 
-    - name: Use updated nimble version on Windows
-      if: contains(matrix.os, 'windows')
+    - name: Build crawler
       run: |
-        del $HOME\.nimble\bin\nimble.exe
-        nimble --version
-
-    - name: Build
-      run: nimble build -y
-    - name: Test
-      run: nimble test -y
+        make -j${ncpu}

From 3de2d27cbc069bee3cfdedbd4424f6239f420091 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 16:40:36 +0200
Subject: [PATCH 27/31] attempt to fix ci

---
 .github/workflows/main.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b1016a4..c4e065b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -26,10 +26,6 @@ jobs:
       with:
         submodules: true
 
-    - uses: iffy/install-nim@v4
-      with:
-        version: ${{ matrix.nim }}
-
     - name: Build Nim and dependencies
       run: |
         which gcc

From f4535b7b088f1c7806be77e495f885ab28c45e28 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 16:48:19 +0200
Subject: [PATCH 28/31] attempting to fix ci

---
 .github/workflows/main.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c4e065b..3a0a7fc 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -30,14 +30,14 @@ jobs:
       run: |
         which gcc
         gcc --version
-        make -j${ncpu} update
+        make update
         ./env.sh nim --version
-        make -j${ncpu}
+        make
 
     - name: Run tests
       run: |
-        make -j${ncpu} test
+        make test
 
     - name: Build crawler
       run: |
-        make -j${ncpu}
+        make

From 59b4b256673ef8950ad2311dd6bdc4d8dbde2fbd Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 17:39:37 +0200
Subject: [PATCH 29/31] Sets permissions for env.sh

---
 env.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 env.sh

diff --git a/env.sh b/env.sh
old mode 100644
new mode 100755
index 697a426..4deb2f1
--- a/env.sh
+++ b/env.sh
@@ -1,7 +1,7 @@
-#!/usr/bin/env bash
-
-# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
-# and we fall back to a Zsh-specific special var to also support Zsh.
-REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
-ABS_PATH="$(cd ${REL_PATH}; pwd)"
-source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
+#!/usr/bin/env bash
+
+# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
+# and we fall back to a Zsh-specific special var to also support Zsh.
+REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
+ABS_PATH="$(cd ${REL_PATH}; pwd)"
+source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

From d71e1f190690f42a5c51fc7b15245dfc2be4d0cd Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 17:53:40 +0200
Subject: [PATCH 30/31] adjusting line endings

---
 .dockerignore                               |  12 +-
 .gitattributes                              |   2 +
 .github/workflows/docker.yml                |  42 +--
 .github/workflows/main.yml                  |  86 ++---
 .gitignore                                  |  16 +-
 Makefile                                    | 394 ++++++++++----------
 README.md                                   |  54 +--
 codexcrawler.nimble                         |  78 ++--
 codexcrawler/config.nim                     |  50 +--
 codexcrawler/services/marketplace/readme.md |   4 +-
 docker/crawler.Dockerfile                   |  72 ++--
 docker/docker-compose.yaml                  |  40 +-
 env.sh                                      |  14 +-
 13 files changed, 433 insertions(+), 431 deletions(-)
 create mode 100644 .gitattributes

diff --git a/.dockerignore b/.dockerignore
index 6427da1..49fd67f 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,6 +1,6 @@
-.github
-build
-docs
-metrics
-nimcache
-tests
+.github
+build
+docs
+metrics
+nimcache
+tests
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..404d135
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+env.sh text eol=lf
+docker/docker-entrypoint.sh text eol=lf
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index f9553d5..07eb2f9 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -1,21 +1,21 @@
-name: Docker
-
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*.*.*'
-  workflow_dispatch:
-
-
-jobs:
-  build-and-push:
-    name: Build and Push
-    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
-    with:
-      docker_file: docker/crawler.Dockerfile
-      dockerhub_repo: codexstorage/codex-network-crawler
-      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
-    secrets: inherit
+name: Docker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: docker/crawler.Dockerfile
+      dockerhub_repo: codexstorage/codex-network-crawler
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
+    secrets: inherit
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3a0a7fc..9dfd963 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1,43 +1,43 @@
-name: CI
-
-on: [push, pull_request]
-
-jobs:
-  linting:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Check `nph` formatting
-        uses: arnetheduck/nph-action@v1
-        with:
-          version: 0.6.1
-          options: "./"
-          fail: true
-          suggest: true
-
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
-        nim: [2.2.4]
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-
-    - name: Build Nim and dependencies
-      run: |
-        which gcc
-        gcc --version
-        make update
-        ./env.sh nim --version
-        make
-
-    - name: Run tests
-      run: |
-        make test
-
-    - name: Build crawler
-      run: |
-        make
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check `nph` formatting
+        uses: arnetheduck/nph-action@v1
+        with:
+          version: 0.6.1
+          options: "./"
+          fail: true
+          suggest: true
+
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        nim: [2.2.4]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Build Nim and dependencies
+      run: |
+        which gcc
+        gcc --version
+        make update
+        ./env.sh nim --version
+        make
+
+    - name: Run tests
+      run: |
+        make test
+
+    - name: Build crawler
+      run: |
+        make
diff --git a/.gitignore b/.gitignore
index fbe4999..332b98e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,9 @@
-*
-!*.*
-!*/
-nimble.paths
-nimbus-build-system.paths
-vendor/*
-*.exe
-crawler_data
+*
+!*.*
+!*/
+nimble.paths
+nimbus-build-system.paths
+vendor/*
+*.exe
+crawler_data
 .update.timestamp
diff --git a/Makefile b/Makefile
index 4268fad..8b3b755 100644
--- a/Makefile
+++ b/Makefile
@@ -1,197 +1,197 @@
-# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
-# either of:
-# - Apache License, version 2.0
-# - MIT license
-# at your option. This file may not be copied, modified, or distributed except
-# according to those terms.
-
-# This is the Nim version used locally and in regular CI builds.
-# Can be a specific version tag, a branch name, or a commit hash.
-# Can be overridden by setting the NIM_COMMIT environment variable
-# before calling make.
-#
-# For readability in CI, if NIM_COMMIT is set to "pinned",
-# this will also default to the version pinned here.
-#
-# If NIM_COMMIT is set to "nimbusbuild", this will use the
-# version pinned by nimbus-build-system.
-PINNED_NIM_VERSION := v2.2.4
-
-ifeq ($(NIM_COMMIT),)
-NIM_COMMIT := $(PINNED_NIM_VERSION)
-else ifeq ($(NIM_COMMIT),pinned)
-NIM_COMMIT := $(PINNED_NIM_VERSION)
-endif
-
-ifeq ($(NIM_COMMIT),nimbusbuild)
-undefine NIM_COMMIT
-else
-export NIM_COMMIT
-endif
-
-SHELL := bash # the shell used internally by Make
-
-# used inside the included makefiles
-BUILD_SYSTEM_DIR := vendor/nimbus-build-system
-
-# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
-# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
-DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
-
-LINK_PCRE := 0
-
-ifeq ($(OS),Windows_NT)
-    ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
-        ARCH = x86_64
-    endif
-    ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
-        ARCH = arm64
-    endif
-else
-    UNAME_P := $(shell uname -m)
-    ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
-        ARCH = x86_64
-    endif
-    ifneq ($(filter $(UNAME_P), aarch64 arm),)
-        ARCH = arm64
-    endif
-endif
-
-ifeq ($(ARCH), x86_64)
-    CXXFLAGS ?= -std=c++17 -mssse3
-else
-    CXXFLAGS ?= -std=c++17
-endif
-export CXXFLAGS
-
-# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
--include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
-
-.PHONY: \
-	all \
-	clean \
-	coverage \
-	deps \
-	libbacktrace \
-	test \
-	update
-
-ifeq ($(NIM_PARAMS),)
-# "variables.mk" was not included, so we update the submodules.
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
-.DEFAULT:
-	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
-		$(GIT_SUBMODULE_UPDATE); \
-		echo
-# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
-# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
-#
-# After restarting, it will execute its original goal, so we don't have to start a child Make here
-# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
-
-else # "variables.mk" was included. Business as usual until the end of this file.
-
-# default target, because it's the first one that doesn't start with '.'
-
-# Builds the crawler binary
-all: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
-
-# must be included after the default target
--include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
-
-# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
-ifeq ($(USE_LIBBACKTRACE), 0)
-NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
-else
-NIM_PARAMS := $(NIM_PARAMS) -d:release
-endif
-
-deps: | deps-common nat-libs
-ifneq ($(USE_LIBBACKTRACE), 0)
-deps: | libbacktrace
-endif
-
-update: | update-common
-
-# detecting the os
-ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
- detected_OS := Windows
-else ifeq ($(strip $(shell uname)),Darwin)
- detected_OS := macOS
-else
- # e.g. Linux
- detected_OS := $(strip $(shell uname))
-endif
-
-# Builds and run a part of the test suite
-test: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-		$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
-
-# nim-libbacktrace
-LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
-libbacktrace:
-ifeq ($(detected_OS), Windows)
-# MSYS2 detection
-ifneq ($(MSYSTEM),)
-	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
-else
-	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
-endif
-else
-	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
-endif
-
-# usual cleaning
-clean: | clean-common
-	rm -rf build
-ifneq ($(USE_LIBBACKTRACE), 0)
-	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
-endif
-
-############
-## Format ##
-############
-.PHONY: build-nph install-nph-hook clean-nph print-nph-path
-
-# Default location for nph binary shall be next to nim binary to make it available on the path.
-NPH:=$(shell dirname $(NIM_BINARY))/nph
-
-build-nph:
-ifeq ("$(wildcard $(NPH))","")
-		$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
-		mv vendor/nph/src/nph $(shell dirname $(NPH))
-		echo "nph utility is available at " $(NPH)
-endif
-
-GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
-
-install-nph-hook: build-nph
-ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
-	cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
-else
-	echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
-	exit 1
-endif
-
-nph/%: build-nph
-	echo -e $(FORMAT_MSG) "nph/$*" && \
-		$(NPH) $*
-
-format:
-	$(NPH) *.nim
-	$(NPH) codexcrawler/
-	$(NPH) tests/
-
-clean-nph:
-	rm -f $(NPH)
-
-# To avoid hardcoding nph binary location in several places
-print-nph-path:
-	echo "$(NPH)"
-
-clean: | clean-nph
-
-endif # "variables.mk" was not included
+# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+# This is the Nim version used locally and in regular CI builds.
+# Can be a specific version tag, a branch name, or a commit hash.
+# Can be overridden by setting the NIM_COMMIT environment variable
+# before calling make.
+#
+# For readability in CI, if NIM_COMMIT is set to "pinned",
+# this will also default to the version pinned here.
+#
+# If NIM_COMMIT is set to "nimbusbuild", this will use the
+# version pinned by nimbus-build-system.
+PINNED_NIM_VERSION := v2.2.4
+
+ifeq ($(NIM_COMMIT),)
+NIM_COMMIT := $(PINNED_NIM_VERSION)
+else ifeq ($(NIM_COMMIT),pinned)
+NIM_COMMIT := $(PINNED_NIM_VERSION)
+endif
+
+ifeq ($(NIM_COMMIT),nimbusbuild)
+undefine NIM_COMMIT
+else
+export NIM_COMMIT
+endif
+
+SHELL := bash # the shell used internally by Make
+
+# used inside the included makefiles
+BUILD_SYSTEM_DIR := vendor/nimbus-build-system
+
+# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
+# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
+DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
+
+LINK_PCRE := 0
+
+ifeq ($(OS),Windows_NT)
+    ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
+        ARCH = x86_64
+    endif
+    ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
+        ARCH = arm64
+    endif
+else
+    UNAME_P := $(shell uname -m)
+    ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
+        ARCH = x86_64
+    endif
+    ifneq ($(filter $(UNAME_P), aarch64 arm),)
+        ARCH = arm64
+    endif
+endif
+
+ifeq ($(ARCH), x86_64)
+    CXXFLAGS ?= -std=c++17 -mssse3
+else
+    CXXFLAGS ?= -std=c++17
+endif
+export CXXFLAGS
+
+# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
+-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
+
+.PHONY: \
+	all \
+	clean \
+	coverage \
+	deps \
+	libbacktrace \
+	test \
+	update
+
+ifeq ($(NIM_PARAMS),)
+# "variables.mk" was not included, so we update the submodules.
+GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+.DEFAULT:
+	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
+		$(GIT_SUBMODULE_UPDATE); \
+		echo
+# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
+# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
+#
+# After restarting, it will execute its original goal, so we don't have to start a child Make here
+# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
+
+else # "variables.mk" was included. Business as usual until the end of this file.
+
+# default target, because it's the first one that doesn't start with '.'
+
+# Builds the crawler binary
+all: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
+
+# must be included after the default target
+-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+
+# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
+ifeq ($(USE_LIBBACKTRACE), 0)
+NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
+else
+NIM_PARAMS := $(NIM_PARAMS) -d:release
+endif
+
+deps: | deps-common nat-libs
+ifneq ($(USE_LIBBACKTRACE), 0)
+deps: | libbacktrace
+endif
+
+update: | update-common
+
+# detecting the os
+ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
+ detected_OS := Windows
+else ifeq ($(strip $(shell uname)),Darwin)
+ detected_OS := macOS
+else
+ # e.g. Linux
+ detected_OS := $(strip $(shell uname))
+endif
+
+# Builds and run a part of the test suite
+test: | build deps
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
+
+# nim-libbacktrace
+LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
+libbacktrace:
+ifeq ($(detected_OS), Windows)
+# MSYS2 detection
+ifneq ($(MSYSTEM),)
+	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
+else
+	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
+endif
+else
+	+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
+endif
+
+# usual cleaning
+clean: | clean-common
+	rm -rf build
+ifneq ($(USE_LIBBACKTRACE), 0)
+	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
+endif
+
+############
+## Format ##
+############
+.PHONY: build-nph install-nph-hook clean-nph print-nph-path
+
+# Default location for nph binary shall be next to nim binary to make it available on the path.
+NPH:=$(shell dirname $(NIM_BINARY))/nph
+
+build-nph:
+ifeq ("$(wildcard $(NPH))","")
+		$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
+		mv vendor/nph/src/nph $(shell dirname $(NPH))
+		echo "nph utility is available at " $(NPH)
+endif
+
+GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
+
+install-nph-hook: build-nph
+ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
+	cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
+else
+	echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
+	exit 1
+endif
+
+nph/%: build-nph
+	echo -e $(FORMAT_MSG) "nph/$*" && \
+		$(NPH) $*
+
+format:
+	$(NPH) *.nim
+	$(NPH) codexcrawler/
+	$(NPH) tests/
+
+clean-nph:
+	rm -f $(NPH)
+
+# To avoid hardcoding nph binary location in several places
+print-nph-path:
+	echo "$(NPH)"
+
+clean: | clean-nph
+
+endif # "variables.mk" was not included
diff --git a/README.md b/README.md
index 68948bb..08fa271 100644
--- a/README.md
+++ b/README.md
@@ -1,27 +1,27 @@
-# Codex Network Crawler
-
-![Crawler](crawler.png)
-
-[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability)
-[![CI (GitHub Actions)](https://github.com/codex-storage/nim-codex-dht/workflows/CI/badge.svg?branch=master)](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster)
-[![codecov](https://codecov.io/gh/codex-storage/nim-codex-dht/branch/master/graph/badge.svg?token=tlmMJgU4l7)](https://codecov.io/gh/codex-storage/nim-codex-dht)
-
-# !! Work in Progress !!
-
-This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as:
-- Number of DHT nodes (alive vs total)
-- P2P connectivity (percentage)
-- Storage contract statistics (created, total size, average size, average duration, pricing information??)
-
-Metrics are published from a scrape target.
-
-# Usage
-
-```sh
-nimble format
-nimble build
-nimble test
-nimble run
-```
+# Codex Network Crawler
+
+![Crawler](crawler.png)
+
+[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability)
+[![CI (GitHub Actions)](https://github.com/codex-storage/nim-codex-dht/workflows/CI/badge.svg?branch=master)](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster)
+[![codecov](https://codecov.io/gh/codex-storage/nim-codex-dht/branch/master/graph/badge.svg?token=tlmMJgU4l7)](https://codecov.io/gh/codex-storage/nim-codex-dht)
+
+# !! Work in Progress !!
+
+This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as:
+- Number of DHT nodes (alive vs total)
+- P2P connectivity (percentage)
+- Storage contract statistics (created, total size, average size, average duration, pricing information??)
+
+Metrics are published from a scrape target.
+
+# Usage
+
+```sh
+nimble format
+nimble build
+nimble test
+nimble run
+```
diff --git a/codexcrawler.nimble b/codexcrawler.nimble
index c14c124..6eb3c19 100644
--- a/codexcrawler.nimble
+++ b/codexcrawler.nimble
@@ -1,39 +1,39 @@
-# Package
-
-version = "0.0.1"
-author = "Codex core contributors"
-description = "Crawler for Codex networks"
-license = "MIT"
-skipDirs = @["tests"]
-bin = @["codexcrawler"]
-binDir = "build"
-
-# Dependencies
-requires "nim >= 2.0.14 & < 3.0.0"
-requires "secp256k1 >= 0.6.0 & < 0.7.0"
-requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
-requires "nimcrypto >= 0.6.2 & < 0.7.0"
-requires "bearssl >= 0.2.5 & < 0.3.0"
-requires "chronicles >= 0.10.2 & < 0.11.0"
-requires "chronos >= 4.0.3 & < 4.1.0"
-# requires "libp2p >= 1.5.0 & < 2.0.0"
-requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
-requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
-
-requires "metrics >= 0.1.0 & < 0.2.0"
-requires "stew >= 0.2.0 & < 0.3.0"
-requires "stint >= 0.8.1 & < 0.9.0"
-requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
-requires "questionable >= 0.10.15 & < 0.11.0"
-requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
-requires "docopt >= 0.7.1 & < 1.0.0"
-requires "nph >= 0.6.1 & < 1.0.0"
-requires "ethers >= 1.0.0 & < 2.0.0"
-
-task format, "Formatting...":
-  exec "nph ./"
-
-task test, "Run tests":
-  exec "nimble install -d -y"
-  withDir "tests":
-    exec "nimble test"
+# Package
+
+version = "0.0.1"
+author = "Codex core contributors"
+description = "Crawler for Codex networks"
+license = "MIT"
+skipDirs = @["tests"]
+bin = @["codexcrawler"]
+binDir = "build"
+
+# Dependencies
+requires "nim >= 2.0.14 & < 3.0.0"
+requires "secp256k1 >= 0.6.0 & < 0.7.0"
+requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
+requires "nimcrypto >= 0.6.2 & < 0.7.0"
+requires "bearssl >= 0.2.5 & < 0.3.0"
+requires "chronicles >= 0.10.2 & < 0.11.0"
+requires "chronos >= 4.0.3 & < 4.1.0"
+# requires "libp2p >= 1.5.0 & < 2.0.0"
+requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
+requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
+
+requires "metrics >= 0.1.0 & < 0.2.0"
+requires "stew >= 0.2.0 & < 0.3.0"
+requires "stint >= 0.8.1 & < 0.9.0"
+requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
+requires "questionable >= 0.10.15 & < 0.11.0"
+requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
+requires "docopt >= 0.7.1 & < 1.0.0"
+requires "nph >= 0.6.1 & < 1.0.0"
+requires "ethers >= 1.0.0 & < 2.0.0"
+
+task format, "Formatting...":
+  exec "nph ./"
+
+task test, "Run tests":
+  exec "nimble install -d -y"
+  withDir "tests":
+    exec "nimble test"
diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim
index ef20ae2..6b21207 100644
--- a/codexcrawler/config.nim
+++ b/codexcrawler/config.nim
@@ -6,31 +6,31 @@ import pkg/codexdht
 import ./utils/version
 
 let doc =
-  """
-Codex Network Crawler. Generates network metrics.
-
-Usage:
-  codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>]  [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
-
-Options:
-  --logLevel=<l>                    Sets log level [default: INFO]
-  --publicIp=<a>                    Public IP address where this instance is reachable.
-  --metricsAddress=<ip>             Listen address of the metrics server [default: 0.0.0.0]
-  --metricsPort=<p>                 Listen HTTP port of the metrics server [default: 8008]
-  --dataDir=<dir>                   Directory for storing data [default: crawler_data]
-  --discoveryPort=<p>               Port used for DHT [default: 8090]
-  --bootNodes=<n>                   Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
-
-  --dhtEnable=<e>                   Set to "1" to enable DHT crawler [default: 1]
-  --stepDelay=<ms>                  Delay in milliseconds per node visit [default: 1000]
-  --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
-  --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
-  --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
-
-  --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
-  --ethProvider=<a>                 Address including http(s) or ws of the eth provider
-  --marketplaceAddress=<a>          Eth address of Codex contracts deployment
-  --requestCheckDelay=<m>           Delay in minutes after which storage contract status is (re)checked [default: 10]
+  """
+Codex Network Crawler. Generates network metrics.
+
+Usage:
+  codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>]  [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
+
+Options:
+  --logLevel=<l>                    Sets log level [default: INFO]
+  --publicIp=<a>                    Public IP address where this instance is reachable.
+  --metricsAddress=<ip>             Listen address of the metrics server [default: 0.0.0.0]
+  --metricsPort=<p>                 Listen HTTP port of the metrics server [default: 8008]
+  --dataDir=<dir>                   Directory for storing data [default: crawler_data]
+  --discoveryPort=<p>               Port used for DHT [default: 8090]
+  --bootNodes=<n>                   Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
+
+  --dhtEnable=<e>                   Set to "1" to enable DHT crawler [default: 1]
+  --stepDelay=<ms>                  Delay in milliseconds per node visit [default: 1000]
+  --revisitDelay=<m>                Delay in minutes after which a node can be revisited [default: 60]
+  --checkDelay=<m>                  Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
+  --expiryDelay=<m>                 Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
+
+  --marketplaceEnable=<e>           Set to "1" to enable marketplace metrics [default: 1]
+  --ethProvider=<a>                 Address including http(s) or ws of the eth provider
+  --marketplaceAddress=<a>          Eth address of Codex contracts deployment
+  --requestCheckDelay=<m>           Delay in minutes after which storage contract status is (re)checked [default: 10]
 """
 
 import strutils
diff --git a/codexcrawler/services/marketplace/readme.md b/codexcrawler/services/marketplace/readme.md
index fd2b28a..bf693cc 100644
--- a/codexcrawler/services/marketplace/readme.md
+++ b/codexcrawler/services/marketplace/readme.md
@@ -1,2 +1,2 @@
-These are copied from nim-codex v0.2.0.
-There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
+These are copied from nim-codex v0.2.0.
+There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile
index 0d68a08..1d453be 100644
--- a/docker/crawler.Dockerfile
+++ b/docker/crawler.Dockerfile
@@ -1,36 +1,36 @@
-# Variables
-ARG BUILDER=ubuntu:24.04
-ARG IMAGE=${BUILDER}
-ARG BUILD_HOME=/src
-ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
-ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
-ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
-ARG APP_HOME=/crawler
-
-# Build
-FROM ${BUILDER} AS builder
-ARG BUILD_HOME
-ARG MAKE_PARALLEL
-ARG NIMFLAGS
-ARG USE_LIBBACKTRACE
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
-
-WORKDIR ${BUILD_HOME}
-COPY . .
-RUN make -j ${MAKE_PARALLEL} update
-RUN make -j ${MAKE_PARALLEL}
-
-# Create
-FROM ${IMAGE}
-ARG BUILD_HOME
-ARG APP_HOME
-
-WORKDIR ${APP_HOME}
-COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
-COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
-RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
-
-ENTRYPOINT ["/docker-entrypoint.sh"]
-CMD ["codexcrawler"]
+# Variables
+ARG BUILDER=ubuntu:24.04
+ARG IMAGE=${BUILDER}
+ARG BUILD_HOME=/src
+ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
+ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
+ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
+ARG APP_HOME=/crawler
+
+# Build
+FROM ${BUILDER} AS builder
+ARG BUILD_HOME
+ARG MAKE_PARALLEL
+ARG NIMFLAGS
+ARG USE_LIBBACKTRACE
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
+
+WORKDIR ${BUILD_HOME}
+COPY . .
+RUN make -j ${MAKE_PARALLEL} update
+RUN make -j ${MAKE_PARALLEL}
+
+# Create
+FROM ${IMAGE}
+ARG BUILD_HOME
+ARG APP_HOME
+
+WORKDIR ${APP_HOME}
+COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
+COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
+RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["codexcrawler"]
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index ebc8c12..71465db 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -1,20 +1,20 @@
-services:
-  crawler1:
-    build:
-      context: ..
-      dockerfile: ./docker/crawler.Dockerfile
-    environment:
-      - CRAWLER_LOGLEVEL=TRACE
-      # - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
-      - CRAWLER_METRICSADDRESS=0.0.0.0
-      - CRAWLER_METRICSPORT=8008
-      - CRAWLER_DATADIR=crawler_data
-      - CRAWLER_DISCPORT=8090
-      - CRAWLER_BOOTNODES=testnet_sprs
-      - CRAWLER_STEPDELAY=3000
-      - CRAWLER_REVISITDELAY=1440
-    ports:
-      - 8008:8008/tcp # Metrics
-      - 8090:8090/udp # DHT discovery
-    volumes:
-      - ./crawler_data:/crawler_data:z
+services:
+  crawler1:
+    build:
+      context: ..
+      dockerfile: ./docker/crawler.Dockerfile
+    environment:
+      - CRAWLER_LOGLEVEL=TRACE
+      # - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
+      - CRAWLER_METRICSADDRESS=0.0.0.0
+      - CRAWLER_METRICSPORT=8008
+      - CRAWLER_DATADIR=crawler_data
+      - CRAWLER_DISCPORT=8090
+      - CRAWLER_BOOTNODES=testnet_sprs
+      - CRAWLER_STEPDELAY=3000
+      - CRAWLER_REVISITDELAY=1440
+    ports:
+      - 8008:8008/tcp # Metrics
+      - 8090:8090/udp # DHT discovery
+    volumes:
+      - ./crawler_data:/crawler_data:z
diff --git a/env.sh b/env.sh
index 4deb2f1..697a426 100755
--- a/env.sh
+++ b/env.sh
@@ -1,7 +1,7 @@
-#!/usr/bin/env bash
-
-# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
-# and we fall back to a Zsh-specific special var to also support Zsh.
-REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
-ABS_PATH="$(cd ${REL_PATH}; pwd)"
-source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
+#!/usr/bin/env bash
+
+# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
+# and we fall back to a Zsh-specific special var to also support Zsh.
+REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
+ABS_PATH="$(cd ${REL_PATH}; pwd)"
+source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

From 4a3c388e5a43b31675ae3ec79e72db778504fac1 Mon Sep 17 00:00:00 2001
From: thatben <thatbenbierens@gmail.com>
Date: Mon, 2 Jun 2025 18:02:41 +0200
Subject: [PATCH 31/31] reverts nimble file

---
 codexcrawler.nimble | 78 ++++++++++++++++++++++-----------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

diff --git a/codexcrawler.nimble b/codexcrawler.nimble
index 6eb3c19..c14c124 100644
--- a/codexcrawler.nimble
+++ b/codexcrawler.nimble
@@ -1,39 +1,39 @@
-# Package
-
-version = "0.0.1"
-author = "Codex core contributors"
-description = "Crawler for Codex networks"
-license = "MIT"
-skipDirs = @["tests"]
-bin = @["codexcrawler"]
-binDir = "build"
-
-# Dependencies
-requires "nim >= 2.0.14 & < 3.0.0"
-requires "secp256k1 >= 0.6.0 & < 0.7.0"
-requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
-requires "nimcrypto >= 0.6.2 & < 0.7.0"
-requires "bearssl >= 0.2.5 & < 0.3.0"
-requires "chronicles >= 0.10.2 & < 0.11.0"
-requires "chronos >= 4.0.3 & < 4.1.0"
-# requires "libp2p >= 1.5.0 & < 2.0.0"
-requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
-requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
-
-requires "metrics >= 0.1.0 & < 0.2.0"
-requires "stew >= 0.2.0 & < 0.3.0"
-requires "stint >= 0.8.1 & < 0.9.0"
-requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
-requires "questionable >= 0.10.15 & < 0.11.0"
-requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
-requires "docopt >= 0.7.1 & < 1.0.0"
-requires "nph >= 0.6.1 & < 1.0.0"
-requires "ethers >= 1.0.0 & < 2.0.0"
-
-task format, "Formatting...":
-  exec "nph ./"
-
-task test, "Run tests":
-  exec "nimble install -d -y"
-  withDir "tests":
-    exec "nimble test"
+# Package
+
+version = "0.0.1"
+author = "Codex core contributors"
+description = "Crawler for Codex networks"
+license = "MIT"
+skipDirs = @["tests"]
+bin = @["codexcrawler"]
+binDir = "build"
+
+# Dependencies
+requires "nim >= 2.0.14 & < 3.0.0"
+requires "secp256k1 >= 0.6.0 & < 0.7.0"
+requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
+requires "nimcrypto >= 0.6.2 & < 0.7.0"
+requires "bearssl >= 0.2.5 & < 0.3.0"
+requires "chronicles >= 0.10.2 & < 0.11.0"
+requires "chronos >= 4.0.3 & < 4.1.0"
+# requires "libp2p >= 1.5.0 & < 2.0.0"
+requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
+requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
+
+requires "metrics >= 0.1.0 & < 0.2.0"
+requires "stew >= 0.2.0 & < 0.3.0"
+requires "stint >= 0.8.1 & < 0.9.0"
+requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
+requires "questionable >= 0.10.15 & < 0.11.0"
+requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
+requires "docopt >= 0.7.1 & < 1.0.0"
+requires "nph >= 0.6.1 & < 1.0.0"
+requires "ethers >= 1.0.0 & < 2.0.0"
+
+task format, "Formatting...":
+  exec "nph ./"
+
+task test, "Run tests":
+  exec "nimble install -d -y"
+  withDir "tests":
+    exec "nimble test"