From bc27eebb8565543b9b31ff8ec42773966e1f0a61 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:08:09 +1100 Subject: [PATCH 1/6] fix pinned deps Leaving nim-datastore as a commit hash until it has a relevant release tag --- codexdht.nimble | 11 ++++++----- codexdht/private/eth/p2p/discoveryv5/encoding.nim | 2 +- codexdht/private/eth/p2p/discoveryv5/ip_vote.nim | 2 +- codexdht/private/eth/p2p/discoveryv5/lru.nim | 2 +- codexdht/private/eth/p2p/discoveryv5/messages.nim | 2 +- .../eth/p2p/discoveryv5/messages_encoding.nim | 1 + codexdht/private/eth/p2p/discoveryv5/node.nim | 2 +- .../eth/p2p/discoveryv5/nodes_verification.nim | 2 +- codexdht/private/eth/p2p/discoveryv5/protocol.nim | 10 +++++----- .../private/eth/p2p/discoveryv5/providers/cache.nim | 2 +- .../eth/p2p/discoveryv5/providers/common.nim | 2 +- .../eth/p2p/discoveryv5/providers/maintenance.nim | 2 +- .../eth/p2p/discoveryv5/providers/manager.nim | 2 +- .../private/eth/p2p/discoveryv5/routing_table.nim | 8 ++++---- codexdht/private/eth/p2p/discoveryv5/sessions.nim | 2 +- tests/discv5/test_discoveryv5.nim | 13 ++++++++----- 16 files changed, 35 insertions(+), 30 deletions(-) diff --git a/codexdht.nimble b/codexdht.nimble index 9afa8c5..961003f 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -7,16 +7,17 @@ license = "MIT" skipDirs = @["tests"] # Dependencies -requires "secp256k1#2acbbdcc0e63002a013fff49f015708522875832" # >= 0.5.2 & < 0.6.0 -requires "protobuf_serialization#5a31137a82c2b6a989c9ed979bb636c7a49f570e" # >= 0.2.0 & < 0.3.0 +requires "nim >= 2.0.14 & < 3.0.0" +requires "secp256k1" # >= 0.5.2 & < 0.6.0 +requires "protobuf_serialization" # >= 0.2.0 & < 0.3.0 requires "nimcrypto >= 0.5.4" requires "bearssl == 0.2.5" requires "chronicles >= 0.10.2 & < 0.11.0" requires "chronos >= 4.0.3 & < 4.1.0" -requires "libp2p == 1.5.0" -requires "metrics#cacfdc12454a0804c65112b9f4f50d1375208dcd" +requires "libp2p >= 1.5.0 & < 2.0.0" +requires "metrics" requires "stew >= 0.2.0" -requires "stint#3236fa68394f1e3a06e2bc34218aacdd2d675923" +requires "stint >= 0.8.1 & < 0.9.0" requires "https://github.com/codex-storage/nim-datastore#a969b9799cb7fd2c2511b6820ded00bced141dea" requires "questionable >= 0.10.15 & < 0.11.0" diff --git a/codexdht/private/eth/p2p/discoveryv5/encoding.nim b/codexdht/private/eth/p2p/discoveryv5/encoding.nim index cdbc30e..2cb7962 100644 --- a/codexdht/private/eth/p2p/discoveryv5/encoding.nim +++ b/codexdht/private/eth/p2p/discoveryv5/encoding.nim @@ -11,7 +11,7 @@ ## https://github.com/ethereum/devp2p/blob/master/discv5/discv5-theory.md#sessions ## -{.push raises: [Defect].} +{.push raises: [].} import std/[hashes, net, options, sugar, tables], diff --git a/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim b/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim index 8a57f8f..2f29fc4 100644 --- a/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim +++ b/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim @@ -15,7 +15,7 @@ ## To select the right address, a majority count is done. This is done over a ## sort of moving window as votes expire after `IpVoteTimeout`. -{.push raises: [Defect].} +{.push raises: [].} import std/[tables, options], diff --git a/codexdht/private/eth/p2p/discoveryv5/lru.nim b/codexdht/private/eth/p2p/discoveryv5/lru.nim index 2fa8a11..ad458f7 100644 --- a/codexdht/private/eth/p2p/discoveryv5/lru.nim +++ b/codexdht/private/eth/p2p/discoveryv5/lru.nim @@ -1,6 +1,6 @@ import std/[tables, lists, options] -{.push raises: [Defect].} +{.push raises: [].} export tables, lists, options diff --git a/codexdht/private/eth/p2p/discoveryv5/messages.nim b/codexdht/private/eth/p2p/discoveryv5/messages.nim index da3cdb0..f6fcbd9 100644 --- a/codexdht/private/eth/p2p/discoveryv5/messages.nim +++ b/codexdht/private/eth/p2p/discoveryv5/messages.nim @@ -10,7 +10,7 @@ ## These messages get protobuf encoded, while in the spec they get RLP encoded. ## -{.push raises: [Defect].} +{.push raises: [].} import std/[hashes, net], diff --git a/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim b/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim index 09f690d..bbc513b 100644 --- a/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim +++ b/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim @@ -14,6 +14,7 @@ import stew/endians2, libp2p/routing_record, libp2p/signed_envelope, + libp2p/protobuf/minprotobuf, "."/[messages, spr, node], ../../../../dht/providers_encoding diff --git a/codexdht/private/eth/p2p/discoveryv5/node.nim b/codexdht/private/eth/p2p/discoveryv5/node.nim index 442d551..cad809f 100644 --- a/codexdht/private/eth/p2p/discoveryv5/node.nim +++ b/codexdht/private/eth/p2p/discoveryv5/node.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/hashes, diff --git a/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim b/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim index d3cecf7..bd2dbd0 100644 --- a/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim +++ b/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim @@ -1,4 +1,4 @@ -{.push raises: [Defect].} +{.push raises: [].} import std/[sets, options], diff --git a/codexdht/private/eth/p2p/discoveryv5/protocol.nim b/codexdht/private/eth/p2p/discoveryv5/protocol.nim index 807e9a1..0089d12 100644 --- a/codexdht/private/eth/p2p/discoveryv5/protocol.nim +++ b/codexdht/private/eth/p2p/discoveryv5/protocol.nim @@ -71,7 +71,7 @@ ## more requests will be needed for a lookup (adding bandwidth and latency). ## This might be a concern for mobile devices. -{.push raises: [Defect].} +{.push raises: [].} import std/[tables, sets, options, math, sequtils, algorithm, strutils], @@ -236,18 +236,18 @@ proc getNode*(d: Protocol, id: NodeId): Option[Node] = ## Get the node with id from the routing table. d.routingTable.getNode(id) -proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] {.raises: Exception.} = +proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] = ## Get a `maxAmount` of random nodes from the local routing table. d.routingTable.randomNodes(maxAmount) proc randomNodes*(d: Protocol, maxAmount: int, - pred: proc(x: Node): bool {.gcsafe, noSideEffect.}): seq[Node] {.raises: Exception.} = + pred: proc(x: Node): bool {.gcsafe, noSideEffect, raises: [].}): seq[Node] = ## Get a `maxAmount` of random nodes from the local routing table with the ## `pred` predicate function applied as filter on the nodes selected. d.routingTable.randomNodes(maxAmount, pred) proc randomNodes*(d: Protocol, maxAmount: int, - enrField: (string, seq[byte])): seq[Node] {.raises: Exception.} = + enrField: (string, seq[byte])): seq[Node] = ## Get a `maxAmount` of random nodes from the local routing table. The ## the nodes selected are filtered by provided `enrField`. d.randomNodes(maxAmount, proc(x: Node): bool = x.record.contains(enrField)) @@ -563,7 +563,7 @@ proc ping*(d: Protocol, toNode: Node): dht_message_requests_outgoing.inc(labelValues = ["invalid_response"]) return err("Invalid response to ping message") else: - # A ping (or the pong) was lost, what should we do? Previous implementation called + # A ping (or the pong) was lost, what should we do? Previous implementation called # d.replaceNode(toNode) immediately, which removed the node. This is too aggressive, # especially if we have a temporary network outage. Although bootstrap nodes are protected # from being removed, everything else would slowly be removed. diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim b/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim index 78c0ee2..b523c91 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/sequtils diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/common.nim b/codexdht/private/eth/p2p/discoveryv5/providers/common.nim index 4edad41..e0ec0cc 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/common.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/common.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/sequtils import std/strutils diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim b/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim index 3536979..3be6943 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/options import std/sequtils diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim b/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim index 44ba094..7eb33b1 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim @@ -19,7 +19,7 @@ import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -{.push raises: [Defect].} +{.push raises: [].} import ./maintenance import ./cache diff --git a/codexdht/private/eth/p2p/discoveryv5/routing_table.nim b/codexdht/private/eth/p2p/discoveryv5/routing_table.nim index 8239ee4..e270315 100644 --- a/codexdht/private/eth/p2p/discoveryv5/routing_table.nim +++ b/codexdht/private/eth/p2p/discoveryv5/routing_table.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/[algorithm, times, sequtils, bitops, sets, options, tables], @@ -21,7 +21,7 @@ declarePublicGauge dht_routing_table_buckets, logScope: topics = "discv5 routingtable" - + type DistanceProc* = proc(a, b: NodeId): NodeId {.raises: [Defect], gcsafe, noSideEffect.} LogDistanceProc* = proc(a, b: NodeId): uint16 {.raises: [Defect], gcsafe, noSideEffect.} @@ -425,7 +425,7 @@ proc addNode*(r: var RoutingTable, n: Node): NodeStatus = (depth mod r.bitsPerHop != 0 and depth != ID_SIZE): r.splitBucket(r.buckets.find(bucket)) return r.addNode(n) # retry adding - + # When bucket doesn't get split the node is added to the replacement cache return r.addReplacement(bucket, n) @@ -555,7 +555,7 @@ proc nodeToRevalidate*(r: RoutingTable): Node = return b.nodes[^1] proc randomNodes*(r: RoutingTable, maxAmount: int, - pred: proc(x: Node): bool {.gcsafe, noSideEffect.} = nil): seq[Node] {.raises: Exception.} = + pred: proc(x: Node): bool {.gcsafe, noSideEffect, raises: [].} = nil): seq[Node] = ## Get a `maxAmount` of random nodes from the routing table with the `pred` ## predicate function applied as filter on the nodes selected. var maxAmount = maxAmount diff --git a/codexdht/private/eth/p2p/discoveryv5/sessions.nim b/codexdht/private/eth/p2p/discoveryv5/sessions.nim index ffcb76f..9c969cf 100644 --- a/codexdht/private/eth/p2p/discoveryv5/sessions.nim +++ b/codexdht/private/eth/p2p/discoveryv5/sessions.nim @@ -16,7 +16,7 @@ ## - the one derived in the key-exchange started by the other node. ## To alleviate this issue, we store two decryption keys in each session. -{.push raises: [Defect].} +{.push raises: [].} import std/options, diff --git a/tests/discv5/test_discoveryv5.nim b/tests/discv5/test_discoveryv5.nim index 3d7bd59..375dc93 100644 --- a/tests/discv5/test_discoveryv5.nim +++ b/tests/discv5/test_discoveryv5.nim @@ -287,7 +287,7 @@ suite "Discovery v5 Tests": await mainNode.closeWait() await testNode.closeWait() - proc testLookupTargets(fast: bool = false) {.async: (raises: [Exception]).} = + proc testLookupTargets(fast: bool = false): Future[bool] {.async.} = const nodeCount = 17 @@ -306,9 +306,9 @@ suite "Discovery v5 Tests": for t in nodes: if n != t: let pong = await n.ping(t.localNode) - check pong.isOk() if pong.isErr(): echo pong.error + return false # check (await n.ping(t.localNode)).isOk() for i in 1 ..< nodeCount: @@ -318,16 +318,19 @@ suite "Discovery v5 Tests": let target = nodes[i] let discovered = await nodes[nodeCount-1].lookup(target.localNode.id, fast = fast) debug "Lookup result", target = target.localNode, discovered - check discovered[0] == target.localNode + if discovered[0] != target.localNode: + return false for node in nodes: await node.closeWait() + return true + test "Lookup targets": - await testLookupTargets() + check await testLookupTargets() test "Lookup targets using traditional findNode": - await testLookupTargets(fast = true) + check await testLookupTargets(fast = true) test "Resolve target": let From a9e17f4a330f2244cd1106de754b86daca32f0ee Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:09:50 +1100 Subject: [PATCH 2/6] remove nim 1.6 from ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21463aa..c46d466 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - nim: [1.6.18, 2.0.14] + nim: [2.0.14] os: [ubuntu-latest, macOS-latest, windows-latest] steps: - name: Checkout From 14d4dd97e931a215fbeed61a03fc59b4ff4e3490 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:15:00 +1100 Subject: [PATCH 3/6] toBytes -> toBytesBE --- codexdht/private/eth/p2p/discoveryv5/sessions.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codexdht/private/eth/p2p/discoveryv5/sessions.nim b/codexdht/private/eth/p2p/discoveryv5/sessions.nim index 9c969cf..ee770bb 100644 --- a/codexdht/private/eth/p2p/discoveryv5/sessions.nim +++ b/codexdht/private/eth/p2p/discoveryv5/sessions.nim @@ -39,7 +39,7 @@ type func makeKey(id: NodeId, address: Address): SessionKey = var pos = 0 - result[pos ..< pos+sizeof(id)] = toBytes(id) + result[pos ..< pos+sizeof(id)] = toBytesBE(id) pos.inc(sizeof(id)) case address.ip.family of IpAddressFamily.IpV4: @@ -47,7 +47,7 @@ func makeKey(id: NodeId, address: Address): SessionKey = of IpAddressFamily.IpV6: result[pos ..< pos+sizeof(address.ip.address_v6)] = address.ip.address_v6 pos.inc(sizeof(address.ip.address_v6)) - result[pos ..< pos+sizeof(address.port)] = toBytes(address.port.uint16) + result[pos ..< pos+sizeof(address.port)] = toBytesBE(address.port.uint16) func swapr*(s: var Sessions, id: NodeId, address: Address) = var value: array[3 * sizeof(AesKey), byte] From c777106e7f29799e819dcd208fe4c7d2f58467a3 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:15:22 +1100 Subject: [PATCH 4/6] set non-major range for remaining deps --- codexdht.nimble | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codexdht.nimble b/codexdht.nimble index 961003f..2d6037c 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -10,13 +10,13 @@ skipDirs = @["tests"] requires "nim >= 2.0.14 & < 3.0.0" requires "secp256k1" # >= 0.5.2 & < 0.6.0 requires "protobuf_serialization" # >= 0.2.0 & < 0.3.0 -requires "nimcrypto >= 0.5.4" -requires "bearssl == 0.2.5" +requires "nimcrypto >= 0.5.4 & < 0.6.0" +requires "bearssl >= 0.2.5 & < 0.3.0" requires "chronicles >= 0.10.2 & < 0.11.0" requires "chronos >= 4.0.3 & < 4.1.0" requires "libp2p >= 1.5.0 & < 2.0.0" requires "metrics" -requires "stew >= 0.2.0" +requires "stew >= 0.2.0 & < 0.3.0" requires "stint >= 0.8.1 & < 0.9.0" requires "https://github.com/codex-storage/nim-datastore#a969b9799cb7fd2c2511b6820ded00bced141dea" requires "questionable >= 0.10.15 & < 0.11.0" From ee33946afb705a96a142f447b78ce1693e869d15 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:19:43 +1100 Subject: [PATCH 5/6] bump nimcrypto --- codexdht.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codexdht.nimble b/codexdht.nimble index 2d6037c..65fb548 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -10,7 +10,7 @@ skipDirs = @["tests"] requires "nim >= 2.0.14 & < 3.0.0" requires "secp256k1" # >= 0.5.2 & < 0.6.0 requires "protobuf_serialization" # >= 0.2.0 & < 0.3.0 -requires "nimcrypto >= 0.5.4 & < 0.6.0" +requires "nimcrypto >= 0.6.0 & < 0.7.0" requires "bearssl >= 0.2.5 & < 0.3.0" requires "chronicles >= 0.10.2 & < 0.11.0" requires "chronos >= 4.0.3 & < 4.1.0" From 89d22c156ec4846e8d6e6dca3cd08ac438bedf55 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:07:51 +1100 Subject: [PATCH 6/6] bump nimcrypto --- codexdht.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codexdht.nimble b/codexdht.nimble index 65fb548..d51e50f 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -10,7 +10,7 @@ skipDirs = @["tests"] requires "nim >= 2.0.14 & < 3.0.0" requires "secp256k1" # >= 0.5.2 & < 0.6.0 requires "protobuf_serialization" # >= 0.2.0 & < 0.3.0 -requires "nimcrypto >= 0.6.0 & < 0.7.0" +requires "nimcrypto >= 0.6.2 & < 0.7.0" requires "bearssl >= 0.2.5 & < 0.3.0" requires "chronicles >= 0.10.2 & < 0.11.0" requires "chronos >= 4.0.3 & < 4.1.0"