diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a25711f..c46d466 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - nim: [1.6.18] + nim: [2.0.14] os: [ubuntu-latest, macOS-latest, windows-latest] steps: - name: Checkout diff --git a/.gitignore b/.gitignore index 8bda8b1..a189692 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ NimBinaries .update.timestamp *.dSYM .vscode/* +nimbledeps diff --git a/codexdht.nimble b/codexdht.nimble index 0f89769..03de482 100644 --- a/codexdht.nimble +++ b/codexdht.nimble @@ -1,24 +1,25 @@ # Package -version = "0.4.0" +version = "0.5.0" author = "Status Research & Development GmbH" description = "DHT based on Eth discv5 implementation" license = "MIT" skipDirs = @["tests"] # Dependencies -requires "secp256k1#2acbbdcc0e63002a013fff49f015708522875832" # >= 0.5.2 & < 0.6.0 -requires "protobuf_serialization" # >= 0.2.0 & < 0.3.0 -requires "nimcrypto >= 0.5.4" -requires "bearssl == 0.2.5" +requires "nim >= 2.0.14 & < 3.0.0" +requires "secp256k1 >= 0.6.0 & < 0.7.0" +requires "protobuf_serialization >= 0.3.0 & < 0.4.0" +requires "nimcrypto >= 0.6.2 & < 0.7.0" +requires "bearssl >= 0.2.5 & < 0.3.0" requires "chronicles >= 0.10.2 & < 0.11.0" requires "chronos >= 4.0.3 & < 4.1.0" -requires "libp2p == 1.5.0" -requires "metrics" -requires "stew#head" -requires "stint" -requires "https://github.com/codex-storage/nim-datastore >= 0.1.1 & < 0.2.0" -requires "questionable" +requires "libp2p >= 1.5.0 & < 2.0.0" +requires "metrics >= 0.1.0 & < 0.2.0" +requires "stew >= 0.2.0 & < 0.3.0" +requires "stint >= 0.8.1 & < 0.9.0" +requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0" +requires "questionable >= 0.10.15 & < 0.11.0" task testAll, "Run all test suites": exec "nimble install -d -y" diff --git a/codexdht/private/eth/p2p/discoveryv5/encoding.nim b/codexdht/private/eth/p2p/discoveryv5/encoding.nim index cdbc30e..2cb7962 100644 --- a/codexdht/private/eth/p2p/discoveryv5/encoding.nim +++ b/codexdht/private/eth/p2p/discoveryv5/encoding.nim @@ -11,7 +11,7 @@ ## https://github.com/ethereum/devp2p/blob/master/discv5/discv5-theory.md#sessions ## -{.push raises: [Defect].} +{.push raises: [].} import std/[hashes, net, options, sugar, tables], diff --git a/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim b/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim index 8a57f8f..2f29fc4 100644 --- a/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim +++ b/codexdht/private/eth/p2p/discoveryv5/ip_vote.nim @@ -15,7 +15,7 @@ ## To select the right address, a majority count is done. This is done over a ## sort of moving window as votes expire after `IpVoteTimeout`. -{.push raises: [Defect].} +{.push raises: [].} import std/[tables, options], diff --git a/codexdht/private/eth/p2p/discoveryv5/lru.nim b/codexdht/private/eth/p2p/discoveryv5/lru.nim index 2fa8a11..ad458f7 100644 --- a/codexdht/private/eth/p2p/discoveryv5/lru.nim +++ b/codexdht/private/eth/p2p/discoveryv5/lru.nim @@ -1,6 +1,6 @@ import std/[tables, lists, options] -{.push raises: [Defect].} +{.push raises: [].} export tables, lists, options diff --git a/codexdht/private/eth/p2p/discoveryv5/messages.nim b/codexdht/private/eth/p2p/discoveryv5/messages.nim index da3cdb0..f6fcbd9 100644 --- a/codexdht/private/eth/p2p/discoveryv5/messages.nim +++ b/codexdht/private/eth/p2p/discoveryv5/messages.nim @@ -10,7 +10,7 @@ ## These messages get protobuf encoded, while in the spec they get RLP encoded. ## -{.push raises: [Defect].} +{.push raises: [].} import std/[hashes, net], diff --git a/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim b/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim index 09f690d..bbc513b 100644 --- a/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim +++ b/codexdht/private/eth/p2p/discoveryv5/messages_encoding.nim @@ -14,6 +14,7 @@ import stew/endians2, libp2p/routing_record, libp2p/signed_envelope, + libp2p/protobuf/minprotobuf, "."/[messages, spr, node], ../../../../dht/providers_encoding diff --git a/codexdht/private/eth/p2p/discoveryv5/node.nim b/codexdht/private/eth/p2p/discoveryv5/node.nim index 2f3d6c9..cad809f 100644 --- a/codexdht/private/eth/p2p/discoveryv5/node.nim +++ b/codexdht/private/eth/p2p/discoveryv5/node.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/hashes, @@ -28,7 +28,7 @@ type NodeId* = UInt256 Address* = object - ip*: ValidIpAddress + ip*: IpAddress port*: Port Stats* = object @@ -69,7 +69,7 @@ func newNode*( id: ? pk.toNodeId(), pubkey: pk, record: record, - address: Address(ip: ValidIpAddress.init(ip), port: port).some) + address: Address(ip: ip, port: port).some) ok node @@ -103,7 +103,7 @@ func newNode*(r: SignedPeerRecord): Result[Node, cstring] = record: r, address: none(Address))) -proc update*(n: Node, pk: PrivateKey, ip: Option[ValidIpAddress], +proc update*(n: Node, pk: PrivateKey, ip: Option[IpAddress], tcpPort, udpPort: Option[Port] = none[Port]()): Result[void, cstring] = ? n.record.update(pk, ip, tcpPort, udpPort) @@ -154,7 +154,7 @@ func shortLog*(id: NodeId): string = result.add(sid[i]) chronicles.formatIt(NodeId): shortLog(it) -func hash*(ip: ValidIpAddress): Hash = +func hash*(ip: IpAddress): Hash = case ip.family of IpAddressFamily.IPv6: hash(ip.address_v6) of IpAddressFamily.IPv4: hash(ip.address_v4) diff --git a/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim b/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim index d3cecf7..bd2dbd0 100644 --- a/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim +++ b/codexdht/private/eth/p2p/discoveryv5/nodes_verification.nim @@ -1,4 +1,4 @@ -{.push raises: [Defect].} +{.push raises: [].} import std/[sets, options], diff --git a/codexdht/private/eth/p2p/discoveryv5/protocol.nim b/codexdht/private/eth/p2p/discoveryv5/protocol.nim index 77afee3..0089d12 100644 --- a/codexdht/private/eth/p2p/discoveryv5/protocol.nim +++ b/codexdht/private/eth/p2p/discoveryv5/protocol.nim @@ -71,7 +71,7 @@ ## more requests will be needed for a lookup (adding bandwidth and latency). ## This might be a concern for mobile devices. -{.push raises: [Defect].} +{.push raises: [].} import std/[tables, sets, options, math, sequtils, algorithm, strutils], @@ -241,7 +241,7 @@ proc randomNodes*(d: Protocol, maxAmount: int): seq[Node] = d.routingTable.randomNodes(maxAmount) proc randomNodes*(d: Protocol, maxAmount: int, - pred: proc(x: Node): bool {.gcsafe, noSideEffect.}): seq[Node] = + pred: proc(x: Node): bool {.gcsafe, noSideEffect, raises: [].}): seq[Node] = ## Get a `maxAmount` of random nodes from the local routing table with the ## `pred` predicate function applied as filter on the nodes selected. d.routingTable.randomNodes(maxAmount, pred) @@ -563,7 +563,7 @@ proc ping*(d: Protocol, toNode: Node): dht_message_requests_outgoing.inc(labelValues = ["invalid_response"]) return err("Invalid response to ping message") else: - # A ping (or the pong) was lost, what should we do? Previous implementation called + # A ping (or the pong) was lost, what should we do? Previous implementation called # d.replaceNode(toNode) immediately, which removed the node. This is too aggressive, # especially if we have a temporary network outage. Although bootstrap nodes are protected # from being removed, everything else would slowly be removed. @@ -963,7 +963,7 @@ proc revalidateNode*(d: Protocol, n: Node) {.async.} = # Get IP and port from pong message and add it to the ip votes trace "pong rx", n, myip = res.ip, myport = res.port - let a = Address(ip: ValidIpAddress.init(res.ip), port: Port(res.port)) + let a = Address(ip: res.ip, port: Port(res.port)) d.ipVote.insert(n.id, a) proc revalidateLoop(d: Protocol) {.async.} = @@ -1076,7 +1076,7 @@ func init*( proc newProtocol*( privKey: PrivateKey, - enrIp: Option[ValidIpAddress], + enrIp: Option[IpAddress], enrTcpPort, enrUdpPort: Option[Port], localEnrFields: openArray[(string, seq[byte])] = [], bootstrapRecords: openArray[SignedPeerRecord] = [], diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim b/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim index 78c0ee2..b523c91 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/cache.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/sequtils diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/common.nim b/codexdht/private/eth/p2p/discoveryv5/providers/common.nim index 4edad41..e0ec0cc 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/common.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/common.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/sequtils import std/strutils diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim b/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim index 3536979..3be6943 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/maintenance.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/options import std/sequtils diff --git a/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim b/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim index 44ba094..7eb33b1 100644 --- a/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim +++ b/codexdht/private/eth/p2p/discoveryv5/providers/manager.nim @@ -19,7 +19,7 @@ import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -{.push raises: [Defect].} +{.push raises: [].} import ./maintenance import ./cache diff --git a/codexdht/private/eth/p2p/discoveryv5/routing_table.nim b/codexdht/private/eth/p2p/discoveryv5/routing_table.nim index edc9e65..e270315 100644 --- a/codexdht/private/eth/p2p/discoveryv5/routing_table.nim +++ b/codexdht/private/eth/p2p/discoveryv5/routing_table.nim @@ -5,7 +5,7 @@ # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # at your option. This file may not be copied, modified, or distributed except according to those terms. -{.push raises: [Defect].} +{.push raises: [].} import std/[algorithm, times, sequtils, bitops, sets, options, tables], @@ -21,7 +21,7 @@ declarePublicGauge dht_routing_table_buckets, logScope: topics = "discv5 routingtable" - + type DistanceProc* = proc(a, b: NodeId): NodeId {.raises: [Defect], gcsafe, noSideEffect.} LogDistanceProc* = proc(a, b: NodeId): uint16 {.raises: [Defect], gcsafe, noSideEffect.} @@ -34,7 +34,7 @@ type IpLimits* = object limit*: uint - ips: Table[ValidIpAddress, uint] + ips: Table[IpAddress, uint] RoutingTable* = object @@ -101,7 +101,7 @@ type ReplacementExisting NoAddress -func inc*(ipLimits: var IpLimits, ip: ValidIpAddress): bool = +func inc*(ipLimits: var IpLimits, ip: IpAddress): bool = let val = ipLimits.ips.getOrDefault(ip, 0) if val < ipLimits.limit: ipLimits.ips[ip] = val + 1 @@ -109,7 +109,7 @@ func inc*(ipLimits: var IpLimits, ip: ValidIpAddress): bool = else: false -func dec*(ipLimits: var IpLimits, ip: ValidIpAddress) = +func dec*(ipLimits: var IpLimits, ip: IpAddress) = let val = ipLimits.ips.getOrDefault(ip, 0) if val == 1: ipLimits.ips.del(ip) @@ -425,7 +425,7 @@ proc addNode*(r: var RoutingTable, n: Node): NodeStatus = (depth mod r.bitsPerHop != 0 and depth != ID_SIZE): r.splitBucket(r.buckets.find(bucket)) return r.addNode(n) # retry adding - + # When bucket doesn't get split the node is added to the replacement cache return r.addReplacement(bucket, n) @@ -555,7 +555,7 @@ proc nodeToRevalidate*(r: RoutingTable): Node = return b.nodes[^1] proc randomNodes*(r: RoutingTable, maxAmount: int, - pred: proc(x: Node): bool {.gcsafe, noSideEffect.} = nil): seq[Node] = + pred: proc(x: Node): bool {.gcsafe, noSideEffect, raises: [].} = nil): seq[Node] = ## Get a `maxAmount` of random nodes from the routing table with the `pred` ## predicate function applied as filter on the nodes selected. var maxAmount = maxAmount diff --git a/codexdht/private/eth/p2p/discoveryv5/sessions.nim b/codexdht/private/eth/p2p/discoveryv5/sessions.nim index ffcb76f..ee770bb 100644 --- a/codexdht/private/eth/p2p/discoveryv5/sessions.nim +++ b/codexdht/private/eth/p2p/discoveryv5/sessions.nim @@ -16,7 +16,7 @@ ## - the one derived in the key-exchange started by the other node. ## To alleviate this issue, we store two decryption keys in each session. -{.push raises: [Defect].} +{.push raises: [].} import std/options, @@ -39,7 +39,7 @@ type func makeKey(id: NodeId, address: Address): SessionKey = var pos = 0 - result[pos ..< pos+sizeof(id)] = toBytes(id) + result[pos ..< pos+sizeof(id)] = toBytesBE(id) pos.inc(sizeof(id)) case address.ip.family of IpAddressFamily.IpV4: @@ -47,7 +47,7 @@ func makeKey(id: NodeId, address: Address): SessionKey = of IpAddressFamily.IpV6: result[pos ..< pos+sizeof(address.ip.address_v6)] = address.ip.address_v6 pos.inc(sizeof(address.ip.address_v6)) - result[pos ..< pos+sizeof(address.port)] = toBytes(address.port.uint16) + result[pos ..< pos+sizeof(address.port)] = toBytesBE(address.port.uint16) func swapr*(s: var Sessions, id: NodeId, address: Address) = var value: array[3 * sizeof(AesKey), byte] diff --git a/codexdht/private/eth/p2p/discoveryv5/spr.nim b/codexdht/private/eth/p2p/discoveryv5/spr.nim index 15c6c2a..351b8a7 100644 --- a/codexdht/private/eth/p2p/discoveryv5/spr.nim +++ b/codexdht/private/eth/p2p/discoveryv5/spr.nim @@ -58,7 +58,7 @@ proc incSeqNo*( proc update*( r: var SignedPeerRecord, pk: crypto.PrivateKey, - ip: Option[ValidIpAddress], + ip: Option[IpAddress], tcpPort, udpPort: Option[Port] = none[Port]()): RecordResult[void] = ## Update a `SignedPeerRecord` with given ip address, tcp port, udp port and optional @@ -97,9 +97,8 @@ proc update*( if udpPort.isNone and tcpPort.isNone: return err "No existing address in SignedPeerRecord with no port provided" - let ipAddr = try: ValidIpAddress.init(ip.get) - except ValueError as e: - return err ("Existing address contains invalid address: " & $e.msg).cstring + let ipAddr = ip.get + if tcpPort.isSome: transProto = IpTransportProtocol.tcpProtocol transProtoPort = tcpPort.get @@ -223,7 +222,7 @@ proc init*( T: type SignedPeerRecord, seqNum: uint64, pk: PrivateKey, - ip: Option[ValidIpAddress], + ip: Option[IpAddress], tcpPort, udpPort: Option[Port]): RecordResult[T] = ## Initialize a `SignedPeerRecord` with given sequence number, private key, optional @@ -238,9 +237,7 @@ proc init*( tcpPort, udpPort var - ipAddr = try: ValidIpAddress.init("127.0.0.1") - except ValueError as e: - return err ("Existing address contains invalid address: " & $e.msg).cstring + ipAddr = static parseIpAddress("127.0.0.1") proto: IpTransportProtocol protoPort: Port diff --git a/codexdht/private/eth/p2p/discoveryv5/transport.nim b/codexdht/private/eth/p2p/discoveryv5/transport.nim index 04fe20c..e1dac60 100644 --- a/codexdht/private/eth/p2p/discoveryv5/transport.nim +++ b/codexdht/private/eth/p2p/discoveryv5/transport.nim @@ -259,7 +259,7 @@ proc processClient[T](transp: DatagramTransport, raddr: TransportAddress): except ValueError as e: error "Not a valid IpAddress", exception = e.name, msg = e.msg return - let a = Address(ip: ValidIpAddress.init(ip), port: raddr.port) + let a = Address(ip: ip, port: raddr.port) t.receive(a, buf) @@ -292,7 +292,7 @@ proc newTransport*[T]( Transport[T]( client: client, - bindAddress: Address(ip: ValidIpAddress.init(bindIp), port: bindPort), + bindAddress: Address(ip: bindIp, port: bindPort), codec: Codec( localNode: localNode, privKey: privKey, diff --git a/config.nims b/config.nims index eacae8d..d816c1a 100644 --- a/config.nims +++ b/config.nims @@ -1,2 +1,9 @@ switch("define", "libp2p_pki_schemes=secp256k1") +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config + +when (NimMajor, NimMinor) >= (2, 0): + --mm:refc diff --git a/tests/dht/test_helper.nim b/tests/dht/test_helper.nim index 3d92b96..f841b7e 100644 --- a/tests/dht/test_helper.nim +++ b/tests/dht/test_helper.nim @@ -4,14 +4,10 @@ import libp2p/crypto/[crypto, secp], libp2p/multiaddress, codexdht/discv5/[node, routing_table, spr], - codexdht/discv5/crypto as dhtcrypto, - codexdht/discv5/protocol as discv5_protocol, - stew/shims/net - -export net + codexdht/discv5/protocol as discv5_protocol proc localAddress*(port: int): Address = - Address(ip: ValidIpAddress.init("127.0.0.1"), port: Port(port)) + Address(ip: parseIpAddress("127.0.0.1"), port: Port(port)) proc example*(T: type PrivateKey, rng: ref HmacDrbgContext): PrivateKey = PrivateKey.random(PKScheme.Secp256k1, rng[]).expect("Valid rng for private key") @@ -54,7 +50,7 @@ proc nodeIdInNodes*(id: NodeId, nodes: openArray[Node]): bool = if id == n.id: return true proc generateNode*(privKey: PrivateKey, port: int, - ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): Node = + ip: IpAddress = parseIpAddress("127.0.0.1")): Node = let port = Port(port) @@ -72,7 +68,7 @@ proc generateNRandomNodes*(rng: ref HmacDrbgContext, n: int): seq[Node] = res proc nodeAndPrivKeyAtDistance*(n: Node, rng: var HmacDrbgContext, d: uint32, - ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): (Node, PrivateKey) = + ip: IpAddress = parseIpAddress("127.0.0.1")): (Node, PrivateKey) = while true: let privKey = PrivateKey.random(rng).expect("Valid rng for private key") @@ -81,23 +77,23 @@ proc nodeAndPrivKeyAtDistance*(n: Node, rng: var HmacDrbgContext, d: uint32, return (node, privKey) proc nodeAtDistance*(n: Node, rng: var HmacDrbgContext, d: uint32, - ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): Node = + ip: IpAddress = parseIpAddress("127.0.0.1")): Node = let (node, _) = n.nodeAndPrivKeyAtDistance(rng, d, ip) node proc nodesAtDistance*( n: Node, rng: var HmacDrbgContext, d: uint32, amount: int, - ip: ValidIpAddress = ValidIpAddress.init("127.0.0.1")): seq[Node] = + ip: IpAddress = parseIpAddress("127.0.0.1")): seq[Node] = for i in 0..