From b2994db0a0c8f67d5ce7cb7bd45b6f2d469385d8 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 3 Jun 2026 12:55:40 -0300 Subject: [PATCH 01/15] Get nim-libp2p 2.0.0 bump to E2E working state (WIP) * deps: json_rpc + web3 via websock-relaxed forks (WIP: revert on upstream fix) * fix: setup() post-build switch services (autonat nil segfault) * fix: unified 2.0.0 connection limits * test: 2.0.0 defaults (identify-push, agent string) * test: drop reboot/reconnect (unsupported Switch restart) --- logos_delivery.nimble | 2 +- nimble.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/logos_delivery.nimble b/logos_delivery.nimble index efe118595..3ffa8fa22 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -42,7 +42,7 @@ requires "nim >= 2.2.4", # RPC & APIs "https://github.com/status-im/nim-json-rpc.git#v0.6.1", "presto", - "web3", + "https://github.com/fcecin/nim-web3-websock040#96f39ea1b633277c1d93eafd62759db1d498e359", # Database "db_connector", "sqlite3_abi", diff --git a/nimble.lock b/nimble.lock index 18ebde258..e9ecc42b8 100644 --- a/nimble.lock +++ b/nimble.lock @@ -538,8 +538,8 @@ }, "web3": { "version": "0.8.0", - "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", - "url": "https://github.com/status-im/nim-web3", + "vcsRevision": "96f39ea1b633277c1d93eafd62759db1d498e359", + "url": "https://github.com/fcecin/nim-web3-websock040", "downloadMethod": "git", "dependencies": [ "nim", @@ -557,7 +557,7 @@ "results" ], "checksums": { - "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" + "sha1": "b5972104bc9223a34ae3758293225264822af893" } }, "dnsdisc": { From f21ef7f384b88457d614a4eca14d08d190a58e2d Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 10 Jun 2026 09:40:51 -0300 Subject: [PATCH 02/15] remove dep forks --- logos_delivery.nimble | 2 +- nimble.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/logos_delivery.nimble b/logos_delivery.nimble index 3ffa8fa22..efe118595 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -42,7 +42,7 @@ requires "nim >= 2.2.4", # RPC & APIs "https://github.com/status-im/nim-json-rpc.git#v0.6.1", "presto", - "https://github.com/fcecin/nim-web3-websock040#96f39ea1b633277c1d93eafd62759db1d498e359", + "web3", # Database "db_connector", "sqlite3_abi", diff --git a/nimble.lock b/nimble.lock index e9ecc42b8..e8ac890a9 100644 --- a/nimble.lock +++ b/nimble.lock @@ -472,7 +472,7 @@ "unittest2" ], "checksums": { - "sha1": "596db0aafcb3c83f5dba6d42993f2276e0d00eb5" + "sha1": "4221c98dbc09ba56d68ca0a36b5c2ad90e39a43b" } }, "lsquic": { @@ -538,8 +538,8 @@ }, "web3": { "version": "0.8.0", - "vcsRevision": "96f39ea1b633277c1d93eafd62759db1d498e359", - "url": "https://github.com/fcecin/nim-web3-websock040", + "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", + "url": "https://github.com/status-im/nim-web3", "downloadMethod": "git", "dependencies": [ "nim", @@ -557,7 +557,7 @@ "results" ], "checksums": { - "sha1": "b5972104bc9223a34ae3758293225264822af893" + "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" } }, "dnsdisc": { From 7fb87c4a75c4f1aa141b97418093a387cb5b1536 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 10 Jun 2026 15:10:55 -0300 Subject: [PATCH 03/15] pin json_rpc, lsquic, boringssl to release tags * json_rpc: to v0.6.1 tag (latest) * lsquic: v0.5.1 (latest; QUIC connection-handling fixes) * boringssl: v0.0.8 (latest) --- nimble.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimble.lock b/nimble.lock index e8ac890a9..18ebde258 100644 --- a/nimble.lock +++ b/nimble.lock @@ -472,7 +472,7 @@ "unittest2" ], "checksums": { - "sha1": "4221c98dbc09ba56d68ca0a36b5c2ad90e39a43b" + "sha1": "596db0aafcb3c83f5dba6d42993f2276e0d00eb5" } }, "lsquic": { From 573da98390e6b73ec5c30191ba8d3f31e60cc161 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 3 Jun 2026 12:55:40 -0300 Subject: [PATCH 04/15] Get nim-libp2p 2.0.0 bump to E2E working state (WIP) * deps: json_rpc + web3 via websock-relaxed forks (WIP: revert on upstream fix) * fix: setup() post-build switch services (autonat nil segfault) * fix: unified 2.0.0 connection limits * test: 2.0.0 defaults (identify-push, agent string) * test: drop reboot/reconnect (unsupported Switch restart) --- logos_delivery.nimble | 2 +- nimble.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/logos_delivery.nimble b/logos_delivery.nimble index efe118595..3ffa8fa22 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -42,7 +42,7 @@ requires "nim >= 2.2.4", # RPC & APIs "https://github.com/status-im/nim-json-rpc.git#v0.6.1", "presto", - "web3", + "https://github.com/fcecin/nim-web3-websock040#96f39ea1b633277c1d93eafd62759db1d498e359", # Database "db_connector", "sqlite3_abi", diff --git a/nimble.lock b/nimble.lock index 18ebde258..e9ecc42b8 100644 --- a/nimble.lock +++ b/nimble.lock @@ -538,8 +538,8 @@ }, "web3": { "version": "0.8.0", - "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", - "url": "https://github.com/status-im/nim-web3", + "vcsRevision": "96f39ea1b633277c1d93eafd62759db1d498e359", + "url": "https://github.com/fcecin/nim-web3-websock040", "downloadMethod": "git", "dependencies": [ "nim", @@ -557,7 +557,7 @@ "results" ], "checksums": { - "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" + "sha1": "b5972104bc9223a34ae3758293225264822af893" } }, "dnsdisc": { From 6ce9925f668a2a830b4d22a1b9a952cc42ace486 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 10 Jun 2026 09:40:51 -0300 Subject: [PATCH 05/15] remove dep forks --- logos_delivery.nimble | 2 +- nimble.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/logos_delivery.nimble b/logos_delivery.nimble index 3ffa8fa22..efe118595 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -42,7 +42,7 @@ requires "nim >= 2.2.4", # RPC & APIs "https://github.com/status-im/nim-json-rpc.git#v0.6.1", "presto", - "https://github.com/fcecin/nim-web3-websock040#96f39ea1b633277c1d93eafd62759db1d498e359", + "web3", # Database "db_connector", "sqlite3_abi", diff --git a/nimble.lock b/nimble.lock index e9ecc42b8..e8ac890a9 100644 --- a/nimble.lock +++ b/nimble.lock @@ -472,7 +472,7 @@ "unittest2" ], "checksums": { - "sha1": "596db0aafcb3c83f5dba6d42993f2276e0d00eb5" + "sha1": "4221c98dbc09ba56d68ca0a36b5c2ad90e39a43b" } }, "lsquic": { @@ -538,8 +538,8 @@ }, "web3": { "version": "0.8.0", - "vcsRevision": "96f39ea1b633277c1d93eafd62759db1d498e359", - "url": "https://github.com/fcecin/nim-web3-websock040", + "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", + "url": "https://github.com/status-im/nim-web3", "downloadMethod": "git", "dependencies": [ "nim", @@ -557,7 +557,7 @@ "results" ], "checksums": { - "sha1": "b5972104bc9223a34ae3758293225264822af893" + "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" } }, "dnsdisc": { From 20d43c577befa1b653e0fa0101bdaae9b76f4731 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 10 Jun 2026 15:10:55 -0300 Subject: [PATCH 06/15] pin json_rpc, lsquic, boringssl to release tags * json_rpc: to v0.6.1 tag (latest) * lsquic: v0.5.1 (latest; QUIC connection-handling fixes) * boringssl: v0.0.8 (latest) --- nimble.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimble.lock b/nimble.lock index e8ac890a9..18ebde258 100644 --- a/nimble.lock +++ b/nimble.lock @@ -472,7 +472,7 @@ "unittest2" ], "checksums": { - "sha1": "4221c98dbc09ba56d68ca0a36b5c2ad90e39a43b" + "sha1": "596db0aafcb3c83f5dba6d42993f2276e0d00eb5" } }, "lsquic": { From 2c345199c6c3af6ebbbe0aa695857c14709dd287 Mon Sep 17 00:00:00 2001 From: SionoiS Date: Wed, 6 May 2026 11:31:37 -0400 Subject: [PATCH 07/15] kademlia discovery first draft --- waku/discovery/waku_kademlia.nim | 201 ++++++++++++++++++ .../kademlia_discovery_conf_builder.nim | 79 +++++++ 2 files changed, 280 insertions(+) create mode 100644 waku/discovery/waku_kademlia.nim create mode 100644 waku/factory/conf_builder/kademlia_discovery_conf_builder.nim diff --git a/waku/discovery/waku_kademlia.nim b/waku/discovery/waku_kademlia.nim new file mode 100644 index 000000000..be5ddb49e --- /dev/null +++ b/waku/discovery/waku_kademlia.nim @@ -0,0 +1,201 @@ +{.push raises: [].} + +import std/sequtils +import + chronos, + chronicles, + results, + stew/byteutils, + libp2p/[peerid, multiaddress, switch], + libp2p/extended_peer_record, + libp2p/crypto/curve25519, + libp2p/protocols/service_discovery, + libp2p/protocols/service_discovery/types as sd_types, + libp2p/protocols/mix/mix_protocol + +import + waku/waku_core, + waku/node/peer_manager, + waku/events/discovery_events, + waku/factory/waku_conf + +logScope: + topics = "waku service discovery" + +const + DefaultServiceDiscoveryInterval* = chronos.seconds(60) + DefaultRandomDiscoveryInterval* = chronos.seconds(60) + +type WakuKademlia* = ref object + protocol*: ServiceDiscovery + peerManager: PeerManager + randomLookupLoop: Future[void] + serviceLookupLoop: Future[void] + randomLookupInterval: Duration + serviceLookupInterval: Duration + servicesToDiscover: seq[string] + servicesToAdvertise: seq[ServiceInfo] + +proc extractMixPubKey(service: ServiceInfo): Opt[Curve25519Key] = + if service.id != MixProtocolID: + return none(Curve25519Key) + + if service.data.len != Curve25519KeySize: + error "invalid mix pub key length", + expected = Curve25519KeySize, + actual = service.data.len, + dataHex = byteutils.toHex(service.data) + return none(Curve25519Key) + + let key = intoCurve25519Key(service.data) + + return some(key) + +proc remotePeerInfoFrom(record: ExtendedPeerRecord): Opt[RemotePeerInfo] = + if record.addresses.len == 0: + error "missing addresses", peerId = record.peerId + return none(RemotePeerInfo) + + let addrs = record.addresses.mapIt(it.address) + if addrs.len == 0: + error "no dialable addresses", peerId = record.peerId + return none(RemotePeerInfo) + + var mixPubKey = none(Curve25519Key) + for service in record.services: + mixPubKey = extractMixPubKey(service).valueOr: + continue + + trace "successfully extracted mix pub key", + peerId = record.peerId, keyHex = byteutils.toHex(mixPubKey.get()) + + break + + return some( + RemotePeerInfo.init( + record.peerId, addrs = addrs, origin = PeerOrigin.Kademlia, mixPubKey = mixPubKey + ) + ) + +proc lookupServicePeers*( + self: WakuKademlia, serviceId: string +): Future[Result[seq[RemotePeerInfo], string]] {.async: (raises: []).} = + if self.protocol.isNil(): + return err("cannot lookup service peers: service discovery not mounted") + + let serviceInfo = ServiceInfo(id: serviceId, data: @[]) + + let lookupCatch = catch: + (await self.protocol.lookup(serviceInfo)) + + let lookupResult = lookupCatch.valueOr: + return err("service peer lookup failed: " & error.msg) + + let advertisements = lookupResult.valueOr: + return err("service peer lookup failed: " & lookupResult.error) + + var discovered: seq[RemotePeerInfo] + for ad in advertisements: + let record = ad.data + let peerInfo = remotePeerInfoFrom(record).valueOr: + continue + + self.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) + discovered.add(peerInfo) + + debug "service lookup complete", serviceId, found = discovered.len + return ok(discovered) + +proc runRandomLookupLoop(self: WakuKademlia) {.async: (raises: [CancelledError]).} = + debug "periodic random lookup started", interval = $self.randomLookupInterval + + while true: + await sleepAsync(self.randomLookupInterval) + + let recordsRes = catch: + (await self.protocol.lookupRandom()) + + let records = recordsRes.valueOr: + error "random lookup failed", error + continue + + var discoveredPeers: seq[RemotePeerInfo] + for record in records: + let peerInfo = remotePeerInfoFrom(record).valueOr: + continue + + self.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) + discoveredPeers.add(peerInfo) + + if discoveredPeers.len > 0: + PeersDiscoveredEvent.emit(peers = discoveredPeers) + + debug "random lookup complete", found = discoveredPeers.len + +proc runServiceLookupLoop(self: WakuKademlia) {.async: (raises: [CancelledError]).} = + debug "periodic service lookup started", + interval = $self.serviceLookupInterval, services = self.servicesToDiscover + + while true: + await sleepAsync(self.serviceLookupInterval) + + if self.servicesToDiscover.len == 0: + continue + + for serviceId in self.servicesToDiscover: + let discovered = (await self.lookupServicePeers(serviceId)).valueOr: + error "service lookup failed", serviceId, error + continue + + if discovered.len > 0: + PeersDiscoveredEvent.emit(peers = discovered) + +proc new*( + T: type WakuKademlia, + switch: Switch, + peerManager: PeerManager, + config: KademliaDiscoveryConf, +): Result[T, string] = + if config.bootstrapNodes.len == 0: + debug "creating service discovery as seed node (no bootstrap nodes)" + + let protocol = ServiceDiscovery.new( + switch, bootstrapNodes = config.bootstrapNodes, services = config.servicesToDiscover + ) + + let self = WakuKademlia( + protocol: protocol, + peerManager: peerManager, + randomLookupInterval: config.randomLookupInterval, + serviceLookupInterval: config.serviceLookupInterval, + servicesToDiscover: config.servicesToDiscover, + servicesToAdvertise: config.servicesToAdvertise, + ) + + return ok(self) + +proc start*(self: WakuKademlia) {.async: (raises: []).} = + for serviceId in self.servicesToDiscover: + discard self.protocol.startDiscovering(serviceId) + + for serviceInfo in self.servicesToAdvertise: + self.protocol.addProvidedService(serviceInfo) + + if self.randomLookupLoop.isNil(): + self.randomLookupLoop = self.runRandomLookupLoop() + + if self.serviceLookupLoop.isNil(): + self.serviceLookupLoop = self.runServiceLookupLoop() + + info "kademlia discovery started" + +proc stop*(self: WakuKademlia) {.async: (raises: []).} = + if not self.serviceLookupLoop.isNil(): + await self.serviceLookupLoop.cancelAndWait() + self.serviceLookupLoop = nil + + if not self.randomLookupLoop.isNil(): + await self.randomLookupLoop.cancelAndWait() + self.randomLookupLoop = nil + + info "kademlia discovery stopped" diff --git a/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim b/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim new file mode 100644 index 000000000..98304db23 --- /dev/null +++ b/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim @@ -0,0 +1,79 @@ +import chronicles, std/options, results +import chronos +import libp2p/[peerid, multiaddress, peerinfo, extended_peer_record] +import waku/factory/waku_conf + +logScope: + topics = "waku conf builder kademlia discovery" + +####################################### +## Kademlia Discovery Config Builder ## +####################################### +type KademliaDiscoveryConfBuilder* = object + enabled*: bool + bootstrapNodes*: seq[string] + servicesToAdvertise*: seq[(string, seq[byte])] + servicesToDiscover*: seq[string] + randomLookupInterval*: Duration + serviceLookupInterval*: Duration + +proc init*(T: type KademliaDiscoveryConfBuilder): KademliaDiscoveryConfBuilder = + KademliaDiscoveryConfBuilder() + +proc withEnabled*(b: var KademliaDiscoveryConfBuilder, enabled: bool) = + b.enabled = enabled + +proc withBootstrapNodes*( + b: var KademliaDiscoveryConfBuilder, bootstrapNodes: seq[string] +) = + b.bootstrapNodes = bootstrapNodes + +proc withServicesToAdvertise*( + b: var KademliaDiscoveryConfBuilder, services: seq[(string, seq[byte])] +) = + b.servicesToAdvertise = services + +proc withServicesToDiscover*( + b: var KademliaDiscoveryConfBuilder, services: seq[string] +) = + b.servicesToDiscover = services + +proc withRandomLookupInterval*( + b: var KademliaDiscoveryConfBuilder, interval: Duration +) = + b.randomLookupInterval = interval + +proc withServiceLookupInterval*( + b: var KademliaDiscoveryConfBuilder, interval: Duration +) = + b.serviceLookupInterval = interval + +proc build*( + b: KademliaDiscoveryConfBuilder +): Result[Option[KademliaDiscoveryConf], string] = + # Kademlia is enabled if explicitly enabled OR if bootstrap nodes are provided + let enabled = b.enabled or b.bootstrapNodes.len > 0 + if not enabled: + return ok(none(KademliaDiscoveryConf)) + + var parsedNodes: seq[(PeerId, seq[MultiAddress])] + for nodeStr in b.bootstrapNodes: + let (peerId, ma) = parseFullAddress(nodeStr).valueOr: + return err("Failed to parse kademlia bootstrap node: " & error) + parsedNodes.add((peerId, @[ma])) + + var servicesToAdvertise: seq[ServiceInfo] + for (serviceId, data) in b.servicesToAdvertise: + servicesToAdvertise.add(ServiceInfo(id: serviceId, data: data)) + + return ok( + some( + KademliaDiscoveryConf( + bootstrapNodes: parsedNodes, + servicesToAdvertise: servicesToAdvertise, + servicesToDiscover: b.servicesToDiscover, + randomLookupInterval: b.randomLookupInterval, + serviceLookupInterval: b.serviceLookupInterval, + ) + ) + ) From eb02a3fb74b1c3b66eaa1b60e7a89c4ae6764572 Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Thu, 11 Jun 2026 08:24:03 -0400 Subject: [PATCH 08/15] update config --- logos_delivery/waku/node/waku_node.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/logos_delivery/waku/node/waku_node.nim b/logos_delivery/waku/node/waku_node.nim index 2ad7dc601..ad0ba2d0a 100644 --- a/logos_delivery/waku/node/waku_node.nim +++ b/logos_delivery/waku/node/waku_node.nim @@ -343,10 +343,11 @@ proc mountKademlia*( return err("WakuKademlia already mounted, skipping") let wk = WakuKademlia.new( - node.switch, node.peerManager, config.bootstrapNodes, config.servicesToAdvertise, - config.servicesToDiscover, config.randomLookupInterval, - config.serviceLookupInterval, node.rng, config.kadDhtConfig, config.discoConfig, - config.clientMode, config.xprPublishing, + node.switch, node.peerManager, + config.bootstrapNodes, config.servicesToAdvertise, config.servicesToDiscover, + config.randomLookupInterval, config.serviceLookupInterval, + node.rng, + config.kadDhtConfig, config.discoConfig, config.clientMode, config.xprPublishing ).valueOr: return err("failed to create service discovery: " & error) From c82638862824cc7b6559f8aae5fb4a2320743401 Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Thu, 11 Jun 2026 12:35:47 -0400 Subject: [PATCH 09/15] chat2disco init --- Makefile | 6 +- apps/chat2disco/chat2disco.nim | 439 ++++++++++++++++++++++++++ apps/chat2disco/config_chat2disco.nim | 103 ++++++ apps/chat2disco/nim.cfg | 4 + logos_delivery.nimble | 11 + 5 files changed, 562 insertions(+), 1 deletion(-) create mode 100644 apps/chat2disco/chat2disco.nim create mode 100644 apps/chat2disco/config_chat2disco.nim create mode 100644 apps/chat2disco/nim.cfg diff --git a/Makefile b/Makefile index daa8a8ad7..60c26751b 100644 --- a/Makefile +++ b/Makefile @@ -219,7 +219,7 @@ testcommon: | build-deps build ########## ## Waku ## ########## -.PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester +.PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2mix chat2disco chat2bridge liteprotocoltester testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ @@ -256,6 +256,10 @@ chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(NIMBLE) chat2mix +chat2disco: | build-deps build deps librln + echo -e $(BUILD_MSG) "build/$@" && \ + $(NIMBLE) chat2disco + rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(NIMBLE) rln_db_inspector diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim new file mode 100644 index 000000000..83151e857 --- /dev/null +++ b/apps/chat2disco/chat2disco.nim @@ -0,0 +1,439 @@ +# chat2disco is a minimal chat app for testing Waku Kademlia service discovery. +# /room subscribes to the pubsub topic of that name and starts + +when not (compileOption("threads")): + {.fatal: "Please, compile this program with the --threads:on option!".} + +{.push raises: [].} + +import std/[strformat, strutils, times, options, sequtils] +import + confutils, + chronicles, + chronos, + eth/keys, + bearssl, + results, + stew/[byteutils], + metrics, + metrics/chronos_httpserver +import + libp2p/[ + switch, + crypto/crypto, + stream/connection, + multiaddress, + peerinfo, + peerid, + protobuf/minprotobuf, + protocols/kademlia/types, + protocols/service_discovery, + protocols/service_discovery/types as sd_types, + ] +import + logos_delivery/waku/[ + waku_core, + waku_enr, + discovery/waku_kademlia, + waku_node, + node/waku_metrics, + node/peer_manager, + factory/builder, + factory/conf_builder/kademlia_discovery_conf_builder, + common/utils/nat, + common/logging, + discovery/autonat_service, + ], + ./config_chat2disco + +import libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub +import libp2p/extended_peer_record # for ServiceInfo + +logScope: + topics = "chat2disco" + +const Help = """ + Commands: /[?|help|connect|nick|room|exit] + help: Prints this help + connect: dials a remote peer + nick: change nickname for current chat session + room : subscribe to pubsub topic and advertise + discover the kademlia service of the same name (kademlia always active) + exit: exits chat session +""" + +const DefaultContentTopic = "/chat2disco/1/room/proto" + +type Chat = ref object + node: WakuNode + transp: StreamTransport + subscribed: bool + connected: bool + started: bool + nick: string + prompt: bool + currentPubsubTopic: string + +type + PrivateKey* = crypto.PrivateKey + Topic* = waku_core.PubsubTopic + +##################### +## chat2 protobufs ## +##################### + +type + SelectResult*[T] = Result[T, string] + + Chat2Message* = object + timestamp*: int64 + nick*: string + payload*: seq[byte] + +proc init*(T: type Chat2Message, buffer: seq[byte]): ProtoResult[T] = + var msg = Chat2Message() + let pb = initProtoBuffer(buffer) + + var timestamp: uint64 + discard ?pb.getField(1, timestamp) + msg.timestamp = int64(timestamp) + + discard ?pb.getField(2, msg.nick) + discard ?pb.getField(3, msg.payload) + + ok(msg) + +proc encode*(message: Chat2Message): ProtoBuffer = + var serialised = initProtoBuffer() + + serialised.write(1, uint64(message.timestamp)) + serialised.write(2, message.nick) + serialised.write(3, message.payload) + + return serialised + +proc `$`*(message: Chat2Message): string = + let time = message.timestamp.fromUnix().local().format("'<'MMM' 'dd,' 'HH:mm'>'") + return time & " " & message.nick & ": " & string.fromBytes(message.payload) + +##################### + +proc connectToNodes(c: Chat, nodes: seq[string]) {.async.} = + echo "Connecting to nodes" + await c.node.connectToNodes(nodes) + c.connected = true + +proc showChatPrompt(c: Chat) = + if not c.prompt: + try: + stdout.write(">> ") + stdout.flushFile() + c.prompt = true + except IOError: + discard + +proc getChatLine(payload: seq[byte]): string = + let pb = Chat2Message.init(payload).valueOr: + return string.fromBytes(payload) + return $pb + +proc printReceivedMessage(c: Chat, pubsubTopic: PubsubTopic, msg: WakuMessage) = + let chatLine = getChatLine(msg.payload) + try: + echo &"[{pubsubTopic}] {chatLine}" + except ValueError: + echo chatLine + c.prompt = false + showChatPrompt(c) + trace "Printing message", chatLine, pubsubTopic, contentTopic = msg.contentTopic + +proc readNick(transp: StreamTransport): Future[string] {.async.} = + stdout.write("Choose a nickname >> ") + stdout.flushFile() + return await transp.readLine() + +proc readRoom(transp: StreamTransport): Future[string] {.async.} = + stdout.write("Choose a room >> ") + stdout.flushFile() + return await transp.readLine() + +proc startMetricsServer( + serverIp: IpAddress, serverPort: Port +): Result[MetricsHttpServerRef, string] = + info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $serverPort + + let server = MetricsHttpServerRef.new($serverIp, serverPort).valueOr: + return err("metrics HTTP server start failed: " & $error) + + try: + waitFor server.start() + except CatchableError: + return err("metrics HTTP server start failed: " & getCurrentExceptionMsg()) + + info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $serverPort + ok(server) + +proc publish(c: Chat, line: string) {.async.} = + if c.currentPubsubTopic.len == 0: + echo "No active room. Use /room to join one first." + return + + let time = getTime().toUnix() + let chat2pb = + Chat2Message(timestamp: time, nick: c.nick, payload: line.toBytes()).encode() + + var message = WakuMessage( + payload: chat2pb.buffer, + contentTopic: DefaultContentTopic, + version: 0, + timestamp: getNanosecondTime(time), + ) + + try: + (await c.node.publish(some(c.currentPubsubTopic), message)).isOkOr: + error "failed to publish message", error = error + except CatchableError: + error "caught error publishing message: ", error = getCurrentExceptionMsg() + +proc joinRoom(c: Chat, roomName: string) {.async.} = + if c.currentPubsubTopic.len > 0 and c.currentPubsubTopic != roomName: + discard c.node.unsubscribe((kind: PubsubSub, topic: c.currentPubsubTopic)) + + c.currentPubsubTopic = roomName + + proc roomHandler( + pubsubTopic: PubsubTopic, msg: WakuMessage + ): Future[void] {.async, gcsafe.} = + c.printReceivedMessage(pubsubTopic, msg) + + c.node.subscribe((kind: PubsubSub, topic: roomName), WakuRelayHandler(roomHandler)).isOkOr: + error "failed to subscribe to pubsub topic for room", + topic = roomName, error = error + + echo "subscribed to pubsub topic: ", roomName + + # Kademlia service discovery + advertisement for the room. + # Kademlia is always mounted (seed node if no --kad-bootstrap-node provided). + let svc = roomName + discard c.node.wakuKademlia.protocol.registerInterest(svc) + let svcInfo = ServiceInfo(id: svc, data: @[]) + c.node.wakuKademlia.protocol.startAdvertising(svcInfo) + echo "advertising and discovering kademlia service: ", svc + + asyncSpawn ( + proc() {.async.} = + let res = await c.node.wakuKademlia.lookupServicePeers(svc) + if res.isOk(): + let peers = res.get() + if peers.len > 0: + echo "discovered room peers via kademlia: ", peers.mapIt($it.peerId) + await c.node.connectToNodes(peers) + else: + error "service lookup failed for room", service = svc, error = res.error + )() + +proc readAndPrint(c: Chat) {.async.} = + while true: + await sleepAsync(100.millis) + +proc writeAndPrint(c: Chat) {.async.} = + while true: + showChatPrompt(c) + + let line = await c.transp.readLine() + if line.startsWith("/help") or line.startsWith("/?") or not c.started: + echo Help + continue + elif line.startsWith("/connect"): + if c.connected: + echo "already connected to at least one peer" + continue + echo "enter address of remote peer" + let address = await c.transp.readLine() + if address.len > 0: + await c.connectToNodes(@[address]) + elif line.startsWith("/nick"): + c.nick = await readNick(c.transp) + echo "You are now known as " & c.nick + elif line.startsWith("/room"): + let parts = line.split(maxsplit = 1) + if parts.len < 2 or parts[1].strip().len == 0: + echo "usage: /room " + continue + let roomName = parts[1].strip() + await c.joinRoom(roomName) + elif line.startsWith("/exit"): + echo "quitting..." + + try: + await c.node.stop() + except: + echo "exception happened when stopping: " & getCurrentExceptionMsg() + + quit(QuitSuccess) + else: + if c.started: + if c.currentPubsubTopic.len > 0: + echo "publishing message to ", c.currentPubsubTopic, ": ", line + await c.publish(line) + else: + echo "Join a room first with /room before sending messages" + else: + try: + if line.startsWith("/") and "p2p" in line: + await c.connectToNodes(@[line]) + except: + echo &"unable to dial remote peer {line}" + echo getCurrentExceptionMsg() + +proc readWriteLoop(c: Chat) {.async.} = + asyncSpawn c.writeAndPrint() + asyncSpawn c.readAndPrint() + +proc readInput(wfd: AsyncFD) {.thread, raises: [Defect, CatchableError].} = + let transp = fromPipe(wfd) + + while true: + let line = stdin.readLine() + discard waitFor transp.write(line & "\r\n") + +{.pop.} + # @TODO confutils.nim(775, 17) Error: can raise an unlisted exception: ref IOError +proc processInput(rfd: AsyncFD, rng: crypto.Rng) {.async.} = + let + transp = fromPipe(rfd) + conf = Chat2Conf.load() + nodekey = + if conf.nodekey.isSome(): + conf.nodekey.get() + else: + PrivateKey.random(Secp256k1, rng).tryGet() + + # set log level + if conf.logLevel != LogLevel.NONE: + setLogLevel(conf.logLevel) + + let (extIp, extTcpPort, extUdpPort) = setupNat( + conf.nat, + clientId, + Port(uint16(conf.tcpPort) + conf.portsShift), + Port(uint16(conf.udpPort) + conf.portsShift), + ).valueOr: + raise newException(ValueError, "setupNat error " & error) + + var enrBuilder = EnrBuilder.init(nodeKey) + + let record = enrBuilder.build().valueOr: + error "failed to create enr record", error = error + quit(QuitFailure) + + let node = block: + var builder = WakuNodeBuilder.init() + builder.withNodeKey(nodeKey) + builder.withRecord(record) + + builder + .withNetworkConfigurationDetails( + conf.listenAddress, + Port(uint16(conf.tcpPort) + conf.portsShift), + extIp, + extTcpPort, + ) + .tryGet() + builder.build().tryGet() + + (await node.mountRelay()).isOkOr: + error "failed to mount relay: ", error = error + quit(QuitFailure) + + await node.mountLibp2pPing() + + var kadBootstrapPeers: seq[(PeerId, seq[MultiAddress])] + for nodeStr in conf.kadBootstrapNodes: + let (peerId, ma) = block: + let r = parseFullAddress(nodeStr) + if r.isErr: + error "Failed to parse kademlia bootstrap node", node = nodeStr, error = r.error + continue + r.get() + kadBootstrapPeers.add((peerId, @[ma])) + + node.mountKademlia( + KademliaDiscoveryConf( + bootstrapNodes: kadBootstrapPeers, + servicesToAdvertise: @[], + servicesToDiscover: @[], + randomLookupInterval: chronos.seconds(60), + serviceLookupInterval: chronos.seconds(60), + kadDhtConfig: KadDHTConfig.new(), + discoConfig: + sd_types.ServiceDiscoveryConfig.new(advertExpiry = chronos.seconds(60)), + clientMode: false, + xprPublishing: false, + ) + ).isOkOr: + error "failed to setup kademlia service discovery", error = error + quit(QuitFailure) + + let autonatService = getAutonatService(rng) + node.switch.services = @[Service(autonatService)] + for service in node.switch.services: + try: + service.setup(node.switch) + except ServiceSetupError as e: + error "failed to set up libp2p switch service", error = e.msg + + await node.start() + + node.peerManager.start() + + let nick = await readNick(transp) + let room = await readRoom(transp) + echo "Welcome, " & nick & "! Joined room '" & room & "'." + + var chat = Chat( + node: node, + transp: transp, + subscribed: false, + connected: false, + started: true, + nick: nick, + prompt: false, + currentPubsubTopic: "", + ) + + let peerInfo = node.switch.peerInfo + let listenStr = $peerInfo.addrs[0] & "/p2p/" & $peerInfo.peerId + echo &"Listening on\n {listenStr}" + + if conf.metricsLogging: + startMetricsLog() + + if conf.metricsServer: + let metricsServer = startMetricsServer( + conf.metricsServerAddress, Port(conf.metricsServerPort + conf.portsShift) + ) + + await chat.joinRoom(room) + + await chat.readWriteLoop() + + runForever() + +proc main(rng: crypto.Rng) {.async.} = + let (rfd, wfd) = createAsyncPipe() + if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe: + raise newException(ValueError, "Could not initialize pipe!") + + var thread: Thread[AsyncFD] + thread.createThread(readInput, wfd) + try: + await processInput(rfd, rng) + except ConfigurationError as e: + raise e + +when isMainModule: + let rng = crypto.newRng() + try: + waitFor(main(rng)) + except CatchableError as e: + raise e diff --git a/apps/chat2disco/config_chat2disco.nim b/apps/chat2disco/config_chat2disco.nim new file mode 100644 index 000000000..9f1a8290a --- /dev/null +++ b/apps/chat2disco/config_chat2disco.nim @@ -0,0 +1,103 @@ +import chronicles, chronos, std/strutils + +import + eth/keys, + libp2p/crypto/crypto, + libp2p/crypto/secp, + libp2p/multiaddress, + nimcrypto/utils, + confutils, + confutils/defs, + confutils/std/net + +type Chat2Conf* = object ## General node config + logLevel* {. + desc: "Sets the log level.", defaultValue: LogLevel.DEBUG, name: "log-level" + .}: LogLevel + + nodekey* {.desc: "P2P node private key as 64 char hex string.", name: "nodekey".}: + Option[crypto.PrivateKey] + + listenAddress* {. + defaultValue: defaultListenAddress(config), + desc: "Listening address for the LibP2P traffic.", + name: "listen-address" + .}: IpAddress + + tcpPort* {.desc: "TCP listening port.", defaultValue: 60000, name: "tcp-port".}: Port + + udpPort* {.desc: "UDP listening port.", defaultValue: 60000, name: "udp-port".}: Port + + portsShift* {. + desc: "Add a shift to all port numbers.", defaultValue: 0, name: "ports-shift" + .}: uint16 + + nat* {. + desc: + "Specify method to use for determining public address. " & + "Must be one of: any, none, upnp, pmp, extip:.", + defaultValue: "any" + .}: string + + ## Metrics config + metricsServer* {. + desc: "Enable the metrics server: true|false", + defaultValue: false, + name: "metrics-server" + .}: bool + + metricsServerAddress* {. + desc: "Listening address of the metrics server.", + defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]), + name: "metrics-server-address" + .}: IpAddress + + metricsServerPort* {. + desc: "Listening HTTP port of the metrics server.", + defaultValue: 8008, + name: "metrics-server-port" + .}: uint16 + + metricsLogging* {. + desc: "Enable metrics logging: true|false", + defaultValue: true, + name: "metrics-logging" + .}: bool + + ## Kademlia Discovery config (core for chat2disco) + kadBootstrapNodes* {. + desc: + "Optional peer multiaddr for kademlia bootstrap (must include /p2p/). Argument may be repeated.", + name: "kad-bootstrap-node" + .}: seq[string] + +proc parseCmdArg*(T: type crypto.PrivateKey, p: string): T = + try: + let key = SkPrivateKey.init(utils.fromHex(p)).tryGet() + result = crypto.PrivateKey(scheme: Secp256k1, skkey: key) + except CatchableError as e: + raise newException(ValueError, "Invalid private key") + +proc completeCmdArg*(T: type crypto.PrivateKey, val: string): seq[string] = + return @[] + +proc parseCmdArg*(T: type IpAddress, p: string): T = + try: + result = parseIpAddress(p) + except CatchableError as e: + raise newException(ValueError, "Invalid IP address") + +proc completeCmdArg*(T: type IpAddress, val: string): seq[string] = + return @[] + +proc parseCmdArg*(T: type Port, p: string): T = + try: + result = Port(parseInt(p)) + except CatchableError as e: + raise newException(ValueError, "Invalid Port number") + +proc completeCmdArg*(T: type Port, val: string): seq[string] = + return @[] + +func defaultListenAddress*(conf: Chat2Conf): IpAddress = + (static parseIpAddress("0.0.0.0")) diff --git a/apps/chat2disco/nim.cfg b/apps/chat2disco/nim.cfg new file mode 100644 index 000000000..2231f2ebe --- /dev/null +++ b/apps/chat2disco/nim.cfg @@ -0,0 +1,4 @@ +-d:chronicles_line_numbers +-d:chronicles_runtime_filtering:on +-d:discv5_protocol_id:d5waku +path = "../.." diff --git a/logos_delivery.nimble b/logos_delivery.nimble index efe118595..9c9ef7b96 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -439,6 +439,17 @@ task chat2mix, "Build example Waku chat mix usage": "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE " # -d:ssl - cause unlisted exception error in libp2p/utility... +task chat2disco, "Build chat2disco (kademlia service discovery test chat)": + # This is a testing/discovery tool, so we want debug logs on stdout by default + # (the config sets the default to DEBUG, and runtime filtering is enabled). + # Unlike the pure chat examples, we do not redirect to a file sink. + + let name = "chat2disco" + buildBinary name, + "apps/chat2disco/", + "-d:chronicles_log_level=TRACE " + # -d:ssl - cause unlisted exception error in libp2p/utility... + task chat2bridge, "Build chat2bridge": let name = "chat2bridge" buildBinary name, "apps/chat2bridge/" From bbfa4cc529b4d1b9f7667922a50f5fd7b4f89ce3 Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Fri, 12 Jun 2026 09:19:14 -0400 Subject: [PATCH 10/15] rebase fix --- waku/discovery/waku_kademlia.nim | 201 ------------------ .../kademlia_discovery_conf_builder.nim | 79 ------- 2 files changed, 280 deletions(-) delete mode 100644 waku/discovery/waku_kademlia.nim delete mode 100644 waku/factory/conf_builder/kademlia_discovery_conf_builder.nim diff --git a/waku/discovery/waku_kademlia.nim b/waku/discovery/waku_kademlia.nim deleted file mode 100644 index be5ddb49e..000000000 --- a/waku/discovery/waku_kademlia.nim +++ /dev/null @@ -1,201 +0,0 @@ -{.push raises: [].} - -import std/sequtils -import - chronos, - chronicles, - results, - stew/byteutils, - libp2p/[peerid, multiaddress, switch], - libp2p/extended_peer_record, - libp2p/crypto/curve25519, - libp2p/protocols/service_discovery, - libp2p/protocols/service_discovery/types as sd_types, - libp2p/protocols/mix/mix_protocol - -import - waku/waku_core, - waku/node/peer_manager, - waku/events/discovery_events, - waku/factory/waku_conf - -logScope: - topics = "waku service discovery" - -const - DefaultServiceDiscoveryInterval* = chronos.seconds(60) - DefaultRandomDiscoveryInterval* = chronos.seconds(60) - -type WakuKademlia* = ref object - protocol*: ServiceDiscovery - peerManager: PeerManager - randomLookupLoop: Future[void] - serviceLookupLoop: Future[void] - randomLookupInterval: Duration - serviceLookupInterval: Duration - servicesToDiscover: seq[string] - servicesToAdvertise: seq[ServiceInfo] - -proc extractMixPubKey(service: ServiceInfo): Opt[Curve25519Key] = - if service.id != MixProtocolID: - return none(Curve25519Key) - - if service.data.len != Curve25519KeySize: - error "invalid mix pub key length", - expected = Curve25519KeySize, - actual = service.data.len, - dataHex = byteutils.toHex(service.data) - return none(Curve25519Key) - - let key = intoCurve25519Key(service.data) - - return some(key) - -proc remotePeerInfoFrom(record: ExtendedPeerRecord): Opt[RemotePeerInfo] = - if record.addresses.len == 0: - error "missing addresses", peerId = record.peerId - return none(RemotePeerInfo) - - let addrs = record.addresses.mapIt(it.address) - if addrs.len == 0: - error "no dialable addresses", peerId = record.peerId - return none(RemotePeerInfo) - - var mixPubKey = none(Curve25519Key) - for service in record.services: - mixPubKey = extractMixPubKey(service).valueOr: - continue - - trace "successfully extracted mix pub key", - peerId = record.peerId, keyHex = byteutils.toHex(mixPubKey.get()) - - break - - return some( - RemotePeerInfo.init( - record.peerId, addrs = addrs, origin = PeerOrigin.Kademlia, mixPubKey = mixPubKey - ) - ) - -proc lookupServicePeers*( - self: WakuKademlia, serviceId: string -): Future[Result[seq[RemotePeerInfo], string]] {.async: (raises: []).} = - if self.protocol.isNil(): - return err("cannot lookup service peers: service discovery not mounted") - - let serviceInfo = ServiceInfo(id: serviceId, data: @[]) - - let lookupCatch = catch: - (await self.protocol.lookup(serviceInfo)) - - let lookupResult = lookupCatch.valueOr: - return err("service peer lookup failed: " & error.msg) - - let advertisements = lookupResult.valueOr: - return err("service peer lookup failed: " & lookupResult.error) - - var discovered: seq[RemotePeerInfo] - for ad in advertisements: - let record = ad.data - let peerInfo = remotePeerInfoFrom(record).valueOr: - continue - - self.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) - discovered.add(peerInfo) - - debug "service lookup complete", serviceId, found = discovered.len - return ok(discovered) - -proc runRandomLookupLoop(self: WakuKademlia) {.async: (raises: [CancelledError]).} = - debug "periodic random lookup started", interval = $self.randomLookupInterval - - while true: - await sleepAsync(self.randomLookupInterval) - - let recordsRes = catch: - (await self.protocol.lookupRandom()) - - let records = recordsRes.valueOr: - error "random lookup failed", error - continue - - var discoveredPeers: seq[RemotePeerInfo] - for record in records: - let peerInfo = remotePeerInfoFrom(record).valueOr: - continue - - self.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) - discoveredPeers.add(peerInfo) - - if discoveredPeers.len > 0: - PeersDiscoveredEvent.emit(peers = discoveredPeers) - - debug "random lookup complete", found = discoveredPeers.len - -proc runServiceLookupLoop(self: WakuKademlia) {.async: (raises: [CancelledError]).} = - debug "periodic service lookup started", - interval = $self.serviceLookupInterval, services = self.servicesToDiscover - - while true: - await sleepAsync(self.serviceLookupInterval) - - if self.servicesToDiscover.len == 0: - continue - - for serviceId in self.servicesToDiscover: - let discovered = (await self.lookupServicePeers(serviceId)).valueOr: - error "service lookup failed", serviceId, error - continue - - if discovered.len > 0: - PeersDiscoveredEvent.emit(peers = discovered) - -proc new*( - T: type WakuKademlia, - switch: Switch, - peerManager: PeerManager, - config: KademliaDiscoveryConf, -): Result[T, string] = - if config.bootstrapNodes.len == 0: - debug "creating service discovery as seed node (no bootstrap nodes)" - - let protocol = ServiceDiscovery.new( - switch, bootstrapNodes = config.bootstrapNodes, services = config.servicesToDiscover - ) - - let self = WakuKademlia( - protocol: protocol, - peerManager: peerManager, - randomLookupInterval: config.randomLookupInterval, - serviceLookupInterval: config.serviceLookupInterval, - servicesToDiscover: config.servicesToDiscover, - servicesToAdvertise: config.servicesToAdvertise, - ) - - return ok(self) - -proc start*(self: WakuKademlia) {.async: (raises: []).} = - for serviceId in self.servicesToDiscover: - discard self.protocol.startDiscovering(serviceId) - - for serviceInfo in self.servicesToAdvertise: - self.protocol.addProvidedService(serviceInfo) - - if self.randomLookupLoop.isNil(): - self.randomLookupLoop = self.runRandomLookupLoop() - - if self.serviceLookupLoop.isNil(): - self.serviceLookupLoop = self.runServiceLookupLoop() - - info "kademlia discovery started" - -proc stop*(self: WakuKademlia) {.async: (raises: []).} = - if not self.serviceLookupLoop.isNil(): - await self.serviceLookupLoop.cancelAndWait() - self.serviceLookupLoop = nil - - if not self.randomLookupLoop.isNil(): - await self.randomLookupLoop.cancelAndWait() - self.randomLookupLoop = nil - - info "kademlia discovery stopped" diff --git a/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim b/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim deleted file mode 100644 index 98304db23..000000000 --- a/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim +++ /dev/null @@ -1,79 +0,0 @@ -import chronicles, std/options, results -import chronos -import libp2p/[peerid, multiaddress, peerinfo, extended_peer_record] -import waku/factory/waku_conf - -logScope: - topics = "waku conf builder kademlia discovery" - -####################################### -## Kademlia Discovery Config Builder ## -####################################### -type KademliaDiscoveryConfBuilder* = object - enabled*: bool - bootstrapNodes*: seq[string] - servicesToAdvertise*: seq[(string, seq[byte])] - servicesToDiscover*: seq[string] - randomLookupInterval*: Duration - serviceLookupInterval*: Duration - -proc init*(T: type KademliaDiscoveryConfBuilder): KademliaDiscoveryConfBuilder = - KademliaDiscoveryConfBuilder() - -proc withEnabled*(b: var KademliaDiscoveryConfBuilder, enabled: bool) = - b.enabled = enabled - -proc withBootstrapNodes*( - b: var KademliaDiscoveryConfBuilder, bootstrapNodes: seq[string] -) = - b.bootstrapNodes = bootstrapNodes - -proc withServicesToAdvertise*( - b: var KademliaDiscoveryConfBuilder, services: seq[(string, seq[byte])] -) = - b.servicesToAdvertise = services - -proc withServicesToDiscover*( - b: var KademliaDiscoveryConfBuilder, services: seq[string] -) = - b.servicesToDiscover = services - -proc withRandomLookupInterval*( - b: var KademliaDiscoveryConfBuilder, interval: Duration -) = - b.randomLookupInterval = interval - -proc withServiceLookupInterval*( - b: var KademliaDiscoveryConfBuilder, interval: Duration -) = - b.serviceLookupInterval = interval - -proc build*( - b: KademliaDiscoveryConfBuilder -): Result[Option[KademliaDiscoveryConf], string] = - # Kademlia is enabled if explicitly enabled OR if bootstrap nodes are provided - let enabled = b.enabled or b.bootstrapNodes.len > 0 - if not enabled: - return ok(none(KademliaDiscoveryConf)) - - var parsedNodes: seq[(PeerId, seq[MultiAddress])] - for nodeStr in b.bootstrapNodes: - let (peerId, ma) = parseFullAddress(nodeStr).valueOr: - return err("Failed to parse kademlia bootstrap node: " & error) - parsedNodes.add((peerId, @[ma])) - - var servicesToAdvertise: seq[ServiceInfo] - for (serviceId, data) in b.servicesToAdvertise: - servicesToAdvertise.add(ServiceInfo(id: serviceId, data: data)) - - return ok( - some( - KademliaDiscoveryConf( - bootstrapNodes: parsedNodes, - servicesToAdvertise: servicesToAdvertise, - servicesToDiscover: b.servicesToDiscover, - randomLookupInterval: b.randomLookupInterval, - serviceLookupInterval: b.serviceLookupInterval, - ) - ) - ) From ec62e139144d492d4b5cb649e8229b58ed0eb19b Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Fri, 12 Jun 2026 09:40:26 -0400 Subject: [PATCH 11/15] fixes --- logos_delivery/waku/node/waku_node.nim | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/logos_delivery/waku/node/waku_node.nim b/logos_delivery/waku/node/waku_node.nim index ad0ba2d0a..af68ef2f8 100644 --- a/logos_delivery/waku/node/waku_node.nim +++ b/logos_delivery/waku/node/waku_node.nim @@ -150,6 +150,11 @@ type import ./subscription_manager +proc selectRandomPeers*(peers: seq[PeerId], numRandomPeers: int): seq[PeerId] = + var randomPeers = peers + shuffle(randomPeers) + return randomPeers[0 ..< min(len(randomPeers), numRandomPeers)] + proc deduceRelayShard( node: WakuNode, contentTopic: ContentTopic, @@ -336,6 +341,8 @@ proc mountMix*( return err(error.msg) return ok() +import logos_delivery/waku/factory/conf_builder/kademlia_discovery_conf_builder + proc mountKademlia*( node: WakuNode, config: KademliaDiscoveryConf ): Result[void, string] = @@ -343,11 +350,10 @@ proc mountKademlia*( return err("WakuKademlia already mounted, skipping") let wk = WakuKademlia.new( - node.switch, node.peerManager, - config.bootstrapNodes, config.servicesToAdvertise, config.servicesToDiscover, - config.randomLookupInterval, config.serviceLookupInterval, - node.rng, - config.kadDhtConfig, config.discoConfig, config.clientMode, config.xprPublishing + node.switch, node.peerManager, config.bootstrapNodes, config.servicesToAdvertise, + config.servicesToDiscover, config.randomLookupInterval, + config.serviceLookupInterval, node.rng, config.kadDhtConfig, config.discoConfig, + config.clientMode, config.xprPublishing, ).valueOr: return err("failed to create service discovery: " & error) @@ -416,11 +422,6 @@ proc reconnectRelayPeers*(node: WakuNode) {.async.} = node.wakuRelay.parameters.pruneBackoff + chronos.seconds(BackoffSlackTime) await node.peerManager.reconnectPeers(WakuRelayCodec, backoffPeriod) -proc selectRandomPeers*(peers: seq[PeerId], numRandomPeers: int): seq[PeerId] = - var randomPeers = peers - shuffle(randomPeers) - return randomPeers[0 ..< min(len(randomPeers), numRandomPeers)] - proc mountRendezvousClient*(node: WakuNode, clusterId: uint16) {.async: (raises: []).} = info "mounting rendezvous client" From 6562b14efe2b200e17170538f5172b8f5f0eb45c Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Mon, 15 Jun 2026 09:24:33 -0400 Subject: [PATCH 12/15] broker hook & --- apps/chat2disco/chat2disco.nim | 48 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim index 83151e857..4be8a76e3 100644 --- a/apps/chat2disco/chat2disco.nim +++ b/apps/chat2disco/chat2disco.nim @@ -46,6 +46,8 @@ import ], ./config_chat2disco +import logos_delivery/waku/events/discovery_events + import libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub import libp2p/extended_peer_record # for ServiceInfo @@ -72,6 +74,7 @@ type Chat = ref object nick: string prompt: bool currentPubsubTopic: string + discoPeersListener: PeersDiscoveredEventListener type PrivateKey* = crypto.PrivateKey @@ -197,6 +200,9 @@ proc publish(c: Chat, line: string) {.async.} = proc joinRoom(c: Chat, roomName: string) {.async.} = if c.currentPubsubTopic.len > 0 and c.currentPubsubTopic != roomName: discard c.node.unsubscribe((kind: PubsubSub, topic: c.currentPubsubTopic)) + if not c.node.wakuKademlia.isNil(): + c.node.wakuKademlia.removeServiceToDiscover(c.currentPubsubTopic) + await c.node.wakuKademlia.removeServiceToAdvertise(c.currentPubsubTopic) c.currentPubsubTopic = roomName @@ -211,25 +217,11 @@ proc joinRoom(c: Chat, roomName: string) {.async.} = echo "subscribed to pubsub topic: ", roomName - # Kademlia service discovery + advertisement for the room. - # Kademlia is always mounted (seed node if no --kad-bootstrap-node provided). - let svc = roomName - discard c.node.wakuKademlia.protocol.registerInterest(svc) - let svcInfo = ServiceInfo(id: svc, data: @[]) - c.node.wakuKademlia.protocol.startAdvertising(svcInfo) - echo "advertising and discovering kademlia service: ", svc - - asyncSpawn ( - proc() {.async.} = - let res = await c.node.wakuKademlia.lookupServicePeers(svc) - if res.isOk(): - let peers = res.get() - if peers.len > 0: - echo "discovered room peers via kademlia: ", peers.mapIt($it.peerId) - await c.node.connectToNodes(peers) - else: - error "service lookup failed for room", service = svc, error = res.error - )() + if not c.node.wakuKademlia.isNil(): + let svcInfo = ServiceInfo(id: roomName, data: @[]) + c.node.wakuKademlia.addServiceToDiscover(roomName) + c.node.wakuKademlia.addServiceToAdvertise(svcInfo) + echo "advertising and discovering kademlia service: ", roomName proc readAndPrint(c: Chat) {.async.} = while true: @@ -262,6 +254,7 @@ proc writeAndPrint(c: Chat) {.async.} = let roomName = parts[1].strip() await c.joinRoom(roomName) elif line.startsWith("/exit"): + await PeersDiscoveredEvent.dropListener(c.discoPeersListener) echo "quitting..." try: @@ -399,8 +392,25 @@ proc processInput(rfd: AsyncFD, rng: crypto.Rng) {.async.} = nick: nick, prompt: false, currentPubsubTopic: "", + discoPeersListener: PeersDiscoveredEventListener(), ) + let listenerHandle = PeersDiscoveredEvent.listen( + proc(event: PeersDiscoveredEvent) {.async: (raises: []).} = + let peers = event.peers + if peers.len > 0: + try: + echo "discovered peers via kademlia: ", peers.mapIt($it.peerId) + await chat.node.connectToNodes(peers) + chat.connected = true + except CatchableError as e: + error "error connecting discovered peers", error = e.msg + ).valueOr: + error "failed to subscribe to PeersDiscoveredEvent", error = error + return + + chat.discoPeersListener = listenerHandle + let peerInfo = node.switch.peerInfo let listenStr = $peerInfo.addrs[0] & "/p2p/" & $peerInfo.peerId echo &"Listening on\n {listenStr}" From 4795d9f08169ae18b2b80c0c1799b9b9d5e76ada Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Thu, 18 Jun 2026 13:23:32 -0400 Subject: [PATCH 13/15] fix rebase --- apps/chat2disco/chat2disco.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim index 4be8a76e3..50a9c2353 100644 --- a/apps/chat2disco/chat2disco.nim +++ b/apps/chat2disco/chat2disco.nim @@ -202,7 +202,9 @@ proc joinRoom(c: Chat, roomName: string) {.async.} = discard c.node.unsubscribe((kind: PubsubSub, topic: c.currentPubsubTopic)) if not c.node.wakuKademlia.isNil(): c.node.wakuKademlia.removeServiceToDiscover(c.currentPubsubTopic) - await c.node.wakuKademlia.removeServiceToAdvertise(c.currentPubsubTopic) + await c.node.wakuKademlia.removeServiceToAdvertise( + ServiceInfo(id: c.currentPubsubTopic, data: @[]) + ) c.currentPubsubTopic = roomName @@ -353,8 +355,6 @@ proc processInput(rfd: AsyncFD, rng: crypto.Rng) {.async.} = node.mountKademlia( KademliaDiscoveryConf( bootstrapNodes: kadBootstrapPeers, - servicesToAdvertise: @[], - servicesToDiscover: @[], randomLookupInterval: chronos.seconds(60), serviceLookupInterval: chronos.seconds(60), kadDhtConfig: KadDHTConfig.new(), From 6debd34de5ce9fce682331d38215b83f9f87a556 Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Mon, 13 Jul 2026 10:53:37 -0400 Subject: [PATCH 14/15] libp2p bump --- apps/chat2disco/chat2disco.nim | 4 +- logos_delivery.nimble | 4 +- .../recv_service/recv_service.nim | 2 +- .../send_service/send_service.nim | 2 +- .../common/rate_limit/request_limiter.nim | 2 +- .../rate_limit/single_token_limiter.nim | 2 +- .../waku/common/rate_limit/timed_map.nim | 1 - logos_delivery/waku/node/waku_node.nim | 1 - logos_delivery/waku/node/waku_node/filter.nim | 3 +- .../waku/node/waku_node/lightpush.nim | 1 - .../waku/node/waku_node/peer_exchange.nim | 3 +- logos_delivery/waku/node/waku_node/ping.nim | 3 +- logos_delivery/waku/node/waku_node/relay.nim | 1 - logos_delivery/waku/node/waku_node/store.nim | 3 +- .../waku/waku_rendezvous/protocol.nim | 3 +- .../waku/waku_store_sync/reconciliation.nim | 1 - .../waku/waku_store_sync/transfer.nim | 1 - nimble.lock | 516 +++++++++--------- 18 files changed, 258 insertions(+), 295 deletions(-) diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim index 50a9c2353..81c1b34a7 100644 --- a/apps/chat2disco/chat2disco.nim +++ b/apps/chat2disco/chat2disco.nim @@ -46,7 +46,7 @@ import ], ./config_chat2disco -import logos_delivery/waku/events/discovery_events +import logos_delivery/waku/api/events/discovery_events import libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub import libp2p/extended_peer_record # for ServiceInfo @@ -357,7 +357,7 @@ proc processInput(rfd: AsyncFD, rng: crypto.Rng) {.async.} = bootstrapNodes: kadBootstrapPeers, randomLookupInterval: chronos.seconds(60), serviceLookupInterval: chronos.seconds(60), - kadDhtConfig: KadDHTConfig.new(), + kadDhtConfig: KadDHTConfig.new(disableBootstrapping = true), discoConfig: sd_types.ServiceDiscoveryConfig.new(advertExpiry = chronos.seconds(60)), clientMode: false, diff --git a/logos_delivery.nimble b/logos_delivery.nimble index 5a3319f8d..ee8edd232 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -28,7 +28,7 @@ requires "nim >= 2.2.4", "toml_serialization", "faststreams", # Networking & P2P - "https://github.com/vacp2p/nim-libp2p.git#v2.0.0", + "https://github.com/vacp2p/nim-libp2p.git#v2.1.4", "eth", "nat_traversal", "dnsdisc", @@ -67,7 +67,7 @@ requires "https://github.com/logos-messaging/nim-sds.git#b12f5ee07c5b764303b51fb requires "https://github.com/NagyZoltanPeter/nim-brokers.git#v3.1.4" -requires "https://github.com/vacp2p/nim-lsquic.git#v0.5.1" +requires "https://github.com/vacp2p/nim-lsquic.git#v0.5.5" requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2" requires "https://github.com/logos-co/nim-libp2p-mix#380513117d556bf8f70066f5e72a7fd74fe36ba6" diff --git a/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim b/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim index d2c369d46..e77729e0c 100644 --- a/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim +++ b/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim @@ -4,7 +4,7 @@ import logos_delivery/waku/compat/option_valueor ## import std/[tables, sequtils, options, sets] -import chronos, chronicles, libp2p/utility +import chronos, chronicles, libp2p/utils/[shortlog, collections] import brokers/broker_context import logos_delivery/waku/[waku_core, waku_core/topics, waku_store/common], diff --git a/logos_delivery/messaging/delivery_service/send_service/send_service.nim b/logos_delivery/messaging/delivery_service/send_service/send_service.nim index 037ad879b..fec78f5aa 100644 --- a/logos_delivery/messaging/delivery_service/send_service/send_service.nim +++ b/logos_delivery/messaging/delivery_service/send_service/send_service.nim @@ -3,7 +3,7 @@ import logos_delivery/waku/compat/option_valueor ## import std/[sequtils, tables, options, typetraits] -import chronos, chronicles, libp2p/utility +import chronos, chronicles, libp2p/utils/[shortlog, collections] import brokers/broker_context import ./[send_processor, relay_processor, lightpush_processor, delivery_task], diff --git a/logos_delivery/waku/common/rate_limit/request_limiter.nim b/logos_delivery/waku/common/rate_limit/request_limiter.nim index 9fccf1c29..8d2f7f92c 100644 --- a/logos_delivery/waku/common/rate_limit/request_limiter.nim +++ b/logos_delivery/waku/common/rate_limit/request_limiter.nim @@ -21,7 +21,7 @@ import chronicles, chronos/timer, libp2p/stream/connection, - libp2p/utility + libp2p/utils/opt import std/times except TimeInterval, Duration, seconds, minutes diff --git a/logos_delivery/waku/common/rate_limit/single_token_limiter.nim b/logos_delivery/waku/common/rate_limit/single_token_limiter.nim index fc4b0acd5..b26ca3775 100644 --- a/logos_delivery/waku/common/rate_limit/single_token_limiter.nim +++ b/logos_delivery/waku/common/rate_limit/single_token_limiter.nim @@ -2,7 +2,7 @@ {.push raises: [].} -import std/[options], chronos/timer, libp2p/stream/connection, libp2p/utility +import std/[options], chronos/timer, libp2p/stream/connection import std/times except TimeInterval, Duration diff --git a/logos_delivery/waku/common/rate_limit/timed_map.nim b/logos_delivery/waku/common/rate_limit/timed_map.nim index b9a5c4cbf..5bab16de7 100644 --- a/logos_delivery/waku/common/rate_limit/timed_map.nim +++ b/logos_delivery/waku/common/rate_limit/timed_map.nim @@ -14,7 +14,6 @@ import std/[hashes, sets] import chronos/timer, results -import libp2p/utility export results diff --git a/logos_delivery/waku/node/waku_node.nim b/logos_delivery/waku/node/waku_node.nim index e386ef18c..7d84e1813 100644 --- a/logos_delivery/waku/node/waku_node.nim +++ b/logos_delivery/waku/node/waku_node.nim @@ -22,7 +22,6 @@ import libp2p/transports/transport, libp2p/transports/tcptransport, libp2p/transports/wstransport, - libp2p/utility, libp2p/utils/offsettedseq, libp2p_mix, libp2p_mix/mix_protocol, diff --git a/logos_delivery/waku/node/waku_node/filter.nim b/logos_delivery/waku/node/waku_node/filter.nim index 4b2ea8494..dd5d4265b 100644 --- a/logos_delivery/waku/node/waku_node/filter.nim +++ b/logos_delivery/waku/node/waku_node/filter.nim @@ -17,8 +17,7 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/builders, libp2p/transports/tcptransport, - libp2p/transports/wstransport, - libp2p/utility + libp2p/transports/wstransport import ../waku_node, diff --git a/logos_delivery/waku/node/waku_node/lightpush.nim b/logos_delivery/waku/node/waku_node/lightpush.nim index 1bb4ddcbb..da5ba8e1a 100644 --- a/logos_delivery/waku/node/waku_node/lightpush.nim +++ b/logos_delivery/waku/node/waku_node/lightpush.nim @@ -17,7 +17,6 @@ import libp2p/builders, libp2p/transports/tcptransport, libp2p/transports/wstransport, - libp2p/utility, libp2p_mix import diff --git a/logos_delivery/waku/node/waku_node/peer_exchange.nim b/logos_delivery/waku/node/waku_node/peer_exchange.nim index cd6e6146c..5e5cc272a 100644 --- a/logos_delivery/waku/node/waku_node/peer_exchange.nim +++ b/logos_delivery/waku/node/waku_node/peer_exchange.nim @@ -15,8 +15,7 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/builders, libp2p/transports/tcptransport, - libp2p/transports/wstransport, - libp2p/utility + libp2p/transports/wstransport import ../waku_node, diff --git a/logos_delivery/waku/node/waku_node/ping.nim b/logos_delivery/waku/node/waku_node/ping.nim index e7e011477..5eb1baeb9 100644 --- a/logos_delivery/waku/node/waku_node/ping.nim +++ b/logos_delivery/waku/node/waku_node/ping.nim @@ -9,8 +9,7 @@ import results, libp2p/protocols/ping, libp2p/builders, - libp2p/transports/tcptransport, - libp2p/utility + libp2p/transports/tcptransport import ../waku_node, ../peer_manager diff --git a/logos_delivery/waku/node/waku_node/relay.nim b/logos_delivery/waku/node/waku_node/relay.nim index 18a3f0f59..8411734df 100644 --- a/logos_delivery/waku/node/waku_node/relay.nim +++ b/logos_delivery/waku/node/waku_node/relay.nim @@ -18,7 +18,6 @@ import libp2p/builders, libp2p/transports/tcptransport, libp2p/transports/wstransport, - libp2p/utility, brokers/broker_context import diff --git a/logos_delivery/waku/node/waku_node/store.nim b/logos_delivery/waku/node/waku_node/store.nim index 189ef9052..3b517a59a 100644 --- a/logos_delivery/waku/node/waku_node/store.nim +++ b/logos_delivery/waku/node/waku_node/store.nim @@ -15,8 +15,7 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/builders, libp2p/transports/tcptransport, - libp2p/transports/wstransport, - libp2p/utility + libp2p/transports/wstransport import ../waku_node, diff --git a/logos_delivery/waku/waku_rendezvous/protocol.nim b/logos_delivery/waku/waku_rendezvous/protocol.nim index 253fa5a12..6b06820e7 100644 --- a/logos_delivery/waku/waku_rendezvous/protocol.nim +++ b/logos_delivery/waku/waku_rendezvous/protocol.nim @@ -11,8 +11,7 @@ import libp2p/protocols/rendezvous/protobuf, libp2p/utils/offsettedseq, libp2p/crypto/curve25519, - libp2p/switch, - libp2p/utility + libp2p/switch import metrics except collect diff --git a/logos_delivery/waku/waku_store_sync/reconciliation.nim b/logos_delivery/waku/waku_store_sync/reconciliation.nim index 6f301d633..f091cccf0 100644 --- a/logos_delivery/waku/waku_store_sync/reconciliation.nim +++ b/logos_delivery/waku/waku_store_sync/reconciliation.nim @@ -8,7 +8,6 @@ import chronicles, chronos, metrics, - libp2p/utility, libp2p/protocols/protocol, libp2p/stream/connection, libp2p/crypto/crypto, diff --git a/logos_delivery/waku/waku_store_sync/transfer.nim b/logos_delivery/waku/waku_store_sync/transfer.nim index aa02adfdc..c53730d86 100644 --- a/logos_delivery/waku/waku_store_sync/transfer.nim +++ b/logos_delivery/waku/waku_store_sync/transfer.nim @@ -7,7 +7,6 @@ import chronicles, chronos, metrics, - libp2p/utility, libp2p/protocols/protocol, libp2p/stream/connection, libp2p/crypto/crypto, diff --git a/nimble.lock b/nimble.lock index 03bb7a434..03c0d0064 100644 --- a/nimble.lock +++ b/nimble.lock @@ -1,91 +1,28 @@ { "version": 2, "packages": { - "nim": { - "version": "2.2.4", - "vcsRevision": "911e0dbb1f76de61fa0215ab1bb85af5334cc9a8", - "url": "https://github.com/nim-lang/Nim.git", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "68bb85cbfb1832ce4db43943911b046c3af3caab" - } - }, - "unittest2": { - "version": "0.2.5", - "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", - "url": "https://github.com/status-im/nim-unittest2", + "boringssl": { + "version": "0.0.10", + "vcsRevision": "084f2c8994137a72655b72745936a05949c768cc", + "url": "https://github.com/vacp2p/nim-boringssl", "downloadMethod": "git", "dependencies": [ "nim" ], "checksums": { - "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + "sha1": "4ac575c18b11c0f06d6381279843aa8508f7940e" } }, - "bearssl": { - "version": "0.2.8", - "vcsRevision": "22c6a76ce015bc07e011562bdcfc51d9446c1e82", - "url": "https://github.com/status-im/nim-bearssl", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unittest2" - ], - "checksums": { - "sha1": "da4dd7ae96d536bdaf42dca9c85d7aed024b6a86" - } - }, - "bearssl_pkey_decoder": { - "version": "#21dd3710df9345ed2ad8bf8f882761e07863b8e0", - "vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0", - "url": "https://github.com/vacp2p/bearssl_pkey_decoder", - "downloadMethod": "git", - "dependencies": [ - "nim", - "bearssl" - ], - "checksums": { - "sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714" - } - }, - "jwt": { - "version": "#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", - "vcsRevision": "057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", - "url": "https://github.com/vacp2p/nim-jwt.git", - "downloadMethod": "git", - "dependencies": [ - "nim", - "bearssl", - "bearssl_pkey_decoder" - ], - "checksums": { - "sha1": "3cd368666fd2bc7f99f253452289e827abcac13c" - } - }, - "testutils": { - "version": "0.8.1", - "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", - "url": "https://github.com/status-im/nim-testutils", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unittest2" - ], - "checksums": { - "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" - } - }, - "db_connector": { - "version": "0.1.0", - "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", - "url": "https://github.com/nim-lang/db_connector", + "npeg": { + "version": "1.3.0", + "vcsRevision": "409f6796d0e880b3f0222c964d1da7de6e450811", + "url": "https://github.com/zevv/npeg", "downloadMethod": "git", "dependencies": [ "nim" ], "checksums": { - "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + "sha1": "64f15c85a059c889cb11c5fe72372677c50da621" } }, "results": { @@ -113,6 +50,96 @@ "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97" } }, + "unicodedb": { + "version": "0.13.2", + "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", + "url": "https://github.com/nitely/nim-unicodedb", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" + } + }, + "regex": { + "version": "0.26.3", + "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", + "url": "https://github.com/nitely/nim-regex", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unicodedb" + ], + "checksums": { + "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" + } + }, + "unittest2": { + "version": "0.2.5", + "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", + "url": "https://github.com/status-im/nim-unittest2", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + } + }, + "testutils": { + "version": "0.8.1", + "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", + "url": "https://github.com/status-im/nim-testutils", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unittest2" + ], + "checksums": { + "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" + } + }, + "bearssl": { + "version": "0.2.10", + "vcsRevision": "25197fec8a3988d1153be7a1a79890b70e69d6db", + "url": "https://github.com/status-im/nim-bearssl", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unittest2" + ], + "checksums": { + "sha1": "e53ab85b8c4c1ce3aa1f4eb251c0ab29e08d4972" + } + }, + "bearssl_pkey_decoder": { + "version": "#d34aa46bf9d0a3ffff810fbd3c4d2fa024eb9368", + "vcsRevision": "d34aa46bf9d0a3ffff810fbd3c4d2fa024eb9368", + "url": "https://github.com/vacp2p/bearssl_pkey_decoder", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl" + ], + "checksums": { + "sha1": "8666edbcb77cb9f97c659114d57c4ba0e7ab74c3" + } + }, + "jwt": { + "version": "#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", + "vcsRevision": "057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", + "url": "https://github.com/vacp2p/nim-jwt.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl", + "bearssl_pkey_decoder" + ], + "checksums": { + "sha1": "3cd368666fd2bc7f99f253452289e827abcac13c" + } + }, "stew": { "version": "0.5.0", "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa", @@ -128,8 +155,8 @@ } }, "zlib": { - "version": "0.1.0", - "vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe", + "version": "0.2.0", + "vcsRevision": "190246aa0bb6569781370964fa2faa474203d6dd", "url": "https://github.com/status-im/nim-zlib", "downloadMethod": "git", "dependencies": [ @@ -138,12 +165,12 @@ "results" ], "checksums": { - "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f" + "sha1": "a8c0c569d82315f3ffc1249ab42b0404e84fddc3" } }, "httputils": { - "version": "0.4.1", - "vcsRevision": "f142cb2e8bd812dd002a6493b6082827bb248592", + "version": "0.4.3", + "vcsRevision": "a9ca86095e262251d8ebafa1c6504fd476c41e43", "url": "https://github.com/status-im/nim-http-utils", "downloadMethod": "git", "dependencies": [ @@ -153,7 +180,7 @@ "unittest2" ], "checksums": { - "sha1": "016774ab31c3afff9a423f7d80584905ee59c570" + "sha1": "dcdd6633bc35695c2b1ac0f0bd1c5ff4e3623cb5" } }, "chronos": { @@ -174,8 +201,8 @@ } }, "metrics": { - "version": "0.2.1", - "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2", + "version": "0.2.2", + "vcsRevision": "b4b70a88fe1755d281366cbc3f22d7515240d192", "url": "https://github.com/status-im/nim-metrics", "downloadMethod": "git", "dependencies": [ @@ -185,12 +212,12 @@ "stew" ], "checksums": { - "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2" + "sha1": "94e0785170b639d0046e7469408bb752197ead8e" } }, "faststreams": { - "version": "0.5.0", - "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e", + "version": "0.5.1", + "vcsRevision": "50889cd16ec8771106cdd0eeea460039e8571e06", "url": "https://github.com/status-im/nim-faststreams", "downloadMethod": "git", "dependencies": [ @@ -199,7 +226,7 @@ "unittest2" ], "checksums": { - "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383" + "sha1": "969ceb3666e807db8fe5c8df63466749822367a9" } }, "snappy": { @@ -208,19 +235,19 @@ "url": "https://github.com/status-im/nim-snappy", "downloadMethod": "git", "dependencies": [ - "nim", "faststreams", "unittest2", "results", - "stew" + "stew", + "testutils" ], "checksums": { "sha1": "e572d60d6a3178c5b1cde2400c51ad771812cd3d" } }, "serialization": { - "version": "0.5.2", - "vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b", + "version": "0.5.3", + "vcsRevision": "4092500cea76154576539371709ae801afbd2a9d", "url": "https://github.com/status-im/nim-serialization", "downloadMethod": "git", "dependencies": [ @@ -230,7 +257,24 @@ "stew" ], "checksums": { - "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8" + "sha1": "c087d26c50da40436599163888532660d6f9e631" + } + }, + "protobuf_serialization": { + "version": "0.5.3", + "vcsRevision": "8406e7287196661614ce6a8e8be20f755376af7f", + "url": "https://github.com/status-im/nim-protobuf-serialization", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "faststreams", + "serialization", + "npeg", + "unittest2" + ], + "checksums": { + "sha1": "3307412f9755f7ec2079e12cf036fc103aa130b0" } }, "toml_serialization": { @@ -254,7 +298,6 @@ "url": "https://github.com/status-im/nim-confutils", "downloadMethod": "git", "dependencies": [ - "nim", "stew", "serialization", "results" @@ -264,18 +307,19 @@ } }, "cbor_serialization": { - "version": "0.3.0", - "vcsRevision": "1664160e04d153573373afddc552b9cbf6fbe4dc", + "version": "0.4.1", + "vcsRevision": "e32d61c54f69b396f47645f9b7373ea2e149013a", "url": "https://github.com/vacp2p/nim-cbor-serialization", "downloadMethod": "git", "dependencies": [ - "nim", "serialization", "stew", - "results" + "npeg", + "results", + "unittest2" ], "checksums": { - "sha1": "ab126eae09a6e39c72972a6a0b83cb06a2ffe8f0" + "sha1": "4782b43d06594ac59730b3939110ba93de7d3156" } }, "json_serialization": { @@ -316,7 +360,6 @@ "url": "https://github.com/status-im/nim-presto", "downloadMethod": "git", "dependencies": [ - "nim", "chronos", "chronicles", "metrics", @@ -350,7 +393,6 @@ "url": "https://github.com/status-im/nim-stint", "downloadMethod": "git", "dependencies": [ - "nim", "stew", "unittest2" ], @@ -364,7 +406,6 @@ "url": "https://github.com/status-im/nim-minilru", "downloadMethod": "git", "dependencies": [ - "nim", "results", "unittest2" ], @@ -372,55 +413,6 @@ "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" } }, - "sqlite3_abi": { - "version": "3.53.0.0", - "vcsRevision": "8240e8e2819dfce1b67fa2733135d01b5cc80ae0", - "url": "https://github.com/arnetheduck/nim-sqlite3-abi", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "fb7a6e6f36fc4eb4dfa6634dbcbf5cd0dfd0ebf0" - } - }, - "dnsclient": { - "version": "0.3.4", - "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", - "url": "https://github.com/ba0f3/dnsclient.nim", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" - } - }, - "unicodedb": { - "version": "0.13.2", - "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", - "url": "https://github.com/nitely/nim-unicodedb", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" - } - }, - "regex": { - "version": "0.26.3", - "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", - "url": "https://github.com/nitely/nim-regex", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unicodedb" - ], - "checksums": { - "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" - } - }, "nimcrypto": { "version": "0.6.4", "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", @@ -433,13 +425,31 @@ "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" } }, + "lsquic": { + "version": "#v0.5.5", + "vcsRevision": "079768948ac03d8f2384e03045a090cb88aeb7d9", + "url": "https://github.com/vacp2p/nim-lsquic.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "zlib", + "stew", + "chronos", + "nimcrypto", + "unittest2", + "chronicles", + "https://github.com/vacp2p/nim-boringssl" + ], + "checksums": { + "sha1": "6742442d2f0f4aa4880b98bb12965beef85e513c" + } + }, "websock": { "version": "0.4.0", "vcsRevision": "387a8eb7e961e8fdd3b1a717d36bc53b55e4dc5d", "url": "https://github.com/status-im/nim-websock", "downloadMethod": "git", "dependencies": [ - "nim", "chronos", "httputils", "chronicles", @@ -454,7 +464,7 @@ } }, "json_rpc": { - "version": "0.6.1", + "version": "#v0.6.1", "vcsRevision": "6f1fff8ba685c9192fab153a9d66484ad9066e78", "url": "https://github.com/status-im/nim-json-rpc.git", "downloadMethod": "git", @@ -475,25 +485,6 @@ "sha1": "596db0aafcb3c83f5dba6d42993f2276e0d00eb5" } }, - "lsquic": { - "version": "0.5.1", - "vcsRevision": "2f01046bf1d513de8b5f8296c3d8bec819ab0cb9", - "url": "https://github.com/vacp2p/nim-lsquic", - "downloadMethod": "git", - "dependencies": [ - "nim", - "zlib", - "stew", - "chronos", - "nimcrypto", - "unittest2", - "chronicles", - "boringssl" - ], - "checksums": { - "sha1": "959df1a9ac2a574d6fe30a4faf37c37b443e1cfb" - } - }, "secp256k1": { "version": "0.6.0.3.2", "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", @@ -509,13 +500,32 @@ "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" } }, + "db_connector": { + "version": "0.1.0", + "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", + "url": "https://github.com/nim-lang/db_connector", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + } + }, + "sqlite3_abi": { + "version": "3.53.3.0", + "vcsRevision": "a4d052efe81472b8d5271ece8e5a3caea90a45eb", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "20571756875f29ef292acf10a71c27139e6bf44e" + } + }, "eth": { "version": "0.9.0", "vcsRevision": "d9135e6c3c5d6d819afdfb566aa8d958756b73a8", "url": "https://github.com/status-im/nim-eth", "downloadMethod": "git", "dependencies": [ - "nim", "nimcrypto", "stint", "secp256k1", @@ -542,7 +552,6 @@ "url": "https://github.com/status-im/nim-web3", "downloadMethod": "git", "dependencies": [ - "nim", "chronicles", "chronos", "bearssl", @@ -561,12 +570,11 @@ } }, "dnsdisc": { - "version": "0.1.0", - "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e", + "version": "0.1.1", + "vcsRevision": "6cb1b7e3922645275043c68e476cac1501a45e55", "url": "https://github.com/status-im/nim-dnsdisc", "downloadMethod": "git", "dependencies": [ - "nim", "bearssl", "chronicles", "chronos", @@ -579,20 +587,29 @@ "results" ], "checksums": { - "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154" + "sha1": "6451cab35990f334a46927f49f9176579460934d" + } + }, + "dnsclient": { + "version": "0.3.4", + "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", + "url": "https://github.com/ba0f3/dnsclient.nim", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" } }, "libp2p": { - "version": "2.0.0", - "vcsRevision": "c43199378f46d0aaf61be1cad1ee1d63e8f665d6", + "version": "#v2.1.4", + "vcsRevision": "3b5ae1da95f0614af06221be7a3bb2aeab03f4c7", "url": "https://github.com/vacp2p/nim-libp2p.git", "downloadMethod": "git", "dependencies": [ "nim", "nimcrypto", - "dnsclient", "bearssl", - "boringssl", + "https://github.com/vacp2p/nim-boringssl", "chronicles", "chronos", "metrics", @@ -604,102 +621,14 @@ "lsquic", "protobuf_serialization", "websock", - "jwt" + "nat_traversal" ], "checksums": { - "sha1": "327dc7a0cb7e9d0be3d6083841bd496c4cbc48dc" - } - }, - "taskpools": { - "version": "0.1.0", - "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", - "url": "https://github.com/status-im/nim-taskpools", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" - } - }, - "sds": { - "version": "#b12f5ee07c5b764303b51fb948b32a4ade1de3b5", - "vcsRevision": "b12f5ee07c5b764303b51fb948b32a4ade1de3b5", - "url": "https://github.com/logos-messaging/nim-sds.git", - "downloadMethod": "git", - "dependencies": [ - "nim", - "chronos", - "libp2p", - "chronicles", - "stew", - "stint", - "metrics", - "results", - "ffi" - ], - "checksums": { - "sha1": "175f65038b9877cdf974b07c5f83081f810d5fbe" - } - }, - "ffi": { - "version": "0.1.3", - "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", - "url": "https://github.com/logos-messaging/nim-ffi", - "downloadMethod": "git", - "dependencies": [ - "nim", - "chronos", - "chronicles", - "taskpools" - ], - "checksums": { - "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" - } - }, - "boringssl": { - "version": "0.0.8", - "vcsRevision": "e77caabae78fbc9aa5b78a0a521181b077c82571", - "url": "https://github.com/vacp2p/nim-boringssl", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "2f603bb6d70683393bbb091bf6bd325d9c52be9f" - } - }, - "protobuf_serialization": { - "version": "0.4.0", - "vcsRevision": "38d24eb3bd93e605fb88199da71d36b1ec0ad60d", - "url": "https://github.com/status-im/nim-protobuf-serialization", - "downloadMethod": "git", - "dependencies": [ - "nim", - "stew", - "faststreams", - "serialization", - "npeg", - "unittest2" - ], - "checksums": { - "sha1": "5a7a80fb8cca29e41899ce9540b74e49c874f8fd" - } - }, - "npeg": { - "version": "1.3.0", - "vcsRevision": "409f6796d0e880b3f0222c964d1da7de6e450811", - "url": "https://github.com/zevv/npeg", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "64f15c85a059c889cb11c5fe72372677c50da621" + "sha1": "48da49700427bf68f1bbece13f8f9b620d0abdcb" } }, "libp2p_mix": { - "version": "0.1.0", + "version": "#380513117d556bf8f70066f5e72a7fd74fe36ba6", "vcsRevision": "380513117d556bf8f70066f5e72a7fd74fe36ba6", "url": "https://github.com/logos-co/nim-libp2p-mix", "downloadMethod": "git", @@ -717,7 +646,52 @@ "checksums": { "sha1": "ccfb0f0160ac15ac970471964c730d57edacad91" } + }, + "taskpools": { + "version": "0.1.0", + "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", + "url": "https://github.com/status-im/nim-taskpools", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" + } + }, + "ffi": { + "version": "#v0.1.3", + "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", + "url": "https://github.com/logos-messaging/nim-ffi", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronos", + "chronicles", + "taskpools" + ], + "checksums": { + "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" + } + }, + "sds": { + "version": "#b12f5ee07c5b764303b51fb948b32a4ade1de3b5", + "vcsRevision": "2ad4432f4eaf5d85771412f8c527eb767ba44ea8", + "url": "https://github.com/logos-messaging/nim-sds.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronos", + "libp2p", + "chronicles", + "stew", + "stint", + "metrics", + "results", + "ffi" + ], + "checksums": { + "sha1": "175f65038b9877cdf974b07c5f83081f810d5fbe" + } } }, "tasks": {} -} \ No newline at end of file +} From 861d3f907118130b1517fb60056c69e3faf0f02e Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Tue, 14 Jul 2026 08:35:37 -0400 Subject: [PATCH 15/15] libp2p 2.2.0 --- Makefile | 29 ++- apps/chat2disco/chat2disco.nim | 4 +- logos_delivery.nimble | 3 +- .../waku/discovery/waku_kademlia.nim | 16 +- logos_delivery/waku/factory/node_factory.nim | 6 +- logos_delivery/waku/waku_relay/message_id.nim | 5 +- logos_delivery/waku/waku_relay/protocol.nim | 27 ++- .../waku/waku_rendezvous/protocol.nim | 4 +- nimble.lock | 21 ++- scripts/build_rln.sh | 17 +- scripts/install_nimble.sh | 6 +- tools/apply-libp2p-2.2-shims.sh | 19 ++ tools/install_from_lock.nim | 175 ++++++++++++++++++ 13 files changed, 295 insertions(+), 37 deletions(-) create mode 100755 tools/apply-libp2p-2.2-shims.sh create mode 100644 tools/install_from_lock.nim diff --git a/Makefile b/Makefile index 60c26751b..c357bdb78 100644 --- a/Makefile +++ b/Makefile @@ -75,11 +75,25 @@ endif %: @true -logos_delivery.nims: - ln -s logos_delivery.nimble $@ +# Prefer a symlink; fall back to a full copy when `ln` is unavailable (some sandboxes). +logos_delivery.nims: logos_delivery.nimble + @if command -v ln >/dev/null 2>&1 && ln -sfn logos_delivery.nimble $@ 2>/dev/null; then \ + : ; \ + else \ + cp -f logos_delivery.nimble $@; \ + fi -$(NIMBLEDEPS_STAMP): nimble.lock | install-nimble build-nph logos_delivery.nims - $(NIMBLE) setup --localdeps +# Install locked deps. `nimble setup` is preferred when it works; on SAT failures +# (seen with libp2p git tags) fall back to tools/install_from_lock.nim which +# clones exact lock revisions, writes nimble.paths, and applies libp2p 2.2 shims. +$(NIMBLEDEPS_STAMP): nimble.lock tools/install_from_lock.nim tools/apply-libp2p-2.2-shims.sh | install-nimble build-nph logos_delivery.nims + @if $(NIMBLE) setup --localdeps; then \ + echo "nimble setup ok"; \ + bash tools/apply-libp2p-2.2-shims.sh || true; \ + else \ + echo "nimble setup failed; installing from nimble.lock via tools/install_from_lock.nim"; \ + nim c -r --hints:off --mm:refc -d:release -o:nimbledeps/install_from_lock tools/install_from_lock.nim; \ + fi touch $@ # Must be phony so the recipe always runs and the sub-make re-evaluates @@ -256,9 +270,14 @@ chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(NIMBLE) chat2mix +# Compile with `nim c` (not `nimble chat2disco`) so a failed nimble SAT resolve +# after deps are already installed cannot block the build. Paths come from +# nimble.paths written by nimble setup or tools/install_from_lock.nim. chat2disco: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(NIMBLE) chat2disco + nim c --out:build/chat2disco --mm:refc --path:. \ + $(NIM_PARAMS) -d:chronicles_log_level=TRACE \ + apps/chat2disco/chat2disco.nim rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim index 81c1b34a7..72b6f100d 100644 --- a/apps/chat2disco/chat2disco.nim +++ b/apps/chat2disco/chat2disco.nim @@ -203,7 +203,7 @@ proc joinRoom(c: Chat, roomName: string) {.async.} = if not c.node.wakuKademlia.isNil(): c.node.wakuKademlia.removeServiceToDiscover(c.currentPubsubTopic) await c.node.wakuKademlia.removeServiceToAdvertise( - ServiceInfo(id: c.currentPubsubTopic, data: @[]) + ServiceInfo(id: c.currentPubsubTopic, data: Opt.none(seq[byte])) ) c.currentPubsubTopic = roomName @@ -220,7 +220,7 @@ proc joinRoom(c: Chat, roomName: string) {.async.} = echo "subscribed to pubsub topic: ", roomName if not c.node.wakuKademlia.isNil(): - let svcInfo = ServiceInfo(id: roomName, data: @[]) + let svcInfo = ServiceInfo(id: roomName, data: Opt.none(seq[byte])) c.node.wakuKademlia.addServiceToDiscover(roomName) c.node.wakuKademlia.addServiceToAdvertise(svcInfo) echo "advertising and discovering kademlia service: ", roomName diff --git a/logos_delivery.nimble b/logos_delivery.nimble index ee8edd232..c5d28bdd1 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -28,7 +28,8 @@ requires "nim >= 2.2.4", "toml_serialization", "faststreams", # Networking & P2P - "https://github.com/vacp2p/nim-libp2p.git#v2.1.4", + "https://github.com/vacp2p/nim-libp2p.git#v2.2.0", + "libbacktrace", # required by nim-libp2p >= 2.2.0 "eth", "nat_traversal", "dnsdisc", diff --git a/logos_delivery/waku/discovery/waku_kademlia.nim b/logos_delivery/waku/discovery/waku_kademlia.nim index e6775530c..753dbf96b 100644 --- a/logos_delivery/waku/discovery/waku_kademlia.nim +++ b/logos_delivery/waku/discovery/waku_kademlia.nim @@ -55,14 +55,18 @@ proc extractMixPubKey*(service: ServiceInfo): Option[Curve25519Key] = if service.id != MixProtocolID: return none(Curve25519Key) - if service.data.len != Curve25519KeySize: - trace "invalid mix pub key length", - expected = Curve25519KeySize, - actual = service.data.len, - dataHex = byteutils.toHex(service.data) + # libp2p >= 2.2: ServiceInfo.data is Opt[seq[byte]] + let data = service.data.valueOr: return none(Curve25519Key) - let key = intoCurve25519Key(service.data) + if data.len != Curve25519KeySize: + trace "invalid mix pub key length", + expected = Curve25519KeySize, + actual = data.len, + dataHex = byteutils.toHex(data) + return none(Curve25519Key) + + let key = intoCurve25519Key(data) return some(key) diff --git a/logos_delivery/waku/factory/node_factory.nim b/logos_delivery/waku/factory/node_factory.nim index daa2e5fd0..e2e602806 100644 --- a/logos_delivery/waku/factory/node_factory.nim +++ b/logos_delivery/waku/factory/node_factory.nim @@ -3,6 +3,7 @@ import std/[options, sequtils], chronicles, chronos, + results, libp2p/peerid, libp2p/protocols/pubsub/gossipsub, libp2p/protocols/connectivity/relay/relay, @@ -175,8 +176,9 @@ proc setupProtocols( var kadConf = conf.kademliaDiscoveryConf.get() if conf.mixConf.isSome(): - let mixService = - ServiceInfo(id: MixProtocolID, data: @(conf.mixConf.get().mixPubKey)) + let mixService = ServiceInfo( + id: MixProtocolID, data: Opt.some(@(conf.mixConf.get().mixPubKey)) + ) kadConf.servicesToAdvertise.incl(mixService) kadConf.servicesToDiscover.incl(mixService.id) diff --git a/logos_delivery/waku/waku_relay/message_id.nim b/logos_delivery/waku/waku_relay/message_id.nim index 655f50c5a..d4b624498 100644 --- a/logos_delivery/waku/waku_relay/message_id.nim +++ b/logos_delivery/waku/waku_relay/message_id.nim @@ -23,7 +23,10 @@ type MessageIdProvider* = pubsub.MsgIdProvider proc defaultMessageIdProvider*( message: messages.Message ): Result[MessageID, ValidationResult] = - let hash = sha256.digest(message.data) + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = message.data.valueOr: + return err(ValidationResult.Reject) + let hash = sha256.digest(data) ok(@(hash.data)) ## Waku message Unique ID provider diff --git a/logos_delivery/waku/waku_relay/protocol.nim b/logos_delivery/waku/waku_relay/protocol.nim index 409b02929..87720749d 100644 --- a/logos_delivery/waku/waku_relay/protocol.nim +++ b/logos_delivery/waku/waku_relay/protocol.nim @@ -268,7 +268,10 @@ proc initRelayObservers(w: WakuRelay) = let msg_id_short = shortLog(msg_id) - let wakuMessage = WakuMessage.decode(msg.data).valueOr: + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = msg.data.valueOr: + return err() + let wakuMessage = WakuMessage.decode(data).valueOr: warn "Error decoding to Waku Message", my_peer_id = w.switch.peerInfo.peerId, msg_id = msg_id_short, @@ -277,7 +280,7 @@ proc initRelayObservers(w: WakuRelay) = error = $error return err() - let msgSize = msg.data.len + msg.topic.len + let msgSize = data.len + msg.topic.len return ok((msg_id_short, msg.topic, wakuMessage, msgSize)) proc updateMetrics( @@ -304,11 +307,16 @@ proc initRelayObservers(w: WakuRelay) = var topicsChanged = false for graft in ctrl.graft: - w.topicHealthDirty.incl(graft.topicID) + # libp2p >= 2.2: topicID is Opt[string] + let topicId = graft.topicID.valueOr: + continue + w.topicHealthDirty.incl(topicId) topicsChanged = true for prune in ctrl.prune: - w.topicHealthDirty.incl(prune.topicID) + let topicId = prune.topicID.valueOr: + continue + w.topicHealthDirty.incl(topicId) topicsChanged = true if topicsChanged: @@ -323,7 +331,9 @@ proc initRelayObservers(w: WakuRelay) = proc onValidated(peer: PubSubPeer, msg: Message, msgId: MessageId) = let msg_id_short = shortLog(msgId) - let wakuMessage = WakuMessage.decode(msg.data).valueOr: + let data = msg.data.valueOr: + return + let wakuMessage = WakuMessage.decode(data).valueOr: warn "onValidated: failed decoding to Waku Message", my_peer_id = w.switch.peerInfo.peerId, msg_id = msg_id_short, @@ -540,7 +550,12 @@ proc generateOrderedValidator(w: WakuRelay): ValidatorHandler {.gcsafe.} = ): Future[ValidationResult] {.async.} = # can be optimized by checking if the message is a WakuMessage without allocating memory # see nim-libp2p protobuf library - let msg = WakuMessage.decode(message.data).valueOr: + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = message.data.valueOr: + error "protocol generateOrderedValidator reject missing data", + pubsubTopic = pubsubTopic + return ValidationResult.Reject + let msg = WakuMessage.decode(data).valueOr: error "protocol generateOrderedValidator reject decode error", pubsubTopic = pubsubTopic, error = $error return ValidationResult.Reject diff --git a/logos_delivery/waku/waku_rendezvous/protocol.nim b/logos_delivery/waku/waku_rendezvous/protocol.nim index 6b06820e7..cde02163f 100644 --- a/logos_delivery/waku/waku_rendezvous/protocol.nim +++ b/logos_delivery/waku/waku_rendezvous/protocol.nim @@ -59,8 +59,8 @@ proc advertise*( ).valueOr: return err("rendezvous advertisement failed: Failed to sign Waku Peer Record: " & $error) - let sprBuff = se.encode().valueOr: - return err("rendezvous advertisement failed: Wrong Signed Peer Record: " & $error) + # libp2p >= 2.2: SignedPayload.encode returns seq[byte] (no longer Result) + let sprBuff = se.encode() # rendezvous.advertise expects already opened connections # must dial first diff --git a/nimble.lock b/nimble.lock index 03c0d0064..66e4d119a 100644 --- a/nimble.lock +++ b/nimble.lock @@ -600,13 +600,26 @@ "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" } }, + "libbacktrace": { + "version": "0.2.0", + "vcsRevision": "af60500cb0383231065f1d6624beeeefa308d085", + "url": "https://github.com/status-im/nim-libbacktrace", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "03deb5b6d5eb047f468b9c31ea1db640f8096bfd" + } + }, "libp2p": { - "version": "#v2.1.4", - "vcsRevision": "3b5ae1da95f0614af06221be7a3bb2aeab03f4c7", + "version": "#v2.2.0", + "vcsRevision": "0e763288c10b6488e5971608f12aa49ab129720e", "url": "https://github.com/vacp2p/nim-libp2p.git", "downloadMethod": "git", "dependencies": [ "nim", + "libbacktrace", "nimcrypto", "bearssl", "https://github.com/vacp2p/nim-boringssl", @@ -624,7 +637,7 @@ "nat_traversal" ], "checksums": { - "sha1": "48da49700427bf68f1bbece13f8f9b620d0abdcb" + "sha1": "48155b6ee4101ff058f93985a09d9480b2073aca" } }, "libp2p_mix": { @@ -674,7 +687,7 @@ }, "sds": { "version": "#b12f5ee07c5b764303b51fb948b32a4ade1de3b5", - "vcsRevision": "2ad4432f4eaf5d85771412f8c527eb767ba44ea8", + "vcsRevision": "b12f5ee07c5b764303b51fb948b32a4ade1de3b5", "url": "https://github.com/logos-messaging/nim-sds.git", "downloadMethod": "git", "dependencies": [ diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index b028885e2..d34acf1bf 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -24,7 +24,12 @@ output_filename=$3 # --- Prefer the prebuilt release asset -------------------------------------- # Host target triple, e.g. x86_64-unknown-linux-gnu / aarch64-apple-darwin. -host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}') +# Prefer rustc's dedicated flag; fall back to parsing -vV without requiring awk. +host_triplet=$(rustc --print host-tuple 2>/dev/null || true) +if [[ -z "${host_triplet}" ]]; then + host_triplet=$(rustc -vV | sed -n 's/^host: //p') +fi +[[ -n "${host_triplet}" ]] || { echo "Could not determine rustc host triple"; exit 1; } tarball="${host_triplet}-stateless-rln.tar.gz" url="https://github.com/vacp2p/zerokit/releases/download/${rln_version}/${tarball}" @@ -45,12 +50,10 @@ echo "No prebuilt asset for ${host_triplet} at ${rln_version}; building from sou # Check if submodule version = version in Makefile cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" -detected_OS=$(uname -s) -if [[ "$detected_OS" == MINGW* || "$detected_OS" == MSYS* ]]; then - submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" | sed -n 's/.*"name":"rln","version":"\([^"]*\)".*/\1/p') -else - submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" | jq -r '.packages[] | select(.name == "rln") | .version') -fi +# Portable: sed works without jq (some sandboxes lack jq/curl/awk). +submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" \ + | sed -n 's/.*"name":"rln","version":"\([^"]*\)".*/\1/p' | head -1) + if [[ "v${submodule_version}" != "${rln_version}" ]]; then echo "Submodule version (v${submodule_version}) does not match version in Makefile (${rln_version})" diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh index dba2d6612..c174b39d2 100755 --- a/scripts/install_nimble.sh +++ b/scripts/install_nimble.sh @@ -37,7 +37,11 @@ PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." mkdir -p "${HOME}/.nimble/bin" - ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + if command -v ln >/dev/null 2>&1 && ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" 2>/dev/null; then + : + else + cp -f "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + fi exit 0 fi diff --git a/tools/apply-libp2p-2.2-shims.sh b/tools/apply-libp2p-2.2-shims.sh new file mode 100755 index 000000000..8f20a5549 --- /dev/null +++ b/tools/apply-libp2p-2.2-shims.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Hack for libp2p 2.2 + older libp2p_mix: restore removed modules as shims. +set -euo pipefail +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +LP=$(ls -dt "$ROOT"/nimbledeps/pkgs2/libp2p-2.2.* 2>/dev/null | head -1) +if [ -z "${LP:-}" ] || [ ! -d "$LP/libp2p" ]; then + echo "libp2p 2.2 package not found under nimbledeps/pkgs2" >&2 + exit 1 +fi +cat > "$LP/libp2p/utility.nim" <<'NIM' +# Shim: libp2p/utility removed in nim-libp2p 2.x; needed by current libp2p_mix pin. +import ./utils/[opt, shortlog, collections] +export opt, shortlog, collections +NIM +cat > "$LP/libp2p/utils/sequninit.nim" <<'NIM' +# Shim: libp2p/utils/sequninit removed; newSeqUninit lives in system. +export system +NIM +echo "Applied libp2p 2.2 shims in $LP" diff --git a/tools/install_from_lock.nim b/tools/install_from_lock.nim new file mode 100644 index 000000000..bec9609d5 --- /dev/null +++ b/tools/install_from_lock.nim @@ -0,0 +1,175 @@ +## Install nimble.lock packages into nimbledeps/pkgs2 and write nimble.paths. +## +## Used when `nimble setup --localdeps` fails (e.g. SAT cannot resolve libp2p +## git pins). Reproducible: clones exact vcsRevision from the lockfile. +## +## Usage (from repo root): +## nim c -r --hints:off tools/install_from_lock.nim +## +## Optional env: +## INSTALL_FROM_LOCK_FORCE=1 re-clone packages even if present + +import std/[json, os, osproc, strutils, algorithm] + +const + PkgsRel = "nimbledeps" / "pkgs2" + LockRel = "nimble.lock" + PathsRel = "nimble.paths" + +proc run(cmd: string): int = + execCmd(cmd) + +proc runOut(cmd: string): tuple[output: string, exitCode: int] = + execCmdEx(cmd) + +proc gitClone(url, dest, rev: string): bool = + if dirExists(dest): + removeDir(dest) + # Shallow clone of a specific commit is awkward; full clone then checkout. + var c = run("git clone --quiet " & quoteShell(url) & " " & quoteShell(dest)) + if c != 0: + let url2 = url.replace(".git", "") + c = run("git clone --quiet " & quoteShell(url2) & " " & quoteShell(dest)) + if c != 0: + return false + if run("git -C " & quoteShell(dest) & " checkout -q " & quoteShell(rev)) != 0: + discard run("git -C " & quoteShell(dest) & " fetch --quiet origin " & quoteShell(rev)) + if run("git -C " & quoteShell(dest) & " checkout -q " & quoteShell(rev)) != 0: + return false + # Submodules (bearssl, boringssl, secp256k1, lsquic, zlib, ...) + discard run( + "git -C " & quoteShell(dest) & " submodule update --init --recursive --quiet" + ) + true + +proc packageVersion(pkgDir: string, fallback: string): string = + result = fallback + for f in walkFiles(pkgDir / "*.nimble"): + for line in readFile(f).splitLines: + let t = line.strip + if t.startsWith("version"): + let parts = t.split('=') + if parts.len >= 2: + return parts[1].strip().strip(chars = {'"', '\''}) + break + +proc writeMeta(pkgDir, url, rev, sha1, ver: string) = + let meta = %*{ + "url": url, + "vcsRevision": rev, + "downloadMethod": "git", + "versions": [ver], + "checksums": {"sha1": sha1} + } + writeFile(pkgDir / "nimblemeta.json", meta.pretty) + +proc installPkg(root, name: string, entry: JsonNode, force: bool): string = + ## Returns final package directory path. + let version = entry["version"].getStr + let rev = entry["vcsRevision"].getStr + let url = entry["url"].getStr + let sha1 = entry["checksums"]["sha1"].getStr + let pkgsDir = root / PkgsRel + createDir(pkgsDir) + + # Prefer existing dir matching name-*-sha1 + var existing = "" + for kind, path in walkDir(pkgsDir): + if kind != pcDir: + continue + let base = path.extractFilename + if base.startsWith(name & "-") and base.endsWith("-" & sha1): + existing = path + break + + if existing.len > 0 and not force and fileExists(existing / "nimblemeta.json"): + # Ensure submodules present (best-effort) + if dirExists(existing / ".git"): + discard run( + "git -C " & quoteShell(existing) & + " submodule update --init --recursive --quiet" + ) + echo " skip ", existing.extractFilename + return existing + + echo " install ", name, " @ ", version, " (", rev[0 .. min(11, rev.high)], ")" + let tmpName = name & "-tmp-" & sha1 + let tmpDest = pkgsDir / tmpName + if not gitClone(url, tmpDest, rev): + echo " FAILED ", name + quit(1) + + var pkgVer = packageVersion(tmpDest, version.replace("#", "")) + if pkgVer.len == 0: + pkgVer = version.replace("#", "") + let finalName = name & "-" & pkgVer & "-" & sha1 + let finalDest = pkgsDir / finalName + if finalDest != tmpDest: + if dirExists(finalDest): + removeDir(finalDest) + moveDir(tmpDest, finalDest) + writeMeta(finalDest, url, rev, sha1, pkgVer) + echo " -> ", finalName + finalDest + +proc writePaths(root: string, dirs: seq[string]) = + var lines = @["--noNimblePath"] + var sorted = dirs + sorted.sort() + for d in sorted: + lines.add("--path:\"" & d & "\"") + if dirExists(d / "src"): + lines.add("--path:\"" & d / "src" & "\"") + writeFile(root / PathsRel, lines.join("\n") & "\n") + echo "Wrote ", root / PathsRel, " (", sorted.len, " packages)" + +proc applyLibp2pShims(root: string) = + ## libp2p_mix still imports removed libp2p modules; restore as thin shims. + var lp = "" + let pkgs = root / PkgsRel + for kind, path in walkDir(pkgs): + if kind != pcDir: + continue + let base = path.extractFilename + if base.startsWith("libp2p-2.2") and dirExists(path / "libp2p"): + lp = path + break + if lp.len == 0: + echo " (no libp2p 2.2 package; skip shims)" + return + writeFile( + lp / "libp2p" / "utility.nim", + """ +# Shim: libp2p/utility removed in nim-libp2p 2.x; needed by current libp2p_mix pin. +import ./utils/[opt, shortlog, collections] +export opt, shortlog, collections +""", + ) + createDir(lp / "libp2p" / "utils") + writeFile( + lp / "libp2p" / "utils" / "sequninit.nim", + """ +# Shim: libp2p/utils/sequninit removed; newSeqUninit lives in system. +export system +""", + ) + echo " applied libp2p 2.2 shims in ", lp.extractFilename + +proc main() = + let root = getCurrentDir() + let lockPath = root / LockRel + if not fileExists(lockPath): + echo "error: ", lockPath, " not found (run from repo root)" + quit(1) + + let force = getEnv("INSTALL_FROM_LOCK_FORCE") == "1" + let lock = parseFile(lockPath) + echo "Installing packages from ", LockRel, " ..." + var dirs: seq[string] + for name, entry in lock["packages"].pairs: + dirs.add(installPkg(root, name, entry, force)) + writePaths(root, dirs) + applyLibp2pShims(root) + echo "Done." + +main()