diff --git a/apps/chat2mix/chat2mix.nim b/apps/chat2mix/chat2mix.nim index 8b786d7b6..15c829d2f 100644 --- a/apps/chat2mix/chat2mix.nim +++ b/apps/chat2mix/chat2mix.nim @@ -465,23 +465,12 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} = kadBootstrapPeers.add((peerId, @[ma])) if kadBootstrapPeers.len > 0: - node.wakuKademlia = WakuKademlia.new( - node.switch, - ExtendedKademliaDiscoveryParams( - bootstrapNodes: kadBootstrapPeers, - mixPubKey: some(mixPubKey), - advertiseMix: false, - ), - node.peerManager, - getMixNodePoolSize = proc(): int {.gcsafe, raises: [].} = - if node.wakuMix.isNil(): - 0 - else: - node.getMixNodePoolSize(), - isNodeStarted = proc(): bool {.gcsafe, raises: [].} = - node.started, - ).valueOr: - error "failed to setup kademlia discovery", error = error + node.mountKademlia( + KademliaDiscoveryConf( + bootstrapNodes: kadBootstrapPeers, servicesToDiscover: @[MixProtocolID] + ) + ).isOkOr: + error "failed to setup service discovery", error = error quit(QuitFailure) #await node.mountRendezvousClient(conf.clusterId) @@ -489,10 +478,6 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} = await node.start() node.peerManager.start() - if not node.wakuKademlia.isNil(): - (await node.wakuKademlia.start(minMixPeers = MinMixNodePoolSize)).isOkOr: - error "failed to start kademlia discovery", error = error - quit(QuitFailure) await node.mountLibp2pPing() #await node.mountPeerExchangeClient() diff --git a/config.nims b/config.nims index 0f6052c9b..ebe501db8 100644 --- a/config.nims +++ b/config.nims @@ -117,7 +117,7 @@ if defined(android): switch("passL", "--sysroot=" & sysRoot) switch("cincludes", sysRoot & "/usr/include/") # begin Nimble config (version 2) +--noNimblePath when withDir(thisDir(), system.fileExists("nimble.paths")): - --noNimblePath include "nimble.paths" # end Nimble config diff --git a/nimble.lock b/nimble.lock index 7c76f7fa9..c52df53d7 100644 --- a/nimble.lock +++ b/nimble.lock @@ -2,90 +2,37 @@ "version": 2, "packages": { "nim": { - "version": "2.2.4", - "vcsRevision": "911e0dbb1f76de61fa0215ab1bb85af5334cc9a8", + "version": "2.2.10", + "vcsRevision": "9fe2137fa2f3f66cf5a44f357d461829ac9e20c4", "url": "https://github.com/nim-lang/Nim.git", "downloadMethod": "git", "dependencies": [], "checksums": { - "sha1": "68bb85cbfb1832ce4db43943911b046c3af3caab" + "sha1": "17ec440fdb89f8903db29a17898af590087d2b64" } }, - "unittest2": { - "version": "0.2.5", - "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", - "url": "https://github.com/status-im/nim-unittest2", + "nimcrypto": { + "version": "0.6.4", + "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", + "url": "https://github.com/cheatfate/nimcrypto", "downloadMethod": "git", "dependencies": [ "nim" ], "checksums": { - "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" } }, - "bearssl": { - "version": "0.2.8", - "vcsRevision": "22c6a76ce015bc07e011562bdcfc51d9446c1e82", - "url": "https://github.com/status-im/nim-bearssl", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unittest2" - ], - "checksums": { - "sha1": "da4dd7ae96d536bdaf42dca9c85d7aed024b6a86" - } - }, - "bearssl_pkey_decoder": { - "version": "#21dd3710df9345ed2ad8bf8f882761e07863b8e0", - "vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0", - "url": "https://github.com/vacp2p/bearssl_pkey_decoder", - "downloadMethod": "git", - "dependencies": [ - "nim", - "bearssl" - ], - "checksums": { - "sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714" - } - }, - "jwt": { - "version": "#18f8378de52b241f321c1f9ea905456e89b95c6f", - "vcsRevision": "18f8378de52b241f321c1f9ea905456e89b95c6f", - "url": "https://github.com/vacp2p/nim-jwt.git", - "downloadMethod": "git", - "dependencies": [ - "nim", - "bearssl", - "bearssl_pkey_decoder" - ], - "checksums": { - "sha1": "bcfd6fc9c5e10a52b87117219b7ab5c98136bc8e" - } - }, - "testutils": { - "version": "0.8.1", - "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", - "url": "https://github.com/status-im/nim-testutils", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unittest2" - ], - "checksums": { - "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" - } - }, - "db_connector": { - "version": "0.1.0", - "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", - "url": "https://github.com/nim-lang/db_connector", + "sqlite3_abi": { + "version": "3.53.0.0", + "vcsRevision": "8240e8e2819dfce1b67fa2733135d01b5cc80ae0", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi", "downloadMethod": "git", "dependencies": [ "nim" ], "checksums": { - "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + "sha1": "fb7a6e6f36fc4eb4dfa6634dbcbf5cd0dfd0ebf0" } }, "results": { @@ -113,6 +60,121 @@ "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97" } }, + "db_connector": { + "version": "0.1.0", + "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", + "url": "https://github.com/nim-lang/db_connector", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + } + }, + "taskpools": { + "version": "0.1.0", + "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", + "url": "https://github.com/status-im/nim-taskpools", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" + } + }, + "dnsclient": { + "version": "0.3.4", + "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", + "url": "https://github.com/ba0f3/dnsclient.nim", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" + } + }, + "unittest2": { + "version": "0.2.5", + "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", + "url": "https://github.com/status-im/nim-unittest2.git", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + } + }, + "minilru": { + "version": "0.1.0", + "vcsRevision": "6dd93feb60f4cded3c05e7af7209cf63fb677893", + "url": "https://github.com/status-im/nim-minilru", + "downloadMethod": "git", + "dependencies": [ + "nim", + "results", + "unittest2" + ], + "checksums": { + "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" + } + }, + "bearssl": { + "version": "0.2.8", + "vcsRevision": "22c6a76ce015bc07e011562bdcfc51d9446c1e82", + "url": "https://github.com/status-im/nim-bearssl", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unittest2" + ], + "checksums": { + "sha1": "da4dd7ae96d536bdaf42dca9c85d7aed024b6a86" + } + }, + "bearssl_pkey_decoder": { + "version": "#d34aa46bf9d0a3ffff810fbd3c4d2fa024eb9368", + "vcsRevision": "d34aa46bf9d0a3ffff810fbd3c4d2fa024eb9368", + "url": "https://github.com/vacp2p/bearssl_pkey_decoder", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl" + ], + "checksums": { + "sha1": "8666edbcb77cb9f97c659114d57c4ba0e7ab74c3" + } + }, + "jwt": { + "version": "#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", + "vcsRevision": "057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", + "url": "https://github.com/vacp2p/nim-jwt.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl", + "bearssl_pkey_decoder" + ], + "checksums": { + "sha1": "3cd368666fd2bc7f99f253452289e827abcac13c" + } + }, + "testutils": { + "version": "0.8.1", + "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", + "url": "https://github.com/status-im/nim-testutils", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unittest2" + ], + "checksums": { + "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" + } + }, "stew": { "version": "0.5.0", "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa", @@ -141,56 +203,24 @@ "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f" } }, - "httputils": { - "version": "0.4.1", - "vcsRevision": "f142cb2e8bd812dd002a6493b6082827bb248592", - "url": "https://github.com/status-im/nim-http-utils", + "secp256k1": { + "version": "0.6.0.3.2", + "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", + "url": "https://github.com/status-im/nim-secp256k1", "downloadMethod": "git", "dependencies": [ "nim", "stew", "results", - "unittest2" + "nimcrypto" ], "checksums": { - "sha1": "016774ab31c3afff9a423f7d80584905ee59c570" - } - }, - "chronos": { - "version": "4.2.2", - "vcsRevision": "45f43a9ad8bd8bcf5903b42f365c1c879bd54240", - "url": "https://github.com/status-im/nim-chronos", - "downloadMethod": "git", - "dependencies": [ - "nim", - "results", - "stew", - "bearssl", - "httputils", - "unittest2" - ], - "checksums": { - "sha1": "3a4c9477df8cef20a04e4f1b54a2d74fdfc2a3d0" - } - }, - "metrics": { - "version": "0.2.1", - "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2", - "url": "https://github.com/status-im/nim-metrics", - "downloadMethod": "git", - "dependencies": [ - "nim", - "chronos", - "results", - "stew" - ], - "checksums": { - "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2" + "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" } }, "faststreams": { - "version": "0.5.0", - "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e", + "version": "0.5.1", + "vcsRevision": "50889cd16ec8771106cdd0eeea460039e8571e06", "url": "https://github.com/status-im/nim-faststreams", "downloadMethod": "git", "dependencies": [ @@ -199,7 +229,7 @@ "unittest2" ], "checksums": { - "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383" + "sha1": "969ceb3666e807db8fe5c8df63466749822367a9" } }, "snappy": { @@ -233,36 +263,6 @@ "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8" } }, - "toml_serialization": { - "version": "0.2.18", - "vcsRevision": "b5b387e6fb2a7cc75d54a269b07cc6218361bd46", - "url": "https://github.com/status-im/nim-toml-serialization", - "downloadMethod": "git", - "dependencies": [ - "nim", - "faststreams", - "serialization", - "stew" - ], - "checksums": { - "sha1": "76ae1c2af5dd092849b41750ff29217980dc9ca3" - } - }, - "confutils": { - "version": "0.1.0", - "vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a", - "url": "https://github.com/status-im/nim-confutils", - "downloadMethod": "git", - "dependencies": [ - "nim", - "stew", - "serialization", - "results" - ], - "checksums": { - "sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e" - } - }, "json_serialization": { "version": "0.4.4", "vcsRevision": "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44", @@ -295,21 +295,34 @@ "sha1": "02febb20d088120b2836d3306cfa21f434f88f65" } }, - "presto": { - "version": "0.1.1", - "vcsRevision": "d66043dd7ede146442e6c39720c76a20bde5225f", - "url": "https://github.com/status-im/nim-presto", + "toml_serialization": { + "version": "0.2.18", + "vcsRevision": "b5b387e6fb2a7cc75d54a269b07cc6218361bd46", + "url": "https://github.com/status-im/nim-toml-serialization", "downloadMethod": "git", "dependencies": [ "nim", - "chronos", - "chronicles", - "metrics", - "results", + "faststreams", + "serialization", "stew" ], "checksums": { - "sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2" + "sha1": "76ae1c2af5dd092849b41750ff29217980dc9ca3" + } + }, + "confutils": { + "version": "0.1.0", + "vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a", + "url": "https://github.com/status-im/nim-confutils", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "serialization", + "results" + ], + "checksums": { + "sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e" } }, "stint": { @@ -326,79 +339,54 @@ "sha1": "d8f871fd617e7857192d4609fe003b48942a8ae5" } }, - "minilru": { - "version": "0.1.0", - "vcsRevision": "6dd93feb60f4cded3c05e7af7209cf63fb677893", - "url": "https://github.com/status-im/nim-minilru", + "httputils": { + "version": "0.4.1", + "vcsRevision": "f142cb2e8bd812dd002a6493b6082827bb248592", + "url": "https://github.com/status-im/nim-http-utils", "downloadMethod": "git", "dependencies": [ "nim", + "stew", "results", "unittest2" ], "checksums": { - "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" + "sha1": "016774ab31c3afff9a423f7d80584905ee59c570" } }, - "sqlite3_abi": { - "version": "3.53.0.0", - "vcsRevision": "8240e8e2819dfce1b67fa2733135d01b5cc80ae0", - "url": "https://github.com/arnetheduck/nim-sqlite3-abi", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "fb7a6e6f36fc4eb4dfa6634dbcbf5cd0dfd0ebf0" - } - }, - "dnsclient": { - "version": "0.3.4", - "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", - "url": "https://github.com/ba0f3/dnsclient.nim", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" - } - }, - "unicodedb": { - "version": "0.13.2", - "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", - "url": "https://github.com/nitely/nim-unicodedb", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" - } - }, - "regex": { - "version": "0.26.3", - "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", - "url": "https://github.com/nitely/nim-regex", + "chronos": { + "version": "4.2.2", + "vcsRevision": "45f43a9ad8bd8bcf5903b42f365c1c879bd54240", + "url": "https://github.com/status-im/nim-chronos", "downloadMethod": "git", "dependencies": [ "nim", - "unicodedb" + "results", + "stew", + "bearssl", + "httputils", + "unittest2" ], "checksums": { - "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" + "sha1": "3a4c9477df8cef20a04e4f1b54a2d74fdfc2a3d0" } }, - "nimcrypto": { - "version": "0.6.4", - "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", - "url": "https://github.com/cheatfate/nimcrypto", + "lsquic": { + "version": "#4fb03ee7bfb39aecb3316889fdcb60bec3d0936f", + "vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f", + "url": "https://github.com/vacp2p/nim-lsquic", "downloadMethod": "git", "dependencies": [ - "nim" + "nim", + "zlib", + "stew", + "chronos", + "nimcrypto", + "unittest2", + "chronicles" ], "checksums": { - "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" + "sha1": "f465fa994346490d0924d162f53d9b5aec62f948" } }, "websock": { @@ -443,37 +431,45 @@ "sha1": "30ff6ead115b88c79862c5c7e37b1c9852eea59f" } }, - "lsquic": { - "version": "0.0.1", - "vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f", - "url": "https://github.com/vacp2p/nim-lsquic", + "metrics": { + "version": "0.2.1", + "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2", + "url": "https://github.com/status-im/nim-metrics", "downloadMethod": "git", "dependencies": [ "nim", - "zlib", - "stew", "chronos", - "nimcrypto", - "unittest2", - "chronicles" + "results", + "stew" ], "checksums": { - "sha1": "f465fa994346490d0924d162f53d9b5aec62f948" + "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2" } }, - "secp256k1": { - "version": "0.6.0.3.2", - "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", - "url": "https://github.com/status-im/nim-secp256k1", + "libp2p": { + "version": "#master", + "vcsRevision": "5733e5d28ea166fb6f287adbc4f52bc26109eb09", + "url": "https://github.com/vacp2p/nim-libp2p.git", "downloadMethod": "git", "dependencies": [ "nim", + "nimcrypto", + "dnsclient", + "bearssl", + "chronicles", + "chronos", + "metrics", + "secp256k1", "stew", + "websock", + "unittest2", "results", - "nimcrypto" + "serialization", + "lsquic", + "jwt" ], "checksums": { - "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" + "sha1": "d5c0d0d7df99b58a72edb76792ea80d3cd5f8062" } }, "eth": { @@ -503,6 +499,28 @@ "sha1": "2e01b0cfff9523d110562af70d19948280f8013e" } }, + "dnsdisc": { + "version": "0.1.0", + "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e", + "url": "https://github.com/status-im/nim-dnsdisc", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl", + "chronicles", + "chronos", + "eth", + "secp256k1", + "stew", + "testutils", + "unittest2", + "nimcrypto", + "results" + ], + "checksums": { + "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154" + } + }, "web3": { "version": "0.8.0", "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", @@ -527,64 +545,21 @@ "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" } }, - "dnsdisc": { - "version": "0.1.0", - "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e", - "url": "https://github.com/status-im/nim-dnsdisc", + "presto": { + "version": "0.1.1", + "vcsRevision": "d66043dd7ede146442e6c39720c76a20bde5225f", + "url": "https://github.com/status-im/nim-presto", "downloadMethod": "git", "dependencies": [ "nim", - "bearssl", - "chronicles", "chronos", - "eth", - "secp256k1", - "stew", - "testutils", - "unittest2", - "nimcrypto", - "results" - ], - "checksums": { - "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154" - } - }, - "libp2p": { - "version": "#ff8d51857b4b79a68468e7bcc27b2026cca02996", - "vcsRevision": "ff8d51857b4b79a68468e7bcc27b2026cca02996", - "url": "https://github.com/vacp2p/nim-libp2p.git", - "downloadMethod": "git", - "dependencies": [ - "nim", - "nimcrypto", - "dnsclient", - "bearssl", "chronicles", - "chronos", "metrics", - "secp256k1", - "stew", - "websock", - "unittest2", "results", - "serialization", - "lsquic", - "jwt" + "stew" ], "checksums": { - "sha1": "fa2a7552c6ec860717b77ce34cf0b7afe4570234" - } - }, - "taskpools": { - "version": "0.1.0", - "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", - "url": "https://github.com/status-im/nim-taskpools", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" + "sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2" } }, "ffi": { @@ -601,6 +576,31 @@ "checksums": { "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" } + }, + "unicodedb": { + "version": "0.13.2", + "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", + "url": "https://github.com/nitely/nim-unicodedb", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" + } + }, + "regex": { + "version": "0.26.3", + "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", + "url": "https://github.com/nitely/nim-regex", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unicodedb" + ], + "checksums": { + "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" + } } }, "tasks": {} diff --git a/waku.nimble b/waku.nimble index 591307c23..2ebae67d9 100644 --- a/waku.nimble +++ b/waku.nimble @@ -27,7 +27,7 @@ requires "nim >= 2.2.4", "toml_serialization", "faststreams", # Networking & P2P - "https://github.com/vacp2p/nim-libp2p.git#ff8d51857b4b79a68468e7bcc27b2026cca02996", + "https://github.com/vacp2p/nim-libp2p.git#master", "eth", "nat_traversal", "dnsdisc", diff --git a/waku/discovery/waku_kademlia.nim b/waku/discovery/waku_kademlia.nim index 94b63a321..be5ddb49e 100644 --- a/waku/discovery/waku_kademlia.nim +++ b/waku/discovery/waku_kademlia.nim @@ -1,6 +1,6 @@ {.push raises: [].} -import std/[options, sequtils] +import std/sequtils import chronos, chronicles, @@ -9,272 +9,193 @@ import libp2p/[peerid, multiaddress, switch], libp2p/extended_peer_record, libp2p/crypto/curve25519, - libp2p/protocols/[kademlia, kad_disco], - libp2p/protocols/kademlia_discovery/types as kad_types, + libp2p/protocols/service_discovery, + libp2p/protocols/service_discovery/types as sd_types, libp2p/protocols/mix/mix_protocol -import waku/waku_core, waku/node/peer_manager +import + waku/waku_core, + waku/node/peer_manager, + waku/events/discovery_events, + waku/factory/waku_conf logScope: - topics = "waku extended kademlia discovery" + topics = "waku service discovery" const - DefaultExtendedKademliaDiscoveryInterval* = chronos.seconds(5) - ExtendedKademliaDiscoveryStartupDelay* = chronos.seconds(5) + DefaultServiceDiscoveryInterval* = chronos.seconds(60) + DefaultRandomDiscoveryInterval* = chronos.seconds(60) -type - MixNodePoolSizeProvider* = proc(): int {.gcsafe, raises: [].} - NodeStartedProvider* = proc(): bool {.gcsafe, raises: [].} +type WakuKademlia* = ref object + protocol*: ServiceDiscovery + peerManager: PeerManager + randomLookupLoop: Future[void] + serviceLookupLoop: Future[void] + randomLookupInterval: Duration + serviceLookupInterval: Duration + servicesToDiscover: seq[string] + servicesToAdvertise: seq[ServiceInfo] - ExtendedKademliaDiscoveryParams* = object - bootstrapNodes*: seq[(PeerId, seq[MultiAddress])] - mixPubKey*: Option[Curve25519Key] - advertiseMix*: bool = false - - WakuKademlia* = ref object - protocol*: KademliaDiscovery - peerManager: PeerManager - discoveryLoop: Future[void] - running*: bool - getMixNodePoolSize: MixNodePoolSizeProvider - isNodeStarted: NodeStartedProvider - -proc new*( - T: type WakuKademlia, - switch: Switch, - params: ExtendedKademliaDiscoveryParams, - peerManager: PeerManager, - getMixNodePoolSize: MixNodePoolSizeProvider = nil, - isNodeStarted: NodeStartedProvider = nil, -): Result[T, string] = - if params.bootstrapNodes.len == 0: - info "creating kademlia discovery as seed node (no bootstrap nodes)" - - let kademlia = KademliaDiscovery.new( - switch, - bootstrapNodes = params.bootstrapNodes, - config = KadDHTConfig.new( - validator = kad_types.ExtEntryValidator(), selector = kad_types.ExtEntrySelector() - ), - codec = ExtendedKademliaDiscoveryCodec, - ) - - try: - switch.mount(kademlia) - except CatchableError: - return err("failed to mount kademlia discovery: " & getCurrentExceptionMsg()) - - # Register services BEFORE starting kademlia so they are included in the - # initial self-signed peer record published to the DHT - if params.advertiseMix: - if params.mixPubKey.isSome(): - let alreadyAdvertising = kademlia.startAdvertising( - ServiceInfo(id: MixProtocolID, data: @(params.mixPubKey.get())) - ) - if alreadyAdvertising: - warn "mix service was already being advertised" - debug "extended kademlia advertising mix service", - keyHex = byteutils.toHex(params.mixPubKey.get()), - bootstrapNodes = params.bootstrapNodes.len - else: - warn "mix advertising enabled but no key provided" - - info "kademlia discovery created", - bootstrapNodes = params.bootstrapNodes.len, advertiseMix = params.advertiseMix - - return ok( - WakuKademlia( - protocol: kademlia, - peerManager: peerManager, - running: false, - getMixNodePoolSize: getMixNodePoolSize, - isNodeStarted: isNodeStarted, - ) - ) - -proc extractMixPubKey(service: ServiceInfo): Option[Curve25519Key] = +proc extractMixPubKey(service: ServiceInfo): Opt[Curve25519Key] = if service.id != MixProtocolID: - trace "service is not mix protocol", - serviceId = service.id, mixProtocolId = MixProtocolID return none(Curve25519Key) if service.data.len != Curve25519KeySize: - warn "invalid mix pub key length from kademlia record", + error "invalid mix pub key length", expected = Curve25519KeySize, actual = service.data.len, dataHex = byteutils.toHex(service.data) return none(Curve25519Key) - debug "found mix protocol service", - dataLen = service.data.len, expectedLen = Curve25519KeySize - let key = intoCurve25519Key(service.data) - debug "successfully extracted mix pub key", keyHex = byteutils.toHex(key) + return some(key) -proc remotePeerInfoFrom(record: ExtendedPeerRecord): Option[RemotePeerInfo] = - debug "processing kademlia record", - peerId = record.peerId, - numAddresses = record.addresses.len, - numServices = record.services.len, - serviceIds = record.services.mapIt(it.id) - +proc remotePeerInfoFrom(record: ExtendedPeerRecord): Opt[RemotePeerInfo] = if record.addresses.len == 0: - trace "kademlia record missing addresses", peerId = record.peerId + error "missing addresses", peerId = record.peerId return none(RemotePeerInfo) let addrs = record.addresses.mapIt(it.address) if addrs.len == 0: - trace "kademlia record produced no dialable addresses", peerId = record.peerId + error "no dialable addresses", peerId = record.peerId return none(RemotePeerInfo) - let protocols = record.services.mapIt(it.id) - var mixPubKey = none(Curve25519Key) for service in record.services: - debug "checking service", - peerId = record.peerId, serviceId = service.id, dataLen = service.data.len - mixPubKey = extractMixPubKey(service) - if mixPubKey.isSome(): - debug "extracted mix public key from service", peerId = record.peerId - break + mixPubKey = extractMixPubKey(service).valueOr: + continue + + trace "successfully extracted mix pub key", + peerId = record.peerId, keyHex = byteutils.toHex(mixPubKey.get()) + + break - if record.services.len > 0 and mixPubKey.isNone(): - debug "record has services but no valid mix key", - peerId = record.peerId, services = record.services.mapIt(it.id) - return none(RemotePeerInfo) return some( RemotePeerInfo.init( - record.peerId, - addrs = addrs, - protocols = protocols, - origin = PeerOrigin.Kademlia, - mixPubKey = mixPubKey, + record.peerId, addrs = addrs, origin = PeerOrigin.Kademlia, mixPubKey = mixPubKey ) ) -proc lookupMixPeers*( - wk: WakuKademlia -): Future[Result[int, string]] {.async: (raises: []).} = - ## Lookup mix peers via kademlia and add them to the peer store. - ## Returns the number of mix peers found and added. - if wk.protocol.isNil(): - return err("cannot lookup mix peers: kademlia not mounted") +proc lookupServicePeers*( + self: WakuKademlia, serviceId: string +): Future[Result[seq[RemotePeerInfo], string]] {.async: (raises: []).} = + if self.protocol.isNil(): + return err("cannot lookup service peers: service discovery not mounted") - let mixService = ServiceInfo(id: MixProtocolID, data: @[]) - var records: seq[ExtendedPeerRecord] - try: - records = await wk.protocol.lookup(mixService) - except CatchableError: - return err("mix peer lookup failed: " & getCurrentExceptionMsg()) + let serviceInfo = ServiceInfo(id: serviceId, data: @[]) - debug "mix peer lookup returned records", numRecords = records.len + let lookupCatch = catch: + (await self.protocol.lookup(serviceInfo)) - var added = 0 - for record in records: - let peerOpt = remotePeerInfoFrom(record) - if peerOpt.isNone(): + let lookupResult = lookupCatch.valueOr: + return err("service peer lookup failed: " & error.msg) + + let advertisements = lookupResult.valueOr: + return err("service peer lookup failed: " & lookupResult.error) + + var discovered: seq[RemotePeerInfo] + for ad in advertisements: + let record = ad.data + let peerInfo = remotePeerInfoFrom(record).valueOr: continue - let peerInfo = peerOpt.get() - if peerInfo.mixPubKey.isNone(): + self.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) + discovered.add(peerInfo) + + debug "service lookup complete", serviceId, found = discovered.len + return ok(discovered) + +proc runRandomLookupLoop(self: WakuKademlia) {.async: (raises: [CancelledError]).} = + debug "periodic random lookup started", interval = $self.randomLookupInterval + + while true: + await sleepAsync(self.randomLookupInterval) + + let recordsRes = catch: + (await self.protocol.lookupRandom()) + + let records = recordsRes.valueOr: + error "random lookup failed", error continue - wk.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) - info "mix peer added via kademlia lookup", - peerId = $peerInfo.peerId, mixPubKey = byteutils.toHex(peerInfo.mixPubKey.get()) - added.inc() - - info "mix peer lookup complete", found = added - return ok(added) - -proc runDiscoveryLoop( - wk: WakuKademlia, interval: Duration, minMixPeers: int -) {.async: (raises: []).} = - info "extended kademlia discovery loop started", interval = interval - - try: - while true: - # Wait for node to be started - if not wk.isNodeStarted.isNil() and not wk.isNodeStarted(): - await sleepAsync(ExtendedKademliaDiscoveryStartupDelay) + var discoveredPeers: seq[RemotePeerInfo] + for record in records: + let peerInfo = remotePeerInfoFrom(record).valueOr: continue - var records: seq[ExtendedPeerRecord] - try: - records = await wk.protocol.randomRecords() - except CatchableError as e: - warn "extended kademlia discovery failed", error = e.msg - await sleepAsync(interval) + self.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) + discoveredPeers.add(peerInfo) + + if discoveredPeers.len > 0: + PeersDiscoveredEvent.emit(peers = discoveredPeers) + + debug "random lookup complete", found = discoveredPeers.len + +proc runServiceLookupLoop(self: WakuKademlia) {.async: (raises: [CancelledError]).} = + debug "periodic service lookup started", + interval = $self.serviceLookupInterval, services = self.servicesToDiscover + + while true: + await sleepAsync(self.serviceLookupInterval) + + if self.servicesToDiscover.len == 0: + continue + + for serviceId in self.servicesToDiscover: + let discovered = (await self.lookupServicePeers(serviceId)).valueOr: + error "service lookup failed", serviceId, error continue - debug "received random records from kademlia", numRecords = records.len + if discovered.len > 0: + PeersDiscoveredEvent.emit(peers = discovered) - var added = 0 - for record in records: - let peerOpt = remotePeerInfoFrom(record) - if peerOpt.isNone(): - continue +proc new*( + T: type WakuKademlia, + switch: Switch, + peerManager: PeerManager, + config: KademliaDiscoveryConf, +): Result[T, string] = + if config.bootstrapNodes.len == 0: + debug "creating service discovery as seed node (no bootstrap nodes)" - let peerInfo = peerOpt.get() - wk.peerManager.addPeer(peerInfo, PeerOrigin.Kademlia) - debug "peer added via extended kademlia discovery", - peerId = $peerInfo.peerId, - addresses = peerInfo.addrs.mapIt($it), - protocols = peerInfo.protocols, - hasMixPubKey = peerInfo.mixPubKey.isSome() - added.inc() + let protocol = ServiceDiscovery.new( + switch, bootstrapNodes = config.bootstrapNodes, services = config.servicesToDiscover + ) - if added > 0: - info "added peers from extended kademlia discovery", count = added + let self = WakuKademlia( + protocol: protocol, + peerManager: peerManager, + randomLookupInterval: config.randomLookupInterval, + serviceLookupInterval: config.serviceLookupInterval, + servicesToDiscover: config.servicesToDiscover, + servicesToAdvertise: config.servicesToAdvertise, + ) - # Targeted mix peer lookup when pool is low - if minMixPeers > 0 and not wk.getMixNodePoolSize.isNil() and - wk.getMixNodePoolSize() < minMixPeers: - debug "mix node pool below threshold, performing targeted lookup", - currentPoolSize = wk.getMixNodePoolSize(), threshold = minMixPeers - let found = (await wk.lookupMixPeers()).valueOr: - warn "targeted mix peer lookup failed", error = error - 0 - if found > 0: - info "found mix peers via targeted kademlia lookup", count = found + return ok(self) - await sleepAsync(interval) - except CancelledError as e: - debug "extended kademlia discovery loop cancelled", error = e.msg - except CatchableError as e: - error "extended kademlia discovery loop failed", error = e.msg +proc start*(self: WakuKademlia) {.async: (raises: []).} = + for serviceId in self.servicesToDiscover: + discard self.protocol.startDiscovering(serviceId) -proc start*( - wk: WakuKademlia, - interval: Duration = DefaultExtendedKademliaDiscoveryInterval, - minMixPeers: int = 0, -): Future[Result[void, string]] {.async: (raises: []).} = - if wk.running: - return err("already running") + for serviceInfo in self.servicesToAdvertise: + self.protocol.addProvidedService(serviceInfo) - try: - await wk.protocol.start() - except CatchableError as e: - return err("failed to start kademlia discovery: " & e.msg) + if self.randomLookupLoop.isNil(): + self.randomLookupLoop = self.runRandomLookupLoop() - wk.discoveryLoop = wk.runDiscoveryLoop(interval, minMixPeers) + if self.serviceLookupLoop.isNil(): + self.serviceLookupLoop = self.runServiceLookupLoop() info "kademlia discovery started" - return ok() -proc stop*(wk: WakuKademlia) {.async: (raises: []).} = - if not wk.running: - return +proc stop*(self: WakuKademlia) {.async: (raises: []).} = + if not self.serviceLookupLoop.isNil(): + await self.serviceLookupLoop.cancelAndWait() + self.serviceLookupLoop = nil - info "Stopping kademlia discovery" + if not self.randomLookupLoop.isNil(): + await self.randomLookupLoop.cancelAndWait() + self.randomLookupLoop = nil - wk.running = false - - if not wk.discoveryLoop.isNil(): - await wk.discoveryLoop.cancelAndWait() - wk.discoveryLoop = nil - - if not wk.protocol.isNil(): - await wk.protocol.stop() - info "Successfully stopped kademlia discovery" + info "kademlia discovery stopped" diff --git a/waku/events/discovery_events.nim b/waku/events/discovery_events.nim new file mode 100644 index 000000000..aaff6122f --- /dev/null +++ b/waku/events/discovery_events.nim @@ -0,0 +1,14 @@ +import libp2p/peerinfo, waku/common/broker/[event_broker, request_broker] + +EventBroker: + # Event emitted when peers are discovered via random or service lookup + type PeersDiscoveredEvent* = object + peers*: seq[RemotePeerInfo] + +RequestBroker: + # Request broker for on-demand service peer lookup + type ServicePeersRequest* = object + serviceId*: string + peers*: seq[RemotePeerInfo] + + proc signature*(serviceId: string): Future[Result[ServicePeersRequest, string]] diff --git a/waku/events/events.nim b/waku/events/events.nim index 46dd4fdd3..824c3aa77 100644 --- a/waku/events/events.nim +++ b/waku/events/events.nim @@ -1,3 +1,3 @@ -import ./[message_events, delivery_events, health_events, peer_events] +import ./[message_events, delivery_events, health_events, peer_events, discovery_events] -export message_events, delivery_events, health_events, peer_events +export message_events, delivery_events, health_events, peer_events, discovery_events diff --git a/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim b/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim index 916d71be1..98304db23 100644 --- a/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim +++ b/waku/factory/conf_builder/kademlia_discovery_conf_builder.nim @@ -1,5 +1,6 @@ import chronicles, std/options, results -import libp2p/[peerid, multiaddress, peerinfo] +import chronos +import libp2p/[peerid, multiaddress, peerinfo, extended_peer_record] import waku/factory/waku_conf logScope: @@ -11,6 +12,10 @@ logScope: type KademliaDiscoveryConfBuilder* = object enabled*: bool bootstrapNodes*: seq[string] + servicesToAdvertise*: seq[(string, seq[byte])] + servicesToDiscover*: seq[string] + randomLookupInterval*: Duration + serviceLookupInterval*: Duration proc init*(T: type KademliaDiscoveryConfBuilder): KademliaDiscoveryConfBuilder = KademliaDiscoveryConfBuilder() @@ -23,6 +28,26 @@ proc withBootstrapNodes*( ) = b.bootstrapNodes = bootstrapNodes +proc withServicesToAdvertise*( + b: var KademliaDiscoveryConfBuilder, services: seq[(string, seq[byte])] +) = + b.servicesToAdvertise = services + +proc withServicesToDiscover*( + b: var KademliaDiscoveryConfBuilder, services: seq[string] +) = + b.servicesToDiscover = services + +proc withRandomLookupInterval*( + b: var KademliaDiscoveryConfBuilder, interval: Duration +) = + b.randomLookupInterval = interval + +proc withServiceLookupInterval*( + b: var KademliaDiscoveryConfBuilder, interval: Duration +) = + b.serviceLookupInterval = interval + proc build*( b: KademliaDiscoveryConfBuilder ): Result[Option[KademliaDiscoveryConf], string] = @@ -37,4 +62,18 @@ proc build*( return err("Failed to parse kademlia bootstrap node: " & error) parsedNodes.add((peerId, @[ma])) - return ok(some(KademliaDiscoveryConf(bootstrapNodes: parsedNodes))) + var servicesToAdvertise: seq[ServiceInfo] + for (serviceId, data) in b.servicesToAdvertise: + servicesToAdvertise.add(ServiceInfo(id: serviceId, data: data)) + + return ok( + some( + KademliaDiscoveryConf( + bootstrapNodes: parsedNodes, + servicesToAdvertise: servicesToAdvertise, + servicesToDiscover: b.servicesToDiscover, + randomLookupInterval: b.randomLookupInterval, + serviceLookupInterval: b.serviceLookupInterval, + ) + ) + ) diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index 52b719b8f..b3e5d30dd 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -7,7 +7,8 @@ import libp2p/protocols/connectivity/relay/relay, libp2p/nameresolving/dnsresolver, libp2p/crypto/crypto, - libp2p/crypto/curve25519 + libp2p/crypto/curve25519, + libp2p/protocols/mix/mix_protocol import ./internal_config, @@ -34,7 +35,8 @@ import ../node/peer_manager/peer_store/waku_peer_storage, ../node/peer_manager/peer_store/migrations as peer_store_sqlite_migrations, ../waku_lightpush_legacy/common, - ../common/rate_limit/setting + ../common/rate_limit/setting, + ../events/discovery_events ## Peer persistence @@ -165,31 +167,30 @@ proc setupProtocols( (await node.mountMix(conf.clusterId, mixConf.mixKey, mixConf.mixnodes)).isOkOr: return err("failed to mount waku mix protocol: " & $error) - # Setup extended kademlia discovery + # Setup service discovery if conf.kademliaDiscoveryConf.isSome(): - let mixPubKey = - if conf.mixConf.isSome(): - some(conf.mixConf.get().mixPubKey) - else: - none(Curve25519Key) + let kadConf = conf.kademliaDiscoveryConf.get() - node.wakuKademlia = WakuKademlia.new( - node.switch, - ExtendedKademliaDiscoveryParams( - bootstrapNodes: conf.kademliaDiscoveryConf.get().bootstrapNodes, - mixPubKey: mixPubKey, - advertiseMix: conf.mixConf.isSome(), - ), - node.peerManager, - getMixNodePoolSize = proc(): int {.gcsafe, raises: [].} = - if node.wakuMix.isNil(): - 0 - else: - node.getMixNodePoolSize(), - isNodeStarted = proc(): bool {.gcsafe, raises: [].} = - node.started, - ).valueOr: - return err("failed to setup kademlia discovery: " & error) + if conf.mixConf.isSome(): + kadConf.servicesToAdvertise.add( + ServiceInfo(id: MixProtocolID, data: @(conf.mixConf.get().mixPubKey)) + ) + + if conf.mixConf.isSome() and MixProtocolID notin kadConf.servicesToDiscover: + kadConf.servicesToDiscover.add(MixProtocolID) + + node.mountKademlia(kadConf).isOkOr: + return err("failed to setup service discovery: " & error) + + # Register ServicePeersRequest provider + ServicePeersRequest.setProvider( + node.brokerCtx, + proc(serviceId: string): Future[Result[ServicePeersRequest, string]] {.async.} = + let peers = (await node.wakuKademlia.lookupServicePeers(serviceId)).valueOr: + return err(error) + return ok(ServicePeersRequest(serviceId: serviceId, peers: peers)), + ).isOkOr: + error "Can't set provider for ServicePeersRequest", error = error if conf.storeServiceConf.isSome(): let storeServiceConf = conf.storeServiceConf.get() @@ -450,11 +451,6 @@ proc startNode*( if conf.relay: node.peerManager.start() - if not node.wakuKademlia.isNil(): - let minMixPeers = if conf.mixConf.isSome(): 4 else: 0 - (await node.wakuKademlia.start(minMixPeers = minMixPeers)).isOkOr: - return err("failed to start kademlia discovery: " & error) - return ok() proc setupNode*( diff --git a/waku/factory/waku_conf.nim b/waku/factory/waku_conf.nim index 4934faccc..8b7b3ecfe 100644 --- a/waku/factory/waku_conf.nim +++ b/waku/factory/waku_conf.nim @@ -1,10 +1,12 @@ import std/[net, options, strutils], chronicles, + chronos, libp2p/crypto/crypto, libp2p/multiaddress, libp2p/crypto/curve25519, libp2p/peerid, + libp2p/extended_peer_record, secp256k1, results @@ -54,7 +56,10 @@ type MixConf* = ref object type KademliaDiscoveryConf* = object bootstrapNodes*: seq[(PeerId, seq[MultiAddress])] - ## Bootstrap nodes for extended kademlia discovery. + servicesToAdvertise*: seq[ServiceInfo] + servicesToDiscover*: seq[string] + randomLookupInterval*: Duration + serviceLookupInterval*: Duration type StoreServiceConf* {.requiresInit.} = object dbMigration*: bool diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 45080d9d0..d2fb151b1 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -138,7 +138,6 @@ type legacyAppHandlers*: Table[PubsubTopic, WakuRelayHandler] ## Kernel API Relay appHandlers (if any) wakuMix*: WakuMix - kademliaDiscoveryLoop*: Future[void] wakuKademlia*: WakuKademlia proc deduceRelayShard( @@ -324,6 +323,24 @@ proc mountMix*( return err(error.msg) return ok() +proc mountKademlia*( + node: WakuNode, config: KademliaDiscoveryConf +): Result[void, string] = + if not node.wakuKademlia.isNil(): + return err("WakuKademlia already mounted, skipping") + + let wk = WakuKademlia.new(node.switch, node.peerManager, config).valueOr: + return err("failed to create service discovery: " & error) + + node.wakuKademlia = wk + + let mountRes = catch: + node.switch.mount(wk.protocol) + mountRes.isOkOr: + return err("failed to mount service discovery: " & error.msg) + + return ok() + ## Waku Sync proc mountStoreSync*( @@ -590,6 +607,10 @@ proc start*(node: WakuNode) {.async.} = node.started = true + if not node.wakuKademlia.isNil(): + (await node.wakuKademlia.start()).isOkOr: + error "failed to start service discovery", error = error + if not node.wakuFilterClient.isNil(): node.wakuFilterClient.registerPushHandler( proc(pubsubTopic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = @@ -611,6 +632,9 @@ proc stop*(node: WakuNode) {.async.} = node.stopProvidersAndListeners() + if not node.wakuKademlia.isNil(): + await node.wakuKademlia.stop() + ## NOTE: This will dispatch gossipsub stop to the WakuRelay.stop method override await node.switch.stop() @@ -632,9 +656,6 @@ proc stop*(node: WakuNode) {.async.} = not node.wakuPeerExchangeClient.pxLoopHandle.isNil(): await node.wakuPeerExchangeClient.pxLoopHandle.cancelAndWait() - if not node.wakuKademlia.isNil(): - await node.wakuKademlia.stop() - if not node.wakuRendezvousClient.isNil(): await node.wakuRendezvousClient.stopWait()