diff --git a/Makefile b/Makefile index daa8a8ad7..c357bdb78 100644 --- a/Makefile +++ b/Makefile @@ -75,11 +75,25 @@ endif %: @true -logos_delivery.nims: - ln -s logos_delivery.nimble $@ +# Prefer a symlink; fall back to a full copy when `ln` is unavailable (some sandboxes). +logos_delivery.nims: logos_delivery.nimble + @if command -v ln >/dev/null 2>&1 && ln -sfn logos_delivery.nimble $@ 2>/dev/null; then \ + : ; \ + else \ + cp -f logos_delivery.nimble $@; \ + fi -$(NIMBLEDEPS_STAMP): nimble.lock | install-nimble build-nph logos_delivery.nims - $(NIMBLE) setup --localdeps +# Install locked deps. `nimble setup` is preferred when it works; on SAT failures +# (seen with libp2p git tags) fall back to tools/install_from_lock.nim which +# clones exact lock revisions, writes nimble.paths, and applies libp2p 2.2 shims. +$(NIMBLEDEPS_STAMP): nimble.lock tools/install_from_lock.nim tools/apply-libp2p-2.2-shims.sh | install-nimble build-nph logos_delivery.nims + @if $(NIMBLE) setup --localdeps; then \ + echo "nimble setup ok"; \ + bash tools/apply-libp2p-2.2-shims.sh || true; \ + else \ + echo "nimble setup failed; installing from nimble.lock via tools/install_from_lock.nim"; \ + nim c -r --hints:off --mm:refc -d:release -o:nimbledeps/install_from_lock tools/install_from_lock.nim; \ + fi touch $@ # Must be phony so the recipe always runs and the sub-make re-evaluates @@ -219,7 +233,7 @@ testcommon: | build-deps build ########## ## Waku ## ########## -.PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester +.PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2mix chat2disco chat2bridge liteprotocoltester testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ @@ -256,6 +270,15 @@ chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(NIMBLE) chat2mix +# Compile with `nim c` (not `nimble chat2disco`) so a failed nimble SAT resolve +# after deps are already installed cannot block the build. Paths come from +# nimble.paths written by nimble setup or tools/install_from_lock.nim. +chat2disco: | build-deps build deps librln + echo -e $(BUILD_MSG) "build/$@" && \ + nim c --out:build/chat2disco --mm:refc --path:. \ + $(NIM_PARAMS) -d:chronicles_log_level=TRACE \ + apps/chat2disco/chat2disco.nim + rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(NIMBLE) rln_db_inspector diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim new file mode 100644 index 000000000..72b6f100d --- /dev/null +++ b/apps/chat2disco/chat2disco.nim @@ -0,0 +1,449 @@ +# chat2disco is a minimal chat app for testing Waku Kademlia service discovery. +# /room subscribes to the pubsub topic of that name and starts + +when not (compileOption("threads")): + {.fatal: "Please, compile this program with the --threads:on option!".} + +{.push raises: [].} + +import std/[strformat, strutils, times, options, sequtils] +import + confutils, + chronicles, + chronos, + eth/keys, + bearssl, + results, + stew/[byteutils], + metrics, + metrics/chronos_httpserver +import + libp2p/[ + switch, + crypto/crypto, + stream/connection, + multiaddress, + peerinfo, + peerid, + protobuf/minprotobuf, + protocols/kademlia/types, + protocols/service_discovery, + protocols/service_discovery/types as sd_types, + ] +import + logos_delivery/waku/[ + waku_core, + waku_enr, + discovery/waku_kademlia, + waku_node, + node/waku_metrics, + node/peer_manager, + factory/builder, + factory/conf_builder/kademlia_discovery_conf_builder, + common/utils/nat, + common/logging, + discovery/autonat_service, + ], + ./config_chat2disco + +import logos_delivery/waku/api/events/discovery_events + +import libp2p/protocols/pubsub/rpc/messages, libp2p/protocols/pubsub/pubsub +import libp2p/extended_peer_record # for ServiceInfo + +logScope: + topics = "chat2disco" + +const Help = """ + Commands: /[?|help|connect|nick|room|exit] + help: Prints this help + connect: dials a remote peer + nick: change nickname for current chat session + room : subscribe to pubsub topic and advertise + discover the kademlia service of the same name (kademlia always active) + exit: exits chat session +""" + +const DefaultContentTopic = "/chat2disco/1/room/proto" + +type Chat = ref object + node: WakuNode + transp: StreamTransport + subscribed: bool + connected: bool + started: bool + nick: string + prompt: bool + currentPubsubTopic: string + discoPeersListener: PeersDiscoveredEventListener + +type + PrivateKey* = crypto.PrivateKey + Topic* = waku_core.PubsubTopic + +##################### +## chat2 protobufs ## +##################### + +type + SelectResult*[T] = Result[T, string] + + Chat2Message* = object + timestamp*: int64 + nick*: string + payload*: seq[byte] + +proc init*(T: type Chat2Message, buffer: seq[byte]): ProtoResult[T] = + var msg = Chat2Message() + let pb = initProtoBuffer(buffer) + + var timestamp: uint64 + discard ?pb.getField(1, timestamp) + msg.timestamp = int64(timestamp) + + discard ?pb.getField(2, msg.nick) + discard ?pb.getField(3, msg.payload) + + ok(msg) + +proc encode*(message: Chat2Message): ProtoBuffer = + var serialised = initProtoBuffer() + + serialised.write(1, uint64(message.timestamp)) + serialised.write(2, message.nick) + serialised.write(3, message.payload) + + return serialised + +proc `$`*(message: Chat2Message): string = + let time = message.timestamp.fromUnix().local().format("'<'MMM' 'dd,' 'HH:mm'>'") + return time & " " & message.nick & ": " & string.fromBytes(message.payload) + +##################### + +proc connectToNodes(c: Chat, nodes: seq[string]) {.async.} = + echo "Connecting to nodes" + await c.node.connectToNodes(nodes) + c.connected = true + +proc showChatPrompt(c: Chat) = + if not c.prompt: + try: + stdout.write(">> ") + stdout.flushFile() + c.prompt = true + except IOError: + discard + +proc getChatLine(payload: seq[byte]): string = + let pb = Chat2Message.init(payload).valueOr: + return string.fromBytes(payload) + return $pb + +proc printReceivedMessage(c: Chat, pubsubTopic: PubsubTopic, msg: WakuMessage) = + let chatLine = getChatLine(msg.payload) + try: + echo &"[{pubsubTopic}] {chatLine}" + except ValueError: + echo chatLine + c.prompt = false + showChatPrompt(c) + trace "Printing message", chatLine, pubsubTopic, contentTopic = msg.contentTopic + +proc readNick(transp: StreamTransport): Future[string] {.async.} = + stdout.write("Choose a nickname >> ") + stdout.flushFile() + return await transp.readLine() + +proc readRoom(transp: StreamTransport): Future[string] {.async.} = + stdout.write("Choose a room >> ") + stdout.flushFile() + return await transp.readLine() + +proc startMetricsServer( + serverIp: IpAddress, serverPort: Port +): Result[MetricsHttpServerRef, string] = + info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $serverPort + + let server = MetricsHttpServerRef.new($serverIp, serverPort).valueOr: + return err("metrics HTTP server start failed: " & $error) + + try: + waitFor server.start() + except CatchableError: + return err("metrics HTTP server start failed: " & getCurrentExceptionMsg()) + + info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $serverPort + ok(server) + +proc publish(c: Chat, line: string) {.async.} = + if c.currentPubsubTopic.len == 0: + echo "No active room. Use /room to join one first." + return + + let time = getTime().toUnix() + let chat2pb = + Chat2Message(timestamp: time, nick: c.nick, payload: line.toBytes()).encode() + + var message = WakuMessage( + payload: chat2pb.buffer, + contentTopic: DefaultContentTopic, + version: 0, + timestamp: getNanosecondTime(time), + ) + + try: + (await c.node.publish(some(c.currentPubsubTopic), message)).isOkOr: + error "failed to publish message", error = error + except CatchableError: + error "caught error publishing message: ", error = getCurrentExceptionMsg() + +proc joinRoom(c: Chat, roomName: string) {.async.} = + if c.currentPubsubTopic.len > 0 and c.currentPubsubTopic != roomName: + discard c.node.unsubscribe((kind: PubsubSub, topic: c.currentPubsubTopic)) + if not c.node.wakuKademlia.isNil(): + c.node.wakuKademlia.removeServiceToDiscover(c.currentPubsubTopic) + await c.node.wakuKademlia.removeServiceToAdvertise( + ServiceInfo(id: c.currentPubsubTopic, data: Opt.none(seq[byte])) + ) + + c.currentPubsubTopic = roomName + + proc roomHandler( + pubsubTopic: PubsubTopic, msg: WakuMessage + ): Future[void] {.async, gcsafe.} = + c.printReceivedMessage(pubsubTopic, msg) + + c.node.subscribe((kind: PubsubSub, topic: roomName), WakuRelayHandler(roomHandler)).isOkOr: + error "failed to subscribe to pubsub topic for room", + topic = roomName, error = error + + echo "subscribed to pubsub topic: ", roomName + + if not c.node.wakuKademlia.isNil(): + let svcInfo = ServiceInfo(id: roomName, data: Opt.none(seq[byte])) + c.node.wakuKademlia.addServiceToDiscover(roomName) + c.node.wakuKademlia.addServiceToAdvertise(svcInfo) + echo "advertising and discovering kademlia service: ", roomName + +proc readAndPrint(c: Chat) {.async.} = + while true: + await sleepAsync(100.millis) + +proc writeAndPrint(c: Chat) {.async.} = + while true: + showChatPrompt(c) + + let line = await c.transp.readLine() + if line.startsWith("/help") or line.startsWith("/?") or not c.started: + echo Help + continue + elif line.startsWith("/connect"): + if c.connected: + echo "already connected to at least one peer" + continue + echo "enter address of remote peer" + let address = await c.transp.readLine() + if address.len > 0: + await c.connectToNodes(@[address]) + elif line.startsWith("/nick"): + c.nick = await readNick(c.transp) + echo "You are now known as " & c.nick + elif line.startsWith("/room"): + let parts = line.split(maxsplit = 1) + if parts.len < 2 or parts[1].strip().len == 0: + echo "usage: /room " + continue + let roomName = parts[1].strip() + await c.joinRoom(roomName) + elif line.startsWith("/exit"): + await PeersDiscoveredEvent.dropListener(c.discoPeersListener) + echo "quitting..." + + try: + await c.node.stop() + except: + echo "exception happened when stopping: " & getCurrentExceptionMsg() + + quit(QuitSuccess) + else: + if c.started: + if c.currentPubsubTopic.len > 0: + echo "publishing message to ", c.currentPubsubTopic, ": ", line + await c.publish(line) + else: + echo "Join a room first with /room before sending messages" + else: + try: + if line.startsWith("/") and "p2p" in line: + await c.connectToNodes(@[line]) + except: + echo &"unable to dial remote peer {line}" + echo getCurrentExceptionMsg() + +proc readWriteLoop(c: Chat) {.async.} = + asyncSpawn c.writeAndPrint() + asyncSpawn c.readAndPrint() + +proc readInput(wfd: AsyncFD) {.thread, raises: [Defect, CatchableError].} = + let transp = fromPipe(wfd) + + while true: + let line = stdin.readLine() + discard waitFor transp.write(line & "\r\n") + +{.pop.} + # @TODO confutils.nim(775, 17) Error: can raise an unlisted exception: ref IOError +proc processInput(rfd: AsyncFD, rng: crypto.Rng) {.async.} = + let + transp = fromPipe(rfd) + conf = Chat2Conf.load() + nodekey = + if conf.nodekey.isSome(): + conf.nodekey.get() + else: + PrivateKey.random(Secp256k1, rng).tryGet() + + # set log level + if conf.logLevel != LogLevel.NONE: + setLogLevel(conf.logLevel) + + let (extIp, extTcpPort, extUdpPort) = setupNat( + conf.nat, + clientId, + Port(uint16(conf.tcpPort) + conf.portsShift), + Port(uint16(conf.udpPort) + conf.portsShift), + ).valueOr: + raise newException(ValueError, "setupNat error " & error) + + var enrBuilder = EnrBuilder.init(nodeKey) + + let record = enrBuilder.build().valueOr: + error "failed to create enr record", error = error + quit(QuitFailure) + + let node = block: + var builder = WakuNodeBuilder.init() + builder.withNodeKey(nodeKey) + builder.withRecord(record) + + builder + .withNetworkConfigurationDetails( + conf.listenAddress, + Port(uint16(conf.tcpPort) + conf.portsShift), + extIp, + extTcpPort, + ) + .tryGet() + builder.build().tryGet() + + (await node.mountRelay()).isOkOr: + error "failed to mount relay: ", error = error + quit(QuitFailure) + + await node.mountLibp2pPing() + + var kadBootstrapPeers: seq[(PeerId, seq[MultiAddress])] + for nodeStr in conf.kadBootstrapNodes: + let (peerId, ma) = block: + let r = parseFullAddress(nodeStr) + if r.isErr: + error "Failed to parse kademlia bootstrap node", node = nodeStr, error = r.error + continue + r.get() + kadBootstrapPeers.add((peerId, @[ma])) + + node.mountKademlia( + KademliaDiscoveryConf( + bootstrapNodes: kadBootstrapPeers, + randomLookupInterval: chronos.seconds(60), + serviceLookupInterval: chronos.seconds(60), + kadDhtConfig: KadDHTConfig.new(disableBootstrapping = true), + discoConfig: + sd_types.ServiceDiscoveryConfig.new(advertExpiry = chronos.seconds(60)), + clientMode: false, + xprPublishing: false, + ) + ).isOkOr: + error "failed to setup kademlia service discovery", error = error + quit(QuitFailure) + + let autonatService = getAutonatService(rng) + node.switch.services = @[Service(autonatService)] + for service in node.switch.services: + try: + service.setup(node.switch) + except ServiceSetupError as e: + error "failed to set up libp2p switch service", error = e.msg + + await node.start() + + node.peerManager.start() + + let nick = await readNick(transp) + let room = await readRoom(transp) + echo "Welcome, " & nick & "! Joined room '" & room & "'." + + var chat = Chat( + node: node, + transp: transp, + subscribed: false, + connected: false, + started: true, + nick: nick, + prompt: false, + currentPubsubTopic: "", + discoPeersListener: PeersDiscoveredEventListener(), + ) + + let listenerHandle = PeersDiscoveredEvent.listen( + proc(event: PeersDiscoveredEvent) {.async: (raises: []).} = + let peers = event.peers + if peers.len > 0: + try: + echo "discovered peers via kademlia: ", peers.mapIt($it.peerId) + await chat.node.connectToNodes(peers) + chat.connected = true + except CatchableError as e: + error "error connecting discovered peers", error = e.msg + ).valueOr: + error "failed to subscribe to PeersDiscoveredEvent", error = error + return + + chat.discoPeersListener = listenerHandle + + let peerInfo = node.switch.peerInfo + let listenStr = $peerInfo.addrs[0] & "/p2p/" & $peerInfo.peerId + echo &"Listening on\n {listenStr}" + + if conf.metricsLogging: + startMetricsLog() + + if conf.metricsServer: + let metricsServer = startMetricsServer( + conf.metricsServerAddress, Port(conf.metricsServerPort + conf.portsShift) + ) + + await chat.joinRoom(room) + + await chat.readWriteLoop() + + runForever() + +proc main(rng: crypto.Rng) {.async.} = + let (rfd, wfd) = createAsyncPipe() + if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe: + raise newException(ValueError, "Could not initialize pipe!") + + var thread: Thread[AsyncFD] + thread.createThread(readInput, wfd) + try: + await processInput(rfd, rng) + except ConfigurationError as e: + raise e + +when isMainModule: + let rng = crypto.newRng() + try: + waitFor(main(rng)) + except CatchableError as e: + raise e diff --git a/apps/chat2disco/config_chat2disco.nim b/apps/chat2disco/config_chat2disco.nim new file mode 100644 index 000000000..9f1a8290a --- /dev/null +++ b/apps/chat2disco/config_chat2disco.nim @@ -0,0 +1,103 @@ +import chronicles, chronos, std/strutils + +import + eth/keys, + libp2p/crypto/crypto, + libp2p/crypto/secp, + libp2p/multiaddress, + nimcrypto/utils, + confutils, + confutils/defs, + confutils/std/net + +type Chat2Conf* = object ## General node config + logLevel* {. + desc: "Sets the log level.", defaultValue: LogLevel.DEBUG, name: "log-level" + .}: LogLevel + + nodekey* {.desc: "P2P node private key as 64 char hex string.", name: "nodekey".}: + Option[crypto.PrivateKey] + + listenAddress* {. + defaultValue: defaultListenAddress(config), + desc: "Listening address for the LibP2P traffic.", + name: "listen-address" + .}: IpAddress + + tcpPort* {.desc: "TCP listening port.", defaultValue: 60000, name: "tcp-port".}: Port + + udpPort* {.desc: "UDP listening port.", defaultValue: 60000, name: "udp-port".}: Port + + portsShift* {. + desc: "Add a shift to all port numbers.", defaultValue: 0, name: "ports-shift" + .}: uint16 + + nat* {. + desc: + "Specify method to use for determining public address. " & + "Must be one of: any, none, upnp, pmp, extip:.", + defaultValue: "any" + .}: string + + ## Metrics config + metricsServer* {. + desc: "Enable the metrics server: true|false", + defaultValue: false, + name: "metrics-server" + .}: bool + + metricsServerAddress* {. + desc: "Listening address of the metrics server.", + defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]), + name: "metrics-server-address" + .}: IpAddress + + metricsServerPort* {. + desc: "Listening HTTP port of the metrics server.", + defaultValue: 8008, + name: "metrics-server-port" + .}: uint16 + + metricsLogging* {. + desc: "Enable metrics logging: true|false", + defaultValue: true, + name: "metrics-logging" + .}: bool + + ## Kademlia Discovery config (core for chat2disco) + kadBootstrapNodes* {. + desc: + "Optional peer multiaddr for kademlia bootstrap (must include /p2p/). Argument may be repeated.", + name: "kad-bootstrap-node" + .}: seq[string] + +proc parseCmdArg*(T: type crypto.PrivateKey, p: string): T = + try: + let key = SkPrivateKey.init(utils.fromHex(p)).tryGet() + result = crypto.PrivateKey(scheme: Secp256k1, skkey: key) + except CatchableError as e: + raise newException(ValueError, "Invalid private key") + +proc completeCmdArg*(T: type crypto.PrivateKey, val: string): seq[string] = + return @[] + +proc parseCmdArg*(T: type IpAddress, p: string): T = + try: + result = parseIpAddress(p) + except CatchableError as e: + raise newException(ValueError, "Invalid IP address") + +proc completeCmdArg*(T: type IpAddress, val: string): seq[string] = + return @[] + +proc parseCmdArg*(T: type Port, p: string): T = + try: + result = Port(parseInt(p)) + except CatchableError as e: + raise newException(ValueError, "Invalid Port number") + +proc completeCmdArg*(T: type Port, val: string): seq[string] = + return @[] + +func defaultListenAddress*(conf: Chat2Conf): IpAddress = + (static parseIpAddress("0.0.0.0")) diff --git a/apps/chat2disco/nim.cfg b/apps/chat2disco/nim.cfg new file mode 100644 index 000000000..2231f2ebe --- /dev/null +++ b/apps/chat2disco/nim.cfg @@ -0,0 +1,4 @@ +-d:chronicles_line_numbers +-d:chronicles_runtime_filtering:on +-d:discv5_protocol_id:d5waku +path = "../.." diff --git a/logos_delivery.nimble b/logos_delivery.nimble index 8c4d32cb5..c5d28bdd1 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -28,7 +28,8 @@ requires "nim >= 2.2.4", "toml_serialization", "faststreams", # Networking & P2P - "https://github.com/vacp2p/nim-libp2p.git#v2.0.0", + "https://github.com/vacp2p/nim-libp2p.git#v2.2.0", + "libbacktrace", # required by nim-libp2p >= 2.2.0 "eth", "nat_traversal", "dnsdisc", @@ -67,7 +68,7 @@ requires "https://github.com/logos-messaging/nim-sds.git#b12f5ee07c5b764303b51fb requires "https://github.com/NagyZoltanPeter/nim-brokers.git#v3.1.4" -requires "https://github.com/vacp2p/nim-lsquic.git#v0.5.1" +requires "https://github.com/vacp2p/nim-lsquic.git#v0.5.5" requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2" requires "https://github.com/logos-co/nim-libp2p-mix#380513117d556bf8f70066f5e72a7fd74fe36ba6" @@ -439,6 +440,17 @@ task chat2mix, "Build example Waku chat mix usage": "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE " # -d:ssl - cause unlisted exception error in libp2p/utility... +task chat2disco, "Build chat2disco (kademlia service discovery test chat)": + # This is a testing/discovery tool, so we want debug logs on stdout by default + # (the config sets the default to DEBUG, and runtime filtering is enabled). + # Unlike the pure chat examples, we do not redirect to a file sink. + + let name = "chat2disco" + buildBinary name, + "apps/chat2disco/", + "-d:chronicles_log_level=TRACE " + # -d:ssl - cause unlisted exception error in libp2p/utility... + task chat2bridge, "Build chat2bridge": let name = "chat2bridge" buildBinary name, "apps/chat2bridge/" diff --git a/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim b/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim index d2c369d46..e77729e0c 100644 --- a/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim +++ b/logos_delivery/messaging/delivery_service/recv_service/recv_service.nim @@ -4,7 +4,7 @@ import logos_delivery/waku/compat/option_valueor ## import std/[tables, sequtils, options, sets] -import chronos, chronicles, libp2p/utility +import chronos, chronicles, libp2p/utils/[shortlog, collections] import brokers/broker_context import logos_delivery/waku/[waku_core, waku_core/topics, waku_store/common], diff --git a/logos_delivery/messaging/delivery_service/send_service/send_service.nim b/logos_delivery/messaging/delivery_service/send_service/send_service.nim index 037ad879b..fec78f5aa 100644 --- a/logos_delivery/messaging/delivery_service/send_service/send_service.nim +++ b/logos_delivery/messaging/delivery_service/send_service/send_service.nim @@ -3,7 +3,7 @@ import logos_delivery/waku/compat/option_valueor ## import std/[sequtils, tables, options, typetraits] -import chronos, chronicles, libp2p/utility +import chronos, chronicles, libp2p/utils/[shortlog, collections] import brokers/broker_context import ./[send_processor, relay_processor, lightpush_processor, delivery_task], diff --git a/logos_delivery/waku/common/rate_limit/request_limiter.nim b/logos_delivery/waku/common/rate_limit/request_limiter.nim index 9fccf1c29..8d2f7f92c 100644 --- a/logos_delivery/waku/common/rate_limit/request_limiter.nim +++ b/logos_delivery/waku/common/rate_limit/request_limiter.nim @@ -21,7 +21,7 @@ import chronicles, chronos/timer, libp2p/stream/connection, - libp2p/utility + libp2p/utils/opt import std/times except TimeInterval, Duration, seconds, minutes diff --git a/logos_delivery/waku/common/rate_limit/single_token_limiter.nim b/logos_delivery/waku/common/rate_limit/single_token_limiter.nim index fc4b0acd5..b26ca3775 100644 --- a/logos_delivery/waku/common/rate_limit/single_token_limiter.nim +++ b/logos_delivery/waku/common/rate_limit/single_token_limiter.nim @@ -2,7 +2,7 @@ {.push raises: [].} -import std/[options], chronos/timer, libp2p/stream/connection, libp2p/utility +import std/[options], chronos/timer, libp2p/stream/connection import std/times except TimeInterval, Duration diff --git a/logos_delivery/waku/common/rate_limit/timed_map.nim b/logos_delivery/waku/common/rate_limit/timed_map.nim index b9a5c4cbf..5bab16de7 100644 --- a/logos_delivery/waku/common/rate_limit/timed_map.nim +++ b/logos_delivery/waku/common/rate_limit/timed_map.nim @@ -14,7 +14,6 @@ import std/[hashes, sets] import chronos/timer, results -import libp2p/utility export results diff --git a/logos_delivery/waku/discovery/waku_kademlia.nim b/logos_delivery/waku/discovery/waku_kademlia.nim index e6775530c..753dbf96b 100644 --- a/logos_delivery/waku/discovery/waku_kademlia.nim +++ b/logos_delivery/waku/discovery/waku_kademlia.nim @@ -55,14 +55,18 @@ proc extractMixPubKey*(service: ServiceInfo): Option[Curve25519Key] = if service.id != MixProtocolID: return none(Curve25519Key) - if service.data.len != Curve25519KeySize: - trace "invalid mix pub key length", - expected = Curve25519KeySize, - actual = service.data.len, - dataHex = byteutils.toHex(service.data) + # libp2p >= 2.2: ServiceInfo.data is Opt[seq[byte]] + let data = service.data.valueOr: return none(Curve25519Key) - let key = intoCurve25519Key(service.data) + if data.len != Curve25519KeySize: + trace "invalid mix pub key length", + expected = Curve25519KeySize, + actual = data.len, + dataHex = byteutils.toHex(data) + return none(Curve25519Key) + + let key = intoCurve25519Key(data) return some(key) diff --git a/logos_delivery/waku/factory/node_factory.nim b/logos_delivery/waku/factory/node_factory.nim index daa2e5fd0..e2e602806 100644 --- a/logos_delivery/waku/factory/node_factory.nim +++ b/logos_delivery/waku/factory/node_factory.nim @@ -3,6 +3,7 @@ import std/[options, sequtils], chronicles, chronos, + results, libp2p/peerid, libp2p/protocols/pubsub/gossipsub, libp2p/protocols/connectivity/relay/relay, @@ -175,8 +176,9 @@ proc setupProtocols( var kadConf = conf.kademliaDiscoveryConf.get() if conf.mixConf.isSome(): - let mixService = - ServiceInfo(id: MixProtocolID, data: @(conf.mixConf.get().mixPubKey)) + let mixService = ServiceInfo( + id: MixProtocolID, data: Opt.some(@(conf.mixConf.get().mixPubKey)) + ) kadConf.servicesToAdvertise.incl(mixService) kadConf.servicesToDiscover.incl(mixService.id) diff --git a/logos_delivery/waku/node/waku_node.nim b/logos_delivery/waku/node/waku_node.nim index db90ab47c..7d84e1813 100644 --- a/logos_delivery/waku/node/waku_node.nim +++ b/logos_delivery/waku/node/waku_node.nim @@ -22,7 +22,6 @@ import libp2p/transports/transport, libp2p/transports/tcptransport, libp2p/transports/wstransport, - libp2p/utility, libp2p/utils/offsettedseq, libp2p_mix, libp2p_mix/mix_protocol, @@ -151,6 +150,11 @@ type import ./subscription_manager +proc selectRandomPeers*(peers: seq[PeerId], numRandomPeers: int): seq[PeerId] = + var randomPeers = peers + shuffle(randomPeers) + return randomPeers[0 ..< min(len(randomPeers), numRandomPeers)] + proc deduceRelayShard( node: WakuNode, contentTopic: ContentTopic, @@ -337,6 +341,8 @@ proc mountMix*( return err(error.msg) return ok() +import logos_delivery/waku/factory/conf_builder/kademlia_discovery_conf_builder + proc mountKademlia*( node: WakuNode, config: KademliaDiscoveryConf ): Result[void, string] = @@ -416,11 +422,6 @@ proc reconnectRelayPeers*(node: WakuNode) {.async.} = node.wakuRelay.parameters.pruneBackoff + chronos.seconds(BackoffSlackTime) await node.peerManager.reconnectPeers(WakuRelayCodec, backoffPeriod) -proc selectRandomPeers*(peers: seq[PeerId], numRandomPeers: int): seq[PeerId] = - var randomPeers = peers - shuffle(randomPeers) - return randomPeers[0 ..< min(len(randomPeers), numRandomPeers)] - proc mountRendezvousClient*(node: WakuNode, clusterId: uint16) {.async: (raises: []).} = info "mounting rendezvous client" diff --git a/logos_delivery/waku/node/waku_node/filter.nim b/logos_delivery/waku/node/waku_node/filter.nim index 4b2ea8494..dd5d4265b 100644 --- a/logos_delivery/waku/node/waku_node/filter.nim +++ b/logos_delivery/waku/node/waku_node/filter.nim @@ -17,8 +17,7 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/builders, libp2p/transports/tcptransport, - libp2p/transports/wstransport, - libp2p/utility + libp2p/transports/wstransport import ../waku_node, diff --git a/logos_delivery/waku/node/waku_node/lightpush.nim b/logos_delivery/waku/node/waku_node/lightpush.nim index 56d39aae3..2f94cbf61 100644 --- a/logos_delivery/waku/node/waku_node/lightpush.nim +++ b/logos_delivery/waku/node/waku_node/lightpush.nim @@ -17,7 +17,6 @@ import libp2p/builders, libp2p/transports/tcptransport, libp2p/transports/wstransport, - libp2p/utility, libp2p_mix import diff --git a/logos_delivery/waku/node/waku_node/peer_exchange.nim b/logos_delivery/waku/node/waku_node/peer_exchange.nim index cd6e6146c..5e5cc272a 100644 --- a/logos_delivery/waku/node/waku_node/peer_exchange.nim +++ b/logos_delivery/waku/node/waku_node/peer_exchange.nim @@ -15,8 +15,7 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/builders, libp2p/transports/tcptransport, - libp2p/transports/wstransport, - libp2p/utility + libp2p/transports/wstransport import ../waku_node, diff --git a/logos_delivery/waku/node/waku_node/ping.nim b/logos_delivery/waku/node/waku_node/ping.nim index e7e011477..5eb1baeb9 100644 --- a/logos_delivery/waku/node/waku_node/ping.nim +++ b/logos_delivery/waku/node/waku_node/ping.nim @@ -9,8 +9,7 @@ import results, libp2p/protocols/ping, libp2p/builders, - libp2p/transports/tcptransport, - libp2p/utility + libp2p/transports/tcptransport import ../waku_node, ../peer_manager diff --git a/logos_delivery/waku/node/waku_node/relay.nim b/logos_delivery/waku/node/waku_node/relay.nim index 77143fabb..9048b9b03 100644 --- a/logos_delivery/waku/node/waku_node/relay.nim +++ b/logos_delivery/waku/node/waku_node/relay.nim @@ -18,7 +18,6 @@ import libp2p/builders, libp2p/transports/tcptransport, libp2p/transports/wstransport, - libp2p/utility, brokers/broker_context import diff --git a/logos_delivery/waku/node/waku_node/store.nim b/logos_delivery/waku/node/waku_node/store.nim index 189ef9052..3b517a59a 100644 --- a/logos_delivery/waku/node/waku_node/store.nim +++ b/logos_delivery/waku/node/waku_node/store.nim @@ -15,8 +15,7 @@ import libp2p/protocols/pubsub/rpc/messages, libp2p/builders, libp2p/transports/tcptransport, - libp2p/transports/wstransport, - libp2p/utility + libp2p/transports/wstransport import ../waku_node, diff --git a/logos_delivery/waku/waku_relay/message_id.nim b/logos_delivery/waku/waku_relay/message_id.nim index 655f50c5a..d4b624498 100644 --- a/logos_delivery/waku/waku_relay/message_id.nim +++ b/logos_delivery/waku/waku_relay/message_id.nim @@ -23,7 +23,10 @@ type MessageIdProvider* = pubsub.MsgIdProvider proc defaultMessageIdProvider*( message: messages.Message ): Result[MessageID, ValidationResult] = - let hash = sha256.digest(message.data) + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = message.data.valueOr: + return err(ValidationResult.Reject) + let hash = sha256.digest(data) ok(@(hash.data)) ## Waku message Unique ID provider diff --git a/logos_delivery/waku/waku_relay/protocol.nim b/logos_delivery/waku/waku_relay/protocol.nim index 409b02929..87720749d 100644 --- a/logos_delivery/waku/waku_relay/protocol.nim +++ b/logos_delivery/waku/waku_relay/protocol.nim @@ -268,7 +268,10 @@ proc initRelayObservers(w: WakuRelay) = let msg_id_short = shortLog(msg_id) - let wakuMessage = WakuMessage.decode(msg.data).valueOr: + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = msg.data.valueOr: + return err() + let wakuMessage = WakuMessage.decode(data).valueOr: warn "Error decoding to Waku Message", my_peer_id = w.switch.peerInfo.peerId, msg_id = msg_id_short, @@ -277,7 +280,7 @@ proc initRelayObservers(w: WakuRelay) = error = $error return err() - let msgSize = msg.data.len + msg.topic.len + let msgSize = data.len + msg.topic.len return ok((msg_id_short, msg.topic, wakuMessage, msgSize)) proc updateMetrics( @@ -304,11 +307,16 @@ proc initRelayObservers(w: WakuRelay) = var topicsChanged = false for graft in ctrl.graft: - w.topicHealthDirty.incl(graft.topicID) + # libp2p >= 2.2: topicID is Opt[string] + let topicId = graft.topicID.valueOr: + continue + w.topicHealthDirty.incl(topicId) topicsChanged = true for prune in ctrl.prune: - w.topicHealthDirty.incl(prune.topicID) + let topicId = prune.topicID.valueOr: + continue + w.topicHealthDirty.incl(topicId) topicsChanged = true if topicsChanged: @@ -323,7 +331,9 @@ proc initRelayObservers(w: WakuRelay) = proc onValidated(peer: PubSubPeer, msg: Message, msgId: MessageId) = let msg_id_short = shortLog(msgId) - let wakuMessage = WakuMessage.decode(msg.data).valueOr: + let data = msg.data.valueOr: + return + let wakuMessage = WakuMessage.decode(data).valueOr: warn "onValidated: failed decoding to Waku Message", my_peer_id = w.switch.peerInfo.peerId, msg_id = msg_id_short, @@ -540,7 +550,12 @@ proc generateOrderedValidator(w: WakuRelay): ValidatorHandler {.gcsafe.} = ): Future[ValidationResult] {.async.} = # can be optimized by checking if the message is a WakuMessage without allocating memory # see nim-libp2p protobuf library - let msg = WakuMessage.decode(message.data).valueOr: + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = message.data.valueOr: + error "protocol generateOrderedValidator reject missing data", + pubsubTopic = pubsubTopic + return ValidationResult.Reject + let msg = WakuMessage.decode(data).valueOr: error "protocol generateOrderedValidator reject decode error", pubsubTopic = pubsubTopic, error = $error return ValidationResult.Reject diff --git a/logos_delivery/waku/waku_rendezvous/protocol.nim b/logos_delivery/waku/waku_rendezvous/protocol.nim index 253fa5a12..cde02163f 100644 --- a/logos_delivery/waku/waku_rendezvous/protocol.nim +++ b/logos_delivery/waku/waku_rendezvous/protocol.nim @@ -11,8 +11,7 @@ import libp2p/protocols/rendezvous/protobuf, libp2p/utils/offsettedseq, libp2p/crypto/curve25519, - libp2p/switch, - libp2p/utility + libp2p/switch import metrics except collect @@ -60,8 +59,8 @@ proc advertise*( ).valueOr: return err("rendezvous advertisement failed: Failed to sign Waku Peer Record: " & $error) - let sprBuff = se.encode().valueOr: - return err("rendezvous advertisement failed: Wrong Signed Peer Record: " & $error) + # libp2p >= 2.2: SignedPayload.encode returns seq[byte] (no longer Result) + let sprBuff = se.encode() # rendezvous.advertise expects already opened connections # must dial first diff --git a/logos_delivery/waku/waku_store_sync/reconciliation.nim b/logos_delivery/waku/waku_store_sync/reconciliation.nim index 6f301d633..f091cccf0 100644 --- a/logos_delivery/waku/waku_store_sync/reconciliation.nim +++ b/logos_delivery/waku/waku_store_sync/reconciliation.nim @@ -8,7 +8,6 @@ import chronicles, chronos, metrics, - libp2p/utility, libp2p/protocols/protocol, libp2p/stream/connection, libp2p/crypto/crypto, diff --git a/logos_delivery/waku/waku_store_sync/transfer.nim b/logos_delivery/waku/waku_store_sync/transfer.nim index aa02adfdc..c53730d86 100644 --- a/logos_delivery/waku/waku_store_sync/transfer.nim +++ b/logos_delivery/waku/waku_store_sync/transfer.nim @@ -7,7 +7,6 @@ import chronicles, chronos, metrics, - libp2p/utility, libp2p/protocols/protocol, libp2p/stream/connection, libp2p/crypto/crypto, diff --git a/nimble.lock b/nimble.lock index 03bb7a434..66e4d119a 100644 --- a/nimble.lock +++ b/nimble.lock @@ -1,91 +1,28 @@ { "version": 2, "packages": { - "nim": { - "version": "2.2.4", - "vcsRevision": "911e0dbb1f76de61fa0215ab1bb85af5334cc9a8", - "url": "https://github.com/nim-lang/Nim.git", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "68bb85cbfb1832ce4db43943911b046c3af3caab" - } - }, - "unittest2": { - "version": "0.2.5", - "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", - "url": "https://github.com/status-im/nim-unittest2", + "boringssl": { + "version": "0.0.10", + "vcsRevision": "084f2c8994137a72655b72745936a05949c768cc", + "url": "https://github.com/vacp2p/nim-boringssl", "downloadMethod": "git", "dependencies": [ "nim" ], "checksums": { - "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + "sha1": "4ac575c18b11c0f06d6381279843aa8508f7940e" } }, - "bearssl": { - "version": "0.2.8", - "vcsRevision": "22c6a76ce015bc07e011562bdcfc51d9446c1e82", - "url": "https://github.com/status-im/nim-bearssl", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unittest2" - ], - "checksums": { - "sha1": "da4dd7ae96d536bdaf42dca9c85d7aed024b6a86" - } - }, - "bearssl_pkey_decoder": { - "version": "#21dd3710df9345ed2ad8bf8f882761e07863b8e0", - "vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0", - "url": "https://github.com/vacp2p/bearssl_pkey_decoder", - "downloadMethod": "git", - "dependencies": [ - "nim", - "bearssl" - ], - "checksums": { - "sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714" - } - }, - "jwt": { - "version": "#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", - "vcsRevision": "057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", - "url": "https://github.com/vacp2p/nim-jwt.git", - "downloadMethod": "git", - "dependencies": [ - "nim", - "bearssl", - "bearssl_pkey_decoder" - ], - "checksums": { - "sha1": "3cd368666fd2bc7f99f253452289e827abcac13c" - } - }, - "testutils": { - "version": "0.8.1", - "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", - "url": "https://github.com/status-im/nim-testutils", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unittest2" - ], - "checksums": { - "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" - } - }, - "db_connector": { - "version": "0.1.0", - "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", - "url": "https://github.com/nim-lang/db_connector", + "npeg": { + "version": "1.3.0", + "vcsRevision": "409f6796d0e880b3f0222c964d1da7de6e450811", + "url": "https://github.com/zevv/npeg", "downloadMethod": "git", "dependencies": [ "nim" ], "checksums": { - "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + "sha1": "64f15c85a059c889cb11c5fe72372677c50da621" } }, "results": { @@ -113,6 +50,96 @@ "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97" } }, + "unicodedb": { + "version": "0.13.2", + "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", + "url": "https://github.com/nitely/nim-unicodedb", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" + } + }, + "regex": { + "version": "0.26.3", + "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", + "url": "https://github.com/nitely/nim-regex", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unicodedb" + ], + "checksums": { + "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" + } + }, + "unittest2": { + "version": "0.2.5", + "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", + "url": "https://github.com/status-im/nim-unittest2", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + } + }, + "testutils": { + "version": "0.8.1", + "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", + "url": "https://github.com/status-im/nim-testutils", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unittest2" + ], + "checksums": { + "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" + } + }, + "bearssl": { + "version": "0.2.10", + "vcsRevision": "25197fec8a3988d1153be7a1a79890b70e69d6db", + "url": "https://github.com/status-im/nim-bearssl", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unittest2" + ], + "checksums": { + "sha1": "e53ab85b8c4c1ce3aa1f4eb251c0ab29e08d4972" + } + }, + "bearssl_pkey_decoder": { + "version": "#d34aa46bf9d0a3ffff810fbd3c4d2fa024eb9368", + "vcsRevision": "d34aa46bf9d0a3ffff810fbd3c4d2fa024eb9368", + "url": "https://github.com/vacp2p/bearssl_pkey_decoder", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl" + ], + "checksums": { + "sha1": "8666edbcb77cb9f97c659114d57c4ba0e7ab74c3" + } + }, + "jwt": { + "version": "#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", + "vcsRevision": "057ec95eb5af0eea9c49bfe9025b3312c95dc5f2", + "url": "https://github.com/vacp2p/nim-jwt.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "bearssl", + "bearssl_pkey_decoder" + ], + "checksums": { + "sha1": "3cd368666fd2bc7f99f253452289e827abcac13c" + } + }, "stew": { "version": "0.5.0", "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa", @@ -128,8 +155,8 @@ } }, "zlib": { - "version": "0.1.0", - "vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe", + "version": "0.2.0", + "vcsRevision": "190246aa0bb6569781370964fa2faa474203d6dd", "url": "https://github.com/status-im/nim-zlib", "downloadMethod": "git", "dependencies": [ @@ -138,12 +165,12 @@ "results" ], "checksums": { - "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f" + "sha1": "a8c0c569d82315f3ffc1249ab42b0404e84fddc3" } }, "httputils": { - "version": "0.4.1", - "vcsRevision": "f142cb2e8bd812dd002a6493b6082827bb248592", + "version": "0.4.3", + "vcsRevision": "a9ca86095e262251d8ebafa1c6504fd476c41e43", "url": "https://github.com/status-im/nim-http-utils", "downloadMethod": "git", "dependencies": [ @@ -153,7 +180,7 @@ "unittest2" ], "checksums": { - "sha1": "016774ab31c3afff9a423f7d80584905ee59c570" + "sha1": "dcdd6633bc35695c2b1ac0f0bd1c5ff4e3623cb5" } }, "chronos": { @@ -174,8 +201,8 @@ } }, "metrics": { - "version": "0.2.1", - "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2", + "version": "0.2.2", + "vcsRevision": "b4b70a88fe1755d281366cbc3f22d7515240d192", "url": "https://github.com/status-im/nim-metrics", "downloadMethod": "git", "dependencies": [ @@ -185,12 +212,12 @@ "stew" ], "checksums": { - "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2" + "sha1": "94e0785170b639d0046e7469408bb752197ead8e" } }, "faststreams": { - "version": "0.5.0", - "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e", + "version": "0.5.1", + "vcsRevision": "50889cd16ec8771106cdd0eeea460039e8571e06", "url": "https://github.com/status-im/nim-faststreams", "downloadMethod": "git", "dependencies": [ @@ -199,7 +226,7 @@ "unittest2" ], "checksums": { - "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383" + "sha1": "969ceb3666e807db8fe5c8df63466749822367a9" } }, "snappy": { @@ -208,19 +235,19 @@ "url": "https://github.com/status-im/nim-snappy", "downloadMethod": "git", "dependencies": [ - "nim", "faststreams", "unittest2", "results", - "stew" + "stew", + "testutils" ], "checksums": { "sha1": "e572d60d6a3178c5b1cde2400c51ad771812cd3d" } }, "serialization": { - "version": "0.5.2", - "vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b", + "version": "0.5.3", + "vcsRevision": "4092500cea76154576539371709ae801afbd2a9d", "url": "https://github.com/status-im/nim-serialization", "downloadMethod": "git", "dependencies": [ @@ -230,7 +257,24 @@ "stew" ], "checksums": { - "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8" + "sha1": "c087d26c50da40436599163888532660d6f9e631" + } + }, + "protobuf_serialization": { + "version": "0.5.3", + "vcsRevision": "8406e7287196661614ce6a8e8be20f755376af7f", + "url": "https://github.com/status-im/nim-protobuf-serialization", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "faststreams", + "serialization", + "npeg", + "unittest2" + ], + "checksums": { + "sha1": "3307412f9755f7ec2079e12cf036fc103aa130b0" } }, "toml_serialization": { @@ -254,7 +298,6 @@ "url": "https://github.com/status-im/nim-confutils", "downloadMethod": "git", "dependencies": [ - "nim", "stew", "serialization", "results" @@ -264,18 +307,19 @@ } }, "cbor_serialization": { - "version": "0.3.0", - "vcsRevision": "1664160e04d153573373afddc552b9cbf6fbe4dc", + "version": "0.4.1", + "vcsRevision": "e32d61c54f69b396f47645f9b7373ea2e149013a", "url": "https://github.com/vacp2p/nim-cbor-serialization", "downloadMethod": "git", "dependencies": [ - "nim", "serialization", "stew", - "results" + "npeg", + "results", + "unittest2" ], "checksums": { - "sha1": "ab126eae09a6e39c72972a6a0b83cb06a2ffe8f0" + "sha1": "4782b43d06594ac59730b3939110ba93de7d3156" } }, "json_serialization": { @@ -316,7 +360,6 @@ "url": "https://github.com/status-im/nim-presto", "downloadMethod": "git", "dependencies": [ - "nim", "chronos", "chronicles", "metrics", @@ -350,7 +393,6 @@ "url": "https://github.com/status-im/nim-stint", "downloadMethod": "git", "dependencies": [ - "nim", "stew", "unittest2" ], @@ -364,7 +406,6 @@ "url": "https://github.com/status-im/nim-minilru", "downloadMethod": "git", "dependencies": [ - "nim", "results", "unittest2" ], @@ -372,55 +413,6 @@ "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" } }, - "sqlite3_abi": { - "version": "3.53.0.0", - "vcsRevision": "8240e8e2819dfce1b67fa2733135d01b5cc80ae0", - "url": "https://github.com/arnetheduck/nim-sqlite3-abi", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "fb7a6e6f36fc4eb4dfa6634dbcbf5cd0dfd0ebf0" - } - }, - "dnsclient": { - "version": "0.3.4", - "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", - "url": "https://github.com/ba0f3/dnsclient.nim", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" - } - }, - "unicodedb": { - "version": "0.13.2", - "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", - "url": "https://github.com/nitely/nim-unicodedb", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" - } - }, - "regex": { - "version": "0.26.3", - "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", - "url": "https://github.com/nitely/nim-regex", - "downloadMethod": "git", - "dependencies": [ - "nim", - "unicodedb" - ], - "checksums": { - "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" - } - }, "nimcrypto": { "version": "0.6.4", "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", @@ -433,13 +425,31 @@ "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" } }, + "lsquic": { + "version": "#v0.5.5", + "vcsRevision": "079768948ac03d8f2384e03045a090cb88aeb7d9", + "url": "https://github.com/vacp2p/nim-lsquic.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "zlib", + "stew", + "chronos", + "nimcrypto", + "unittest2", + "chronicles", + "https://github.com/vacp2p/nim-boringssl" + ], + "checksums": { + "sha1": "6742442d2f0f4aa4880b98bb12965beef85e513c" + } + }, "websock": { "version": "0.4.0", "vcsRevision": "387a8eb7e961e8fdd3b1a717d36bc53b55e4dc5d", "url": "https://github.com/status-im/nim-websock", "downloadMethod": "git", "dependencies": [ - "nim", "chronos", "httputils", "chronicles", @@ -454,7 +464,7 @@ } }, "json_rpc": { - "version": "0.6.1", + "version": "#v0.6.1", "vcsRevision": "6f1fff8ba685c9192fab153a9d66484ad9066e78", "url": "https://github.com/status-im/nim-json-rpc.git", "downloadMethod": "git", @@ -475,25 +485,6 @@ "sha1": "596db0aafcb3c83f5dba6d42993f2276e0d00eb5" } }, - "lsquic": { - "version": "0.5.1", - "vcsRevision": "2f01046bf1d513de8b5f8296c3d8bec819ab0cb9", - "url": "https://github.com/vacp2p/nim-lsquic", - "downloadMethod": "git", - "dependencies": [ - "nim", - "zlib", - "stew", - "chronos", - "nimcrypto", - "unittest2", - "chronicles", - "boringssl" - ], - "checksums": { - "sha1": "959df1a9ac2a574d6fe30a4faf37c37b443e1cfb" - } - }, "secp256k1": { "version": "0.6.0.3.2", "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", @@ -509,13 +500,32 @@ "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" } }, + "db_connector": { + "version": "0.1.0", + "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", + "url": "https://github.com/nim-lang/db_connector", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + } + }, + "sqlite3_abi": { + "version": "3.53.3.0", + "vcsRevision": "a4d052efe81472b8d5271ece8e5a3caea90a45eb", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "20571756875f29ef292acf10a71c27139e6bf44e" + } + }, "eth": { "version": "0.9.0", "vcsRevision": "d9135e6c3c5d6d819afdfb566aa8d958756b73a8", "url": "https://github.com/status-im/nim-eth", "downloadMethod": "git", "dependencies": [ - "nim", "nimcrypto", "stint", "secp256k1", @@ -542,7 +552,6 @@ "url": "https://github.com/status-im/nim-web3", "downloadMethod": "git", "dependencies": [ - "nim", "chronicles", "chronos", "bearssl", @@ -561,12 +570,11 @@ } }, "dnsdisc": { - "version": "0.1.0", - "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e", + "version": "0.1.1", + "vcsRevision": "6cb1b7e3922645275043c68e476cac1501a45e55", "url": "https://github.com/status-im/nim-dnsdisc", "downloadMethod": "git", "dependencies": [ - "nim", "bearssl", "chronicles", "chronos", @@ -579,20 +587,42 @@ "results" ], "checksums": { - "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154" + "sha1": "6451cab35990f334a46927f49f9176579460934d" + } + }, + "dnsclient": { + "version": "0.3.4", + "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", + "url": "https://github.com/ba0f3/dnsclient.nim", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" + } + }, + "libbacktrace": { + "version": "0.2.0", + "vcsRevision": "af60500cb0383231065f1d6624beeeefa308d085", + "url": "https://github.com/status-im/nim-libbacktrace", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "03deb5b6d5eb047f468b9c31ea1db640f8096bfd" } }, "libp2p": { - "version": "2.0.0", - "vcsRevision": "c43199378f46d0aaf61be1cad1ee1d63e8f665d6", + "version": "#v2.2.0", + "vcsRevision": "0e763288c10b6488e5971608f12aa49ab129720e", "url": "https://github.com/vacp2p/nim-libp2p.git", "downloadMethod": "git", "dependencies": [ "nim", + "libbacktrace", "nimcrypto", - "dnsclient", "bearssl", - "boringssl", + "https://github.com/vacp2p/nim-boringssl", "chronicles", "chronos", "metrics", @@ -604,10 +634,30 @@ "lsquic", "protobuf_serialization", "websock", - "jwt" + "nat_traversal" ], "checksums": { - "sha1": "327dc7a0cb7e9d0be3d6083841bd496c4cbc48dc" + "sha1": "48155b6ee4101ff058f93985a09d9480b2073aca" + } + }, + "libp2p_mix": { + "version": "#380513117d556bf8f70066f5e72a7fd74fe36ba6", + "vcsRevision": "380513117d556bf8f70066f5e72a7fd74fe36ba6", + "url": "https://github.com/logos-co/nim-libp2p-mix", + "downloadMethod": "git", + "dependencies": [ + "nim", + "libp2p", + "chronicles", + "chronos", + "metrics", + "nimcrypto", + "stew", + "results", + "unittest2" + ], + "checksums": { + "sha1": "ccfb0f0160ac15ac970471964c730d57edacad91" } }, "taskpools": { @@ -615,13 +665,26 @@ "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", "url": "https://github.com/status-im/nim-taskpools", "downloadMethod": "git", - "dependencies": [ - "nim" - ], + "dependencies": [], "checksums": { "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" } }, + "ffi": { + "version": "#v0.1.3", + "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", + "url": "https://github.com/logos-messaging/nim-ffi", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronos", + "chronicles", + "taskpools" + ], + "checksums": { + "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" + } + }, "sds": { "version": "#b12f5ee07c5b764303b51fb948b32a4ade1de3b5", "vcsRevision": "b12f5ee07c5b764303b51fb948b32a4ade1de3b5", @@ -641,83 +704,7 @@ "checksums": { "sha1": "175f65038b9877cdf974b07c5f83081f810d5fbe" } - }, - "ffi": { - "version": "0.1.3", - "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", - "url": "https://github.com/logos-messaging/nim-ffi", - "downloadMethod": "git", - "dependencies": [ - "nim", - "chronos", - "chronicles", - "taskpools" - ], - "checksums": { - "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" - } - }, - "boringssl": { - "version": "0.0.8", - "vcsRevision": "e77caabae78fbc9aa5b78a0a521181b077c82571", - "url": "https://github.com/vacp2p/nim-boringssl", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "2f603bb6d70683393bbb091bf6bd325d9c52be9f" - } - }, - "protobuf_serialization": { - "version": "0.4.0", - "vcsRevision": "38d24eb3bd93e605fb88199da71d36b1ec0ad60d", - "url": "https://github.com/status-im/nim-protobuf-serialization", - "downloadMethod": "git", - "dependencies": [ - "nim", - "stew", - "faststreams", - "serialization", - "npeg", - "unittest2" - ], - "checksums": { - "sha1": "5a7a80fb8cca29e41899ce9540b74e49c874f8fd" - } - }, - "npeg": { - "version": "1.3.0", - "vcsRevision": "409f6796d0e880b3f0222c964d1da7de6e450811", - "url": "https://github.com/zevv/npeg", - "downloadMethod": "git", - "dependencies": [ - "nim" - ], - "checksums": { - "sha1": "64f15c85a059c889cb11c5fe72372677c50da621" - } - }, - "libp2p_mix": { - "version": "0.1.0", - "vcsRevision": "380513117d556bf8f70066f5e72a7fd74fe36ba6", - "url": "https://github.com/logos-co/nim-libp2p-mix", - "downloadMethod": "git", - "dependencies": [ - "nim", - "libp2p", - "chronicles", - "chronos", - "metrics", - "nimcrypto", - "stew", - "results", - "unittest2" - ], - "checksums": { - "sha1": "ccfb0f0160ac15ac970471964c730d57edacad91" - } } }, "tasks": {} -} \ No newline at end of file +} diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index b028885e2..d34acf1bf 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -24,7 +24,12 @@ output_filename=$3 # --- Prefer the prebuilt release asset -------------------------------------- # Host target triple, e.g. x86_64-unknown-linux-gnu / aarch64-apple-darwin. -host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}') +# Prefer rustc's dedicated flag; fall back to parsing -vV without requiring awk. +host_triplet=$(rustc --print host-tuple 2>/dev/null || true) +if [[ -z "${host_triplet}" ]]; then + host_triplet=$(rustc -vV | sed -n 's/^host: //p') +fi +[[ -n "${host_triplet}" ]] || { echo "Could not determine rustc host triple"; exit 1; } tarball="${host_triplet}-stateless-rln.tar.gz" url="https://github.com/vacp2p/zerokit/releases/download/${rln_version}/${tarball}" @@ -45,12 +50,10 @@ echo "No prebuilt asset for ${host_triplet} at ${rln_version}; building from sou # Check if submodule version = version in Makefile cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" -detected_OS=$(uname -s) -if [[ "$detected_OS" == MINGW* || "$detected_OS" == MSYS* ]]; then - submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" | sed -n 's/.*"name":"rln","version":"\([^"]*\)".*/\1/p') -else - submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" | jq -r '.packages[] | select(.name == "rln") | .version') -fi +# Portable: sed works without jq (some sandboxes lack jq/curl/awk). +submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" \ + | sed -n 's/.*"name":"rln","version":"\([^"]*\)".*/\1/p' | head -1) + if [[ "v${submodule_version}" != "${rln_version}" ]]; then echo "Submodule version (v${submodule_version}) does not match version in Makefile (${rln_version})" diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh index dba2d6612..c174b39d2 100755 --- a/scripts/install_nimble.sh +++ b/scripts/install_nimble.sh @@ -37,7 +37,11 @@ PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." mkdir -p "${HOME}/.nimble/bin" - ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + if command -v ln >/dev/null 2>&1 && ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" 2>/dev/null; then + : + else + cp -f "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + fi exit 0 fi diff --git a/tools/apply-libp2p-2.2-shims.sh b/tools/apply-libp2p-2.2-shims.sh new file mode 100755 index 000000000..8f20a5549 --- /dev/null +++ b/tools/apply-libp2p-2.2-shims.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Hack for libp2p 2.2 + older libp2p_mix: restore removed modules as shims. +set -euo pipefail +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +LP=$(ls -dt "$ROOT"/nimbledeps/pkgs2/libp2p-2.2.* 2>/dev/null | head -1) +if [ -z "${LP:-}" ] || [ ! -d "$LP/libp2p" ]; then + echo "libp2p 2.2 package not found under nimbledeps/pkgs2" >&2 + exit 1 +fi +cat > "$LP/libp2p/utility.nim" <<'NIM' +# Shim: libp2p/utility removed in nim-libp2p 2.x; needed by current libp2p_mix pin. +import ./utils/[opt, shortlog, collections] +export opt, shortlog, collections +NIM +cat > "$LP/libp2p/utils/sequninit.nim" <<'NIM' +# Shim: libp2p/utils/sequninit removed; newSeqUninit lives in system. +export system +NIM +echo "Applied libp2p 2.2 shims in $LP" diff --git a/tools/install_from_lock.nim b/tools/install_from_lock.nim new file mode 100644 index 000000000..bec9609d5 --- /dev/null +++ b/tools/install_from_lock.nim @@ -0,0 +1,175 @@ +## Install nimble.lock packages into nimbledeps/pkgs2 and write nimble.paths. +## +## Used when `nimble setup --localdeps` fails (e.g. SAT cannot resolve libp2p +## git pins). Reproducible: clones exact vcsRevision from the lockfile. +## +## Usage (from repo root): +## nim c -r --hints:off tools/install_from_lock.nim +## +## Optional env: +## INSTALL_FROM_LOCK_FORCE=1 re-clone packages even if present + +import std/[json, os, osproc, strutils, algorithm] + +const + PkgsRel = "nimbledeps" / "pkgs2" + LockRel = "nimble.lock" + PathsRel = "nimble.paths" + +proc run(cmd: string): int = + execCmd(cmd) + +proc runOut(cmd: string): tuple[output: string, exitCode: int] = + execCmdEx(cmd) + +proc gitClone(url, dest, rev: string): bool = + if dirExists(dest): + removeDir(dest) + # Shallow clone of a specific commit is awkward; full clone then checkout. + var c = run("git clone --quiet " & quoteShell(url) & " " & quoteShell(dest)) + if c != 0: + let url2 = url.replace(".git", "") + c = run("git clone --quiet " & quoteShell(url2) & " " & quoteShell(dest)) + if c != 0: + return false + if run("git -C " & quoteShell(dest) & " checkout -q " & quoteShell(rev)) != 0: + discard run("git -C " & quoteShell(dest) & " fetch --quiet origin " & quoteShell(rev)) + if run("git -C " & quoteShell(dest) & " checkout -q " & quoteShell(rev)) != 0: + return false + # Submodules (bearssl, boringssl, secp256k1, lsquic, zlib, ...) + discard run( + "git -C " & quoteShell(dest) & " submodule update --init --recursive --quiet" + ) + true + +proc packageVersion(pkgDir: string, fallback: string): string = + result = fallback + for f in walkFiles(pkgDir / "*.nimble"): + for line in readFile(f).splitLines: + let t = line.strip + if t.startsWith("version"): + let parts = t.split('=') + if parts.len >= 2: + return parts[1].strip().strip(chars = {'"', '\''}) + break + +proc writeMeta(pkgDir, url, rev, sha1, ver: string) = + let meta = %*{ + "url": url, + "vcsRevision": rev, + "downloadMethod": "git", + "versions": [ver], + "checksums": {"sha1": sha1} + } + writeFile(pkgDir / "nimblemeta.json", meta.pretty) + +proc installPkg(root, name: string, entry: JsonNode, force: bool): string = + ## Returns final package directory path. + let version = entry["version"].getStr + let rev = entry["vcsRevision"].getStr + let url = entry["url"].getStr + let sha1 = entry["checksums"]["sha1"].getStr + let pkgsDir = root / PkgsRel + createDir(pkgsDir) + + # Prefer existing dir matching name-*-sha1 + var existing = "" + for kind, path in walkDir(pkgsDir): + if kind != pcDir: + continue + let base = path.extractFilename + if base.startsWith(name & "-") and base.endsWith("-" & sha1): + existing = path + break + + if existing.len > 0 and not force and fileExists(existing / "nimblemeta.json"): + # Ensure submodules present (best-effort) + if dirExists(existing / ".git"): + discard run( + "git -C " & quoteShell(existing) & + " submodule update --init --recursive --quiet" + ) + echo " skip ", existing.extractFilename + return existing + + echo " install ", name, " @ ", version, " (", rev[0 .. min(11, rev.high)], ")" + let tmpName = name & "-tmp-" & sha1 + let tmpDest = pkgsDir / tmpName + if not gitClone(url, tmpDest, rev): + echo " FAILED ", name + quit(1) + + var pkgVer = packageVersion(tmpDest, version.replace("#", "")) + if pkgVer.len == 0: + pkgVer = version.replace("#", "") + let finalName = name & "-" & pkgVer & "-" & sha1 + let finalDest = pkgsDir / finalName + if finalDest != tmpDest: + if dirExists(finalDest): + removeDir(finalDest) + moveDir(tmpDest, finalDest) + writeMeta(finalDest, url, rev, sha1, pkgVer) + echo " -> ", finalName + finalDest + +proc writePaths(root: string, dirs: seq[string]) = + var lines = @["--noNimblePath"] + var sorted = dirs + sorted.sort() + for d in sorted: + lines.add("--path:\"" & d & "\"") + if dirExists(d / "src"): + lines.add("--path:\"" & d / "src" & "\"") + writeFile(root / PathsRel, lines.join("\n") & "\n") + echo "Wrote ", root / PathsRel, " (", sorted.len, " packages)" + +proc applyLibp2pShims(root: string) = + ## libp2p_mix still imports removed libp2p modules; restore as thin shims. + var lp = "" + let pkgs = root / PkgsRel + for kind, path in walkDir(pkgs): + if kind != pcDir: + continue + let base = path.extractFilename + if base.startsWith("libp2p-2.2") and dirExists(path / "libp2p"): + lp = path + break + if lp.len == 0: + echo " (no libp2p 2.2 package; skip shims)" + return + writeFile( + lp / "libp2p" / "utility.nim", + """ +# Shim: libp2p/utility removed in nim-libp2p 2.x; needed by current libp2p_mix pin. +import ./utils/[opt, shortlog, collections] +export opt, shortlog, collections +""", + ) + createDir(lp / "libp2p" / "utils") + writeFile( + lp / "libp2p" / "utils" / "sequninit.nim", + """ +# Shim: libp2p/utils/sequninit removed; newSeqUninit lives in system. +export system +""", + ) + echo " applied libp2p 2.2 shims in ", lp.extractFilename + +proc main() = + let root = getCurrentDir() + let lockPath = root / LockRel + if not fileExists(lockPath): + echo "error: ", lockPath, " not found (run from repo root)" + quit(1) + + let force = getEnv("INSTALL_FROM_LOCK_FORCE") == "1" + let lock = parseFile(lockPath) + echo "Installing packages from ", LockRel, " ..." + var dirs: seq[string] + for name, entry in lock["packages"].pairs: + dirs.add(installPkg(root, name, entry, force)) + writePaths(root, dirs) + applyLibp2pShims(root) + echo "Done." + +main()