From 861d3f907118130b1517fb60056c69e3faf0f02e Mon Sep 17 00:00:00 2001 From: Simon-Pierre Date: Tue, 14 Jul 2026 08:35:37 -0400 Subject: [PATCH] libp2p 2.2.0 --- Makefile | 29 ++- apps/chat2disco/chat2disco.nim | 4 +- logos_delivery.nimble | 3 +- .../waku/discovery/waku_kademlia.nim | 16 +- logos_delivery/waku/factory/node_factory.nim | 6 +- logos_delivery/waku/waku_relay/message_id.nim | 5 +- logos_delivery/waku/waku_relay/protocol.nim | 27 ++- .../waku/waku_rendezvous/protocol.nim | 4 +- nimble.lock | 21 ++- scripts/build_rln.sh | 17 +- scripts/install_nimble.sh | 6 +- tools/apply-libp2p-2.2-shims.sh | 19 ++ tools/install_from_lock.nim | 175 ++++++++++++++++++ 13 files changed, 295 insertions(+), 37 deletions(-) create mode 100755 tools/apply-libp2p-2.2-shims.sh create mode 100644 tools/install_from_lock.nim diff --git a/Makefile b/Makefile index 60c26751b..c357bdb78 100644 --- a/Makefile +++ b/Makefile @@ -75,11 +75,25 @@ endif %: @true -logos_delivery.nims: - ln -s logos_delivery.nimble $@ +# Prefer a symlink; fall back to a full copy when `ln` is unavailable (some sandboxes). +logos_delivery.nims: logos_delivery.nimble + @if command -v ln >/dev/null 2>&1 && ln -sfn logos_delivery.nimble $@ 2>/dev/null; then \ + : ; \ + else \ + cp -f logos_delivery.nimble $@; \ + fi -$(NIMBLEDEPS_STAMP): nimble.lock | install-nimble build-nph logos_delivery.nims - $(NIMBLE) setup --localdeps +# Install locked deps. `nimble setup` is preferred when it works; on SAT failures +# (seen with libp2p git tags) fall back to tools/install_from_lock.nim which +# clones exact lock revisions, writes nimble.paths, and applies libp2p 2.2 shims. +$(NIMBLEDEPS_STAMP): nimble.lock tools/install_from_lock.nim tools/apply-libp2p-2.2-shims.sh | install-nimble build-nph logos_delivery.nims + @if $(NIMBLE) setup --localdeps; then \ + echo "nimble setup ok"; \ + bash tools/apply-libp2p-2.2-shims.sh || true; \ + else \ + echo "nimble setup failed; installing from nimble.lock via tools/install_from_lock.nim"; \ + nim c -r --hints:off --mm:refc -d:release -o:nimbledeps/install_from_lock tools/install_from_lock.nim; \ + fi touch $@ # Must be phony so the recipe always runs and the sub-make re-evaluates @@ -256,9 +270,14 @@ chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(NIMBLE) chat2mix +# Compile with `nim c` (not `nimble chat2disco`) so a failed nimble SAT resolve +# after deps are already installed cannot block the build. Paths come from +# nimble.paths written by nimble setup or tools/install_from_lock.nim. chat2disco: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(NIMBLE) chat2disco + nim c --out:build/chat2disco --mm:refc --path:. \ + $(NIM_PARAMS) -d:chronicles_log_level=TRACE \ + apps/chat2disco/chat2disco.nim rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ diff --git a/apps/chat2disco/chat2disco.nim b/apps/chat2disco/chat2disco.nim index 81c1b34a7..72b6f100d 100644 --- a/apps/chat2disco/chat2disco.nim +++ b/apps/chat2disco/chat2disco.nim @@ -203,7 +203,7 @@ proc joinRoom(c: Chat, roomName: string) {.async.} = if not c.node.wakuKademlia.isNil(): c.node.wakuKademlia.removeServiceToDiscover(c.currentPubsubTopic) await c.node.wakuKademlia.removeServiceToAdvertise( - ServiceInfo(id: c.currentPubsubTopic, data: @[]) + ServiceInfo(id: c.currentPubsubTopic, data: Opt.none(seq[byte])) ) c.currentPubsubTopic = roomName @@ -220,7 +220,7 @@ proc joinRoom(c: Chat, roomName: string) {.async.} = echo "subscribed to pubsub topic: ", roomName if not c.node.wakuKademlia.isNil(): - let svcInfo = ServiceInfo(id: roomName, data: @[]) + let svcInfo = ServiceInfo(id: roomName, data: Opt.none(seq[byte])) c.node.wakuKademlia.addServiceToDiscover(roomName) c.node.wakuKademlia.addServiceToAdvertise(svcInfo) echo "advertising and discovering kademlia service: ", roomName diff --git a/logos_delivery.nimble b/logos_delivery.nimble index ee8edd232..c5d28bdd1 100644 --- a/logos_delivery.nimble +++ b/logos_delivery.nimble @@ -28,7 +28,8 @@ requires "nim >= 2.2.4", "toml_serialization", "faststreams", # Networking & P2P - "https://github.com/vacp2p/nim-libp2p.git#v2.1.4", + "https://github.com/vacp2p/nim-libp2p.git#v2.2.0", + "libbacktrace", # required by nim-libp2p >= 2.2.0 "eth", "nat_traversal", "dnsdisc", diff --git a/logos_delivery/waku/discovery/waku_kademlia.nim b/logos_delivery/waku/discovery/waku_kademlia.nim index e6775530c..753dbf96b 100644 --- a/logos_delivery/waku/discovery/waku_kademlia.nim +++ b/logos_delivery/waku/discovery/waku_kademlia.nim @@ -55,14 +55,18 @@ proc extractMixPubKey*(service: ServiceInfo): Option[Curve25519Key] = if service.id != MixProtocolID: return none(Curve25519Key) - if service.data.len != Curve25519KeySize: - trace "invalid mix pub key length", - expected = Curve25519KeySize, - actual = service.data.len, - dataHex = byteutils.toHex(service.data) + # libp2p >= 2.2: ServiceInfo.data is Opt[seq[byte]] + let data = service.data.valueOr: return none(Curve25519Key) - let key = intoCurve25519Key(service.data) + if data.len != Curve25519KeySize: + trace "invalid mix pub key length", + expected = Curve25519KeySize, + actual = data.len, + dataHex = byteutils.toHex(data) + return none(Curve25519Key) + + let key = intoCurve25519Key(data) return some(key) diff --git a/logos_delivery/waku/factory/node_factory.nim b/logos_delivery/waku/factory/node_factory.nim index daa2e5fd0..e2e602806 100644 --- a/logos_delivery/waku/factory/node_factory.nim +++ b/logos_delivery/waku/factory/node_factory.nim @@ -3,6 +3,7 @@ import std/[options, sequtils], chronicles, chronos, + results, libp2p/peerid, libp2p/protocols/pubsub/gossipsub, libp2p/protocols/connectivity/relay/relay, @@ -175,8 +176,9 @@ proc setupProtocols( var kadConf = conf.kademliaDiscoveryConf.get() if conf.mixConf.isSome(): - let mixService = - ServiceInfo(id: MixProtocolID, data: @(conf.mixConf.get().mixPubKey)) + let mixService = ServiceInfo( + id: MixProtocolID, data: Opt.some(@(conf.mixConf.get().mixPubKey)) + ) kadConf.servicesToAdvertise.incl(mixService) kadConf.servicesToDiscover.incl(mixService.id) diff --git a/logos_delivery/waku/waku_relay/message_id.nim b/logos_delivery/waku/waku_relay/message_id.nim index 655f50c5a..d4b624498 100644 --- a/logos_delivery/waku/waku_relay/message_id.nim +++ b/logos_delivery/waku/waku_relay/message_id.nim @@ -23,7 +23,10 @@ type MessageIdProvider* = pubsub.MsgIdProvider proc defaultMessageIdProvider*( message: messages.Message ): Result[MessageID, ValidationResult] = - let hash = sha256.digest(message.data) + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = message.data.valueOr: + return err(ValidationResult.Reject) + let hash = sha256.digest(data) ok(@(hash.data)) ## Waku message Unique ID provider diff --git a/logos_delivery/waku/waku_relay/protocol.nim b/logos_delivery/waku/waku_relay/protocol.nim index 409b02929..87720749d 100644 --- a/logos_delivery/waku/waku_relay/protocol.nim +++ b/logos_delivery/waku/waku_relay/protocol.nim @@ -268,7 +268,10 @@ proc initRelayObservers(w: WakuRelay) = let msg_id_short = shortLog(msg_id) - let wakuMessage = WakuMessage.decode(msg.data).valueOr: + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = msg.data.valueOr: + return err() + let wakuMessage = WakuMessage.decode(data).valueOr: warn "Error decoding to Waku Message", my_peer_id = w.switch.peerInfo.peerId, msg_id = msg_id_short, @@ -277,7 +280,7 @@ proc initRelayObservers(w: WakuRelay) = error = $error return err() - let msgSize = msg.data.len + msg.topic.len + let msgSize = data.len + msg.topic.len return ok((msg_id_short, msg.topic, wakuMessage, msgSize)) proc updateMetrics( @@ -304,11 +307,16 @@ proc initRelayObservers(w: WakuRelay) = var topicsChanged = false for graft in ctrl.graft: - w.topicHealthDirty.incl(graft.topicID) + # libp2p >= 2.2: topicID is Opt[string] + let topicId = graft.topicID.valueOr: + continue + w.topicHealthDirty.incl(topicId) topicsChanged = true for prune in ctrl.prune: - w.topicHealthDirty.incl(prune.topicID) + let topicId = prune.topicID.valueOr: + continue + w.topicHealthDirty.incl(topicId) topicsChanged = true if topicsChanged: @@ -323,7 +331,9 @@ proc initRelayObservers(w: WakuRelay) = proc onValidated(peer: PubSubPeer, msg: Message, msgId: MessageId) = let msg_id_short = shortLog(msgId) - let wakuMessage = WakuMessage.decode(msg.data).valueOr: + let data = msg.data.valueOr: + return + let wakuMessage = WakuMessage.decode(data).valueOr: warn "onValidated: failed decoding to Waku Message", my_peer_id = w.switch.peerInfo.peerId, msg_id = msg_id_short, @@ -540,7 +550,12 @@ proc generateOrderedValidator(w: WakuRelay): ValidatorHandler {.gcsafe.} = ): Future[ValidationResult] {.async.} = # can be optimized by checking if the message is a WakuMessage without allocating memory # see nim-libp2p protobuf library - let msg = WakuMessage.decode(message.data).valueOr: + # libp2p >= 2.2: Message.data is Opt[seq[byte]] + let data = message.data.valueOr: + error "protocol generateOrderedValidator reject missing data", + pubsubTopic = pubsubTopic + return ValidationResult.Reject + let msg = WakuMessage.decode(data).valueOr: error "protocol generateOrderedValidator reject decode error", pubsubTopic = pubsubTopic, error = $error return ValidationResult.Reject diff --git a/logos_delivery/waku/waku_rendezvous/protocol.nim b/logos_delivery/waku/waku_rendezvous/protocol.nim index 6b06820e7..cde02163f 100644 --- a/logos_delivery/waku/waku_rendezvous/protocol.nim +++ b/logos_delivery/waku/waku_rendezvous/protocol.nim @@ -59,8 +59,8 @@ proc advertise*( ).valueOr: return err("rendezvous advertisement failed: Failed to sign Waku Peer Record: " & $error) - let sprBuff = se.encode().valueOr: - return err("rendezvous advertisement failed: Wrong Signed Peer Record: " & $error) + # libp2p >= 2.2: SignedPayload.encode returns seq[byte] (no longer Result) + let sprBuff = se.encode() # rendezvous.advertise expects already opened connections # must dial first diff --git a/nimble.lock b/nimble.lock index 03c0d0064..66e4d119a 100644 --- a/nimble.lock +++ b/nimble.lock @@ -600,13 +600,26 @@ "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" } }, + "libbacktrace": { + "version": "0.2.0", + "vcsRevision": "af60500cb0383231065f1d6624beeeefa308d085", + "url": "https://github.com/status-im/nim-libbacktrace", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "03deb5b6d5eb047f468b9c31ea1db640f8096bfd" + } + }, "libp2p": { - "version": "#v2.1.4", - "vcsRevision": "3b5ae1da95f0614af06221be7a3bb2aeab03f4c7", + "version": "#v2.2.0", + "vcsRevision": "0e763288c10b6488e5971608f12aa49ab129720e", "url": "https://github.com/vacp2p/nim-libp2p.git", "downloadMethod": "git", "dependencies": [ "nim", + "libbacktrace", "nimcrypto", "bearssl", "https://github.com/vacp2p/nim-boringssl", @@ -624,7 +637,7 @@ "nat_traversal" ], "checksums": { - "sha1": "48da49700427bf68f1bbece13f8f9b620d0abdcb" + "sha1": "48155b6ee4101ff058f93985a09d9480b2073aca" } }, "libp2p_mix": { @@ -674,7 +687,7 @@ }, "sds": { "version": "#b12f5ee07c5b764303b51fb948b32a4ade1de3b5", - "vcsRevision": "2ad4432f4eaf5d85771412f8c527eb767ba44ea8", + "vcsRevision": "b12f5ee07c5b764303b51fb948b32a4ade1de3b5", "url": "https://github.com/logos-messaging/nim-sds.git", "downloadMethod": "git", "dependencies": [ diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index b028885e2..d34acf1bf 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -24,7 +24,12 @@ output_filename=$3 # --- Prefer the prebuilt release asset -------------------------------------- # Host target triple, e.g. x86_64-unknown-linux-gnu / aarch64-apple-darwin. -host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}') +# Prefer rustc's dedicated flag; fall back to parsing -vV without requiring awk. +host_triplet=$(rustc --print host-tuple 2>/dev/null || true) +if [[ -z "${host_triplet}" ]]; then + host_triplet=$(rustc -vV | sed -n 's/^host: //p') +fi +[[ -n "${host_triplet}" ]] || { echo "Could not determine rustc host triple"; exit 1; } tarball="${host_triplet}-stateless-rln.tar.gz" url="https://github.com/vacp2p/zerokit/releases/download/${rln_version}/${tarball}" @@ -45,12 +50,10 @@ echo "No prebuilt asset for ${host_triplet} at ${rln_version}; building from sou # Check if submodule version = version in Makefile cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" -detected_OS=$(uname -s) -if [[ "$detected_OS" == MINGW* || "$detected_OS" == MSYS* ]]; then - submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" | sed -n 's/.*"name":"rln","version":"\([^"]*\)".*/\1/p') -else - submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" | jq -r '.packages[] | select(.name == "rln") | .version') -fi +# Portable: sed works without jq (some sandboxes lack jq/curl/awk). +submodule_version=$(cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" \ + | sed -n 's/.*"name":"rln","version":"\([^"]*\)".*/\1/p' | head -1) + if [[ "v${submodule_version}" != "${rln_version}" ]]; then echo "Submodule version (v${submodule_version}) does not match version in Makefile (${rln_version})" diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh index dba2d6612..c174b39d2 100755 --- a/scripts/install_nimble.sh +++ b/scripts/install_nimble.sh @@ -37,7 +37,11 @@ PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." mkdir -p "${HOME}/.nimble/bin" - ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + if command -v ln >/dev/null 2>&1 && ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" 2>/dev/null; then + : + else + cp -f "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + fi exit 0 fi diff --git a/tools/apply-libp2p-2.2-shims.sh b/tools/apply-libp2p-2.2-shims.sh new file mode 100755 index 000000000..8f20a5549 --- /dev/null +++ b/tools/apply-libp2p-2.2-shims.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Hack for libp2p 2.2 + older libp2p_mix: restore removed modules as shims. +set -euo pipefail +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +LP=$(ls -dt "$ROOT"/nimbledeps/pkgs2/libp2p-2.2.* 2>/dev/null | head -1) +if [ -z "${LP:-}" ] || [ ! -d "$LP/libp2p" ]; then + echo "libp2p 2.2 package not found under nimbledeps/pkgs2" >&2 + exit 1 +fi +cat > "$LP/libp2p/utility.nim" <<'NIM' +# Shim: libp2p/utility removed in nim-libp2p 2.x; needed by current libp2p_mix pin. +import ./utils/[opt, shortlog, collections] +export opt, shortlog, collections +NIM +cat > "$LP/libp2p/utils/sequninit.nim" <<'NIM' +# Shim: libp2p/utils/sequninit removed; newSeqUninit lives in system. +export system +NIM +echo "Applied libp2p 2.2 shims in $LP" diff --git a/tools/install_from_lock.nim b/tools/install_from_lock.nim new file mode 100644 index 000000000..bec9609d5 --- /dev/null +++ b/tools/install_from_lock.nim @@ -0,0 +1,175 @@ +## Install nimble.lock packages into nimbledeps/pkgs2 and write nimble.paths. +## +## Used when `nimble setup --localdeps` fails (e.g. SAT cannot resolve libp2p +## git pins). Reproducible: clones exact vcsRevision from the lockfile. +## +## Usage (from repo root): +## nim c -r --hints:off tools/install_from_lock.nim +## +## Optional env: +## INSTALL_FROM_LOCK_FORCE=1 re-clone packages even if present + +import std/[json, os, osproc, strutils, algorithm] + +const + PkgsRel = "nimbledeps" / "pkgs2" + LockRel = "nimble.lock" + PathsRel = "nimble.paths" + +proc run(cmd: string): int = + execCmd(cmd) + +proc runOut(cmd: string): tuple[output: string, exitCode: int] = + execCmdEx(cmd) + +proc gitClone(url, dest, rev: string): bool = + if dirExists(dest): + removeDir(dest) + # Shallow clone of a specific commit is awkward; full clone then checkout. + var c = run("git clone --quiet " & quoteShell(url) & " " & quoteShell(dest)) + if c != 0: + let url2 = url.replace(".git", "") + c = run("git clone --quiet " & quoteShell(url2) & " " & quoteShell(dest)) + if c != 0: + return false + if run("git -C " & quoteShell(dest) & " checkout -q " & quoteShell(rev)) != 0: + discard run("git -C " & quoteShell(dest) & " fetch --quiet origin " & quoteShell(rev)) + if run("git -C " & quoteShell(dest) & " checkout -q " & quoteShell(rev)) != 0: + return false + # Submodules (bearssl, boringssl, secp256k1, lsquic, zlib, ...) + discard run( + "git -C " & quoteShell(dest) & " submodule update --init --recursive --quiet" + ) + true + +proc packageVersion(pkgDir: string, fallback: string): string = + result = fallback + for f in walkFiles(pkgDir / "*.nimble"): + for line in readFile(f).splitLines: + let t = line.strip + if t.startsWith("version"): + let parts = t.split('=') + if parts.len >= 2: + return parts[1].strip().strip(chars = {'"', '\''}) + break + +proc writeMeta(pkgDir, url, rev, sha1, ver: string) = + let meta = %*{ + "url": url, + "vcsRevision": rev, + "downloadMethod": "git", + "versions": [ver], + "checksums": {"sha1": sha1} + } + writeFile(pkgDir / "nimblemeta.json", meta.pretty) + +proc installPkg(root, name: string, entry: JsonNode, force: bool): string = + ## Returns final package directory path. + let version = entry["version"].getStr + let rev = entry["vcsRevision"].getStr + let url = entry["url"].getStr + let sha1 = entry["checksums"]["sha1"].getStr + let pkgsDir = root / PkgsRel + createDir(pkgsDir) + + # Prefer existing dir matching name-*-sha1 + var existing = "" + for kind, path in walkDir(pkgsDir): + if kind != pcDir: + continue + let base = path.extractFilename + if base.startsWith(name & "-") and base.endsWith("-" & sha1): + existing = path + break + + if existing.len > 0 and not force and fileExists(existing / "nimblemeta.json"): + # Ensure submodules present (best-effort) + if dirExists(existing / ".git"): + discard run( + "git -C " & quoteShell(existing) & + " submodule update --init --recursive --quiet" + ) + echo " skip ", existing.extractFilename + return existing + + echo " install ", name, " @ ", version, " (", rev[0 .. min(11, rev.high)], ")" + let tmpName = name & "-tmp-" & sha1 + let tmpDest = pkgsDir / tmpName + if not gitClone(url, tmpDest, rev): + echo " FAILED ", name + quit(1) + + var pkgVer = packageVersion(tmpDest, version.replace("#", "")) + if pkgVer.len == 0: + pkgVer = version.replace("#", "") + let finalName = name & "-" & pkgVer & "-" & sha1 + let finalDest = pkgsDir / finalName + if finalDest != tmpDest: + if dirExists(finalDest): + removeDir(finalDest) + moveDir(tmpDest, finalDest) + writeMeta(finalDest, url, rev, sha1, pkgVer) + echo " -> ", finalName + finalDest + +proc writePaths(root: string, dirs: seq[string]) = + var lines = @["--noNimblePath"] + var sorted = dirs + sorted.sort() + for d in sorted: + lines.add("--path:\"" & d & "\"") + if dirExists(d / "src"): + lines.add("--path:\"" & d / "src" & "\"") + writeFile(root / PathsRel, lines.join("\n") & "\n") + echo "Wrote ", root / PathsRel, " (", sorted.len, " packages)" + +proc applyLibp2pShims(root: string) = + ## libp2p_mix still imports removed libp2p modules; restore as thin shims. + var lp = "" + let pkgs = root / PkgsRel + for kind, path in walkDir(pkgs): + if kind != pcDir: + continue + let base = path.extractFilename + if base.startsWith("libp2p-2.2") and dirExists(path / "libp2p"): + lp = path + break + if lp.len == 0: + echo " (no libp2p 2.2 package; skip shims)" + return + writeFile( + lp / "libp2p" / "utility.nim", + """ +# Shim: libp2p/utility removed in nim-libp2p 2.x; needed by current libp2p_mix pin. +import ./utils/[opt, shortlog, collections] +export opt, shortlog, collections +""", + ) + createDir(lp / "libp2p" / "utils") + writeFile( + lp / "libp2p" / "utils" / "sequninit.nim", + """ +# Shim: libp2p/utils/sequninit removed; newSeqUninit lives in system. +export system +""", + ) + echo " applied libp2p 2.2 shims in ", lp.extractFilename + +proc main() = + let root = getCurrentDir() + let lockPath = root / LockRel + if not fileExists(lockPath): + echo "error: ", lockPath, " not found (run from repo root)" + quit(1) + + let force = getEnv("INSTALL_FROM_LOCK_FORCE") == "1" + let lock = parseFile(lockPath) + echo "Installing packages from ", LockRel, " ..." + var dirs: seq[string] + for name, entry in lock["packages"].pairs: + dirs.add(installPkg(root, name, entry, force)) + writePaths(root, dirs) + applyLibp2pShims(root) + echo "Done." + +main()