From fee2b386a69c1033f26dda0fb5409ea0bf4b657d Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Thu, 7 May 2026 15:59:35 +0200 Subject: [PATCH 1/6] enhance README and Makefile Co-authored-by: Copilot --- Makefile | 4 ++++ README.md | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index be9e14027..2fd5f8811 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,8 @@ endif ########## ## Main ## ########## +# The Makefile automatically bootstraps dependency setup when needed for build and test targets. +# `make update` is an explicit refresh target and is not required manually for normal builds. .PHONY: all test update clean examples deps nimble install-nim install-nimble # default target @@ -77,6 +79,8 @@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims $(MAKE) rebuild-nat-libs-nimbledeps touch $@ +# `update` refreshes the local dependency stamp and lockfile. +# Most targets already depend on $(NIMBLEDEPS_STAMP) and will update automatically as needed. update: rm -f $(NIMBLEDEPS_STAMP) $(MAKE) $(NIMBLEDEPS_STAMP) diff --git a/README.md b/README.md index 8833ae131..6c542324d 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,13 @@ The standard developer tools, including a C compiler, GNU Make, Bash, and Git. M You'll also need an installation of Rust and its toolchain (specifically `rustc` and `cargo`). The easiest way to install these, is using `rustup`: +Supported toolchain versions: +- Nim 2.2.4 +- Nimble 0.22.3 + +These versions are enforced by the repository configuration and the Makefile. + +Rust: ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` @@ -33,8 +40,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ### Wakunode ```bash -# The first `make` invocation will update all Git submodules. -# You'll run `make update` after each `git pull` in the future to keep those submodules updated. +# The first `make` invocation will initialize the local dependency state. make wakunode2 # Build with custom compilation flags. Do not use NIM_PARAMS unless you know what you are doing. From 6754d61ff17284d6481632256765269a6201a884 Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Thu, 7 May 2026 19:24:09 +0200 Subject: [PATCH 2/6] simplify Makefile to allow build wakunode2 from scratch --- BearSSL.mk | 2 +- Makefile | 55 ++++++++++++++++++++++++++---------------------------- Nat.mk | 2 +- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/BearSSL.mk b/BearSSL.mk index 355e46563..5c2aeac8e 100644 --- a/BearSSL.mk +++ b/BearSSL.mk @@ -38,7 +38,7 @@ endif + [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \ "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true -rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps +rebuild-bearssl-nimbledeps: ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) endif diff --git a/Makefile b/Makefile index 2fd5f8811..5cc55ef2d 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,7 @@ endif ## Main ## ########## # The Makefile automatically bootstraps dependency setup when needed for build and test targets. -# `make update` is an explicit refresh target and is not required manually for normal builds. -.PHONY: all test update clean examples deps nimble install-nim install-nimble +.PHONY: all test clean examples deps nimble install-nim install-nimble # default target all: | wakunode2 libwaku liblogosdelivery @@ -75,16 +74,14 @@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims $(MAKE) install-nimble nimble setup --localdeps $(MAKE) build-nph - $(MAKE) rebuild-bearssl-nimbledeps - $(MAKE) rebuild-nat-libs-nimbledeps touch $@ -# `update` refreshes the local dependency stamp and lockfile. -# Most targets already depend on $(NIMBLEDEPS_STAMP) and will update automatically as needed. -update: - rm -f $(NIMBLEDEPS_STAMP) - $(MAKE) $(NIMBLEDEPS_STAMP) - nimble lock +# Must be phony so the recipe always runs and the sub-make re-evaluates +# BEARSSL_NIMBLEDEPS_DIR / NAT_TRAVERSAL_NIMBLEDEPS_DIR (parse-time variables) +# after nimble setup has populated nimbledeps/. +.PHONY: build-deps +build-deps: | $(NIMBLEDEPS_STAMP) + $(MAKE) rebuild-bearssl-nimbledeps rebuild-nat-libs-nimbledeps clean: rm -rf build 2> /dev/null || true @@ -207,7 +204,7 @@ clean: | clean-librln ################# .PHONY: testcommon -testcommon: | $(NIMBLEDEPS_STAMP) build +testcommon: | build-deps build echo -e $(BUILD_MSG) "build/$@" && \ nimble testcommon @@ -216,59 +213,59 @@ testcommon: | $(NIMBLEDEPS_STAMP) build ########## .PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester -testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln +testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble test -wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln +wakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble wakunode2 -benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln +benchmarks: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble benchmarks -testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln +testwakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble testwakunode2 -example2: | $(NIMBLEDEPS_STAMP) build deps librln +example2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble example2 -chat2: | $(NIMBLEDEPS_STAMP) build deps librln +chat2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2 -chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln +chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2mix -rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln +rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble rln_db_inspector -chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln +chat2bridge: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2bridge -liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln +liteprotocoltester: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble liteprotocoltester -lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln +lightpushwithmix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble lightpushwithmix -api_example: | $(NIMBLEDEPS_STAMP) build deps librln +api_example: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims -build/%: | $(NIMBLEDEPS_STAMP) build deps librln +build/%: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$*" && \ nimble buildone $* -compile-test: | $(NIMBLEDEPS_STAMP) build deps librln +compile-test: | build-deps build deps librln echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \ nimble buildTest $(TEST_FILE) && \ nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\"" @@ -280,11 +277,11 @@ compile-test: | $(NIMBLEDEPS_STAMP) build deps librln tools: networkmonitor wakucanary -wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln +wakucanary: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble wakucanary -networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln +networkmonitor: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble networkmonitor @@ -428,10 +425,10 @@ else ifeq ($(detected_OS),Linux) BUILD_COMMAND := $(BUILD_COMMAND)Linux endif -libwaku: | $(NIMBLEDEPS_STAMP) librln +libwaku: | build-deps librln nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble -liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln +liblogosdelivery: | build-deps librln nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble logosdelivery_example: | build liblogosdelivery diff --git a/Nat.mk b/Nat.mk index 90d0b2ead..596ef1fb5 100644 --- a/Nat.mk +++ b/Nat.mk @@ -39,7 +39,7 @@ endif + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \ "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true -rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps +rebuild-nat-libs-nimbledeps: ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),) $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first) endif From 78ee490ccaf38414c2f9088982818db8746a4cb2 Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Thu, 7 May 2026 19:39:40 +0200 Subject: [PATCH 3/6] fix build from scratch. See Claude comment: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The script looks correct now. Here's the summary of what changed and why: scripts/install_nim.sh — two changes: Moved NIM_DEST definition to the top (line 20), before the early-exit check that references it. Replaced the early-exit condition. The old logic checked nim --version in PATH and exited if it matched — including a silent exit for "newer version already installed". The problem: on the Linux machine, nim 2.2.4 was installed by the system package manager at a non-standard path (e.g. /usr/bin/nim or /nix/store/…). The script saw the right version and exited early, but ~/.nim/nim-2.2.4/ was never created. Later, when nimble fell back to downloading nimble 0.22.3 source and compiling it, nim looked for its stdlib at ~/.nim/nim-2.2.4/lib/system.nim — which didn't exist. The new logic: only skip if ${NIM_DEST}/lib/system.nim actually exists (i.e. we installed it there). If it does, re-create the ~/.nimble/bin/ symlinks just in case they're stale, then exit. Otherwise, always download and install to ~/.nim/nim-2.2.4/, creating a self-contained installation the rest of the build can rely on. --- scripts/install_nim.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh index c8d0f439d..4c386d018 100755 --- a/scripts/install_nim.sh +++ b/scripts/install_nim.sh @@ -17,26 +17,28 @@ if [ -z "${NIM_VERSION}" ]; then exit 1 fi -# Check if the right version is already installed -nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) -if [ "${nim_ver}" = "${NIM_VERSION}" ]; then - echo "Nim ${NIM_VERSION} already installed, skipping." +NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" + +# Check if nim is already installed at our expected location (not just anywhere in PATH). +# Checking PATH version is not sufficient: a system-installed nim of the right version +# won't have its stdlib at ${NIM_DEST}/lib/, causing downstream compilation failures. +if [ -f "${NIM_DEST}/lib/system.nim" ]; then + echo "Nim ${NIM_VERSION} already installed at ${NIM_DEST}, re-linking binaries." + mkdir -p "${HOME}/.nimble/bin" + for bin_path in "${NIM_DEST}/bin/"*; do + ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" + done exit 0 fi -if [ -n "${nim_ver}" ]; then - newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1) - if [ "${newer}" = "${nim_ver}" ]; then - echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2 - echo "WARNING: The build will proceed but may behave differently." >&2 - exit 0 - fi +nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) +if [ -n "${nim_ver}" ] && [ "${nim_ver}" != "${NIM_VERSION}" ]; then + echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2 fi OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/') ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/') -NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz" WORK_DIR=$(mktemp -d) trap 'rm -rf "${WORK_DIR}"' EXIT From dab1a267f6a71e27719b6fcfa435c192c850c785 Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Thu, 7 May 2026 19:45:01 +0200 Subject: [PATCH 4/6] fix in install_nim.sh --- scripts/install_nim.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh index 4c386d018..b4015feb8 100755 --- a/scripts/install_nim.sh +++ b/scripts/install_nim.sh @@ -50,9 +50,7 @@ if [ "${HTTP_STATUS}" = "200" ]; then echo "Downloading pre-built binary from ${BINARY_URL}..." curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz" tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}" - rm -rf "${NIM_DEST}" - mkdir -p "${HOME}/.nim" - cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}" + SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}" else echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..." SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz" @@ -60,11 +58,15 @@ else tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}" cd "${WORK_DIR}/Nim-${NIM_VERSION}" sh build_all.sh - rm -rf "${NIM_DEST}" - mkdir -p "${HOME}/.nim" - cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}" + SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}" fi +# rm -rf can fail with "Directory not empty" on overlay filesystems (e.g. Docker). +# Using cp -r src/. dst/ handles both cases: dst absent (clean) or partially present. +rm -rf "${NIM_DEST}" 2>/dev/null || true +mkdir -p "${NIM_DEST}" +cp -r "${SRC_DIR}/." "${NIM_DEST}/" + mkdir -p "${HOME}/.nimble/bin" for bin_path in "${NIM_DEST}/bin/"*; do ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" From fbe2e19fe7294e34a4914c1b2b8fc9fa2721e5dd Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Thu, 7 May 2026 20:23:03 +0200 Subject: [PATCH 5/6] start using install_nimble.sh script --- Makefile | 7 +--- scripts/install_nimble.sh | 70 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100755 scripts/install_nimble.sh diff --git a/Makefile b/Makefile index 5cc55ef2d..26e5101c6 100644 --- a/Makefile +++ b/Makefile @@ -97,12 +97,7 @@ install-nim: scripts/install_nim.sh $(REQUIRED_NIM_VERSION) install-nimble: install-nim - @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ - if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \ - echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \ - else \ - cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \ - fi + scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION) build: mkdir -p build diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh new file mode 100755 index 000000000..e7f575f59 --- /dev/null +++ b/scripts/install_nimble.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Installs a specific nimble version without using `nimble install nimble`. +# +# `nimble install nimble` is inherently fragile: +# - ETXTBSY: overwriting the running nimble binary in pkgs2/ +# - JSON parse failures with older nimble versions reading packages_official.json +# +# Strategy: +# 1. If the right version is already at ~/.nimble/bin/nimble → done. +# 2. If a previously-compiled binary exists in pkgs2/ → re-link it. +# 3. Otherwise: clone the nimble git repo, init submodules, build with nim, +# and atomically replace the target (mv avoids ETXTBSY on the old binary). + +set -e + +NIMBLE_VERSION="${1:-}" +if [ -z "${NIMBLE_VERSION}" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +NIMBLE_BIN="${HOME}/.nimble/bin/nimble" + +# 1. Already installed at the right version? +if [ -x "${NIMBLE_BIN}" ]; then + nimble_ver=$("${NIMBLE_BIN}" --version 2>/dev/null \ + | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + if [ "${nimble_ver}" = "${NIMBLE_VERSION}" ]; then + echo "Nimble ${NIMBLE_VERSION} already installed, skipping." + exit 0 + fi +fi + +# 2. Already compiled into pkgs2/ from a previous (possibly partial) run? +PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble \ + 2>/dev/null | head -1 || true) +if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then + echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." + mkdir -p "${HOME}/.nimble/bin" + ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + exit 0 +fi + +# 3. Build from source. +NIM_BIN="${HOME}/.nimble/bin/nim" +if [ ! -x "${NIM_BIN}" ]; then + NIM_BIN="$(command -v nim)" +fi + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "${WORK_DIR}"' EXIT + +echo "Cloning nimble v${NIMBLE_VERSION} with submodules..." +git clone --depth=1 --branch "v${NIMBLE_VERSION}" \ + --recurse-submodules --shallow-submodules \ + https://github.com/nim-lang/nimble.git \ + "${WORK_DIR}/nimble" + +echo "Building nimble ${NIMBLE_VERSION} with $("${NIM_BIN}" --version | head -1)..." +cd "${WORK_DIR}/nimble" +# nim reads nim.cfg / config.nims in the current dir, which sets vendor paths. +"${NIM_BIN}" c -d:release --path:src \ + -o:"${WORK_DIR}/nimble_new" src/nimble.nim + +mkdir -p "${HOME}/.nimble/bin" +# Atomic rename: avoids ETXTBSY when the old binary at NIMBLE_BIN is still running. +cp "${WORK_DIR}/nimble_new" "${NIMBLE_BIN}.new.$$" +mv -f "${NIMBLE_BIN}.new.$$" "${NIMBLE_BIN}" + +echo "Nimble ${NIMBLE_VERSION} installed to ${NIMBLE_BIN}" From b7a549b57eaf976542ca0f492fe6e0d8cab16c0d Mon Sep 17 00:00:00 2001 From: Ivan FB Date: Fri, 8 May 2026 18:41:06 +0200 Subject: [PATCH 6/6] make make comment about recommended versions --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c542324d..7d49a3ff2 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,10 @@ The standard developer tools, including a C compiler, GNU Make, Bash, and Git. M You'll also need an installation of Rust and its toolchain (specifically `rustc` and `cargo`). The easiest way to install these, is using `rustup`: -Supported toolchain versions: +Recommended and tested toolchain versions: - Nim 2.2.4 - Nimble 0.22.3 -These versions are enforced by the repository configuration and the Makefile. - Rust: ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh