Merge b7a549b57eaf976542ca0f492fe6e0d8cab16c0d into 71a369ffad2ed15a3d18c9662c0da0db53e7f5d4

BearSSL.mk

@@ -38,7 +38,7 @@ endif
 	+ [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \
 		"$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true
 
-rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps
+rebuild-bearssl-nimbledeps:
 ifeq ($(BEARSSL_NIMBLEDEPS_DIR),)
 	$(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first)
 endif

Makefile

@@ -42,7 +42,8 @@ endif
 ##########
 ## Main ##
 ##########
-.PHONY: all test update clean examples deps nimble install-nim install-nimble
+# The Makefile automatically bootstraps dependency setup when needed for build and test targets.
+.PHONY: all test clean examples deps nimble install-nim install-nimble
 
 # default target
 all: | wakunode2 libwaku liblogosdelivery
@@ -73,14 +74,14 @@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims
 	$(MAKE) install-nimble
 	nimble setup --localdeps
 	$(MAKE) build-nph
-	$(MAKE) rebuild-bearssl-nimbledeps
-	$(MAKE) rebuild-nat-libs-nimbledeps
 	touch $@
 
-update:
-	rm -f $(NIMBLEDEPS_STAMP)
-	$(MAKE) $(NIMBLEDEPS_STAMP)
-	nimble lock
+# Must be phony so the recipe always runs and the sub-make re-evaluates
+# BEARSSL_NIMBLEDEPS_DIR / NAT_TRAVERSAL_NIMBLEDEPS_DIR (parse-time variables)
+# after nimble setup has populated nimbledeps/.
+.PHONY: build-deps
+build-deps: | $(NIMBLEDEPS_STAMP)
+	$(MAKE) rebuild-bearssl-nimbledeps rebuild-nat-libs-nimbledeps
 
 clean:
 	rm -rf build 2> /dev/null || true
@@ -96,12 +97,7 @@ install-nim:
 	scripts/install_nim.sh $(REQUIRED_NIM_VERSION)
 
 install-nimble: install-nim
-	@nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \
-	if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \
-	  echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \
-	else \
-	  cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \
-	fi
+	scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION)
 
 build:
 	mkdir -p build
@@ -203,7 +199,7 @@ clean: | clean-librln
 #################
 .PHONY: testcommon
 
-testcommon: | $(NIMBLEDEPS_STAMP) build
+testcommon: | build-deps build
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble testcommon
 
@@ -212,59 +208,59 @@ testcommon: | $(NIMBLEDEPS_STAMP) build
 ##########
 .PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester
 
-testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln
+testwaku: | build-deps build rln-deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble test
 
-wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln
+wakunode2: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble wakunode2
 
-benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln
+benchmarks: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble benchmarks
 
-testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln
+testwakunode2: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble testwakunode2
 
-example2: | $(NIMBLEDEPS_STAMP) build deps librln
+example2: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble example2
 
-chat2: | $(NIMBLEDEPS_STAMP) build deps librln
+chat2: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble chat2
 
-chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln
+chat2mix: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble chat2mix
 
-rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln
+rln-db-inspector: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble rln_db_inspector
 
-chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln
+chat2bridge: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble chat2bridge
 
-liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln
+liteprotocoltester: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble liteprotocoltester
 
-lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln
+lightpushwithmix: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble lightpushwithmix
 
-api_example: | $(NIMBLEDEPS_STAMP) build deps librln
+api_example: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims
 
-build/%: | $(NIMBLEDEPS_STAMP) build deps librln
+build/%: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$*" && \
 		nimble buildone $*
 
-compile-test: | $(NIMBLEDEPS_STAMP) build deps librln
+compile-test: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \
 		nimble buildTest $(TEST_FILE) && \
 		nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\""
@@ -276,11 +272,11 @@ compile-test: | $(NIMBLEDEPS_STAMP) build deps librln
 
 tools: networkmonitor wakucanary
 
-wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln
+wakucanary: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble wakucanary
 
-networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln
+networkmonitor: | build-deps build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		nimble networkmonitor
 
@@ -424,10 +420,10 @@ else ifeq ($(detected_OS),Linux)
 	BUILD_COMMAND := $(BUILD_COMMAND)Linux
 endif
 
-libwaku: | $(NIMBLEDEPS_STAMP) librln
+libwaku: | build-deps librln
 	nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble
 
-liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln
+liblogosdelivery: | build-deps librln
 	nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble
 
 logosdelivery_example: | build liblogosdelivery

Nat.mk

@@ -39,7 +39,7 @@ endif
 	+ [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \
 		"$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true
 
-rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps
+rebuild-nat-libs-nimbledeps:
 ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),)
 	$(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first)
 endif

README.md

@@ -26,6 +26,11 @@ The standard developer tools, including a C compiler, GNU Make, Bash, and Git. M
 You'll also need an installation of Rust and its toolchain (specifically `rustc` and `cargo`).
 The easiest way to install these, is using `rustup`:
 
+Recommended and tested toolchain versions:
+- Nim 2.2.4
+- Nimble 0.22.3
+
+Rust:
 ```bash
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 ```
@@ -33,8 +38,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 ### Wakunode
 
 ```bash
-# The first `make` invocation will update all Git submodules.
-# You'll run `make update` after each `git pull` in the future to keep those submodules updated.
+# The first `make` invocation will initialize the local dependency state.
 make wakunode2
 
 # Build with custom compilation flags. Do not use NIM_PARAMS unless you know what you are doing.

scripts/install_nim.sh

@@ -17,26 +17,28 @@ if [ -z "${NIM_VERSION}" ]; then
   exit 1
 fi
 
-# Check if the right version is already installed
-nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
-if [ "${nim_ver}" = "${NIM_VERSION}" ]; then
-  echo "Nim ${NIM_VERSION} already installed, skipping."
+NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}"
+
+# Check if nim is already installed at our expected location (not just anywhere in PATH).
+# Checking PATH version is not sufficient: a system-installed nim of the right version
+# won't have its stdlib at ${NIM_DEST}/lib/, causing downstream compilation failures.
+if [ -f "${NIM_DEST}/lib/system.nim" ]; then
+  echo "Nim ${NIM_VERSION} already installed at ${NIM_DEST}, re-linking binaries."
+  mkdir -p "${HOME}/.nimble/bin"
+  for bin_path in "${NIM_DEST}/bin/"*; do
+    ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")"
+  done
   exit 0
 fi
 
-if [ -n "${nim_ver}" ]; then
-  newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1)
-  if [ "${newer}" = "${nim_ver}" ]; then
-    echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2
-    echo "WARNING: The build will proceed but may behave differently." >&2
-    exit 0
-  fi
+nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
+if [ -n "${nim_ver}" ] && [ "${nim_ver}" != "${NIM_VERSION}" ]; then
+  echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2
 fi
 
 OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/')
 ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/')
 
-NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}"
 BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz"
 WORK_DIR=$(mktemp -d)
 trap 'rm -rf "${WORK_DIR}"' EXIT
@@ -48,9 +50,7 @@ if [ "${HTTP_STATUS}" = "200" ]; then
   echo "Downloading pre-built binary from ${BINARY_URL}..."
   curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz"
   tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}"
-  rm -rf "${NIM_DEST}"
-  mkdir -p "${HOME}/.nim"
-  cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}"
+  SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}"
 else
   echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..."
   SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz"
@@ -58,11 +58,15 @@ else
   tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}"
   cd "${WORK_DIR}/Nim-${NIM_VERSION}"
   sh build_all.sh
-  rm -rf "${NIM_DEST}"
-  mkdir -p "${HOME}/.nim"
-  cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}"
+  SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}"
 fi
 
+# rm -rf can fail with "Directory not empty" on overlay filesystems (e.g. Docker).
+# Using cp -r src/. dst/ handles both cases: dst absent (clean) or partially present.
+rm -rf "${NIM_DEST}" 2>/dev/null || true
+mkdir -p "${NIM_DEST}"
+cp -r "${SRC_DIR}/." "${NIM_DEST}/"
+
 mkdir -p "${HOME}/.nimble/bin"
 for bin_path in "${NIM_DEST}/bin/"*; do
   ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")"

scripts/install_nimble.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Installs a specific nimble version without using `nimble install nimble`.
+#
+# `nimble install nimble` is inherently fragile:
+#   - ETXTBSY: overwriting the running nimble binary in pkgs2/
+#   - JSON parse failures with older nimble versions reading packages_official.json
+#
+# Strategy:
+#   1. If the right version is already at ~/.nimble/bin/nimble → done.
+#   2. If a previously-compiled binary exists in pkgs2/ → re-link it.
+#   3. Otherwise: clone the nimble git repo, init submodules, build with nim,
+#      and atomically replace the target (mv avoids ETXTBSY on the old binary).
+
+set -e
+
+NIMBLE_VERSION="${1:-}"
+if [ -z "${NIMBLE_VERSION}" ]; then
+  echo "Usage: $0 <nimble-version>" >&2
+  exit 1
+fi
+
+NIMBLE_BIN="${HOME}/.nimble/bin/nimble"
+
+# 1. Already installed at the right version?
+if [ -x "${NIMBLE_BIN}" ]; then
+  nimble_ver=$("${NIMBLE_BIN}" --version 2>/dev/null \
+    | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
+  if [ "${nimble_ver}" = "${NIMBLE_VERSION}" ]; then
+    echo "Nimble ${NIMBLE_VERSION} already installed, skipping."
+    exit 0
+  fi
+fi
+
+# 2. Already compiled into pkgs2/ from a previous (possibly partial) run?
+PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble \
+  2>/dev/null | head -1 || true)
+if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then
+  echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}."
+  mkdir -p "${HOME}/.nimble/bin"
+  ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}"
+  exit 0
+fi
+
+# 3. Build from source.
+NIM_BIN="${HOME}/.nimble/bin/nim"
+if [ ! -x "${NIM_BIN}" ]; then
+  NIM_BIN="$(command -v nim)"
+fi
+
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "${WORK_DIR}"' EXIT
+
+echo "Cloning nimble v${NIMBLE_VERSION} with submodules..."
+git clone --depth=1 --branch "v${NIMBLE_VERSION}" \
+  --recurse-submodules --shallow-submodules \
+  https://github.com/nim-lang/nimble.git \
+  "${WORK_DIR}/nimble"
+
+echo "Building nimble ${NIMBLE_VERSION} with $("${NIM_BIN}" --version | head -1)..."
+cd "${WORK_DIR}/nimble"
+# nim reads nim.cfg / config.nims in the current dir, which sets vendor paths.
+"${NIM_BIN}" c -d:release --path:src \
+  -o:"${WORK_DIR}/nimble_new" src/nimble.nim
+
+mkdir -p "${HOME}/.nimble/bin"
+# Atomic rename: avoids ETXTBSY when the old binary at NIMBLE_BIN is still running.
+cp "${WORK_DIR}/nimble_new" "${NIMBLE_BIN}.new.$$"
+mv -f "${NIMBLE_BIN}.new.$$" "${NIMBLE_BIN}"
+
+echo "Nimble ${NIMBLE_VERSION} installed to ${NIMBLE_BIN}"