diff --git a/BearSSL.mk b/BearSSL.mk index 355e46563..5c2aeac8e 100644 --- a/BearSSL.mk +++ b/BearSSL.mk @@ -38,7 +38,7 @@ endif + [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \ "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true -rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps +rebuild-bearssl-nimbledeps: ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) endif diff --git a/Makefile b/Makefile index be9e14027..26e5101c6 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,8 @@ endif ########## ## Main ## ########## -.PHONY: all test update clean examples deps nimble install-nim install-nimble +# The Makefile automatically bootstraps dependency setup when needed for build and test targets. +.PHONY: all test clean examples deps nimble install-nim install-nimble # default target all: | wakunode2 libwaku liblogosdelivery @@ -73,14 +74,14 @@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims $(MAKE) install-nimble nimble setup --localdeps $(MAKE) build-nph - $(MAKE) rebuild-bearssl-nimbledeps - $(MAKE) rebuild-nat-libs-nimbledeps touch $@ -update: - rm -f $(NIMBLEDEPS_STAMP) - $(MAKE) $(NIMBLEDEPS_STAMP) - nimble lock +# Must be phony so the recipe always runs and the sub-make re-evaluates +# BEARSSL_NIMBLEDEPS_DIR / NAT_TRAVERSAL_NIMBLEDEPS_DIR (parse-time variables) +# after nimble setup has populated nimbledeps/. +.PHONY: build-deps +build-deps: | $(NIMBLEDEPS_STAMP) + $(MAKE) rebuild-bearssl-nimbledeps rebuild-nat-libs-nimbledeps clean: rm -rf build 2> /dev/null || true @@ -96,12 +97,7 @@ install-nim: scripts/install_nim.sh $(REQUIRED_NIM_VERSION) install-nimble: install-nim - @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ - if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \ - echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \ - else \ - cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \ - fi + scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION) build: mkdir -p build @@ -203,7 +199,7 @@ clean: | clean-librln ################# .PHONY: testcommon -testcommon: | $(NIMBLEDEPS_STAMP) build +testcommon: | build-deps build echo -e $(BUILD_MSG) "build/$@" && \ nimble testcommon @@ -212,59 +208,59 @@ testcommon: | $(NIMBLEDEPS_STAMP) build ########## .PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester -testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln +testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble test -wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln +wakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble wakunode2 -benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln +benchmarks: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble benchmarks -testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln +testwakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble testwakunode2 -example2: | $(NIMBLEDEPS_STAMP) build deps librln +example2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble example2 -chat2: | $(NIMBLEDEPS_STAMP) build deps librln +chat2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2 -chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln +chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2mix -rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln +rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble rln_db_inspector -chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln +chat2bridge: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2bridge -liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln +liteprotocoltester: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble liteprotocoltester -lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln +lightpushwithmix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble lightpushwithmix -api_example: | $(NIMBLEDEPS_STAMP) build deps librln +api_example: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims -build/%: | $(NIMBLEDEPS_STAMP) build deps librln +build/%: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$*" && \ nimble buildone $* -compile-test: | $(NIMBLEDEPS_STAMP) build deps librln +compile-test: | build-deps build deps librln echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \ nimble buildTest $(TEST_FILE) && \ nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\"" @@ -276,11 +272,11 @@ compile-test: | $(NIMBLEDEPS_STAMP) build deps librln tools: networkmonitor wakucanary -wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln +wakucanary: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble wakucanary -networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln +networkmonitor: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble networkmonitor @@ -424,10 +420,10 @@ else ifeq ($(detected_OS),Linux) BUILD_COMMAND := $(BUILD_COMMAND)Linux endif -libwaku: | $(NIMBLEDEPS_STAMP) librln +libwaku: | build-deps librln nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble -liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln +liblogosdelivery: | build-deps librln nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble logosdelivery_example: | build liblogosdelivery diff --git a/Nat.mk b/Nat.mk index 90d0b2ead..596ef1fb5 100644 --- a/Nat.mk +++ b/Nat.mk @@ -39,7 +39,7 @@ endif + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \ "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true -rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps +rebuild-nat-libs-nimbledeps: ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),) $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first) endif diff --git a/README.md b/README.md index 8833ae131..7d49a3ff2 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ The standard developer tools, including a C compiler, GNU Make, Bash, and Git. M You'll also need an installation of Rust and its toolchain (specifically `rustc` and `cargo`). The easiest way to install these, is using `rustup`: +Recommended and tested toolchain versions: +- Nim 2.2.4 +- Nimble 0.22.3 + +Rust: ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` @@ -33,8 +38,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ### Wakunode ```bash -# The first `make` invocation will update all Git submodules. -# You'll run `make update` after each `git pull` in the future to keep those submodules updated. +# The first `make` invocation will initialize the local dependency state. make wakunode2 # Build with custom compilation flags. Do not use NIM_PARAMS unless you know what you are doing. diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh index c8d0f439d..b4015feb8 100755 --- a/scripts/install_nim.sh +++ b/scripts/install_nim.sh @@ -17,26 +17,28 @@ if [ -z "${NIM_VERSION}" ]; then exit 1 fi -# Check if the right version is already installed -nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) -if [ "${nim_ver}" = "${NIM_VERSION}" ]; then - echo "Nim ${NIM_VERSION} already installed, skipping." +NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" + +# Check if nim is already installed at our expected location (not just anywhere in PATH). +# Checking PATH version is not sufficient: a system-installed nim of the right version +# won't have its stdlib at ${NIM_DEST}/lib/, causing downstream compilation failures. +if [ -f "${NIM_DEST}/lib/system.nim" ]; then + echo "Nim ${NIM_VERSION} already installed at ${NIM_DEST}, re-linking binaries." + mkdir -p "${HOME}/.nimble/bin" + for bin_path in "${NIM_DEST}/bin/"*; do + ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" + done exit 0 fi -if [ -n "${nim_ver}" ]; then - newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1) - if [ "${newer}" = "${nim_ver}" ]; then - echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2 - echo "WARNING: The build will proceed but may behave differently." >&2 - exit 0 - fi +nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) +if [ -n "${nim_ver}" ] && [ "${nim_ver}" != "${NIM_VERSION}" ]; then + echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2 fi OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/') ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/') -NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz" WORK_DIR=$(mktemp -d) trap 'rm -rf "${WORK_DIR}"' EXIT @@ -48,9 +50,7 @@ if [ "${HTTP_STATUS}" = "200" ]; then echo "Downloading pre-built binary from ${BINARY_URL}..." curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz" tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}" - rm -rf "${NIM_DEST}" - mkdir -p "${HOME}/.nim" - cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}" + SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}" else echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..." SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz" @@ -58,11 +58,15 @@ else tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}" cd "${WORK_DIR}/Nim-${NIM_VERSION}" sh build_all.sh - rm -rf "${NIM_DEST}" - mkdir -p "${HOME}/.nim" - cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}" + SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}" fi +# rm -rf can fail with "Directory not empty" on overlay filesystems (e.g. Docker). +# Using cp -r src/. dst/ handles both cases: dst absent (clean) or partially present. +rm -rf "${NIM_DEST}" 2>/dev/null || true +mkdir -p "${NIM_DEST}" +cp -r "${SRC_DIR}/." "${NIM_DEST}/" + mkdir -p "${HOME}/.nimble/bin" for bin_path in "${NIM_DEST}/bin/"*; do ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh new file mode 100755 index 000000000..e7f575f59 --- /dev/null +++ b/scripts/install_nimble.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Installs a specific nimble version without using `nimble install nimble`. +# +# `nimble install nimble` is inherently fragile: +# - ETXTBSY: overwriting the running nimble binary in pkgs2/ +# - JSON parse failures with older nimble versions reading packages_official.json +# +# Strategy: +# 1. If the right version is already at ~/.nimble/bin/nimble → done. +# 2. If a previously-compiled binary exists in pkgs2/ → re-link it. +# 3. Otherwise: clone the nimble git repo, init submodules, build with nim, +# and atomically replace the target (mv avoids ETXTBSY on the old binary). + +set -e + +NIMBLE_VERSION="${1:-}" +if [ -z "${NIMBLE_VERSION}" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +NIMBLE_BIN="${HOME}/.nimble/bin/nimble" + +# 1. Already installed at the right version? +if [ -x "${NIMBLE_BIN}" ]; then + nimble_ver=$("${NIMBLE_BIN}" --version 2>/dev/null \ + | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + if [ "${nimble_ver}" = "${NIMBLE_VERSION}" ]; then + echo "Nimble ${NIMBLE_VERSION} already installed, skipping." + exit 0 + fi +fi + +# 2. Already compiled into pkgs2/ from a previous (possibly partial) run? +PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble \ + 2>/dev/null | head -1 || true) +if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then + echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." + mkdir -p "${HOME}/.nimble/bin" + ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + exit 0 +fi + +# 3. Build from source. +NIM_BIN="${HOME}/.nimble/bin/nim" +if [ ! -x "${NIM_BIN}" ]; then + NIM_BIN="$(command -v nim)" +fi + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "${WORK_DIR}"' EXIT + +echo "Cloning nimble v${NIMBLE_VERSION} with submodules..." +git clone --depth=1 --branch "v${NIMBLE_VERSION}" \ + --recurse-submodules --shallow-submodules \ + https://github.com/nim-lang/nimble.git \ + "${WORK_DIR}/nimble" + +echo "Building nimble ${NIMBLE_VERSION} with $("${NIM_BIN}" --version | head -1)..." +cd "${WORK_DIR}/nimble" +# nim reads nim.cfg / config.nims in the current dir, which sets vendor paths. +"${NIM_BIN}" c -d:release --path:src \ + -o:"${WORK_DIR}/nimble_new" src/nimble.nim + +mkdir -p "${HOME}/.nimble/bin" +# Atomic rename: avoids ETXTBSY when the old binary at NIMBLE_BIN is still running. +cp "${WORK_DIR}/nimble_new" "${NIMBLE_BIN}.new.$$" +mv -f "${NIMBLE_BIN}.new.$$" "${NIMBLE_BIN}" + +echo "Nimble ${NIMBLE_VERSION} installed to ${NIMBLE_BIN}"