From 064d01cf7f70c841284cac8eaf663264a331325c Mon Sep 17 00:00:00 2001 From: stubbsta Date: Thu, 21 May 2026 09:34:55 +0200 Subject: [PATCH 1/7] Remove makefile target update --- BearSSL.mk | 15 +++------ Makefile | 60 ++++++++++++++++----------------- Nat.mk | 19 +++-------- scripts/install_nimble.sh | 70 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 57 deletions(-) create mode 100644 scripts/install_nimble.sh diff --git a/BearSSL.mk b/BearSSL.mk index 355e46563..65e4f72a7 100644 --- a/BearSSL.mk +++ b/BearSSL.mk @@ -9,7 +9,7 @@ ## bearssl (nimbledeps) ## ########################### # Rebuilds libbearssl.a from the package installed by nimble under -# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP). +# nimbledeps/pkgs2/. Invoked via $(NIMBLEDEPS_STAMP) / build-deps. # # BEARSSL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that # depend on it must be invoked via a recursive $(MAKE) call so the sub-make @@ -29,18 +29,11 @@ else PORTABLE_BEARSSL_CFLAGS := -W -Wall -Os -fPIC endif -.PHONY: clean-bearssl-nimbledeps rebuild-bearssl-nimbledeps +.PHONY: rebuild-bearssl-nimbledeps -clean-bearssl-nimbledeps: +rebuild-bearssl-nimbledeps: ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) - $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) -endif - + [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \ - "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true - -rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps -ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) - $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) + $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make build-deps' first) endif @echo "Rebuilding bearssl from $(BEARSSL_CSOURCES_DIR)" + "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" CFLAGS="$(PORTABLE_BEARSSL_CFLAGS)" lib \ No newline at end of file diff --git a/Makefile b/Makefile index be9e14027..26e5101c6 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,8 @@ endif ########## ## Main ## ########## -.PHONY: all test update clean examples deps nimble install-nim install-nimble +# The Makefile automatically bootstraps dependency setup when needed for build and test targets. +.PHONY: all test clean examples deps nimble install-nim install-nimble # default target all: | wakunode2 libwaku liblogosdelivery @@ -73,14 +74,14 @@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims $(MAKE) install-nimble nimble setup --localdeps $(MAKE) build-nph - $(MAKE) rebuild-bearssl-nimbledeps - $(MAKE) rebuild-nat-libs-nimbledeps touch $@ -update: - rm -f $(NIMBLEDEPS_STAMP) - $(MAKE) $(NIMBLEDEPS_STAMP) - nimble lock +# Must be phony so the recipe always runs and the sub-make re-evaluates +# BEARSSL_NIMBLEDEPS_DIR / NAT_TRAVERSAL_NIMBLEDEPS_DIR (parse-time variables) +# after nimble setup has populated nimbledeps/. +.PHONY: build-deps +build-deps: | $(NIMBLEDEPS_STAMP) + $(MAKE) rebuild-bearssl-nimbledeps rebuild-nat-libs-nimbledeps clean: rm -rf build 2> /dev/null || true @@ -96,12 +97,7 @@ install-nim: scripts/install_nim.sh $(REQUIRED_NIM_VERSION) install-nimble: install-nim - @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ - if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \ - echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \ - else \ - cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \ - fi + scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION) build: mkdir -p build @@ -203,7 +199,7 @@ clean: | clean-librln ################# .PHONY: testcommon -testcommon: | $(NIMBLEDEPS_STAMP) build +testcommon: | build-deps build echo -e $(BUILD_MSG) "build/$@" && \ nimble testcommon @@ -212,59 +208,59 @@ testcommon: | $(NIMBLEDEPS_STAMP) build ########## .PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester -testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln +testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble test -wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln +wakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble wakunode2 -benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln +benchmarks: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble benchmarks -testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln +testwakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble testwakunode2 -example2: | $(NIMBLEDEPS_STAMP) build deps librln +example2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble example2 -chat2: | $(NIMBLEDEPS_STAMP) build deps librln +chat2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2 -chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln +chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2mix -rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln +rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble rln_db_inspector -chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln +chat2bridge: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble chat2bridge -liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln +liteprotocoltester: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble liteprotocoltester -lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln +lightpushwithmix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble lightpushwithmix -api_example: | $(NIMBLEDEPS_STAMP) build deps librln +api_example: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims -build/%: | $(NIMBLEDEPS_STAMP) build deps librln +build/%: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$*" && \ nimble buildone $* -compile-test: | $(NIMBLEDEPS_STAMP) build deps librln +compile-test: | build-deps build deps librln echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \ nimble buildTest $(TEST_FILE) && \ nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\"" @@ -276,11 +272,11 @@ compile-test: | $(NIMBLEDEPS_STAMP) build deps librln tools: networkmonitor wakucanary -wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln +wakucanary: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble wakucanary -networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln +networkmonitor: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble networkmonitor @@ -424,10 +420,10 @@ else ifeq ($(detected_OS),Linux) BUILD_COMMAND := $(BUILD_COMMAND)Linux endif -libwaku: | $(NIMBLEDEPS_STAMP) librln +libwaku: | build-deps librln nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble -liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln +liblogosdelivery: | build-deps librln nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble logosdelivery_example: | build liblogosdelivery diff --git a/Nat.mk b/Nat.mk index 90d0b2ead..1161121ba 100644 --- a/Nat.mk +++ b/Nat.mk @@ -9,7 +9,7 @@ ## nat-libs (nimbledeps) ## ########################### # Builds miniupnpc and libnatpmp from the package installed by nimble under -# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP). +# nimbledeps/pkgs2/. Invoked via $(NIMBLEDEPS_STAMP) / build-deps. # # NAT_TRAVERSAL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that # depend on it must be invoked via a recursive $(MAKE) call so the sub-make @@ -28,20 +28,11 @@ else PORTABLE_NAT_MARCH := endif -.PHONY: clean-cross-nimbledeps rebuild-nat-libs-nimbledeps +.PHONY: rebuild-nat-libs-nimbledeps -clean-cross-nimbledeps: +rebuild-nat-libs-nimbledeps: ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),) - $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first) -endif - + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" ] && \ - "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" CC=$(CC) clean $(HANDLE_OUTPUT) || true - + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \ - "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true - -rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps -ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),) - $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first) + $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make build-deps' first) endif @echo "Rebuilding nat-libs from $(NAT_TRAVERSAL_NIMBLEDEPS_DIR)" ifeq ($(OS), Windows_NT) @@ -58,4 +49,4 @@ else + "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -fPIC $(PORTABLE_NAT_MARCH) -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \ -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \ CC=$(CC) libnatpmp.a $(HANDLE_OUTPUT) -endif +endif \ No newline at end of file diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh new file mode 100644 index 000000000..dba2d6612 --- /dev/null +++ b/scripts/install_nimble.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Installs a specific nimble version without using `nimble install nimble`. +# +# `nimble install nimble` is inherently fragile: +# - ETXTBSY: overwriting the running nimble binary in pkgs2/ +# - JSON parse failures with older nimble versions reading packages_official.json +# +# Strategy: +# 1. If the right version is already at ~/.nimble/bin/nimble → done. +# 2. If a previously-compiled binary exists in pkgs2/ → re-link it. +# 3. Otherwise: clone the nimble git repo, init submodules, build with nim, +# and atomically replace the target (mv avoids ETXTBSY on the old binary). + +set -e + +NIMBLE_VERSION="${1:-}" +if [ -z "${NIMBLE_VERSION}" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +NIMBLE_BIN="${HOME}/.nimble/bin/nimble" + +# 1. Already installed at the right version? +if [ -x "${NIMBLE_BIN}" ]; then + nimble_ver=$("${NIMBLE_BIN}" --version 2>/dev/null \ + | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + if [ "${nimble_ver}" = "${NIMBLE_VERSION}" ]; then + echo "Nimble ${NIMBLE_VERSION} already installed, skipping." + exit 0 + fi +fi + +# 2. Already compiled into pkgs2/ from a previous (possibly partial) run? +PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble \ + 2>/dev/null | head -1 || true) +if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then + echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." + mkdir -p "${HOME}/.nimble/bin" + ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}" + exit 0 +fi + +# 3. Build from source. +NIM_BIN="${HOME}/.nimble/bin/nim" +if [ ! -x "${NIM_BIN}" ]; then + NIM_BIN="$(command -v nim)" +fi + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "${WORK_DIR}"' EXIT + +echo "Cloning nimble v${NIMBLE_VERSION} with submodules..." +git clone --depth=1 --branch "v${NIMBLE_VERSION}" \ + --recurse-submodules --shallow-submodules \ + https://github.com/nim-lang/nimble.git \ + "${WORK_DIR}/nimble" + +echo "Building nimble ${NIMBLE_VERSION} with $("${NIM_BIN}" --version | head -1)..." +cd "${WORK_DIR}/nimble" +# nim reads nim.cfg / config.nims in the current dir, which sets vendor paths. +"${NIM_BIN}" c -d:release --path:src \ + -o:"${WORK_DIR}/nimble_new" src/nimble.nim + +mkdir -p "${HOME}/.nimble/bin" +# Atomic rename: avoids ETXTBSY when the old binary at NIMBLE_BIN is still running. +cp "${WORK_DIR}/nimble_new" "${NIMBLE_BIN}.new.$$" +mv -f "${NIMBLE_BIN}.new.$$" "${NIMBLE_BIN}" + +echo "Nimble ${NIMBLE_VERSION} installed to ${NIMBLE_BIN}" \ No newline at end of file From 9ba2e456971401c7e854dd0af95e34653928b917 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Thu, 21 May 2026 11:11:34 +0200 Subject: [PATCH 2/7] fix: set execute permission on install_nimble.sh --- scripts/install_nimble.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/install_nimble.sh diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh old mode 100644 new mode 100755 From 78677770f84783bea528df410ebc13ec506240b0 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Thu, 21 May 2026 12:25:12 +0200 Subject: [PATCH 3/7] improve install_nim script --- scripts/install_nim.sh | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh index c8d0f439d..ce71c0345 100755 --- a/scripts/install_nim.sh +++ b/scripts/install_nim.sh @@ -17,26 +17,28 @@ if [ -z "${NIM_VERSION}" ]; then exit 1 fi -# Check if the right version is already installed -nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) -if [ "${nim_ver}" = "${NIM_VERSION}" ]; then - echo "Nim ${NIM_VERSION} already installed, skipping." +NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" + +# Check if nim is already installed at our expected location (not just anywhere in PATH). +# Checking PATH version is not sufficient: a system-installed nim of the right version +# won't have its stdlib at ${NIM_DEST}/lib/, causing downstream compilation failures. +if [ -f "${NIM_DEST}/lib/system.nim" ]; then + echo "Nim ${NIM_VERSION} already installed at ${NIM_DEST}, re-linking binaries." + mkdir -p "${HOME}/.nimble/bin" + for bin_path in "${NIM_DEST}/bin/"*; do + ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" + done exit 0 fi -if [ -n "${nim_ver}" ]; then - newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1) - if [ "${newer}" = "${nim_ver}" ]; then - echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2 - echo "WARNING: The build will proceed but may behave differently." >&2 - exit 0 - fi +nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) +if [ -n "${nim_ver}" ] && [ "${nim_ver}" != "${NIM_VERSION}" ]; then + echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2 fi OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/') ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/') -NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz" WORK_DIR=$(mktemp -d) trap 'rm -rf "${WORK_DIR}"' EXIT @@ -48,9 +50,7 @@ if [ "${HTTP_STATUS}" = "200" ]; then echo "Downloading pre-built binary from ${BINARY_URL}..." curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz" tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}" - rm -rf "${NIM_DEST}" - mkdir -p "${HOME}/.nim" - cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}" + SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}" else echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..." SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz" @@ -58,15 +58,19 @@ else tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}" cd "${WORK_DIR}/Nim-${NIM_VERSION}" sh build_all.sh - rm -rf "${NIM_DEST}" - mkdir -p "${HOME}/.nim" - cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}" + SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}" fi +# rm -rf can fail with "Directory not empty" on overlay filesystems (e.g. Docker). +# Using cp -r src/. dst/ handles both cases: dst absent (clean) or partially present. +rm -rf "${NIM_DEST}" 2>/dev/null || true +mkdir -p "${NIM_DEST}" +cp -r "${SRC_DIR}/." "${NIM_DEST}/" + mkdir -p "${HOME}/.nimble/bin" for bin_path in "${NIM_DEST}/bin/"*; do ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" done echo "Nim ${NIM_VERSION} installed to ${NIM_DEST}" -echo "Binaries symlinked in ~/.nimble/bin — ensure it is in your PATH." +echo "Binaries symlinked in ~/.nimble/bin — ensure it is in your PATH." \ No newline at end of file From 3863932676e8b4b19a3e7685582dfd3188c708f8 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Thu, 21 May 2026 13:19:04 +0200 Subject: [PATCH 4/7] skip second nim install on Windows --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 26e5101c6..a140ff104 100644 --- a/Makefile +++ b/Makefile @@ -94,10 +94,14 @@ REQUIRED_NIM_VERSION := $(shell grep -E '^const RequiredNimVersion\s*=' waku. REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const RequiredNimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') install-nim: +ifneq ($(detected_OS),Windows) scripts/install_nim.sh $(REQUIRED_NIM_VERSION) +endif install-nimble: install-nim +ifneq ($(detected_OS),Windows) scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION) +endif build: mkdir -p build From c3cf9ac0065ba2dc6ddd3c80f0d63efa72b21357 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Thu, 21 May 2026 14:24:51 +0200 Subject: [PATCH 5/7] fix path check in install-nim --- scripts/install_nim.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh index ce71c0345..42aa88ecd 100755 --- a/scripts/install_nim.sh +++ b/scripts/install_nim.sh @@ -19,9 +19,18 @@ fi NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" -# Check if nim is already installed at our expected location (not just anywhere in PATH). -# Checking PATH version is not sufficient: a system-installed nim of the right version -# won't have its stdlib at ${NIM_DEST}/lib/, causing downstream compilation failures. +# 1. A matching Nim is already on PATH (e.g. provided by CI's setup-nim-action, +# choosenim, or a previous run of this script). Use it as-is: installing over it +# would symlink a freshly downloaded Nim into ~/.nimble/bin (first on PATH) and +# shadow a known-good toolchain, which has caused C-backend build failures. +nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) +if [ "${nim_ver}" = "${NIM_VERSION}" ]; then + echo "Nim ${NIM_VERSION} already on PATH ($(command -v nim)), skipping install." + exit 0 +fi + +# 2. Already installed at our expected location from a previous run, but not on PATH. +# Re-link binaries into ~/.nimble/bin. if [ -f "${NIM_DEST}/lib/system.nim" ]; then echo "Nim ${NIM_VERSION} already installed at ${NIM_DEST}, re-linking binaries." mkdir -p "${HOME}/.nimble/bin" @@ -31,8 +40,7 @@ if [ -f "${NIM_DEST}/lib/system.nim" ]; then exit 0 fi -nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) -if [ -n "${nim_ver}" ] && [ "${nim_ver}" != "${NIM_VERSION}" ]; then +if [ -n "${nim_ver}" ]; then echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2 fi From fbd862f36f62be0ac231644aff33a3211fa41891 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Mon, 25 May 2026 09:47:26 +0200 Subject: [PATCH 6/7] Makefile workfile reordering --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e63017ea5..0515ef3ec 100644 --- a/Makefile +++ b/Makefile @@ -70,10 +70,8 @@ endif waku.nims: ln -s waku.nimble $@ -$(NIMBLEDEPS_STAMP): nimble.lock | waku.nims - $(MAKE) install-nimble +$(NIMBLEDEPS_STAMP): nimble.lock | install-nimble build-nph waku.nims nimble setup --localdeps - $(MAKE) build-nph touch $@ # Must be phony so the recipe always runs and the sub-make re-evaluates From 1b46759d9d60d5712bf078e34ea5a496a86f4c1e Mon Sep 17 00:00:00 2001 From: stubbsta Date: Tue, 26 May 2026 10:51:43 +0200 Subject: [PATCH 7/7] Update build_windows script to use nimble --- Makefile | 4 --- scripts/build_windows.sh | 53 ++++++++++++++++++--------------- scripts/install_nim.sh | 62 ++++++++++++++++++++++++++------------- scripts/install_nimble.sh | 16 ++++++---- 4 files changed, 81 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index 0515ef3ec..43bcf61cd 100644 --- a/Makefile +++ b/Makefile @@ -92,14 +92,10 @@ REQUIRED_NIM_VERSION := $(shell grep -E '^const RequiredNimVersion\s*=' waku. REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const RequiredNimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') install-nim: -ifneq ($(detected_OS),Windows) scripts/install_nim.sh $(REQUIRED_NIM_VERSION) -endif install-nimble: install-nim -ifneq ($(detected_OS),Windows) scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION) -endif build: mkdir -p build diff --git a/scripts/build_windows.sh b/scripts/build_windows.sh index c9d4bd860..efbefda0e 100755 --- a/scripts/build_windows.sh +++ b/scripts/build_windows.sh @@ -2,6 +2,12 @@ echo "- - - - - - - - - - Windows Setup Script - - - - - - - - - -" +# Mirrors the steps in .github/workflows/windows-build.yml so a local MSYS2 +# build matches CI. Builds go through the nimble build system: Nim/Nimble are +# installed via scripts/install_nim.sh + install_nimble.sh, dependencies are +# fetched into nimbledeps/ by `nimble setup --localdeps`, and the nat-libs and +# bearssl C sources are rebuilt from there. + success_count=0 failure_count=0 @@ -18,39 +24,38 @@ execute_command() { } echo "1. -.-.-.-- Set PATH -.-.-.-" -export PATH="/c/msys64/usr/bin:/c/msys64/mingw64/bin:/c/msys64/usr/lib:/c/msys64/mingw64/lib:$PATH" +export PATH="$HOME/.nimble/bin:/c/msys64/usr/bin:/c/msys64/mingw64/bin:/c/msys64/usr/lib:/c/msys64/mingw64/lib:$PATH" echo "2. -.-.-.- Verify dependencies -.-.-.-" -execute_command "which gcc g++ make cmake cargo upx rustc python" +execute_command "which gcc g++ make cmake cargo upx rustc python nasm" echo "3. -.-.-.- Updating submodules -.-.-.-" execute_command "git submodule update --init --recursive" -echo "4. -.-.-.- Creating tmp directory -.-.-.-" +echo "4. -.-.-.- Installing nasm -.-.-.-" +execute_command "bash scripts/install_nasm_in_windows.sh" + +echo "5. -.-.-.- Installing Nim and Nimble -.-.-.-" +execute_command "make install-nimble" + +echo "6. -.-.-.- Patch nimble.lock for Windows nim checksum -.-.-.-" +# nimble.exe uses Windows Git (core.autocrlf=true by default), which converts +# LF->CRLF on checkout. This changes the SHA1 of the nim package source tree +# relative to the Linux-computed checksum stored in nimble.lock. Patch the lock +# file with the Windows-computed checksum before nimble reads it. +execute_command "sed -i 's/68bb85cbfb1832ce4db43943911b046c3af3caab/a092a045d3a427d127a5334a6e59c76faff54686/g' nimble.lock" + +echo "7. -.-.-.- Installing nimble deps -.-.-.-" +execute_command "nimble setup --localdeps -y" +execute_command "make rebuild-nat-libs-nimbledeps CC=gcc" +execute_command "make rebuild-bearssl-nimbledeps CC=gcc" +execute_command "touch nimbledeps/.nimble-setup" + +echo "8. -.-.-.- Creating tmp directory -.-.-.-" execute_command "mkdir -p tmp" -echo "5. -.-.-.- Building Nim -.-.-.-" -cd vendor/nimbus-build-system/vendor/Nim -execute_command "./build_all.bat" -cd ../../../.. - -echo "6. -.-.-.- Building libunwind -.-.-.-" -cd vendor/nim-libbacktrace -execute_command "make all V=1 -j8" -cd ../../ - -echo "7. -.-.-.- Building miniupnpc -.-.-.- " -cd vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc -execute_command "make -f Makefile.mingw CC=gcc CXX=g++ libminiupnpc.a V=1 -j8" -cd ../../../../.. - -echo "8. -.-.-.- Building libnatpmp -.-.-.- " -cd ./vendor/nim-nat-traversal/vendor/libnatpmp-upstream -make CC="gcc -fPIC -D_WIN32_WINNT=0x0600 -DNATPMP_STATICLIB" libnatpmp.a V=1 -j8 -cd ../../../../ - echo "9. -.-.-.- Building wakunode2 -.-.-.- " -execute_command "make wakunode2 LOG_LEVEL=DEBUG V=1 -j8" +execute_command "make wakunode2 LOG_LEVEL=DEBUG V=3 -j8" echo "10. -.-.-.- Building libwaku -.-.-.- " execute_command "make libwaku STATIC=0 LOG_LEVEL=DEBUG V=1 -j8" diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh index 42aa88ecd..3802fb85b 100755 --- a/scripts/install_nim.sh +++ b/scripts/install_nim.sh @@ -5,8 +5,9 @@ # Installs to ~/.nim/nim-/ and symlinks binaries into ~/.nimble/bin/, # which is the idiomatic Nim location already on PATH. # -# Pre-built binaries are downloaded from nim-lang.org when available. -# Falls back to building from source otherwise (e.g. macOS on older releases). +# Pre-built binaries are downloaded from nim-lang.org: a .zip on Windows +# (nim-_x64.zip), a .tar.xz on Linux/macOS. On Unix, falls back to +# building from source when no pre-built tarball exists. set -e @@ -44,30 +45,49 @@ if [ -n "${nim_ver}" ]; then echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2 fi -OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/') -ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/') - -BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz" WORK_DIR=$(mktemp -d) trap 'rm -rf "${WORK_DIR}"' EXIT -echo "Checking for pre-built Nim ${NIM_VERSION} (${OS}_${ARCH})..." -HTTP_STATUS=$(curl -sI "${BINARY_URL}" | head -1 | grep -oE '[0-9]{3}' || true) +case "$(uname -s)" in +MINGW* | MSYS* | CYGWIN*) + # Windows: Nim ships as a zip named nim-_.zip + # (no OS segment, .zip rather than .tar.xz, and no source-build fallback). + case "$(uname -m)" in + x86_64 | amd64) WIN_ARCH=x64 ;; + *) WIN_ARCH=x32 ;; + esac + BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}_${WIN_ARCH}.zip" -if [ "${HTTP_STATUS}" = "200" ]; then - echo "Downloading pre-built binary from ${BINARY_URL}..." - curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz" - tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}" + echo "Downloading pre-built Nim ${NIM_VERSION} (windows_${WIN_ARCH}) from ${BINARY_URL}..." + curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.zip" + unzip -q "${WORK_DIR}/nim.zip" -d "${WORK_DIR}" SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}" -else - echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..." - SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz" - curl -fL "${SRC_URL}" -o "${WORK_DIR}/nim-src.tar.gz" - tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}" - cd "${WORK_DIR}/Nim-${NIM_VERSION}" - sh build_all.sh - SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}" -fi + ;; +*) + OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/') + ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/') + + BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz" + + echo "Checking for pre-built Nim ${NIM_VERSION} (${OS}_${ARCH})..." + HTTP_STATUS=$(curl -sI "${BINARY_URL}" | head -1 | grep -oE '[0-9]{3}' || true) + + if [ "${HTTP_STATUS}" = "200" ]; then + echo "Downloading pre-built binary from ${BINARY_URL}..." + curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz" + tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}" + SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}" + else + echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..." + SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz" + curl -fL "${SRC_URL}" -o "${WORK_DIR}/nim-src.tar.gz" + tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}" + cd "${WORK_DIR}/Nim-${NIM_VERSION}" + sh build_all.sh + SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}" + fi + ;; +esac # rm -rf can fail with "Directory not empty" on overlay filesystems (e.g. Docker). # Using cp -r src/. dst/ handles both cases: dst absent (clean) or partially present. diff --git a/scripts/install_nimble.sh b/scripts/install_nimble.sh index dba2d6612..c43011e7b 100755 --- a/scripts/install_nimble.sh +++ b/scripts/install_nimble.sh @@ -19,7 +19,13 @@ if [ -z "${NIMBLE_VERSION}" ]; then exit 1 fi -NIMBLE_BIN="${HOME}/.nimble/bin/nimble" +# On Windows (MSYS2) the binaries carry a .exe extension. +EXE="" +case "$(uname -s)" in +MINGW* | MSYS* | CYGWIN*) EXE=".exe" ;; +esac + +NIMBLE_BIN="${HOME}/.nimble/bin/nimble${EXE}" # 1. Already installed at the right version? if [ -x "${NIMBLE_BIN}" ]; then @@ -32,7 +38,7 @@ if [ -x "${NIMBLE_BIN}" ]; then fi # 2. Already compiled into pkgs2/ from a previous (possibly partial) run? -PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble \ +PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble${EXE} \ 2>/dev/null | head -1 || true) if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}." @@ -42,7 +48,7 @@ if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then fi # 3. Build from source. -NIM_BIN="${HOME}/.nimble/bin/nim" +NIM_BIN="${HOME}/.nimble/bin/nim${EXE}" if [ ! -x "${NIM_BIN}" ]; then NIM_BIN="$(command -v nim)" fi @@ -60,11 +66,11 @@ echo "Building nimble ${NIMBLE_VERSION} with $("${NIM_BIN}" --version | head -1) cd "${WORK_DIR}/nimble" # nim reads nim.cfg / config.nims in the current dir, which sets vendor paths. "${NIM_BIN}" c -d:release --path:src \ - -o:"${WORK_DIR}/nimble_new" src/nimble.nim + -o:"${WORK_DIR}/nimble_new${EXE}" src/nimble.nim mkdir -p "${HOME}/.nimble/bin" # Atomic rename: avoids ETXTBSY when the old binary at NIMBLE_BIN is still running. -cp "${WORK_DIR}/nimble_new" "${NIMBLE_BIN}.new.$$" +cp "${WORK_DIR}/nimble_new${EXE}" "${NIMBLE_BIN}.new.$$" mv -f "${NIMBLE_BIN}.new.$$" "${NIMBLE_BIN}" echo "Nimble ${NIMBLE_VERSION} installed to ${NIMBLE_BIN}" \ No newline at end of file