diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ddf904ef..922901f27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,12 +89,16 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-v2-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | - nimble setup --localdeps -y + # Use system nim instead of the nim pinned in nimble.lock: nim's + # source tree checksums differently across platforms, so the locked + # checksum is unreliable. --useSystemNim skips the locked nim while + # still verifying every other locked dependency. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps make rebuild-bearssl-nimbledeps touch nimbledeps/.nimble-setup @@ -142,12 +146,16 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-v2-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | - nimble setup --localdeps -y + # Use system nim instead of the nim pinned in nimble.lock: nim's + # source tree checksums differently across platforms, so the locked + # checksum is unreliable. --useSystemNim skips the locked nim while + # still verifying every other locked dependency. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps make rebuild-bearssl-nimbledeps touch nimbledeps/.nimble-setup @@ -207,12 +215,16 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-v2-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | - nimble setup --localdeps -y + # Use system nim instead of the nim pinned in nimble.lock: nim's + # source tree checksums differently across platforms, so the locked + # checksum is unreliable. --useSystemNim skips the locked nim while + # still verifying every other locked dependency. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps make rebuild-bearssl-nimbledeps touch nimbledeps/.nimble-setup diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 0ff427d87..618d77aa3 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -15,6 +15,11 @@ env: NPROC: 2 MAKEFLAGS: "-j${NPROC}" NIMFLAGS: "--parallelBuild:${NPROC}" + # nim builds read compile flags from NIM_PARAMS (getNimParams in waku.nimble), + # not NIMFLAGS, so -d:disableMarchNative must live here or config.nims applies + # -march=native and secp256k1 fails to compile (see #3916, which set this in + # ci.yml but missed this workflow). + NIM_PARAMS: "-d:disableMarchNative" NIM_VERSION: '2.2.4' NIMBLE_VERSION: '0.22.3' diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 50f1602cd..059cc954f 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -24,6 +24,20 @@ jobs: MSYSTEM: MINGW64 steps: + - name: Configure Git to keep LF line endings + # Windows Git defaults to core.autocrlf=true, which converts LF→CRLF when + # nimble clones dependency packages into nimbledeps/. The CRLF conversion + # changes the SHA1 of the package source tree relative to the + # Linux-computed checksums stored in nimble.lock, so nimble decides the + # local copy is invalid and re-downloads on every subsequent invocation + # — and these retries can hang indefinitely on Windows runners. + # Disabling autocrlf globally makes nimble's child git clones produce + # the same tree (and SHA1) as on Linux. + shell: pwsh + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Checkout code uses: actions/checkout@v4 @@ -52,6 +66,17 @@ jobs: mingw-w64-x86_64-clang mingw-w64-x86_64-nasm + - name: Configure Git in MSYS2 to keep LF line endings + # The autocrlf=false above only configures Git for Windows. nimble clones + # its dependency packages from within the MSYS2 shell, whose git reads a + # separate global config ($HOME/.gitconfig under the MSYS2 root). Without + # repeating the setting here, CRLF conversion still alters dependency + # source trees, so their SHA1 no longer matches nimble.lock and nimble + # re-downloads (and hangs) on every invocation. + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - name: Manually install nasm run: | bash scripts/install_nasm_in_windows.sh @@ -80,19 +105,18 @@ jobs: cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y echo "$HOME/.nimble/bin" >> $GITHUB_PATH - - name: Patch nimble.lock for Windows nim checksum - # nimble.exe uses Windows Git (core.autocrlf=true by default), which converts LF→CRLF - # on checkout. This changes the SHA1 of the nim package source tree relative to the - # Linux-computed checksum stored in nimble.lock. Patch the lock file with the - # Windows-computed checksum before nimble reads it. - run: | - sed -i 's/68bb85cbfb1832ce4db43943911b046c3af3caab/a092a045d3a427d127a5334a6e59c76faff54686/g' nimble.lock - - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' run: | export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH" - nimble setup --localdeps -y + # Use the CI-installed system nim instead of the nim pinned in + # nimble.lock. nim's source tree checks out differently on Windows + # (its .gitattributes forces line endings on some files), so its + # checksum never matches the Linux-computed value in the lock — not + # even with autocrlf disabled. --useSystemNim makes nimble ignore the + # locked nim (no download, no checksum) while still verifying every + # other locked dependency. + nimble setup --localdeps -y --useSystemNim make rebuild-nat-libs-nimbledeps CC=gcc make rebuild-bearssl-nimbledeps CC=gcc touch nimbledeps/.nimble-setup diff --git a/Makefile b/Makefile index ea1bf66f0..a9d1cb2fe 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,14 @@ export PATH := $(HOME)/.nimble/bin:$(PATH) # NIM binary location NIM_BINARY := $(shell which nim 2>/dev/null) NPH := $(HOME)/.nimble/bin/nph -NIMBLE := $(HOME)/.nimble/bin/nimble +# Resolve nimble via PATH (not an absolute path): on the MSYS2 Windows runner +# $(HOME)/.nimble/bin does not hold the binary, so a hardcoded path is "not +# found", whereas PATH is populated for every platform (see export PATH above +# and the workflow's GITHUB_PATH). --useSystemNim makes every invocation reuse +# the nim already on PATH: nimble re-resolves the locked deps before each task, +# and since setup runs with --localdeps the locked nim is never installed, so +# without this nimble re-clones nim-lang/Nim — a fetch that hangs on Windows. +NIMBLE := nimble --useSystemNim NIMBLEDEPS_STAMP := nimbledeps/.nimble-setup # Compilation parameters @@ -204,7 +211,7 @@ clean: | clean-librln testcommon: | build-deps build echo -e $(BUILD_MSG) "build/$@" && \ - nimble testcommon + $(NIMBLE) testcommon ########## ## Waku ## @@ -213,47 +220,47 @@ testcommon: | build-deps build testwaku: | build-deps build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble test + $(NIMBLE) test wakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble wakunode2 + $(NIMBLE) wakunode2 benchmarks: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble benchmarks + $(NIMBLE) benchmarks testwakunode2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble testwakunode2 + $(NIMBLE) testwakunode2 example2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble example2 + $(NIMBLE) example2 chat2: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble chat2 + $(NIMBLE) chat2 chat2mix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble chat2mix + $(NIMBLE) chat2mix rln-db-inspector: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble rln_db_inspector + $(NIMBLE) rln_db_inspector chat2bridge: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble chat2bridge + $(NIMBLE) chat2bridge liteprotocoltester: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble liteprotocoltester + $(NIMBLE) liteprotocoltester lightpushwithmix: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble lightpushwithmix + $(NIMBLE) lightpushwithmix api_example: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ @@ -261,12 +268,12 @@ api_example: | build-deps build deps librln build/%: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$*" && \ - nimble buildone $* + $(NIMBLE) buildone $* compile-test: | build-deps build deps librln echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \ - nimble buildTest $(TEST_FILE) && \ - nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\"" + $(NIMBLE) buildTest $(TEST_FILE) && \ + $(NIMBLE) execTest $(TEST_FILE) "\"$(TEST_NAME)\"" ################ ## Waku tools ## @@ -277,11 +284,11 @@ tools: networkmonitor wakucanary wakucanary: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble wakucanary + $(NIMBLE) wakucanary networkmonitor: | build-deps build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - nimble networkmonitor + $(NIMBLE) networkmonitor ############ ## Format ## @@ -327,7 +334,7 @@ clean: docs: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - nimble doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims + $(NIMBLE) doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims coverage: echo -e $(BUILD_MSG) "build/$@" && \ @@ -424,10 +431,10 @@ else ifeq ($(detected_OS),Linux) endif libwaku: | build-deps librln - nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble + $(NIMBLE) --verbose libwaku$(BUILD_COMMAND) waku.nimble liblogosdelivery: | build-deps librln - nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble + $(NIMBLE) --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble logosdelivery_example: | build liblogosdelivery @echo -e $(BUILD_MSG) "build/$@" @@ -502,7 +509,7 @@ endif build-libwaku-for-android-arch: ifneq ($(findstring /nix/store,$(LIBRLN_FILE)),) mkdir -p $(CURDIR)/build/android/$(ABIDIR)/ - CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) nimble libWakuAndroid + CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) $(NIMBLE) libWakuAndroid else ./scripts/build_rln_android.sh $(CURDIR)/build $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(CROSS_TARGET) $(ABIDIR) endif @@ -559,7 +566,7 @@ else endif build-libwaku-for-ios-arch: - IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) nimble libWakuIOS + IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) $(NIMBLE) libWakuIOS libwaku-ios-device: IOS_ARCH=arm64 libwaku-ios-device: IOS_SDK=iphoneos diff --git a/waku.nimble b/waku.nimble index 99f649758..8641b0a51 100644 --- a/waku.nimble +++ b/waku.nimble @@ -59,7 +59,7 @@ requires "nim >= 2.2.4", "unittest2" # Packages not on nimble (use git URLs) -requires "https://github.com/logos-messaging/nim-ffi" +requires "https://github.com/logos-messaging/nim-ffi#06111de155253b34e47ed2aaed1d61d08d62cc1b" requires "https://github.com/logos-messaging/nim-sds.git#2e9a7683f0e180bf112135fae3a3803eed8490d4"