From f5762af4c4834972d98cdda961b796c7e30613bf Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:42:14 +0200 Subject: [PATCH 01/25] Start using nimble and deprecate vendor dependencies (#3798) Co-authored-by: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Co-authored-by: Darshan K <35736874+darshankabariya@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bump_dependencies.md | 46 +- .github/ISSUE_TEMPLATE/prepare_release.md | 1 + .github/workflows/ci-daily.yml | 2 +- .github/workflows/ci-nix.yml | 15 +- .github/workflows/ci.yml | 109 ++-- .github/workflows/container-image.yml | 36 +- .github/workflows/pre-release.yml | 4 +- .github/workflows/windows-build.yml | 70 ++- .gitignore | 6 - .gitmodules | 187 ------ BearSSL.mk | 39 ++ Dockerfile | 3 +- Makefile | 446 +++++++-------- Nat.mk | 54 ++ apps/wakunode2/wakunode2.nim | 3 +- config.nims | 11 +- env.sh | 8 - flake.lock | 57 +- flake.nix | 112 ++-- nimble.lock | 551 ++++++++++++++++++ nix/default.nix | 212 +++---- nix/deps.nix | 272 +++++++++ nix/nimble.nix | 12 - nix/shell.nix | 3 +- tests/tools/test_confutils_envvar.nim | 10 +- tests/waku_core/test_peers.nim | 1 + tools/confutils/cli_args.nim | 13 +- tools/gen-nix-deps.sh | 80 +++ vendor/db_connector | 1 - vendor/dnsclient.nim | 1 - vendor/nim-bearssl | 1 - vendor/nim-chronicles | 1 - vendor/nim-chronos | 1 - vendor/nim-confutils | 1 - vendor/nim-dnsdisc | 1 - vendor/nim-eth | 1 - vendor/nim-faststreams | 1 - vendor/nim-ffi | 1 - vendor/nim-http-utils | 1 - vendor/nim-json-rpc | 1 - vendor/nim-json-serialization | 1 - vendor/nim-jwt | 1 - vendor/nim-libbacktrace | 1 - vendor/nim-libp2p | 1 - vendor/nim-lsquic | 1 - vendor/nim-metrics | 1 - vendor/nim-minilru | 1 - vendor/nim-nat-traversal | 1 - vendor/nim-presto | 1 - vendor/nim-regex | 1 - vendor/nim-results | 1 - vendor/nim-secp256k1 | 1 - vendor/nim-serialization | 1 - vendor/nim-sqlite3-abi | 1 - vendor/nim-stew | 1 - vendor/nim-stint | 1 - vendor/nim-taskpools | 1 - vendor/nim-testutils | 1 - vendor/nim-toml-serialization | 1 - vendor/nim-unicodedb | 1 - vendor/nim-unittest2 | 1 - vendor/nim-web3 | 1 - vendor/nim-websock | 1 - vendor/nim-zlib | 1 - vendor/nimbus-build-system | 1 - vendor/nimcrypto | 1 - vendor/nph | 1 - waku.nimble | 602 ++++++++++++-------- waku/api/api_conf.nim | 12 +- 69 files changed, 1916 insertions(+), 1100 deletions(-) create mode 100644 BearSSL.mk create mode 100644 Nat.mk delete mode 100755 env.sh create mode 100644 nimble.lock create mode 100644 nix/deps.nix delete mode 100644 nix/nimble.nix create mode 100755 tools/gen-nix-deps.sh delete mode 160000 vendor/db_connector delete mode 160000 vendor/dnsclient.nim delete mode 160000 vendor/nim-bearssl delete mode 160000 vendor/nim-chronicles delete mode 160000 vendor/nim-chronos delete mode 160000 vendor/nim-confutils delete mode 160000 vendor/nim-dnsdisc delete mode 160000 vendor/nim-eth delete mode 160000 vendor/nim-faststreams delete mode 160000 vendor/nim-ffi delete mode 160000 vendor/nim-http-utils delete mode 160000 vendor/nim-json-rpc delete mode 160000 vendor/nim-json-serialization delete mode 160000 vendor/nim-jwt delete mode 160000 vendor/nim-libbacktrace delete mode 160000 vendor/nim-libp2p delete mode 160000 vendor/nim-lsquic delete mode 160000 vendor/nim-metrics delete mode 160000 vendor/nim-minilru delete mode 160000 vendor/nim-nat-traversal delete mode 160000 vendor/nim-presto delete mode 160000 vendor/nim-regex delete mode 160000 vendor/nim-results delete mode 160000 vendor/nim-secp256k1 delete mode 160000 vendor/nim-serialization delete mode 160000 vendor/nim-sqlite3-abi delete mode 160000 vendor/nim-stew delete mode 160000 vendor/nim-stint delete mode 160000 vendor/nim-taskpools delete mode 160000 vendor/nim-testutils delete mode 160000 vendor/nim-toml-serialization delete mode 160000 vendor/nim-unicodedb delete mode 160000 vendor/nim-unittest2 delete mode 160000 vendor/nim-web3 delete mode 160000 vendor/nim-websock delete mode 160000 vendor/nim-zlib delete mode 160000 vendor/nimbus-build-system delete mode 160000 vendor/nimcrypto delete mode 160000 vendor/nph diff --git a/.github/ISSUE_TEMPLATE/bump_dependencies.md b/.github/ISSUE_TEMPLATE/bump_dependencies.md index 0413cbfd2..59f46f08b 100644 --- a/.github/ISSUE_TEMPLATE/bump_dependencies.md +++ b/.github/ISSUE_TEMPLATE/bump_dependencies.md @@ -1,7 +1,7 @@ --- name: Bump dependencies -about: Bump vendor dependencies for release -title: 'Bump vendor dependencies for release 0.0.0' +about: Bump dependencies for release +title: 'Bump dependencies for release 0.X.0' labels: dependencies assignees: '' @@ -9,40 +9,10 @@ assignees: '' -Update `nwaku` "vendor" dependencies. +### Bumped items +- [ ] Update nimble dependencies + 1. Edit manually waku.nimble. For some dependencies, we want to bump versions manually and use a pinned version, f.e., nim-libp2p and all its dependencies. + 2. Run `nimble lock` (make sure `nimble --version` shows the Nimble version pinned in waku.nimble) + 3. Run `./tools/gen-nix-deps.sh nimble.lock nix/deps.nix` to update nix deps -### Items to bump -- [ ] dnsclient.nim ( update to the latest tag version ) -- [ ] nim-bearssl -- [ ] nimbus-build-system -- [ ] nim-chronicles -- [ ] nim-chronos -- [ ] nim-confutils -- [ ] nimcrypto -- [ ] nim-dnsdisc -- [ ] nim-eth -- [ ] nim-faststreams -- [ ] nim-http-utils -- [ ] nim-json-rpc -- [ ] nim-json-serialization -- [ ] nim-libbacktrace -- [ ] nim-libp2p ( update to the latest tag version ) -- [ ] nim-metrics -- [ ] nim-nat-traversal -- [ ] nim-presto -- [ ] nim-regex ( update to the latest tag version ) -- [ ] nim-results -- [ ] nim-secp256k1 -- [ ] nim-serialization -- [ ] nim-sqlite3-abi ( update to the latest tag version ) -- [ ] nim-stew -- [ ] nim-stint -- [ ] nim-taskpools ( update to the latest tag version ) -- [ ] nim-testutils ( update to the latest tag version ) -- [ ] nim-toml-serialization -- [ ] nim-unicodedb -- [ ] nim-unittest2 ( update to the latest tag version ) -- [ ] nim-web3 ( update to the latest tag version ) -- [ ] nim-websock ( update to the latest tag version ) -- [ ] nim-zlib -- [ ] zerokit ( this should be kept in version `v0.7.0` ) +- [ ] Update vendor/zerokit dependency. diff --git a/.github/ISSUE_TEMPLATE/prepare_release.md b/.github/ISSUE_TEMPLATE/prepare_release.md index 83456e79a..de67b3eaf 100644 --- a/.github/ISSUE_TEMPLATE/prepare_release.md +++ b/.github/ISSUE_TEMPLATE/prepare_release.md @@ -18,6 +18,7 @@ For detailed info on the release process refer to https://github.com/logos-messa All items below are to be completed by the owner of the given release. - [ ] Create release branch with major and minor only ( e.g. release/v0.X ) if it doesn't exist. +- [ ] Update the `version` field in `waku.nimble` to match the release version (e.g. `version = "0.X.0"`). - [ ] Assign release candidate tag to the release branch HEAD (e.g. `v0.X.0-rc.0`, `v0.X.0-rc.1`, ... `v0.X.0-rc.N`). - [ ] Generate and edit release notes in CHANGELOG.md. diff --git a/.github/workflows/ci-daily.yml b/.github/workflows/ci-daily.yml index b442014a6..a4cf39340 100644 --- a/.github/workflows/ci-daily.yml +++ b/.github/workflows/ci-daily.yml @@ -40,7 +40,7 @@ jobs: run: make update - name: Build binaries - run: make V=1 QUICK_AND_DIRTY_COMPILER=1 examples tools + run: make V=1 examples tools - name: Notify Discord if: always() diff --git a/.github/workflows/ci-nix.yml b/.github/workflows/ci-nix.yml index 8fc7ac985..7319f64aa 100644 --- a/.github/workflows/ci-nix.yml +++ b/.github/workflows/ci-nix.yml @@ -16,14 +16,7 @@ jobs: - aarch64-darwin - x86_64-linux nixpkg: - - libwaku - - libwaku-android-arm64 - - wakucanary - - exclude: - # Android SDK limitation - - system: aarch64-darwin - nixpkg: libwaku-android-arm64 + - liblogosdelivery include: - system: aarch64-darwin @@ -36,12 +29,10 @@ jobs: runs-on: ${{ matrix.runs_on }} steps: - uses: actions/checkout@v4 - with: - submodules: recursive - - name: 'Run Nix build for {{ matrix.nixpkg }}' + - name: 'Run Nix build for ${{ matrix.nixpkg }}' shell: bash - run: nix build -L '.?submodules=1#${{ matrix.nixpkg }}' + run: nix build -L '.#${{ matrix.nixpkg }}' - name: 'Show result contents' shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c84f5c6f..b2de4e50e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,8 @@ env: NPROC: 2 MAKEFLAGS: "-j${NPROC}" NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none" + NIM_VERSION: '2.2.4' + NIMBLE_VERSION: '0.18.2' jobs: changes: # changes detection @@ -30,9 +32,9 @@ jobs: filters: | common: - '.github/workflows/**' - - 'vendor/**' - - 'Makefile' + - 'nimble.lock' - 'waku.nimble' + - 'Makefile' - 'library/**' v2: - 'waku/**' @@ -63,24 +65,36 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Get submodules hash - id: submodules - run: | - echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT + - name: Install Nim ${{ env.NIM_VERSION }} + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: ${{ env.NIM_VERSION }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache submodules + - name: Install Nimble ${{ env.NIMBLE_VERSION }} + run: | + cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y + echo "$HOME/.nimble/bin" >> $GITHUB_PATH + + - name: Cache nimble deps + id: cache-nimbledeps uses: actions/cache@v3 with: path: | - vendor/ - .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + nimbledeps/ + nimble.paths + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} - - name: Make update - run: make update + - name: Install nimble deps + if: steps.cache-nimbledeps.outputs.cache-hit != 'true' + run: | + nimble setup --localdeps -y + make rebuild-nat-libs-nimbledeps + make rebuild-bearssl-nimbledeps + touch nimbledeps/.nimble-setup - name: Build binaries - run: make V=1 QUICK_AND_DIRTY_COMPILER=1 all + run: make V=1 all build-windows: needs: changes @@ -104,21 +118,33 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Get submodules hash - id: submodules - run: | - echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT + - name: Install Nim ${{ env.NIM_VERSION }} + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: ${{ env.NIM_VERSION }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache submodules + - name: Install Nimble ${{ env.NIMBLE_VERSION }} + run: | + cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y + echo "$HOME/.nimble/bin" >> $GITHUB_PATH + + - name: Cache nimble deps + id: cache-nimbledeps uses: actions/cache@v3 with: path: | - vendor/ - .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + nimbledeps/ + nimble.paths + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} - - name: Make update - run: make update + - name: Install nimble deps + if: steps.cache-nimbledeps.outputs.cache-hit != 'true' + run: | + nimble setup --localdeps -y + make rebuild-nat-libs-nimbledeps + make rebuild-bearssl-nimbledeps + touch nimbledeps/.nimble-setup - name: Run tests run: | @@ -132,13 +158,13 @@ jobs: export NIMFLAGS="--colors:off -d:chronicles_colors:none" export USE_LIBBACKTRACE=0 - make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 POSTGRES=$postgres_enabled test - make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 POSTGRES=$postgres_enabled testwakunode2 + make V=1 POSTGRES=$postgres_enabled test + make V=1 POSTGRES=$postgres_enabled testwakunode2 build-docker-image: needs: changes if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' || needs.changes.outputs.docker == 'true' }} - uses: logos-messaging/logos-delivery/.github/workflows/container-image.yml@10dc3d3eb4b6a3d4313f7b2cc4a85a925e9ce039 + uses: ./.github/workflows/container-image.yml secrets: inherit nwaku-nwaku-interop-tests: @@ -171,18 +197,33 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Get submodules hash - id: submodules - run: | - echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT + - name: Install Nim ${{ env.NIM_VERSION }} + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: ${{ env.NIM_VERSION }} + repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache submodules + - name: Install Nimble ${{ env.NIMBLE_VERSION }} + run: | + cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y + echo "$HOME/.nimble/bin" >> $GITHUB_PATH + + - name: Cache nimble deps + id: cache-nimbledeps uses: actions/cache@v3 with: path: | - vendor/ - .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + nimbledeps/ + nimble.paths + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + + - name: Install nimble deps + if: steps.cache-nimbledeps.outputs.cache-hit != 'true' + run: | + nimble setup --localdeps -y + make rebuild-nat-libs-nimbledeps + make rebuild-bearssl-nimbledeps + touch nimbledeps/.nimble-setup - name: Build nph run: | diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 2bc08be2f..ae132a477 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -15,6 +15,8 @@ env: NPROC: 2 MAKEFLAGS: "-j${NPROC}" NIMFLAGS: "--parallelBuild:${NPROC}" + NIM_VERSION: '2.2.4' + NIMBLE_VERSION: '0.18.2' # This workflow should not run for outside contributors # If org secrets are not available, we'll avoid building and publishing the docker image and we'll pass the workflow @@ -46,28 +48,42 @@ jobs: if: ${{ steps.secrets.outcome == 'success' }} uses: actions/checkout@v4 - - name: Get submodules hash - id: submodules + - name: Install Nim ${{ env.NIM_VERSION }} + if: ${{ steps.secrets.outcome == 'success' }} + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: ${{ env.NIM_VERSION }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Nimble ${{ env.NIMBLE_VERSION }} if: ${{ steps.secrets.outcome == 'success' }} run: | - echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT + cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y + echo "$HOME/.nimble/bin" >> $GITHUB_PATH - - name: Cache submodules + - name: Cache nimble deps if: ${{ steps.secrets.outcome == 'success' }} + id: cache-nimbledeps uses: actions/cache@v3 with: path: | - vendor/ - .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + nimbledeps/ + nimble.paths + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + + - name: Install nimble deps + if: ${{ steps.secrets.outcome == 'success' && steps.cache-nimbledeps.outputs.cache-hit != 'true' }} + run: | + nimble setup --localdeps -y + make rebuild-nat-libs-nimbledeps + make rebuild-bearssl-nimbledeps + touch nimbledeps/.nimble-setup - name: Build binaries id: build if: ${{ steps.secrets.outcome == 'success' }} run: | - make update - - make -j${NPROC} V=1 QUICK_AND_DIRTY_COMPILER=1 NIMFLAGS="-d:disableMarchNative -d:postgres -d:chronicles_colors:none" wakunode2 + make -j${NPROC} V=1 NIMFLAGS="-d:disableMarchNative -d:postgres -d:chronicles_colors:none" wakunode2 SHORT_REF=$(git rev-parse --short HEAD) diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index e145e28ae..e3c8bb575 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -63,10 +63,10 @@ jobs: run: | OS=$([[ "${{runner.os}}" == "macOS" ]] && echo "macosx" || echo "linux") - make QUICK_AND_DIRTY_COMPILER=1 V=1 CI=false NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \ + make V=1 CI=false NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \ update - make QUICK_AND_DIRTY_COMPILER=1 V=1 CI=false\ + make V=1 CI=false\ NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" \ wakunode2\ chat2\ diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 9c1b1eab0..09ef05a5d 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -7,20 +7,25 @@ on: required: true type: string +env: + NPROC: 4 + NIM_VERSION: '2.2.4' + NIMBLE_VERSION: '0.18.2' + jobs: build: runs-on: windows-latest defaults: run: - shell: msys2 {0} + shell: msys2 {0} env: MSYSTEM: MINGW64 steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup MSYS2 uses: msys2/setup-msys2@v2 @@ -51,50 +56,61 @@ jobs: run: | bash scripts/install_nasm_in_windows.sh source $HOME/.bashrc - + - name: Add UPX to PATH run: | echo "/usr/bin:$PATH" >> $GITHUB_PATH echo "/mingw64/bin:$PATH" >> $GITHUB_PATH echo "/usr/lib:$PATH" >> $GITHUB_PATH - echo "/mingw64/lib:$PATH" >> $GITHUB_PATH + echo "/mingw64/lib:$PATH" >> $GITHUB_PATH - name: Verify dependencies run: | which upx gcc g++ make cmake cargo rustc python nasm - - name: Updating submodules - run: git submodule update --init --recursive + - name: Install Nim ${{ env.NIM_VERSION }} + uses: jiro4989/setup-nim-action@v2 + with: + nim-version: ${{ env.NIM_VERSION }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Nimble ${{ env.NIMBLE_VERSION }} + run: | + export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$PATH" + cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y + echo "$HOME/.nimble/bin" >> $GITHUB_PATH + + - name: Cache nimble deps + id: cache-nimbledeps + uses: actions/cache@v4 + with: + path: | + nimbledeps/ + nimble.paths + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + + - name: Install nimble deps + if: steps.cache-nimbledeps.outputs.cache-hit != 'true' + run: | + export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH" + nimble setup --localdeps -y + make rebuild-nat-libs-nimbledeps CC=gcc + make rebuild-bearssl-nimbledeps CC=gcc + touch nimbledeps/.nimble-setup - name: Creating tmp directory run: mkdir -p tmp - - name: Building Nim - run: | - cd vendor/nimbus-build-system/vendor/Nim - ./build_all.bat - cd ../../../.. - - - name: Building miniupnpc - run: | - cd vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc - make -f Makefile.mingw CC=gcc CXX=g++ libminiupnpc.a V=1 - cd ../../../../.. - - - name: Building libnatpmp - run: | - cd ./vendor/nim-nat-traversal/vendor/libnatpmp-upstream - make CC="gcc -fPIC -D_WIN32_WINNT=0x0600 -DNATPMP_STATICLIB" libnatpmp.a V=1 - cd ../../../../ - - name: Building wakunode2.exe run: | - make wakunode2 LOG_LEVEL=DEBUG V=3 -j8 + export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH" + make wakunode2 V=3 -j${{ env.NPROC }} - name: Building libwaku.dll run: | - make libwaku STATIC=0 LOG_LEVEL=DEBUG V=1 -j - + export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH" + make libwaku STATIC=0 V=1 -j + - name: Check Executable run: | if [ -f "./build/wakunode2.exe" ]; then diff --git a/.gitignore b/.gitignore index 5222a0d5e..188090b19 100644 --- a/.gitignore +++ b/.gitignore @@ -3,9 +3,6 @@ # Executables shall be put in an ignored build/ directory /build -# Nimble packages -/vendor/.nimble - # Generated Files *.generated.nim @@ -45,9 +42,6 @@ node_modules/ rlnKeystore.json *.tar.gz -# Nimbus Build System -nimbus-build-system.paths - # sqlite db *.db *.db-shm diff --git a/.gitmodules b/.gitmodules index 6a63491e3..ac07235b8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,197 +1,10 @@ -[submodule "vendor/nim-eth"] - path = vendor/nim-eth - url = https://github.com/status-im/nim-eth.git - ignore = dirty - branch = master -[submodule "vendor/nim-secp256k1"] - path = vendor/nim-secp256k1 - url = https://github.com/status-im/nim-secp256k1.git - ignore = dirty - branch = master -[submodule "vendor/nim-libp2p"] - path = vendor/nim-libp2p - url = https://github.com/vacp2p/nim-libp2p.git - ignore = dirty - branch = master -[submodule "vendor/nim-stew"] - path = vendor/nim-stew - url = https://github.com/status-im/nim-stew.git - ignore = dirty - branch = master -[submodule "vendor/nimbus-build-system"] - path = vendor/nimbus-build-system - url = https://github.com/status-im/nimbus-build-system.git - ignore = dirty - branch = master -[submodule "vendor/nim-nat-traversal"] - path = vendor/nim-nat-traversal - url = https://github.com/status-im/nim-nat-traversal.git - ignore = dirty - branch = master -[submodule "vendor/nim-libbacktrace"] - path = vendor/nim-libbacktrace - url = https://github.com/status-im/nim-libbacktrace.git - ignore = dirty - branch = master -[submodule "vendor/nim-confutils"] - path = vendor/nim-confutils - url = https://github.com/status-im/nim-confutils.git - ignore = dirty - branch = master -[submodule "vendor/nim-chronicles"] - path = vendor/nim-chronicles - url = https://github.com/status-im/nim-chronicles.git - ignore = dirty - branch = master -[submodule "vendor/nim-faststreams"] - path = vendor/nim-faststreams - url = https://github.com/status-im/nim-faststreams.git - ignore = dirty - branch = master -[submodule "vendor/nim-chronos"] - path = vendor/nim-chronos - url = https://github.com/status-im/nim-chronos.git - ignore = dirty - branch = master -[submodule "vendor/nim-json-serialization"] - path = vendor/nim-json-serialization - url = https://github.com/status-im/nim-json-serialization.git - ignore = dirty - branch = master -[submodule "vendor/nim-serialization"] - path = vendor/nim-serialization - url = https://github.com/status-im/nim-serialization.git - ignore = dirty - branch = master -[submodule "vendor/nimcrypto"] - path = vendor/nimcrypto - url = https://github.com/cheatfate/nimcrypto.git - ignore = dirty - branch = master -[submodule "vendor/nim-metrics"] - path = vendor/nim-metrics - url = https://github.com/status-im/nim-metrics.git - ignore = dirty - branch = master -[submodule "vendor/nim-stint"] - path = vendor/nim-stint - url = https://github.com/status-im/nim-stint.git - ignore = dirty - branch = master -[submodule "vendor/nim-json-rpc"] - path = vendor/nim-json-rpc - url = https://github.com/status-im/nim-json-rpc.git - ignore = dirty - branch = master -[submodule "vendor/nim-http-utils"] - path = vendor/nim-http-utils - url = https://github.com/status-im/nim-http-utils.git - ignore = dirty - branch = master -[submodule "vendor/nim-bearssl"] - path = vendor/nim-bearssl - url = https://github.com/status-im/nim-bearssl.git - ignore = dirty - branch = master -[submodule "vendor/nim-sqlite3-abi"] - path = vendor/nim-sqlite3-abi - url = https://github.com/arnetheduck/nim-sqlite3-abi.git - ignore = dirty - branch = master -[submodule "vendor/nim-web3"] - path = vendor/nim-web3 - url = https://github.com/status-im/nim-web3.git -[submodule "vendor/nim-testutils"] - path = vendor/nim-testutils - url = https://github.com/status-im/nim-testutils.git - ignore = untracked - branch = master -[submodule "vendor/nim-unittest2"] - path = vendor/nim-unittest2 - url = https://github.com/status-im/nim-unittest2.git - ignore = untracked - branch = master -[submodule "vendor/nim-websock"] - path = vendor/nim-websock - url = https://github.com/status-im/nim-websock.git - ignore = untracked - branch = main -[submodule "vendor/nim-zlib"] - path = vendor/nim-zlib - url = https://github.com/status-im/nim-zlib.git - ignore = untracked - branch = master -[submodule "vendor/nim-dnsdisc"] - path = vendor/nim-dnsdisc - url = https://github.com/status-im/nim-dnsdisc.git - ignore = untracked - branch = main -[submodule "vendor/dnsclient.nim"] - path = vendor/dnsclient.nim - url = https://github.com/ba0f3/dnsclient.nim.git - ignore = untracked - branch = master -[submodule "vendor/nim-toml-serialization"] - path = vendor/nim-toml-serialization - url = https://github.com/status-im/nim-toml-serialization.git -[submodule "vendor/nim-presto"] - path = vendor/nim-presto - url = https://github.com/status-im/nim-presto.git - ignore = untracked - branch = master [submodule "vendor/zerokit"] path = vendor/zerokit url = https://github.com/vacp2p/zerokit.git ignore = dirty branch = v0.5.1 -[submodule "vendor/nim-regex"] - path = vendor/nim-regex - url = https://github.com/nitely/nim-regex.git - ignore = untracked - branch = master -[submodule "vendor/nim-unicodedb"] - path = vendor/nim-unicodedb - url = https://github.com/nitely/nim-unicodedb.git - ignore = untracked - branch = master -[submodule "vendor/nim-taskpools"] - path = vendor/nim-taskpools - url = https://github.com/status-im/nim-taskpools.git - ignore = untracked - branch = stable -[submodule "vendor/nim-results"] - ignore = untracked - branch = master - path = vendor/nim-results - url = https://github.com/arnetheduck/nim-results.git -[submodule "vendor/db_connector"] - path = vendor/db_connector - url = https://github.com/nim-lang/db_connector.git - ignore = untracked - branch = devel -[submodule "vendor/nph"] - ignore = untracked - branch = master - path = vendor/nph - url = https://github.com/arnetheduck/nph.git -[submodule "vendor/nim-minilru"] - path = vendor/nim-minilru - url = https://github.com/status-im/nim-minilru.git - ignore = untracked - branch = master [submodule "vendor/waku-rlnv2-contract"] path = vendor/waku-rlnv2-contract url = https://github.com/logos-messaging/waku-rlnv2-contract.git ignore = untracked branch = master -[submodule "vendor/nim-lsquic"] - path = vendor/nim-lsquic - url = https://github.com/vacp2p/nim-lsquic -[submodule "vendor/nim-jwt"] - path = vendor/nim-jwt - url = https://github.com/vacp2p/nim-jwt.git -[submodule "vendor/nim-ffi"] - path = vendor/nim-ffi - url = https://github.com/logos-messaging/nim-ffi/ - ignore = untracked - branch = master diff --git a/BearSSL.mk b/BearSSL.mk new file mode 100644 index 000000000..98e933ebd --- /dev/null +++ b/BearSSL.mk @@ -0,0 +1,39 @@ +# Copyright (c) 2022 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +########################### +## bearssl (nimbledeps) ## +########################### +# Rebuilds libbearssl.a from the package installed by nimble under +# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP). +# +# BEARSSL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that +# depend on it must be invoked via a recursive $(MAKE) call so the sub-make +# re-evaluates the variable after nimble setup has populated nimbledeps/. +# +# `ls -dt` (sort by modification time, newest first) is used to pick the +# latest installed version and is portable across Linux, macOS, and +# Windows (MSYS/MinGW). + +BEARSSL_NIMBLEDEPS_DIR := $(shell ls -dt $(CURDIR)/nimbledeps/pkgs2/bearssl-* 2>/dev/null | head -1) +BEARSSL_CSOURCES_DIR := $(BEARSSL_NIMBLEDEPS_DIR)/bearssl/csources + +.PHONY: clean-bearssl-nimbledeps rebuild-bearssl-nimbledeps + +clean-bearssl-nimbledeps: +ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) + $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) +endif + + [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \ + "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true + +rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps +ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) + $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) +endif + @echo "Rebuilding bearssl from $(BEARSSL_CSOURCES_DIR)" + + "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" lib \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5b16b9eee..412d0977a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ FROM rustlang/rust:nightly-alpine3.19 AS nim-build ARG NIMFLAGS ARG MAKE_TARGET=wakunode2 ARG NIM_COMMIT -ARG LOG_LEVEL=TRACE ARG HEAPTRACK_BUILD=0 # Get build tools and required header files @@ -27,7 +26,7 @@ RUN if [ "$HEAPTRACK_BUILD" = "1" ]; then \ RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT} # Build the final node binary -RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}" +RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET NIMFLAGS="${NIMFLAGS}" # PRODUCTION IMAGE ------------------------------------------------------------- diff --git a/Makefile b/Makefile index afd2389d2..cabeec80f 100644 --- a/Makefile +++ b/Makefile @@ -4,28 +4,13 @@ # - MIT license # at your option. This file may not be copied, modified, or distributed except # according to those terms. -export BUILD_SYSTEM_DIR := vendor/nimbus-build-system -export EXCLUDED_NIM_PACKAGES := vendor/nim-dnsdisc/vendor + +include Nat.mk +include BearSSL.mk + LINK_PCRE := 0 FORMAT_MSG := "\\x1B[95mFormatting:\\x1B[39m" -# we don't want an error here, so we can handle things later, in the ".DEFAULT" target --include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk - - -ifeq ($(NIM_PARAMS),) -# "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := git submodule update --init --recursive -.DEFAULT: - +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ - $(GIT_SUBMODULE_UPDATE); \ - echo -# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: -# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles -# -# After restarting, it will execute its original goal, so we don't have to start a child Make here -# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? - -else # "variables.mk" was included. Business as usual until the end of this file. +BUILD_MSG := "Building:" # Determine the OS detected_OS := $(shell uname -s) @@ -33,29 +18,31 @@ ifneq (,$(findstring MINGW,$(detected_OS))) detected_OS := Windows endif +# NIM binary location +NIM_BINARY := $(shell which nim) +NPH := $(HOME)/.nimble/bin/nph +NIMBLEDEPS_STAMP := nimbledeps/.nimble-setup + +# Compilation parameters +NIM_PARAMS ?= + ifeq ($(detected_OS),Windows) - # Update MINGW_PATH to standard MinGW location MINGW_PATH = /mingw64 NIM_PARAMS += --passC:"-I$(MINGW_PATH)/include" NIM_PARAMS += --passL:"-L$(MINGW_PATH)/lib" - NIM_PARAMS += --passL:"-Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc" - NIM_PARAMS += --passL:"-Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream" - - LIBS = -lws2_32 -lbcrypt -liphlpapi -luserenv -lntdll -lminiupnpc -lnatpmp -lpq + LIBS = -lws2_32 -lbcrypt -liphlpapi -luserenv -lntdll -lpq NIM_PARAMS += $(foreach lib,$(LIBS),--passL:"$(lib)") NIM_PARAMS += --passL:"-Wl,--allow-multiple-definition" - export PATH := /c/msys64/usr/bin:/c/msys64/mingw64/bin:/c/msys64/usr/lib:/c/msys64/mingw64/lib:$(PATH) - endif ########## ## Main ## ########## -.PHONY: all test update clean examples +.PHONY: all test update clean examples deps nimble -# default target, because it's the first one that doesn't start with '.' -all: | wakunode2 libwaku +# default target +all: | wakunode2 libwaku liblogosdelivery examples: | example2 chat2 chat2bridge @@ -71,102 +58,116 @@ ifeq ($(strip $(test_file)),) else $(MAKE) compile-test TEST_FILE="$(test_file)" TEST_NAME="$(call test_name)" endif -# this prevents make from erroring on unknown targets like "Index" + +# this prevents make from erroring on unknown targets %: @true waku.nims: ln -s waku.nimble $@ -update: | update-common - rm -rf waku.nims && \ - $(MAKE) waku.nims $(HANDLE_OUTPUT) +$(NIMBLEDEPS_STAMP): nimble.lock | waku.nims + @if ! command -v nimble > /dev/null 2>&1; then $(MAKE) install-nimble; fi + nimble setup --localdeps $(MAKE) build-nph + $(MAKE) rebuild-bearssl-nimbledeps + touch $@ + +update: + rm -f $(NIMBLEDEPS_STAMP) + $(MAKE) $(NIMBLEDEPS_STAMP) + nimble lock clean: - rm -rf build + rm -rf build 2> /dev/null || true + rm -rf nimbledeps 2> /dev/null || true + rm nimble.lock 2> /dev/null || true + rm -fr nimcache 2> /dev/null || true + rm nimble.paths 2> /dev/null || true + nimble clean -# must be included after the default target --include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk +REQUIRED_NIM_VERSION := $(shell grep -E '^const NimVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') +REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const NimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') + +install-nim: + $(eval NIM_OS := $(shell uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/')) + $(eval NIM_ARCH := $(shell uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/')) + $(eval NIM_INSTALL_DIR := $(HOME)/.nim_runtime) + @nim_ver=$$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ + if [ "$$nim_ver" = "$(REQUIRED_NIM_VERSION)" ]; then \ + echo "nim $(REQUIRED_NIM_VERSION) already installed, skipping."; \ + else \ + curl -L "https://github.com/nim-lang/Nim/releases/download/v$(REQUIRED_NIM_VERSION)/nim-$(REQUIRED_NIM_VERSION)-$(NIM_OS)_$(NIM_ARCH).tar.xz" \ + -o /tmp/nim-$(REQUIRED_NIM_VERSION).tar.xz && \ + tar -xJf /tmp/nim-$(REQUIRED_NIM_VERSION).tar.xz -C /tmp && \ + mkdir -p $(NIM_INSTALL_DIR) && \ + cd /tmp/nim-$(REQUIRED_NIM_VERSION) && ./install.sh $(NIM_INSTALL_DIR); \ + fi + +install-nimble: install-nim + @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ + if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \ + echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \ + else \ + cd /tmp && PATH="$(HOME)/.nim_runtime/bin:$$PATH" \ + nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \ + fi + +build: + @nim_ver=$$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ + if [ "$$nim_ver" != "$(REQUIRED_NIM_VERSION)" ]; then \ + echo "Error: Nim $(REQUIRED_NIM_VERSION) is required, but found '$$nim_ver'"; \ + exit 1; \ + fi + @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ + if [ "$$nimble_ver" != "$(REQUIRED_NIMBLE_VERSION)" ]; then \ + echo "Error: Nimble $(REQUIRED_NIMBLE_VERSION) is required, but found '$$nimble_ver'"; \ + exit 1; \ + fi + mkdir -p build + +nimble: + echo "Inside nimble target, checking for nimble..." && \ + command -v nimble >/dev/null 2>&1 || { \ + mv nimbledeps nimbledeps_backup 2>/dev/null || true; \ + echo "choosenim not found, installing ..."; \ + curl -sSf https://nim-lang.org/choosenim/init.sh | sh; \ + mv nimbledeps_backup nimbledeps 2>/dev/null || true; \ + } ## Possible values: prod; debug TARGET ?= prod ## Git version GIT_VERSION ?= $(shell git describe --abbrev=6 --always --tags) -## Compilation parameters. If defined in the CLI the assignments won't be executed NIM_PARAMS := $(NIM_PARAMS) -d:git_version=\"$(GIT_VERSION)\" ## Heaptracker options HEAPTRACKER ?= 0 HEAPTRACKER_INJECT ?= 0 ifeq ($(HEAPTRACKER), 1) -# Assumes Nim's lib/system/alloc.nim is patched! TARGET := debug-with-heaptrack - ifeq ($(HEAPTRACKER_INJECT), 1) -# the Nim compiler will load 'libheaptrack_inject.so' HEAPTRACK_PARAMS := -d:heaptracker -d:heaptracker_inject NIM_PARAMS := $(NIM_PARAMS) -d:heaptracker -d:heaptracker_inject else -# the Nim compiler will load 'libheaptrack_preload.so' HEAPTRACK_PARAMS := -d:heaptracker NIM_PARAMS := $(NIM_PARAMS) -d:heaptracker endif - -endif -## end of Heaptracker options - -################## -## Dependencies ## -################## -.PHONY: deps libbacktrace - -FOUNDRY_VERSION := 1.5.0 -PNPM_VERSION := 10.23.0 - - -rustup: -ifeq (, $(shell which cargo)) -# Install Rustup if it's not installed -# -y: Assume "yes" for all prompts -# --default-toolchain stable: Install the stable toolchain - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable endif -rln-deps: rustup - ./scripts/install_rln_tests_dependencies.sh $(FOUNDRY_VERSION) $(PNPM_VERSION) - -deps: | deps-common nat-libs waku.nims - - -### nim-libbacktrace - -# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims +# Debug/Release mode ifeq ($(DEBUG), 0) -NIM_PARAMS := $(NIM_PARAMS) -d:release -d:lto_incremental -d:strip +NIM_PARAMS := $(NIM_PARAMS) -d:release else NIM_PARAMS := $(NIM_PARAMS) -d:debug endif -ifeq ($(USE_LIBBACKTRACE), 0) NIM_PARAMS := $(NIM_PARAMS) -d:disable_libbacktrace -endif # enable experimental exit is dest feature in libp2p mix NIM_PARAMS := $(NIM_PARAMS) -d:libp2p_mix_experimental_exit_is_dest -libbacktrace: - + $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 - -clean-libbacktrace: - + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) - -# Extend deps and clean targets -ifneq ($(USE_LIBBACKTRACE), 0) -deps: | libbacktrace -endif - ifeq ($(POSTGRES), 1) NIM_PARAMS := $(NIM_PARAMS) -d:postgres -d:nimDebugDlOpen endif @@ -175,14 +176,26 @@ ifeq ($(DEBUG_DISCV5), 1) NIM_PARAMS := $(NIM_PARAMS) -d:debugDiscv5 endif -clean: | clean-libbacktrace +# Export NIM_PARAMS so nimble can access it +export NIM_PARAMS -### Create nimble links (used when building with Nix) +################## +## Dependencies ## +################## +.PHONY: deps -nimbus-build-system-nimble-dir: - NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \ - PWD_CMD="$(PWD)" \ - $(CURDIR)/scripts/generate_nimble_links.sh +FOUNDRY_VERSION := 1.5.0 +PNPM_VERSION := 10.23.0 + +rustup: +ifeq (, $(shell which cargo)) + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable +endif + +rln-deps: rustup + ./scripts/install_rln_tests_dependencies.sh $(FOUNDRY_VERSION) $(PNPM_VERSION) + +deps: | nimble ################## ## RLN ## @@ -199,8 +212,9 @@ LIBRLN_FILE ?= librln_$(LIBRLN_VERSION).a endif $(LIBRLN_FILE): + git submodule update --init vendor/zerokit echo -e $(BUILD_MSG) "$@" && \ - ./scripts/build_rln.sh $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(LIBRLN_FILE) + bash scripts/build_rln.sh $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(LIBRLN_FILE) librln: | $(LIBRLN_FILE) $(eval NIM_PARAMS += --passL:$(LIBRLN_FILE) --passL:-lm) @@ -209,7 +223,6 @@ clean-librln: cargo clean --manifest-path vendor/zerokit/rln/Cargo.toml rm -f $(LIBRLN_FILE) -# Extend clean target clean: | clean-librln ################# @@ -217,74 +230,71 @@ clean: | clean-librln ################# .PHONY: testcommon -testcommon: | build deps +testcommon: | build echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testcommon $(NIM_PARAMS) waku.nims - + nimble testcommon ########## ## Waku ## ########## .PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester -# install rln-deps only for the testwaku target -testwaku: | build deps rln-deps librln +testwaku: | build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim test -d:os=$(shell uname) $(NIM_PARAMS) waku.nims + nimble test -wakunode2: | build deps librln +wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - \ - $(ENV_SCRIPT) nim wakunode2 $(NIM_PARAMS) waku.nims + nimble wakunode2 -benchmarks: | build deps librln +benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim benchmarks $(NIM_PARAMS) waku.nims + nimble benchmarks -testwakunode2: | build deps librln +testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testwakunode2 $(NIM_PARAMS) waku.nims + nimble testwakunode2 -example2: | build deps librln +example2: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim example2 $(NIM_PARAMS) waku.nims + nimble example2 -chat2: | build deps librln +chat2: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim chat2 $(NIM_PARAMS) waku.nims + nimble chat2 -chat2mix: | build deps librln +chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim chat2mix $(NIM_PARAMS) waku.nims + nimble chat2mix -rln-db-inspector: | build deps librln +rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim rln_db_inspector $(NIM_PARAMS) waku.nims + nimble rln_db_inspector -chat2bridge: | build deps librln +chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim chat2bridge $(NIM_PARAMS) waku.nims + nimble chat2bridge -liteprotocoltester: | build deps librln +liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim liteprotocoltester $(NIM_PARAMS) waku.nims + nimble liteprotocoltester -lightpushwithmix: | build deps librln +lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim lightpushwithmix $(NIM_PARAMS) waku.nims + nimble lightpushwithmix -api_example: | build deps librln +api_example: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims -build/%: | build deps librln +build/%: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$*" && \ - $(ENV_SCRIPT) nim buildone $(NIM_PARAMS) waku.nims $* + nimble buildone $* -compile-test: | build deps librln +compile-test: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \ - $(ENV_SCRIPT) nim buildTest $(NIM_PARAMS) waku.nims $(TEST_FILE) && \ - $(ENV_SCRIPT) nim execTest $(NIM_PARAMS) waku.nims $(TEST_FILE) "\"$(TEST_NAME)\""; \ + nimble buildTest $(TEST_FILE) && \ + nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\"" ################ ## Waku tools ## @@ -293,29 +303,30 @@ compile-test: | build deps librln tools: networkmonitor wakucanary -wakucanary: | build deps librln +wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim wakucanary $(NIM_PARAMS) waku.nims + nimble wakucanary -networkmonitor: | build deps librln +networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim networkmonitor $(NIM_PARAMS) waku.nims + nimble networkmonitor ############ ## Format ## ############ -.PHONY: build-nph install-nph clean-nph print-nph-path - -# Default location for nph binary shall be next to nim binary to make it available on the path. -NPH:=$(shell dirname $(NIM_BINARY))/nph +.PHONY: build-nph install-nph print-nph-path build-nph: | build deps -ifeq ("$(wildcard $(NPH))","") - $(ENV_SCRIPT) nim c --skipParentCfg:on vendor/nph/src/nph.nim && \ - mv vendor/nph/src/nph $(shell dirname $(NPH)) - echo "nph utility is available at " $(NPH) +ifneq ($(detected_OS),Windows) + if command -v nph > /dev/null 2>&1; then \ + echo "nph already installed, skipping"; \ + else \ + echo "Installing nph globally"; \ + (cd /tmp && nimble install nph@0.7.0 --accept -g); \ + fi + command -v nph else - echo "nph utility already exists at " $(NPH) + echo "Skipping nph build on Windows (nph is only used on Unix-like systems)" endif GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit @@ -332,39 +343,30 @@ nph/%: | build-nph echo -e $(FORMAT_MSG) "nph/$*" && \ $(NPH) $* -clean-nph: - rm -f $(NPH) - -# To avoid hardcoding nph binary location in several places print-nph-path: - echo "$(NPH)" + @echo "$(NPH)" -clean: | clean-nph +clean: ################### ## Documentation ## ################### .PHONY: docs coverage -# TODO: Remove unused target docs: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims + nimble doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims coverage: echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) ./scripts/run_cov.sh -y - + ./scripts/run_cov.sh -y ##################### ## Container image ## ##################### -# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics -# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker DOCKER_IMAGE_NIMFLAGS ?= -d:chronicles_colors:none -d:insecure -d:postgres DOCKER_IMAGE_NIMFLAGS := $(DOCKER_IMAGE_NIMFLAGS) $(HEAPTRACK_PARAMS) -# build a docker image for the fleet docker-image: MAKE_TARGET ?= wakunode2 docker-image: DOCKER_IMAGE_TAG ?= $(MAKE_TARGET)-$(GIT_VERSION) docker-image: DOCKER_IMAGE_NAME ?= wakuorg/nwaku:$(DOCKER_IMAGE_TAG) @@ -372,8 +374,6 @@ docker-image: docker build \ --build-arg="MAKE_TARGET=$(MAKE_TARGET)" \ --build-arg="NIMFLAGS=$(DOCKER_IMAGE_NIMFLAGS)" \ - --build-arg="NIM_COMMIT=$(DOCKER_NIM_COMMIT)" \ - --build-arg="LOG_LEVEL=$(LOG_LEVEL)" \ --build-arg="HEAPTRACK_BUILD=$(HEAPTRACKER)" \ --label="commit=$(shell git rev-parse HEAD)" \ --label="version=$(GIT_VERSION)" \ @@ -384,7 +384,7 @@ docker-quick-image: MAKE_TARGET ?= wakunode2 docker-quick-image: DOCKER_IMAGE_TAG ?= $(MAKE_TARGET)-$(GIT_VERSION) docker-quick-image: DOCKER_IMAGE_NAME ?= wakuorg/nwaku:$(DOCKER_IMAGE_TAG) docker-quick-image: NIM_PARAMS := $(NIM_PARAMS) -d:chronicles_colors:none -d:insecure -d:postgres --passL:$(LIBRLN_FILE) --passL:-lm -docker-quick-image: | build deps librln wakunode2 +docker-quick-image: | build librln wakunode2 docker build \ --build-arg="MAKE_TARGET=$(MAKE_TARGET)" \ --tag $(DOCKER_IMAGE_NAME) \ @@ -398,20 +398,14 @@ docker-push: #################################### ## Container lite-protocol-tester ## #################################### -# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics -# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker DOCKER_LPT_NIMFLAGS ?= -d:chronicles_colors:none -d:insecure -# build a docker image for the fleet docker-liteprotocoltester: DOCKER_LPT_TAG ?= latest docker-liteprotocoltester: DOCKER_LPT_NAME ?= wakuorg/liteprotocoltester:$(DOCKER_LPT_TAG) -# --no-cache docker-liteprotocoltester: docker build \ --build-arg="MAKE_TARGET=liteprotocoltester" \ --build-arg="NIMFLAGS=$(DOCKER_LPT_NIMFLAGS)" \ - --build-arg="NIM_COMMIT=$(DOCKER_NIM_COMMIT)" \ - --build-arg="LOG_LEVEL=TRACE" \ --label="commit=$(shell git rev-parse HEAD)" \ --label="version=$(GIT_VERSION)" \ --target $(if $(filter deploy,$(DOCKER_LPT_TAG)),deployment_lpt,standalone_lpt) \ @@ -430,39 +424,38 @@ docker-quick-liteprotocoltester: | liteprotocoltester docker-liteprotocoltester-push: docker push $(DOCKER_LPT_NAME) - ################ ## C Bindings ## ################ .PHONY: cbindings cwaku_example libwaku liblogosdelivery liblogosdelivery_example +detected_OS ?= Linux +ifeq ($(OS),Windows_NT) +detected_OS := Windows +else +detected_OS := $(shell uname -s) +endif + +BUILD_COMMAND ?= Dynamic STATIC ?= 0 -LIBWAKU_BUILD_COMMAND ?= libwakuDynamic -LIBLOGOSDELIVERY_BUILD_COMMAND ?= liblogosdeliveryDynamic +ifeq ($(STATIC), 1) + BUILD_COMMAND = Static +endif ifeq ($(detected_OS),Windows) - LIB_EXT_DYNAMIC = dll - LIB_EXT_STATIC = lib + BUILD_COMMAND := $(BUILD_COMMAND)Windows else ifeq ($(detected_OS),Darwin) - LIB_EXT_DYNAMIC = dylib - LIB_EXT_STATIC = a + BUILD_COMMAND := $(BUILD_COMMAND)Mac + export IOS_SDK_PATH := $(shell xcrun --sdk iphoneos --show-sdk-path) else ifeq ($(detected_OS),Linux) - LIB_EXT_DYNAMIC = so - LIB_EXT_STATIC = a + BUILD_COMMAND := $(BUILD_COMMAND)Linux endif -LIB_EXT := $(LIB_EXT_DYNAMIC) -ifeq ($(STATIC), 1) - LIB_EXT = $(LIB_EXT_STATIC) - LIBWAKU_BUILD_COMMAND = libwakuStatic - LIBLOGOSDELIVERY_BUILD_COMMAND = liblogosdeliveryStatic -endif +libwaku: | $(NIMBLEDEPS_STAMP) librln + nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble -libwaku: | build deps librln - echo -e $(BUILD_MSG) "build/$@.$(LIB_EXT)" && $(ENV_SCRIPT) nim $(LIBWAKU_BUILD_COMMAND) $(NIM_PARAMS) waku.nims $@.$(LIB_EXT) - -liblogosdelivery: | build deps librln - echo -e $(BUILD_MSG) "build/$@.$(LIB_EXT)" && $(ENV_SCRIPT) nim $(LIBLOGOSDELIVERY_BUILD_COMMAND) $(NIM_PARAMS) waku.nims $@.$(LIB_EXT) +liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln + nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble logosdelivery_example: | build liblogosdelivery @echo -e $(BUILD_MSG) "build/$@" @@ -492,17 +485,35 @@ else ifeq ($(detected_OS),Windows) -lws2_32 endif +cwaku_example: | build libwaku + echo -e $(BUILD_MSG) "build/$@" && \ + cc -o "build/$@" \ + ./examples/cbindings/waku_example.c \ + ./examples/cbindings/base64.c \ + -lwaku -Lbuild/ \ + -pthread -ldl -lm + +cppwaku_example: | build libwaku + echo -e $(BUILD_MSG) "build/$@" && \ + g++ -o "build/$@" \ + ./examples/cpp/waku.cpp \ + ./examples/cpp/base64.cpp \ + -lwaku -Lbuild/ \ + -pthread -ldl -lm + +nodejswaku: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + node-gyp build --directory=examples/nodejs/ + ##################### ## Mobile Bindings ## ##################### .PHONY: libwaku-android \ - libwaku-android-precheck \ - libwaku-android-arm64 \ - libwaku-android-amd64 \ - libwaku-android-x86 \ - libwaku-android-arm \ - rebuild-nat-libs \ - build-libwaku-for-android-arch + libwaku-android-precheck \ + libwaku-android-arm64 \ + libwaku-android-amd64 \ + libwaku-android-x86 \ + libwaku-android-arm ANDROID_TARGET ?= 30 ifeq ($(detected_OS),Darwin) @@ -511,22 +522,19 @@ else ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/linux-x86_64 endif -rebuild-nat-libs: | clean-cross nat-libs - libwaku-android-precheck: ifndef ANDROID_NDK_HOME - $(error ANDROID_NDK_HOME is not set) + $(error ANDROID_NDK_HOME is not set) endif build-libwaku-for-android-arch: ifneq ($(findstring /nix/store,$(LIBRLN_FILE)),) mkdir -p $(CURDIR)/build/android/$(ABIDIR)/ - cp $(LIBRLN_FILE) $(CURDIR)/build/android/$(ABIDIR)/ + CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) nimble libWakuAndroid else ./scripts/build_rln_android.sh $(CURDIR)/build $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(CROSS_TARGET) $(ABIDIR) endif - $(MAKE) rebuild-nat-libs CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_COMPILER) - CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) $(ENV_SCRIPT) nim libWakuAndroid $(NIM_PARAMS) waku.nims + $(MAKE) rebuild-nat-libs-nimbledeps CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_COMPILER) libwaku-android-arm64: ANDROID_ARCH=aarch64-linux-android libwaku-android-arm64: CPU=arm64 @@ -550,29 +558,23 @@ libwaku-android-arm: ANDROID_ARCH=armv7a-linux-androideabi libwaku-android-arm: CPU=arm libwaku-android-arm: ABIDIR=armeabi-v7a libwaku-android-arm: | libwaku-android-precheck build deps -# cross-rs target architecture name does not match the one used in android $(MAKE) build-libwaku-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) CROSS_TARGET=armv7-linux-androideabi CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_COMPILER=$(ANDROID_ARCH)$(ANDROID_TARGET)-clang libwaku-android: $(MAKE) libwaku-android-amd64 $(MAKE) libwaku-android-arm64 $(MAKE) libwaku-android-x86 -# This target is disabled because on recent versions of cross-rs complain with the following error -# relocation R_ARM_THM_ALU_PREL_11_0 cannot be used against symbol 'stack_init_trampoline_return'; recompile with -fPIC -# It's likely this architecture is not used so we might just not support it. -# $(MAKE) libwaku-android-arm ################# ## iOS Bindings # ################# .PHONY: libwaku-ios-precheck \ - libwaku-ios-device \ - libwaku-ios-simulator \ - libwaku-ios + libwaku-ios-device \ + libwaku-ios-simulator \ + libwaku-ios IOS_DEPLOYMENT_TARGET ?= 18.0 -# Get SDK paths dynamically using xcrun define get_ios_sdk_path $(shell xcrun --sdk $(1) --show-sdk-path 2>/dev/null) endef @@ -584,59 +586,25 @@ else $(error iOS builds are only supported on macOS) endif -# Build for iOS architecture build-libwaku-for-ios-arch: - IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) $(ENV_SCRIPT) nim libWakuIOS $(NIM_PARAMS) waku.nims + IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) nimble libWakuIOS -# iOS device (arm64) libwaku-ios-device: IOS_ARCH=arm64 libwaku-ios-device: IOS_SDK=iphoneos libwaku-ios-device: IOS_SDK_PATH=$(call get_ios_sdk_path,iphoneos) libwaku-ios-device: | libwaku-ios-precheck build deps $(MAKE) build-libwaku-for-ios-arch IOS_ARCH=$(IOS_ARCH) IOS_SDK=$(IOS_SDK) IOS_SDK_PATH=$(IOS_SDK_PATH) -# iOS simulator (arm64 - Apple Silicon Macs) libwaku-ios-simulator: IOS_ARCH=arm64 libwaku-ios-simulator: IOS_SDK=iphonesimulator libwaku-ios-simulator: IOS_SDK_PATH=$(call get_ios_sdk_path,iphonesimulator) libwaku-ios-simulator: | libwaku-ios-precheck build deps $(MAKE) build-libwaku-for-ios-arch IOS_ARCH=$(IOS_ARCH) IOS_SDK=$(IOS_SDK) IOS_SDK_PATH=$(IOS_SDK_PATH) -# Build all iOS targets libwaku-ios: $(MAKE) libwaku-ios-device $(MAKE) libwaku-ios-simulator -cwaku_example: | build libwaku - echo -e $(BUILD_MSG) "build/$@" && \ - cc -o "build/$@" \ - ./examples/cbindings/waku_example.c \ - ./examples/cbindings/base64.c \ - -lwaku -Lbuild/ \ - -pthread -ldl -lm \ - -lminiupnpc -Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/ \ - -lnatpmp -Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream/ \ - vendor/nim-libbacktrace/libbacktrace_wrapper.o \ - vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a - -cppwaku_example: | build libwaku - echo -e $(BUILD_MSG) "build/$@" && \ - g++ -o "build/$@" \ - ./examples/cpp/waku.cpp \ - ./examples/cpp/base64.cpp \ - -lwaku -Lbuild/ \ - -pthread -ldl -lm \ - -lminiupnpc -Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/ \ - -lnatpmp -Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream/ \ - vendor/nim-libbacktrace/libbacktrace_wrapper.o \ - vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a - -nodejswaku: | build deps - echo -e $(BUILD_MSG) "build/$@" && \ - node-gyp build --directory=examples/nodejs/ - -endif # "variables.mk" was not included - ################### # Release Targets # ################### @@ -649,6 +617,4 @@ release-notes: -u $(shell id -u) \ docker.io/wakuorg/sv4git:latest \ release-notes |\ - sed -E 's@#([0-9]+)@[#\1](https://github.com/waku-org/nwaku/issues/\1)@g' -# I could not get the tool to replace issue ids with links, so using sed for now, -# asked here: https://github.com/bvieira/sv4git/discussions/101 + sed -E 's@#([0-9]+)@[#\1](https://github.com/waku-org/nwaku/issues/\1)@g' \ No newline at end of file diff --git a/Nat.mk b/Nat.mk new file mode 100644 index 000000000..31ad4e018 --- /dev/null +++ b/Nat.mk @@ -0,0 +1,54 @@ +# Copyright (c) 2022 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +########################### +## nat-libs (nimbledeps) ## +########################### +# Builds miniupnpc and libnatpmp from the package installed by nimble under +# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP). +# +# NAT_TRAVERSAL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that +# depend on it must be invoked via a recursive $(MAKE) call so the sub-make +# re-evaluates the variable after nimble setup has populated nimbledeps/. +# +# `ls -dt` (sort by modification time, newest first) is used to pick the +# latest installed version and is portable across Linux, macOS, and +# Windows (MSYS/MinGW). + +NAT_TRAVERSAL_NIMBLEDEPS_DIR := $(shell ls -dt $(CURDIR)/nimbledeps/pkgs2/nat_traversal-* 2>/dev/null | head -1) + +.PHONY: clean-cross-nimbledeps rebuild-nat-libs-nimbledeps + +clean-cross-nimbledeps: +ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),) + $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first) +endif + + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" ] && \ + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" CC=$(CC) clean $(HANDLE_OUTPUT) || true + + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \ + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true + +rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps +ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),) + $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first) +endif + @echo "Rebuilding nat-libs from $(NAT_TRAVERSAL_NIMBLEDEPS_DIR)" +ifeq ($(OS), Windows_NT) + + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc/libminiupnpc.a" ] || \ + PATH=".;$${PATH}" "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" \ + -f Makefile.mingw CC=$(CC) CFLAGS="-Os -fPIC" libminiupnpc.a $(HANDLE_OUTPUT) + + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \ + OS=mingw CC=$(CC) \ + CFLAGS="-Wall -Wno-cpp -Os -fPIC -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \ + libnatpmp.a $(HANDLE_OUTPUT) +else + + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" \ + CC=$(CC) CFLAGS="-Os -fPIC" build/libminiupnpc.a $(HANDLE_OUTPUT) + + "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -fPIC -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \ + -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \ + CC=$(CC) libnatpmp.a $(HANDLE_OUTPUT) +endif diff --git a/apps/wakunode2/wakunode2.nim b/apps/wakunode2/wakunode2.nim index c8132ff4e..484adf68f 100644 --- a/apps/wakunode2/wakunode2.nim +++ b/apps/wakunode2/wakunode2.nim @@ -5,7 +5,6 @@ import chronicles, chronos, metrics, - libbacktrace, system/ansi_c, libp2p/crypto/crypto import @@ -88,7 +87,7 @@ when isMainModule: when defined(posix): proc handleSigsegv(signal: cint) {.noconv.} = # Require --debugger:native - fatal "Shutting down after receiving SIGSEGV", stacktrace = getBacktrace() + fatal "Shutting down after receiving SIGSEGV" # Not available in -d:release mode writeStackTrace() diff --git a/config.nims b/config.nims index 0655bf092..329384ac4 100644 --- a/config.nims +++ b/config.nims @@ -9,12 +9,6 @@ if defined(windows): switch("passL", "rln.lib") switch("define", "postgres=false") - # Automatically add all vendor subdirectories - for dir in walkDir("./vendor"): - if dir.kind == pcDir: - switch("path", dir.path) - switch("path", dir.path / "src") - # disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries switch("passL", "-Wl,--no-insert-timestamp") # increase stack size @@ -121,3 +115,8 @@ if defined(android): switch("passC", "--sysroot=" & sysRoot) switch("passL", "--sysroot=" & sysRoot) switch("cincludes", sysRoot & "/usr/include/") +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + --noNimblePath + include "nimble.paths" +# end Nimble config diff --git a/env.sh b/env.sh deleted file mode 100755 index f90ba9a74..000000000 --- a/env.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file -# and we fall back to a Zsh-specific special var to also support Zsh. -REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" -ABS_PATH="$(cd ${REL_PATH}; pwd)" -source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh - diff --git a/flake.lock b/flake.lock index b927e8807..9b5db728d 100644 --- a/flake.lock +++ b/flake.lock @@ -2,27 +2,48 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1757590060, - "narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=", + "lastModified": 1770464364, + "narHash": "sha256-z5NJPSBwsLf/OfD8WTmh79tlSU8XgIbwmk6qB1/TFzY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", + "rev": "23d72dabcb3b12469f57b37170fcbc1789bd7457", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "0ef228213045d2cdb5a169a95d63ded38670b293", + "rev": "23d72dabcb3b12469f57b37170fcbc1789bd7457", "type": "github" } }, "root": { "inputs": { "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay", "zerokit": "zerokit" } }, "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1775099554, + "narHash": "sha256-3xBsGnGDLOFtnPZ1D3j2LU19wpAlYefRKTlkv648rU0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "8d6387ed6d8e6e6672fd3ed4b61b59d44b124d99", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_2": { "inputs": { "nixpkgs": [ "zerokit", @@ -30,11 +51,11 @@ ] }, "locked": { - "lastModified": 1748399823, - "narHash": "sha256-kahD8D5hOXOsGbNdoLLnqCL887cjHkx98Izc37nDjlA=", + "lastModified": 1771211437, + "narHash": "sha256-lcNK438i4DGtyA+bPXXyVLHVmJjYpVKmpux9WASa3ro=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "d68a69dc71bc19beb3479800392112c2f6218159", + "rev": "c62195b3d6e1bb11e0c2fb2a494117d3b55d410f", "type": "github" }, "original": { @@ -48,21 +69,21 @@ "nixpkgs": [ "nixpkgs" ], - "rust-overlay": "rust-overlay" + "rust-overlay": "rust-overlay_2" }, "locked": { - "lastModified": 1762211504, - "narHash": "sha256-SbDoBElFYJ4cYebltxlO2lYnz6qOaDAVY6aNJ5bqHDE=", - "ref": "refs/heads/master", - "rev": "3160d9504d07791f2fc9b610948a6cf9a58ed488", - "revCount": 342, - "type": "git", - "url": "https://github.com/vacp2p/zerokit" + "lastModified": 1771279884, + "narHash": "sha256-tzkQPwSl4vPTUo1ixHh6NCENjsBDroMKTjifg2q8QX8=", + "owner": "vacp2p", + "repo": "zerokit", + "rev": "53b18098e6d5d046e3eb1ac338a8f4f651432477", + "type": "github" }, "original": { - "rev": "3160d9504d07791f2fc9b610948a6cf9a58ed488", - "type": "git", - "url": "https://github.com/vacp2p/zerokit" + "owner": "vacp2p", + "repo": "zerokit", + "rev": "53b18098e6d5d046e3eb1ac338a8f4f651432477", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 13ca5e618..57592722b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,92 +1,70 @@ { - description = "Logos Messaging Nim build flake"; + description = "logos-delivery nim build flake"; nixConfig = { extra-substituters = [ "https://nix-cache.status.im/" ]; - extra-trusted-public-keys = [ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" ]; + extra-trusted-public-keys = [ + "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" + ]; }; inputs = { - # Ensure Nix fetches git submodules (vendor/*) when evaluating this flake. - # Requires Nix >= 2.27. Consumers no longer need '?submodules=1' in the URL. - self.submodules = true; + # Pinning the commit to use same commit across different projects. + # A commit from nixpkgs 25.11 release: https://github.com/NixOS/nixpkgs/tree/release-25.11 + nixpkgs.url = "github:NixOS/nixpkgs?rev=23d72dabcb3b12469f57b37170fcbc1789bd7457"; - # We are pinning the commit because ultimately we want to use same commit across different projects. - # A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11 - nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293"; - # WARNING: Remember to update commit and use 'nix flake update' to update flake.lock. + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + # External flake input: Zerokit pinned to a specific commit. + # Update the rev here when a new zerokit version is needed. zerokit = { - url = "git+https://github.com/vacp2p/zerokit?rev=3160d9504d07791f2fc9b610948a6cf9a58ed488"; + url = "github:vacp2p/zerokit/53b18098e6d5d046e3eb1ac338a8f4f651432477"; inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = { self, nixpkgs, zerokit }: + outputs = { self, nixpkgs, rust-overlay, zerokit }: let - stableSystems = [ + systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" - "x86_64-windows" "i686-linux" - "i686-windows" + "x86_64-windows" ]; - forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system); + forAllSystems = nixpkgs.lib.genAttrs systems; - pkgsFor = forAllSystems ( - system: import nixpkgs { - inherit system; - config = { - android_sdk.accept_license = true; - allowUnfree = true; + pkgsFor = system: import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + in { + packages = forAllSystems (system: + let + pkgs = pkgsFor system; + mkPkg = zerokitRln: import ./nix/default.nix { + inherit pkgs zerokitRln; + src = ./.; }; - overlays = [ - (final: prev: { - androidEnvCustom = prev.callPackage ./nix/pkgs/android-sdk { }; - androidPkgs = final.androidEnvCustom.pkgs; - androidShell = final.androidEnvCustom.shell; - }) - ]; + in rec { + liblogosdelivery = mkPkg zerokit.packages.${system}.rln; + default = liblogosdelivery; } ); - in rec { - packages = forAllSystems (system: let - pkgs = pkgsFor.${system}; - in rec { - libwaku-android-arm64 = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["libwaku-android-arm64"]; - abidir = "arm64-v8a"; - zerokitRln = zerokit.packages.${system}.rln-android-arm64; - }; - - libwaku = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["libwaku"]; - zerokitRln = zerokit.packages.${system}.rln; - }; - - wakucanary = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["wakucanary"]; - zerokitRln = zerokit.packages.${system}.rln; - }; - - liblogosdelivery = pkgs.callPackage ./nix/default.nix { - inherit stableSystems; - src = self; - targets = ["liblogosdelivery"]; - zerokitRln = zerokit.packages.${system}.rln; - }; - - default = libwaku; - }); - - devShells = forAllSystems (system: { - default = pkgsFor.${system}.callPackage ./nix/shell.nix {}; - }); + devShells = forAllSystems (system: + let + pkgs = pkgsFor system; + in { + default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + nim-2_2 + nimble + ]; + }; + } + ); }; } diff --git a/nimble.lock b/nimble.lock new file mode 100644 index 000000000..96f64baf3 --- /dev/null +++ b/nimble.lock @@ -0,0 +1,551 @@ +{ + "version": 2, + "packages": { + "unittest2": { + "version": "0.2.5", + "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", + "url": "https://github.com/status-im/nim-unittest2", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" + } + }, + "bearssl": { + "version": "0.2.7", + "vcsRevision": "3b341f30d8c619b9a75c154243f9a55468a404e2", + "url": "https://github.com/status-im/nim-bearssl", + "downloadMethod": "git", + "dependencies": [ + "unittest2" + ], + "checksums": { + "sha1": "a85aab15b1b9a8b2438e9a128ac2eba41227da79" + } + }, + "bearssl_pkey_decoder": { + "version": "0.1.0", + "vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0", + "url": "https://github.com/vacp2p/bearssl_pkey_decoder", + "downloadMethod": "git", + "dependencies": [ + "bearssl" + ], + "checksums": { + "sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714" + } + }, + "results": { + "version": "0.5.1", + "vcsRevision": "df8113dda4c2d74d460a8fa98252b0b771bf1f27", + "url": "https://github.com/arnetheduck/nim-results", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "a9c011f74bc9ed5c91103917b9f382b12e82a9e7" + } + }, + "stew": { + "version": "0.5.0", + "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa", + "url": "https://github.com/status-im/nim-stew", + "downloadMethod": "git", + "dependencies": [ + "results", + "unittest2" + ], + "checksums": { + "sha1": "db22942939773ab7d5a0f2b2668c237240c67dd6" + } + }, + "faststreams": { + "version": "0.5.0", + "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e", + "url": "https://github.com/status-im/nim-faststreams", + "downloadMethod": "git", + "dependencies": [ + "stew", + "unittest2" + ], + "checksums": { + "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383" + } + }, + "serialization": { + "version": "0.5.2", + "vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b", + "url": "https://github.com/status-im/nim-serialization", + "downloadMethod": "git", + "dependencies": [ + "faststreams", + "unittest2", + "stew" + ], + "checksums": { + "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8" + } + }, + "json_serialization": { + "version": "0.4.4", + "vcsRevision": "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44", + "url": "https://github.com/status-im/nim-json-serialization", + "downloadMethod": "git", + "dependencies": [ + "faststreams", + "serialization", + "stew", + "results" + ], + "checksums": { + "sha1": "8b3115354104858a0ac9019356fb29720529c2bd" + } + }, + "testutils": { + "version": "0.8.1", + "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d", + "url": "https://github.com/status-im/nim-testutils", + "downloadMethod": "git", + "dependencies": [ + "unittest2" + ], + "checksums": { + "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" + } + }, + "chronicles": { + "version": "0.12.2", + "vcsRevision": "27ec507429a4eb81edc20f28292ee8ec420be05b", + "url": "https://github.com/status-im/nim-chronicles", + "downloadMethod": "git", + "dependencies": [ + "faststreams", + "serialization", + "json_serialization", + "testutils" + ], + "checksums": { + "sha1": "02febb20d088120b2836d3306cfa21f434f88f65" + } + }, + "httputils": { + "version": "0.4.1", + "vcsRevision": "f142cb2e8bd812dd002a6493b6082827bb248592", + "url": "https://github.com/status-im/nim-http-utils", + "downloadMethod": "git", + "dependencies": [ + "stew", + "results", + "unittest2" + ], + "checksums": { + "sha1": "016774ab31c3afff9a423f7d80584905ee59c570" + } + }, + "chronos": { + "version": "4.2.2", + "vcsRevision": "45f43a9ad8bd8bcf5903b42f365c1c879bd54240", + "url": "https://github.com/status-im/nim-chronos", + "downloadMethod": "git", + "dependencies": [ + "results", + "stew", + "bearssl", + "httputils", + "unittest2" + ], + "checksums": { + "sha1": "3a4c9477df8cef20a04e4f1b54a2d74fdfc2a3d0" + } + }, + "confutils": { + "version": "0.1.0", + "vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a", + "url": "https://github.com/status-im/nim-confutils", + "downloadMethod": "git", + "dependencies": [ + "stew", + "serialization", + "results" + ], + "checksums": { + "sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e" + } + }, + "db_connector": { + "version": "0.1.0", + "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", + "url": "https://github.com/nim-lang/db_connector", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + } + }, + "dnsclient": { + "version": "0.3.4", + "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", + "url": "https://github.com/ba0f3/dnsclient.nim", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" + } + }, + "nimcrypto": { + "version": "0.6.4", + "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", + "url": "https://github.com/cheatfate/nimcrypto", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" + } + }, + "stint": { + "version": "0.8.2", + "vcsRevision": "470b7892561b5179ab20bd389a69217d6213fe58", + "url": "https://github.com/status-im/nim-stint", + "downloadMethod": "git", + "dependencies": [ + "stew", + "unittest2" + ], + "checksums": { + "sha1": "d8f871fd617e7857192d4609fe003b48942a8ae5" + } + }, + "secp256k1": { + "version": "0.6.0.3.2", + "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", + "url": "https://github.com/status-im/nim-secp256k1", + "downloadMethod": "git", + "dependencies": [ + "stew", + "results", + "nimcrypto" + ], + "checksums": { + "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" + } + }, + "nat_traversal": { + "version": "0.0.1", + "vcsRevision": "860e18c37667b5dd005b94c63264560c35d88004", + "url": "https://github.com/status-im/nim-nat-traversal", + "downloadMethod": "git", + "dependencies": [ + "results" + ], + "checksums": { + "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97" + } + }, + "metrics": { + "version": "0.2.1", + "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2", + "url": "https://github.com/status-im/nim-metrics", + "downloadMethod": "git", + "dependencies": [ + "chronos", + "results", + "stew" + ], + "checksums": { + "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2" + } + }, + "sqlite3_abi": { + "version": "3.52.0.0", + "vcsRevision": "4b79c5e1882b7fc6c00aec311daf1ed50ad653d5", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "b56b489a7cb01eef8821d66d38d411923a14316d" + } + }, + "minilru": { + "version": "0.1.0", + "vcsRevision": "6dd93feb60f4cded3c05e7af7209cf63fb677893", + "url": "https://github.com/status-im/nim-minilru", + "downloadMethod": "git", + "dependencies": [ + "results", + "unittest2" + ], + "checksums": { + "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" + } + }, + "snappy": { + "version": "0.1.0", + "vcsRevision": "00bfcef94f8ef6981df5d5b994897f6695badfb2", + "url": "https://github.com/status-im/nim-snappy", + "downloadMethod": "git", + "dependencies": [ + "faststreams", + "unittest2", + "results", + "stew" + ], + "checksums": { + "sha1": "e572d60d6a3178c5b1cde2400c51ad771812cd3d" + } + }, + "eth": { + "version": "0.9.0", + "vcsRevision": "d9135e6c3c5d6d819afdfb566aa8d958756b73a8", + "url": "https://github.com/status-im/nim-eth", + "downloadMethod": "git", + "dependencies": [ + "nimcrypto", + "stint", + "secp256k1", + "chronos", + "chronicles", + "stew", + "nat_traversal", + "metrics", + "sqlite3_abi", + "confutils", + "testutils", + "unittest2", + "results", + "minilru", + "snappy" + ], + "checksums": { + "sha1": "2e01b0cfff9523d110562af70d19948280f8013e" + } + }, + "dnsdisc": { + "version": "0.1.0", + "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e", + "url": "https://github.com/status-im/nim-dnsdisc", + "downloadMethod": "git", + "dependencies": [ + "bearssl", + "chronicles", + "chronos", + "eth", + "secp256k1", + "stew", + "testutils", + "unittest2", + "nimcrypto", + "results" + ], + "checksums": { + "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154" + } + }, + "taskpools": { + "version": "0.1.0", + "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", + "url": "https://github.com/status-im/nim-taskpools", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" + } + }, + "ffi": { + "version": "0.1.3", + "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", + "url": "https://github.com/logos-messaging/nim-ffi", + "downloadMethod": "git", + "dependencies": [ + "chronos", + "chronicles", + "taskpools" + ], + "checksums": { + "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" + } + }, + "zlib": { + "version": "0.1.0", + "vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe", + "url": "https://github.com/status-im/nim-zlib", + "downloadMethod": "git", + "dependencies": [ + "stew", + "results" + ], + "checksums": { + "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f" + } + }, + "websock": { + "version": "0.2.2", + "vcsRevision": "3918ce3900c83e1cc7496232a307709f195f7acd", + "url": "https://github.com/status-im/nim-websock", + "downloadMethod": "git", + "dependencies": [ + "chronos", + "httputils", + "chronicles", + "stew", + "nimcrypto", + "bearssl", + "results", + "zlib" + ], + "checksums": { + "sha1": "3c424661eff56c925b01e1cd1a911ff744e72962" + } + }, + "json_rpc": { + "version": "0.5.4", + "vcsRevision": "b6e40a776fa2d00b97a9366761fb7da18f31ae5c", + "url": "https://github.com/status-im/nim-json-rpc", + "downloadMethod": "git", + "dependencies": [ + "stew", + "nimcrypto", + "stint", + "chronos", + "httputils", + "chronicles", + "websock", + "serialization", + "json_serialization", + "unittest2" + ], + "checksums": { + "sha1": "d8e8be795fcf098f4ce03b5826f6b3153f6a6e07" + } + }, + "jwt": { + "version": "0.2", + "vcsRevision": "18f8378de52b241f321c1f9ea905456e89b95c6f", + "url": "https://github.com/vacp2p/nim-jwt.git", + "downloadMethod": "git", + "dependencies": [ + "bearssl", + "bearssl_pkey_decoder" + ], + "checksums": { + "sha1": "bcfd6fc9c5e10a52b87117219b7ab5c98136bc8e" + } + }, + "lsquic": { + "version": "0.0.1", + "vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f", + "url": "https://github.com/vacp2p/nim-lsquic", + "downloadMethod": "git", + "dependencies": [ + "zlib", + "stew", + "chronos", + "nimcrypto", + "unittest2", + "chronicles" + ], + "checksums": { + "sha1": "f465fa994346490d0924d162f53d9b5aec62f948" + } + }, + "libp2p": { + "version": "1.15.2", + "vcsRevision": "ff8d51857b4b79a68468e7bcc27b2026cca02996", + "url": "https://github.com/vacp2p/nim-libp2p.git", + "downloadMethod": "git", + "dependencies": [ + "nimcrypto", + "dnsclient", + "bearssl", + "chronicles", + "chronos", + "metrics", + "secp256k1", + "stew", + "websock", + "unittest2", + "results", + "serialization", + "lsquic", + "jwt" + ], + "checksums": { + "sha1": "fa2a7552c6ec860717b77ce34cf0b7afe4570234" + } + }, + "presto": { + "version": "0.1.1", + "vcsRevision": "d66043dd7ede146442e6c39720c76a20bde5225f", + "url": "https://github.com/status-im/nim-presto", + "downloadMethod": "git", + "dependencies": [ + "chronos", + "chronicles", + "metrics", + "results", + "stew" + ], + "checksums": { + "sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2" + } + }, + "unicodedb": { + "version": "0.13.2", + "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", + "url": "https://github.com/nitely/nim-unicodedb", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" + } + }, + "regex": { + "version": "0.26.3", + "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", + "url": "https://github.com/nitely/nim-regex", + "downloadMethod": "git", + "dependencies": [ + "unicodedb" + ], + "checksums": { + "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" + } + }, + "toml_serialization": { + "version": "0.2.18", + "vcsRevision": "b5b387e6fb2a7cc75d54a269b07cc6218361bd46", + "url": "https://github.com/status-im/nim-toml-serialization", + "downloadMethod": "git", + "dependencies": [ + "faststreams", + "serialization", + "stew" + ], + "checksums": { + "sha1": "76ae1c2af5dd092849b41750ff29217980dc9ca3" + } + }, + "web3": { + "version": "0.8.0", + "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", + "url": "https://github.com/status-im/nim-web3", + "downloadMethod": "git", + "dependencies": [ + "chronicles", + "chronos", + "bearssl", + "eth", + "faststreams", + "json_rpc", + "serialization", + "json_serialization", + "nimcrypto", + "stew", + "stint", + "results" + ], + "checksums": { + "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" + } + } + }, + "tasks": {} +} diff --git a/nix/default.nix b/nix/default.nix index 816d0aed8..f90b8185e 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,149 +1,101 @@ -{ - pkgs, - src ? ../., - targets ? ["libwaku-android-arm64"], - verbosity ? 1, - useSystemNim ? true, - quickAndDirty ? true, - stableSystems ? [ - "x86_64-linux" "aarch64-linux" - ], - abidir ? null, - zerokitRln, -}: - -assert pkgs.lib.assertMsg (builtins.pathExists "${src}/vendor/nimbus-build-system/scripts") - "Unable to build without submodules. Append '?submodules=1#' to the URI."; +{ pkgs, src, zerokitRln }: let - inherit (pkgs) stdenv lib writeScriptBin callPackage; + deps = import ./deps.nix { inherit pkgs; }; - androidManifest = ""; + # nat_traversal is excluded from the static pathArgs; it is handled + # separately in buildPhase (its bundled C libs must be compiled first). + otherDeps = builtins.removeAttrs deps [ "nat_traversal" ]; - tools = pkgs.callPackage ./tools.nix {}; - version = tools.findKeyValue "^version = \"([a-f0-9.-]+)\"$" ../waku.nimble; - revision = lib.substring 0 8 (src.rev or src.dirtyRev or "00000000"); - copyLibwaku = lib.elem "libwaku" targets; - copyLiblogosdelivery = lib.elem "liblogosdelivery" targets; - copyWakunode2 = lib.elem "wakunode2" targets; - hasKnownInstallTarget = copyLibwaku || copyLiblogosdelivery || copyWakunode2; + # Some packages (e.g. regex, unicodedb) put their .nim files under src/ + # while others use the repo root. Pass both so the compiler finds either layout. + pathArgs = + builtins.concatStringsSep " " + (builtins.concatMap (p: [ "--path:${p}" "--path:${p}/src" ]) + (builtins.attrValues otherDeps)); -in stdenv.mkDerivation { - pname = "logos-messaging-nim"; - version = "${version}-${revision}"; + libExt = + if pkgs.stdenv.hostPlatform.isWindows then "dll" + else if pkgs.stdenv.hostPlatform.isDarwin then "dylib" + else "so"; +in +pkgs.stdenv.mkDerivation { + pname = "liblogosdelivery"; + version = "dev"; inherit src; - # Runtime dependencies - buildInputs = with pkgs; [ - openssl gmp zip - ]; + nativeBuildInputs = with pkgs; [ + nim-2_2 + git + gnumake + which + ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.cctools ]; - # Dependencies that should only exist in the build environment. - nativeBuildInputs = let - # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'. - fakeGit = writeScriptBin "git" "echo ${version}"; - in with pkgs; [ - cmake which zerokitRln nim-unwrapped-2_2 fakeGit - ] ++ lib.optionals stdenv.isDarwin [ - pkgs.darwin.cctools gcc # Necessary for libbacktrace - ]; + buildInputs = [ zerokitRln ]; - # Environment variables required for Android builds - ANDROID_SDK_ROOT = "${pkgs.androidPkgs.sdk}"; - ANDROID_NDK_HOME = "${pkgs.androidPkgs.ndk}"; - NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}"; - XDG_CACHE_HOME = "/tmp"; + buildPhase = '' + export HOME=$TMPDIR + export XDG_CACHE_HOME=$TMPDIR/.cache + export NIMBLE_DIR=$TMPDIR/.nimble + export NIMCACHE=$TMPDIR/nimcache - makeFlags = targets ++ [ - "V=${toString verbosity}" - "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}" - "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}" - "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}" - "LIBRLN_FILE=${zerokitRln}/lib/librln.${if abidir != null then "so" else "a"}" - "POSTGRES=1" - ]; + mkdir -p build $NIMCACHE - configurePhase = '' - patchShebangs . vendor/nimbus-build-system > /dev/null + # nat_traversal bundles C sub-libraries that must be compiled before linking. + # Copy the fetchgit store path to a writable tmpdir, build, then pass to nim. + NAT_TRAV=$TMPDIR/nat_traversal + cp -r ${deps.nat_traversal} $NAT_TRAV + chmod -R +w $NAT_TRAV - # build_nim.sh guards "rm -rf dist/checksums" with NIX_BUILD_TOP != "/build", - # but on macOS the nix sandbox uses /private/tmp/... so the check fails and - # dist/checksums (provided via preBuild) gets deleted. Fix the check to skip - # the removal whenever NIX_BUILD_TOP is set (i.e. any nix build). - substituteInPlace vendor/nimbus-build-system/scripts/build_nim.sh \ - --replace 'if [[ "''${NIX_BUILD_TOP}" != "/build" ]]; then' \ - 'if [[ -z "''${NIX_BUILD_TOP}" ]]; then' + make -C $NAT_TRAV/vendor/miniupnp/miniupnpc \ + CFLAGS="-Os -fPIC" build/libminiupnpc.a - make nimbus-build-system-paths - make nimbus-build-system-nimble-dir + make -C $NAT_TRAV/vendor/libnatpmp-upstream \ + CFLAGS="-Wall -Os -fPIC -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4" libnatpmp.a + + echo "== Building liblogosdelivery (dynamic) ==" + nim c \ + --noNimblePath \ + ${pathArgs} \ + --path:$NAT_TRAV \ + --path:$NAT_TRAV/src \ + --passL:"-L${zerokitRln}/lib -lrln" \ + --define:disable_libbacktrace \ + --out:build/liblogosdelivery.${libExt} \ + --app:lib \ + --threads:on \ + --opt:size \ + --noMain \ + --mm:refc \ + --header \ + --nimMainPrefix:liblogosdelivery \ + --nimcache:$NIMCACHE \ + liblogosdelivery/liblogosdelivery.nim + + echo "== Building liblogosdelivery (static) ==" + nim c \ + --noNimblePath \ + ${pathArgs} \ + --path:$NAT_TRAV \ + --path:$NAT_TRAV/src \ + --passL:"-L${zerokitRln}/lib -lrln" \ + --define:disable_libbacktrace \ + --out:build/liblogosdelivery.a \ + --app:staticlib \ + --threads:on \ + --opt:size \ + --noMain \ + --mm:refc \ + --nimMainPrefix:liblogosdelivery \ + --nimcache:$NIMCACHE \ + liblogosdelivery/liblogosdelivery.nim ''; - # For the Nim v2.2.4 built with NBS we added sat and zippy - preBuild = lib.optionalString (!useSystemNim) '' - pushd vendor/nimbus-build-system/vendor/Nim - mkdir dist - mkdir -p dist/nimble/vendor/sat - mkdir -p dist/nimble/vendor/checksums - mkdir -p dist/nimble/vendor/zippy - - cp -r ${callPackage ./nimble.nix {}}/. dist/nimble - cp -r ${callPackage ./checksums.nix {}}/. dist/checksums - cp -r ${callPackage ./csources.nix {}}/. csources_v2 - cp -r ${callPackage ./sat.nix {}}/. dist/nimble/vendor/sat - cp -r ${callPackage ./checksums.nix {}}/. dist/nimble/vendor/checksums - cp -r ${callPackage ./zippy.nix {}}/. dist/nimble/vendor/zippy - chmod 777 -R dist/nimble csources_v2 - popd - ''; - - installPhase = if abidir != null then '' - mkdir -p $out/jni - cp -r ./build/android/${abidir}/* $out/jni/ - echo '${androidManifest}' > $out/jni/AndroidManifest.xml - cd $out && zip -r libwaku.aar * - '' else '' - mkdir -p $out/bin $out/include - - # Copy artifacts from build directory (created by Make during buildPhase) - # Note: build/ is in the source tree, not result/ (which is a post-build symlink) - if [ -d build ]; then - ${lib.optionalString copyLibwaku '' - cp build/libwaku.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true - ''} - - ${lib.optionalString copyLiblogosdelivery '' - cp build/liblogosdelivery.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true - ''} - - ${lib.optionalString copyWakunode2 '' - cp build/wakunode2 $out/bin/ 2>/dev/null || true - ''} - - ${lib.optionalString (!hasKnownInstallTarget) '' - cp build/lib*.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true - ''} - fi - - # Copy header files - ${lib.optionalString copyLibwaku '' - cp library/libwaku.h $out/include/ 2>/dev/null || true - ''} - - ${lib.optionalString copyLiblogosdelivery '' + installPhase = '' + mkdir -p $out/lib $out/include + cp build/liblogosdelivery.${libExt} $out/lib/ 2>/dev/null || true + cp build/liblogosdelivery.a $out/lib/ 2>/dev/null || true cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true - ''} - - ${lib.optionalString (!hasKnownInstallTarget) '' - cp library/libwaku.h $out/include/ 2>/dev/null || true - cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true - ''} ''; - - meta = with pkgs.lib; { - description = "NWaku derivation to build libwaku for mobile targets using Android NDK and Rust."; - homepage = "https://github.com/status-im/nwaku"; - license = licenses.mit; - platforms = stableSystems; - }; } diff --git a/nix/deps.nix b/nix/deps.nix new file mode 100644 index 000000000..2f30a572c --- /dev/null +++ b/nix/deps.nix @@ -0,0 +1,272 @@ +# AUTOGENERATED from nimble.lock — do not edit manually. +# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix +{ pkgs }: + +{ + unittest2 = pkgs.fetchgit { + url = "https://github.com/status-im/nim-unittest2"; + rev = "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189"; + sha256 = "1n8n36kad50m97b64y7bzzknz9n7szffxhp0bqpk3g2v7zpda8sw"; + fetchSubmodules = true; + }; + + bearssl = pkgs.fetchgit { + url = "https://github.com/status-im/nim-bearssl"; + rev = "3b341f30d8c619b9a75c154243f9a55468a404e2"; + sha256 = "059avc2dh39vv9c3a1qayah98fjm5pw04r3dn2bqrgs6vf7licmv"; + fetchSubmodules = true; + }; + + bearssl_pkey_decoder = pkgs.fetchgit { + url = "https://github.com/vacp2p/bearssl_pkey_decoder"; + rev = "21dd3710df9345ed2ad8bf8f882761e07863b8e0"; + sha256 = "0bl3f147zmkazbhdkr4cj1nipf9rqiw3g4hh1j424k9hpl55zdpg"; + fetchSubmodules = true; + }; + + results = pkgs.fetchgit { + url = "https://github.com/arnetheduck/nim-results"; + rev = "df8113dda4c2d74d460a8fa98252b0b771bf1f27"; + sha256 = "1h7amas16sbhlr7zb7n3jb5434k98ji375vzw72k1fsc86vnmcr9"; + fetchSubmodules = true; + }; + + stew = pkgs.fetchgit { + url = "https://github.com/status-im/nim-stew"; + rev = "4382b18f04b3c43c8409bfcd6b62063773b2bbaa"; + sha256 = "0mx9g5m636h3sk5pllcpylk51brf7lx91izx3gc23k3ih3hrxyk2"; + fetchSubmodules = true; + }; + + faststreams = pkgs.fetchgit { + url = "https://github.com/status-im/nim-faststreams"; + rev = "ce27581a3e881f782f482cb66dc5b07a02bd615e"; + sha256 = "0y6bw2scnmr8cxj4fg18w7f34l2bh9qwg5nhlgd84m9fpr5bqarn"; + fetchSubmodules = true; + }; + + serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-serialization"; + rev = "b0f2fa32960ea532a184394b0f27be37bd80248b"; + sha256 = "0wip1fjx7ka39ck1g1xvmyarzq1p5dlngpqil6zff8k8z5skiz27"; + fetchSubmodules = true; + }; + + json_serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-json-serialization"; + rev = "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44"; + sha256 = "0i8sq51nqj8lshf6bfixaz9a7sq0ahsbvq3chkxdvv4khsqvam91"; + fetchSubmodules = true; + }; + + testutils = pkgs.fetchgit { + url = "https://github.com/status-im/nim-testutils"; + rev = "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d"; + sha256 = "1vbkr6i5yxhc2ai3b7rbglhmyc98f99x874fqdp6a152a6kqgwxy"; + fetchSubmodules = true; + }; + + chronicles = pkgs.fetchgit { + url = "https://github.com/status-im/nim-chronicles"; + rev = "27ec507429a4eb81edc20f28292ee8ec420be05b"; + sha256 = "1xx9fcfwgcaizq3s7i3s03mclz253r5j8va38l9ycl19fcbc96z9"; + fetchSubmodules = true; + }; + + httputils = pkgs.fetchgit { + url = "https://github.com/status-im/nim-http-utils"; + rev = "f142cb2e8bd812dd002a6493b6082827bb248592"; + sha256 = "03msj4zdxraz4qx9cidb17g7v0asazxv91nng6xxbzjxz0qaqxw6"; + fetchSubmodules = true; + }; + + chronos = pkgs.fetchgit { + url = "https://github.com/status-im/nim-chronos"; + rev = "45f43a9ad8bd8bcf5903b42f365c1c879bd54240"; + sha256 = "1v1n59zfzznp97pvwgs9kf136bqmv4x2s2y9f24msspa7qv27w39"; + fetchSubmodules = true; + }; + + confutils = pkgs.fetchgit { + url = "https://github.com/status-im/nim-confutils"; + rev = "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a"; + sha256 = "18bj1ilx10jm2vmqx2wy2xl9rzy7alymi2m4n9jgpa4sbxnfh0x3"; + fetchSubmodules = true; + }; + + db_connector = pkgs.fetchgit { + url = "https://github.com/nim-lang/db_connector"; + rev = "29450a2063970712422e1ab857695c12d80112a6"; + sha256 = "11dna09ccdhj3pzpqa04j7a95ibx907z6n1ff33yf0n92qa4x59z"; + fetchSubmodules = true; + }; + + dnsclient = pkgs.fetchgit { + url = "https://github.com/ba0f3/dnsclient.nim"; + rev = "23214235d4784d24aceed99bbfe153379ea557c8"; + sha256 = "03mf3lw5c0m5nq9ppa49nylrl8ibkv2zzlc0wyhqg7w09kz6hks6"; + fetchSubmodules = true; + }; + + nimcrypto = pkgs.fetchgit { + url = "https://github.com/cheatfate/nimcrypto"; + rev = "721fb99ee099b632eb86dfad1f0d96ee87583774"; + sha256 = "178vzb3q8wzjq295ik2pd25rrqf32w381ck76hm5x2d8qnzfmkkc"; + fetchSubmodules = true; + }; + + stint = pkgs.fetchgit { + url = "https://github.com/status-im/nim-stint"; + rev = "470b7892561b5179ab20bd389a69217d6213fe58"; + sha256 = "1isfwmbj98qfi5pm9acy0yyvq0vlz38nxp30xl43jx2mmaga2w22"; + fetchSubmodules = true; + }; + + secp256k1 = pkgs.fetchgit { + url = "https://github.com/status-im/nim-secp256k1"; + rev = "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15"; + sha256 = "1qjrmwbngb73f6r1fznvig53nyal7wj41d1cmqfksrmivk2sgrn2"; + fetchSubmodules = true; + }; + + nat_traversal = pkgs.fetchgit { + url = "https://github.com/status-im/nim-nat-traversal"; + rev = "860e18c37667b5dd005b94c63264560c35d88004"; + sha256 = "0319k5bbl468phwfnvlrh7725sc80rnf7m9gyj0i3cb5hb9q78bs"; + fetchSubmodules = true; + }; + + metrics = pkgs.fetchgit { + url = "https://github.com/status-im/nim-metrics"; + rev = "a1296caf3ebb5f30f51a5feae7749a30df2824c2"; + sha256 = "02vxqy20g8012ks939ac25ksc25k727q84si0p2cmihy5bw1a3qm"; + fetchSubmodules = true; + }; + + sqlite3_abi = pkgs.fetchgit { + url = "https://github.com/arnetheduck/nim-sqlite3-abi"; + rev = "4b79c5e1882b7fc6c00aec311daf1ed50ad653d5"; + sha256 = "0qa6p2vnxmf6r2w19mfydr5rzv7bg1lfxccnpdhk0akzxnc7i5gy"; + fetchSubmodules = true; + }; + + minilru = pkgs.fetchgit { + url = "https://github.com/status-im/nim-minilru"; + rev = "6dd93feb60f4cded3c05e7af7209cf63fb677893"; + sha256 = "1xgx4j56ais3hk8b51zhnfs9q85g2afkp3y1j9ky5iziqvcs2sml"; + fetchSubmodules = true; + }; + + snappy = pkgs.fetchgit { + url = "https://github.com/status-im/nim-snappy"; + rev = "00bfcef94f8ef6981df5d5b994897f6695badfb2"; + sha256 = "117mam97mkjjj1hs8svc07679k5ayww9yigi74yq8dyqm6fpbl6l"; + fetchSubmodules = true; + }; + + eth = pkgs.fetchgit { + url = "https://github.com/status-im/nim-eth"; + rev = "d9135e6c3c5d6d819afdfb566aa8d958756b73a8"; + sha256 = "15r6aszalnbk6mkyfbv5rnz5vcf1mmgj6yg332wry53xsd2ipg7r"; + fetchSubmodules = true; + }; + + dnsdisc = pkgs.fetchgit { + url = "https://github.com/status-im/nim-dnsdisc"; + rev = "38f2e0f52c0a8f032ef4530835e519d550706d9e"; + sha256 = "0dk787ny49n41bmzhlrvm87giwajr01gwdw9nlmphch89rdqpxxn"; + fetchSubmodules = true; + }; + + taskpools = pkgs.fetchgit { + url = "https://github.com/status-im/nim-taskpools"; + rev = "9e8ccc754631ac55ac2fd495e167e74e86293edb"; + sha256 = "1y78l33vdjxmb9dkr455pbphxa73rgdsh8m9gpkf4d9b1wm1yivy"; + fetchSubmodules = true; + }; + + ffi = pkgs.fetchgit { + url = "https://github.com/logos-messaging/nim-ffi"; + rev = "06111de155253b34e47ed2aaed1d61d08d62cc1b"; + sha256 = "0rb0d2i519amgsp7q0bn6m5465z1vwj4rab89529pyiivh3fgh8j"; + fetchSubmodules = true; + }; + + zlib = pkgs.fetchgit { + url = "https://github.com/status-im/nim-zlib"; + rev = "e680f269fb01af2c34a2ba879ff281795a5258fe"; + sha256 = "1xw9f1gjsgqihdg7kdkbaq1wankgnx2vn9l3ihc6nqk2jzv5bvk5"; + fetchSubmodules = true; + }; + + websock = pkgs.fetchgit { + url = "https://github.com/status-im/nim-websock"; + rev = "3918ce3900c83e1cc7496232a307709f195f7acd"; + sha256 = "16zvdjyasfpb04708d072rpvg12pyz3gmszi3md5brmlhbc3x8jp"; + fetchSubmodules = true; + }; + + json_rpc = pkgs.fetchgit { + url = "https://github.com/status-im/nim-json-rpc"; + rev = "b6e40a776fa2d00b97a9366761fb7da18f31ae5c"; + sha256 = "0c86glijpzcxdb5fagdk98hm9dmsrgw179nn3ixbapl48pvly9nr"; + fetchSubmodules = true; + }; + + jwt = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-jwt.git"; + rev = "18f8378de52b241f321c1f9ea905456e89b95c6f"; + sha256 = "1986czmszdxj6g9yr7xn1fx8y2y9mwpb3f1bn9nc6973qawsdm0p"; + fetchSubmodules = true; + }; + + lsquic = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-lsquic"; + rev = "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f"; + sha256 = "0qdhcd4hyp185szc9sv3jvwdwc9zp3j0syy7glxv13k9bchfmkfg"; + fetchSubmodules = true; + }; + + libp2p = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-libp2p.git"; + rev = "ff8d51857b4b79a68468e7bcc27b2026cca02996"; + sha256 = "08y4s0zhqzsd780bwaixfqbi79km0mcq5g8nyw7awfvcbjqsa53l"; + fetchSubmodules = true; + }; + + presto = pkgs.fetchgit { + url = "https://github.com/status-im/nim-presto"; + rev = "d66043dd7ede146442e6c39720c76a20bde5225f"; + sha256 = "1hrppcak32aigrdv3mqk124w81yy9jv1prs57vqqhfj83gl930vi"; + fetchSubmodules = true; + }; + + unicodedb = pkgs.fetchgit { + url = "https://github.com/nitely/nim-unicodedb"; + rev = "66f2458710dc641dd4640368f9483c8a0ec70561"; + sha256 = "092z3glgdb7rmwajm7dmqzvralkm7ixighixk8ycf8sf17zm72ck"; + fetchSubmodules = true; + }; + + regex = pkgs.fetchgit { + url = "https://github.com/nitely/nim-regex"; + rev = "4593305ed1e49731fc75af1dc572dd2559aad19c"; + sha256 = "1b666qws5sva3n5allin0ycvnqlzdjd7xzprpdvv632ccqddzcl9"; + fetchSubmodules = true; + }; + + toml_serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-toml-serialization"; + rev = "b5b387e6fb2a7cc75d54a269b07cc6218361bd46"; + sha256 = "175swdj01rz57h1hvflkyaz4x76qbfn0174ysrk3qk385i1zlg5z"; + fetchSubmodules = true; + }; + + web3 = pkgs.fetchgit { + url = "https://github.com/status-im/nim-web3"; + rev = "cdfe5601d2812a58e54faf53ee634452d01e5918"; + sha256 = "1j52vcqw868qs40bh4wzfw5cvvnywp2q0dnzhfajh31xws98jc27"; + fetchSubmodules = true; + }; + +} diff --git a/nix/nimble.nix b/nix/nimble.nix deleted file mode 100644 index 337ecd672..000000000 --- a/nix/nimble.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: - -let - tools = pkgs.callPackage ./tools.nix {}; - sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; -in pkgs.fetchFromGitHub { - owner = "nim-lang"; - repo = "nimble"; - rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".*$" sourceFile; - # WARNING: Requires manual updates when Nim compiler version changes. - hash = "sha256-8iutVgNzDtttZ7V+7S11KfLEuwhKA9TsgS51mlUI08k="; -} diff --git a/nix/shell.nix b/nix/shell.nix index 3b83ac93d..80e3b7930 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -14,6 +14,7 @@ pkgs.mkShell { rustup rustc cmake - nim-unwrapped-2_2 + nim-2_2 + nimble ]; } diff --git a/tests/tools/test_confutils_envvar.nim b/tests/tools/test_confutils_envvar.nim index ed559ad0b..76d9ddd31 100644 --- a/tests/tools/test_confutils_envvar.nim +++ b/tests/tools/test_confutils_envvar.nim @@ -19,7 +19,7 @@ type TestConf = object Option[InputFile] listenAddress* {. - defaultValue: parseIpAddress("127.0.0.1"), + defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127u8, 0, 0, 1]), desc: "Listening address", name: "listen-address" .}: IpAddress @@ -62,9 +62,15 @@ suite "nim-confutils - envvar": ## Then check confLoadRes.isOk() + let parsedIpAddress = + try: + parseIpAddress(listenAddress) + except ValueError: + IpAddress(family: IpAddressFamily.IPv4, address_v4: [0u8, 0, 0, 0]) + let conf = confLoadRes.get() check: - conf.listenAddress == parseIpAddress(listenAddress) + conf.listenAddress == parsedIpAddress conf.tcpPort == Port(8080) conf.configFile.isSome() diff --git a/tests/waku_core/test_peers.nim b/tests/waku_core/test_peers.nim index 59ae2e2f3..0ba3e5b04 100644 --- a/tests/waku_core/test_peers.nim +++ b/tests/waku_core/test_peers.nim @@ -1,5 +1,6 @@ {.used.} +import std/options import results, testutils/unittests, diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index a99ba43ee..5c1934712 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -251,7 +251,10 @@ type WakuNodeConf* = object dnsAddrsNameServers* {. desc: "DNS name server IPs to query for DNS multiaddrs resolution. Argument may be repeated.", - defaultValue: @[parseIpAddress("1.1.1.1"), parseIpAddress("1.0.0.1")], + defaultValue: @[ + IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1]), + IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 0, 0, 1]), + ], name: "dns-addrs-name-server" .}: seq[IpAddress] @@ -480,7 +483,8 @@ with the drawback of consuming some more bandwidth.""", restAddress* {. desc: "Listening address of the REST HTTP server.", - defaultValue: parseIpAddress("127.0.0.1"), + defaultValue: + IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]), name: "rest-address" .}: IpAddress @@ -520,7 +524,8 @@ with the drawback of consuming some more bandwidth.""", metricsServerAddress* {. desc: "Listening address of the metrics server.", - defaultValue: parseIpAddress("127.0.0.1"), + defaultValue: + IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]), name: "metrics-server-address" .}: IpAddress @@ -774,7 +779,7 @@ proc completeCmdArg*(T: type IpAddress, val: string): seq[string] = proc defaultListenAddress*(): IpAddress = # TODO: Should probably listen on both ipv4 and ipv6 by default. - (static parseIpAddress("0.0.0.0")) + (static IpAddress(family: IpAddressFamily.IPv4, address_v4: [0'u8, 0, 0, 0])) proc defaultColocationLimit*(): int = return DefaultColocationLimit diff --git a/tools/gen-nix-deps.sh b/tools/gen-nix-deps.sh new file mode 100755 index 000000000..9bb43e638 --- /dev/null +++ b/tools/gen-nix-deps.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Generates nix/deps.nix from nimble.lock using nix-prefetch-git. +# Usage: ./tools/gen-nix-deps.sh [nimble.lock] [nix/deps.nix] +set -euo pipefail + +usage() { + cat < + +Example: + $0 nimble.lock nix/deps.nix +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage; exit 0 +fi + +if [[ $# -ne 2 ]]; then + usage; exit 1 +fi + +LOCKFILE="$1" +OUTFILE="$2" + +command -v jq >/dev/null || { echo "error: jq required"; exit 1; } +command -v nix-prefetch-git >/dev/null || { echo "error: nix-prefetch-git required"; exit 1; } + +if [[ ! -f "$LOCKFILE" ]]; then + echo "[!] $LOCKFILE not found" + echo "[*] Generating $LOCKFILE via 'nimble lock'" + nimble lock +fi + +echo "[*] Generating $OUTFILE from $LOCKFILE" +mkdir -p "$(dirname "$OUTFILE")" + +cat > "$OUTFILE" <<'EOF' +# AUTOGENERATED from nimble.lock — do not edit manually. +# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix +{ pkgs }: + +{ +EOF + +jq -c ' + .packages + | to_entries[] + | select(.value.downloadMethod == "git") + | select(.key != "nim" and .key != "nimble") +' "$LOCKFILE" | while read -r entry; do + name=$(jq -r '.key' <<<"$entry") + url=$(jq -r '.value.url' <<<"$entry") + rev=$(jq -r '.value.vcsRevision' <<<"$entry") + + echo " [*] Prefetching $name @ $rev" + + sha=$(nix-prefetch-git \ + --url "$url" \ + --rev "$rev" \ + --fetch-submodules \ + | jq -r '.sha256') + + cat >> "$OUTFILE" <> "$OUTFILE" <<'EOF' +} +EOF + +echo "[✓] Wrote $OUTFILE" diff --git a/vendor/db_connector b/vendor/db_connector deleted file mode 160000 index 74aef399e..000000000 --- a/vendor/db_connector +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 74aef399e5c232f95c9fc5c987cebac846f09d62 diff --git a/vendor/dnsclient.nim b/vendor/dnsclient.nim deleted file mode 160000 index 23214235d..000000000 --- a/vendor/dnsclient.nim +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 23214235d4784d24aceed99bbfe153379ea557c8 diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl deleted file mode 160000 index 11e798b62..000000000 --- a/vendor/nim-bearssl +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 11e798b62b8e6beabe958e048e9e24c7e0f9ee63 diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles deleted file mode 160000 index 54f5b7260..000000000 --- a/vendor/nim-chronicles +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 54f5b726025e8c7385e3a6529d3aa27454c6e6ff diff --git a/vendor/nim-chronos b/vendor/nim-chronos deleted file mode 160000 index 85af4db76..000000000 --- a/vendor/nim-chronos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 85af4db764ecd3573c4704139560df3943216cf1 diff --git a/vendor/nim-confutils b/vendor/nim-confutils deleted file mode 160000 index e214b3992..000000000 --- a/vendor/nim-confutils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e214b3992a31acece6a9aada7d0a1ad37c928f3b diff --git a/vendor/nim-dnsdisc b/vendor/nim-dnsdisc deleted file mode 160000 index 203abd2b3..000000000 --- a/vendor/nim-dnsdisc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 203abd2b3e758e0ea3ae325769b20a7e1bcd1010 diff --git a/vendor/nim-eth b/vendor/nim-eth deleted file mode 160000 index d9135e6c3..000000000 --- a/vendor/nim-eth +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d9135e6c3c5d6d819afdfb566aa8d958756b73a8 diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams deleted file mode 160000 index ce27581a3..000000000 --- a/vendor/nim-faststreams +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ce27581a3e881f782f482cb66dc5b07a02bd615e diff --git a/vendor/nim-ffi b/vendor/nim-ffi deleted file mode 160000 index 06111de15..000000000 --- a/vendor/nim-ffi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 06111de155253b34e47ed2aaed1d61d08d62cc1b diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils deleted file mode 160000 index f142cb2e8..000000000 --- a/vendor/nim-http-utils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f142cb2e8bd812dd002a6493b6082827bb248592 diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc deleted file mode 160000 index 9665c2650..000000000 --- a/vendor/nim-json-rpc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9665c265035f49f5ff94bbffdeadde68e19d6221 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization deleted file mode 160000 index c343b0e24..000000000 --- a/vendor/nim-json-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c343b0e243d9e17e2c40f3a8a24340f7c4a71d44 diff --git a/vendor/nim-jwt b/vendor/nim-jwt deleted file mode 160000 index 18f8378de..000000000 --- a/vendor/nim-jwt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 18f8378de52b241f321c1f9ea905456e89b95c6f diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace deleted file mode 160000 index d8bd4ce5c..000000000 --- a/vendor/nim-libbacktrace +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d8bd4ce5c46bb6d2f984f6b3f3d7380897d95ecb diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p deleted file mode 160000 index ff8d51857..000000000 --- a/vendor/nim-libp2p +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ff8d51857b4b79a68468e7bcc27b2026cca02996 diff --git a/vendor/nim-lsquic b/vendor/nim-lsquic deleted file mode 160000 index 4fb03ee7b..000000000 --- a/vendor/nim-lsquic +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4fb03ee7bfb39aecb3316889fdcb60bec3d0936f diff --git a/vendor/nim-metrics b/vendor/nim-metrics deleted file mode 160000 index a1296caf3..000000000 --- a/vendor/nim-metrics +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a1296caf3ebb5f30f51a5feae7749a30df2824c2 diff --git a/vendor/nim-minilru b/vendor/nim-minilru deleted file mode 160000 index 0c4b2bce9..000000000 --- a/vendor/nim-minilru +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0c4b2bce959591f0a862e9b541ba43c6d0cf3476 diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal deleted file mode 160000 index 860e18c37..000000000 --- a/vendor/nim-nat-traversal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 860e18c37667b5dd005b94c63264560c35d88004 diff --git a/vendor/nim-presto b/vendor/nim-presto deleted file mode 160000 index d66043dd7..000000000 --- a/vendor/nim-presto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d66043dd7ede146442e6c39720c76a20bde5225f diff --git a/vendor/nim-regex b/vendor/nim-regex deleted file mode 160000 index 4593305ed..000000000 --- a/vendor/nim-regex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4593305ed1e49731fc75af1dc572dd2559aad19c diff --git a/vendor/nim-results b/vendor/nim-results deleted file mode 160000 index df8113dda..000000000 --- a/vendor/nim-results +++ /dev/null @@ -1 +0,0 @@ -Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27 diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1 deleted file mode 160000 index 9dd3df621..000000000 --- a/vendor/nim-secp256k1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9dd3df62124aae79d564da636bb22627c53c7676 diff --git a/vendor/nim-serialization b/vendor/nim-serialization deleted file mode 160000 index b0f2fa329..000000000 --- a/vendor/nim-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b0f2fa32960ea532a184394b0f27be37bd80248b diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi deleted file mode 160000 index 89ba51f55..000000000 --- a/vendor/nim-sqlite3-abi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 89ba51f557414d3a3e17ab3df8270e1bdaa3ca2a diff --git a/vendor/nim-stew b/vendor/nim-stew deleted file mode 160000 index b66168735..000000000 --- a/vendor/nim-stew +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b66168735d6f3841c5239c3169d3fe5fe98b1257 diff --git a/vendor/nim-stint b/vendor/nim-stint deleted file mode 160000 index 470b78925..000000000 --- a/vendor/nim-stint +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 470b7892561b5179ab20bd389a69217d6213fe58 diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools deleted file mode 160000 index 9e8ccc754..000000000 --- a/vendor/nim-taskpools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9e8ccc754631ac55ac2fd495e167e74e86293edb diff --git a/vendor/nim-testutils b/vendor/nim-testutils deleted file mode 160000 index e4d37dc16..000000000 --- a/vendor/nim-testutils +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e4d37dc1652d5c63afb89907efb5a5e812261797 diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization deleted file mode 160000 index b5b387e6f..000000000 --- a/vendor/nim-toml-serialization +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b5b387e6fb2a7cc75d54a269b07cc6218361bd46 diff --git a/vendor/nim-unicodedb b/vendor/nim-unicodedb deleted file mode 160000 index 66f245871..000000000 --- a/vendor/nim-unicodedb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 66f2458710dc641dd4640368f9483c8a0ec70561 diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2 deleted file mode 160000 index 26f2ef3ae..000000000 --- a/vendor/nim-unittest2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189 diff --git a/vendor/nim-web3 b/vendor/nim-web3 deleted file mode 160000 index 81ee8ce47..000000000 --- a/vendor/nim-web3 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 81ee8ce479d86acb73be7c4f365328e238d9b4a3 diff --git a/vendor/nim-websock b/vendor/nim-websock deleted file mode 160000 index 35ae76f15..000000000 --- a/vendor/nim-websock +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 35ae76f1559e835c80f9c1a3943bf995d3dd9eb5 diff --git a/vendor/nim-zlib b/vendor/nim-zlib deleted file mode 160000 index daa8723fd..000000000 --- a/vendor/nim-zlib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit daa8723fd32299d4ca621c837430c29a5a11e19a diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system deleted file mode 160000 index e6c2c9da3..000000000 --- a/vendor/nimbus-build-system +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e6c2c9da39c2d368d9cf420ac22692e99715d22c diff --git a/vendor/nimcrypto b/vendor/nimcrypto deleted file mode 160000 index 721fb99ee..000000000 --- a/vendor/nimcrypto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 721fb99ee099b632eb86dfad1f0d96ee87583774 diff --git a/vendor/nph b/vendor/nph deleted file mode 160000 index 2cacf6cc2..000000000 --- a/vendor/nph +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2cacf6cc28116e4046e0b67a13545af5c4e756bd diff --git a/waku.nimble b/waku.nimble index cbbe6aa97..a6e824528 100644 --- a/waku.nimble +++ b/waku.nimble @@ -4,81 +4,161 @@ import os mode = ScriptMode.Verbose ### Package -version = "0.36.0" +version = "0.37.4" author = "Status Research & Development GmbH" description = "Waku, Private P2P Messaging for Resource-Restricted Devices" license = "MIT or Apache License 2.0" #bin = @["build/waku"] +## This indicates the nim compiler version we are currently working on. It may compile with others +## but we haven't tested. +const NimVersion = "2.2.4" +## This is the underlying nimble version that gets installed after doing `choosenim 2.2.4`. +const NimbleVersion = "0.18.2" + ### Dependencies requires "nim >= 2.2.4", + "chronos >= 4.2.0", + "taskpools", + # Logging & Configuration "chronicles", "confutils", - "chronos", - "dnsdisc", - "eth", - "json_rpc", - "libbacktrace", - "nimcrypto", + # Serialization "serialization", + "json_serialization", + "toml_serialization", + "faststreams", + # Networking & P2P + "https://github.com/vacp2p/nim-libp2p.git#ff8d51857b4b79a68468e7bcc27b2026cca02996", + "eth", + "nat_traversal", + "dnsdisc", + "dnsclient", + "httputils >= 0.4.1", + "websock >= 0.2.1", + # Cryptography + "nimcrypto == 0.6.4", # 0.6.4 used in libp2p. Version 0.7.3 makes test to crash on Ubuntu. + "secp256k1", + "bearssl", + # RPC & APIs + "json_rpc", + "presto", + "web3", + # Database + "db_connector", + "sqlite3_abi", + # Utilities "stew", "stint", "metrics", - "libp2p >= 1.15.0", - "web3", - "presto", "regex", + "unicodedb", "results", - "db_connector", "minilru", - "lsquic", - "jwt", - "ffi" + "zlib", + # Debug & Testing + "testutils", + "unittest2" + +# Packages not on nimble (use git URLs) +requires "https://github.com/logos-messaging/nim-ffi" + +requires "https://github.com/vacp2p/nim-lsquic" +requires "https://github.com/vacp2p/nim-jwt.git#18f8378de52b241f321c1f9ea905456e89b95c6f" + +proc getMyCPU(): string = + ## Need to set cpu more explicit manner to avoid arch issues between dependencies + when defined(macosx) and defined(arm64): + return " --cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" " + elif defined(macosx) and defined(amd64): + return " --cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" " + elif defined(arm64): + return " --cpu:arm64 " + elif defined(amd64): + return " --cpu:amd64 " + +proc getNimParams(): string = + return " " & getEnv("NIM_PARAMS") & " " ### Helper functions -proc buildModule(filePath, params = "", lang = "c"): bool = +proc buildModule(filePath, params = ""): bool = if not dirExists "build": mkDir "build" - # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims" - var extra_params = params - for i in 2 ..< paramCount() - 1: - extra_params &= " " & paramStr(i) if not fileExists(filePath): echo "File to build not found: " & filePath return false - exec "nim " & lang & " --out:build/" & filepath & ".bin --mm:refc " & extra_params & + exec "nim c --out:build/" & filepath & ".bin --mm:refc " & getMyCPU() & getNimParams() & " " & params & " " & filePath # exec will raise exception if anything goes wrong return true -proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = +proc buildBinary(name: string, srcDir = "./", params = "") = if not dirExists "build": mkDir "build" - # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims" - var extra_params = params - for i in 2 ..< paramCount(): - extra_params &= " " & paramStr(i) - exec "nim " & lang & " --out:build/" & name & " --mm:refc " & extra_params & " " & + exec "nim c --out:build/" & name & " --mm:refc " & getMyCPU() & getNimParams() & " " & params & " " & srcDir & name & ".nim" proc buildLibrary(lib_name: string, srcDir = "./", params = "", `type` = "static", srcFile = "libwaku.nim", mainPrefix = "libwaku") = if not dirExists "build": mkDir "build" - # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims" - var extra_params = params - for i in 2 ..< (paramCount() - 1): - extra_params &= " " & paramStr(i) + if `type` == "static": exec "nim c" & " --out:build/" & lib_name & " --threads:on --app:staticlib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:on -d:discv5_protocol_id=d5waku " & - extra_params & " " & srcDir & srcFile + getMyCPU() & getNimParams() & srcDir & "/" & srcFile else: exec "nim c" & " --out:build/" & lib_name & " --threads:on --app:lib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:off -d:discv5_protocol_id=d5waku " & - extra_params & " " & srcDir & srcFile + getMyCPU() & getNimParams() & " " & srcDir & "/" & srcFile + +proc buildLibDynamicWindows(libName: string, folderName: string) = + buildLibrary libName & ".dll", folderName, + """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, + "dynamic", libName & ".nim", libname + +proc buildLibDynamicLinux(libName: string, folderName: string) = + buildLibrary libName & ".so", folderName, + """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, + "dynamic", libName & ".nim", libname + +proc buildLibDynamicMac(libName: string, folderName: string) = + let sdkPath = staticExec("xcrun --show-sdk-path").strip() + when defined(arm64): + let archFlags = "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" + elif defined(amd64): + let archFlags = "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" + else: + {.error: "Unsupported macOS architecture".} + buildLibrary libName & ".dylib", folderName, + archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE", + "dynamic", libName & ".nim", libname + +proc buildLibStaticWindows(libName: string, folderName: string) = + buildLibrary libName & ".lib", folderName, + """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, + "static", libName & ".nim", libname + +proc buildLibStaticLinux(libName: string, folderName: string) = + buildLibrary libName & ".a", folderName, + """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """, + "static", libName & ".nim", libname + +proc buildLibStaticMac(libName: string, folderName: string) = + let sdkPath = staticExec("xcrun --show-sdk-path").strip() + when defined(arm64): + let archFlags = "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" + elif defined(amd64): + let archFlags = "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\"" + else: + {.error: "Unsupported macOS architecture".} + buildLibrary libName & ".a", folderName, + archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE", + "static", libName & ".nim", libname + +### Mobile Android proc buildMobileAndroid(srcDir = ".", params = "") = let cpu = getEnv("CPU") @@ -88,16 +168,206 @@ proc buildMobileAndroid(srcDir = ".", params = "") = if not dirExists outDir: mkDir outDir - var extra_params = params - for i in 2 ..< paramCount(): - extra_params &= " " & paramStr(i) - exec "nim c" & " --out:" & outDir & - "/libwaku.so --threads:on --app:lib --opt:speed --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" & - outdir & " --passL:-lrln --passL:-llog --cpu:" & cpu & " --os:android -d:androidNDK " & - extra_params & " " & srcDir & "/libwaku.nim" + "/liblogosdelivery.so --threads:on --app:lib --opt:speed --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" & + outdir & " --passL:-lrln --passL:-llog --cpu:" & cpu & " --nimMainPrefix:liblogosdelivery --os:android -d:androidNDK " & params & + getNimParams() & " " & srcDir & "/liblogosdelivery.nim" -proc test(name: string, params = "-d:chronicles_log_level=DEBUG", lang = "c") = +task libLogosDeliveryAndroid, "Build the mobile bindings for Android": + let srcDir = "./library" + buildMobileAndroid srcDir, "-d:chronicles_log_level=ERROR" + +### Mobile iOS + +import std/sequtils + +proc buildMobileIOS(srcDir = ".", params = "") = + echo "Building iOS liblogosdelivery library" + + let iosArch = getEnv("IOS_ARCH") + let iosSdk = getEnv("IOS_SDK") + let sdkPath = getEnv("IOS_SDK_PATH") + + if sdkPath.len == 0: + quit "Error: IOS_SDK_PATH not set. Set it to the path of the iOS SDK" + + # Get nimble package paths + let bearsslPath = gorge("nimble path bearssl").strip() + let secp256k1Path = gorge("nimble path secp256k1").strip() + let natTraversalPath = gorge("nimble path nat_traversal").strip() + + # Get Nim standard library path + let nimPath = gorge("nim --fullhelp 2>&1 | head -1 | sed 's/.*\\[//' | sed 's/\\].*//'").strip() + let nimLibPath = nimPath.parentDir.parentDir / "lib" + + # Use SDK name in path to differentiate device vs simulator + let outDir = "build/ios/" & iosSdk & "-" & iosArch + if not dirExists outDir: + mkDir outDir + + var extra_params = params + let args = commandLineParams() + for arg in args: + extra_params &= " " & arg + + let cpu = if iosArch == "arm64": "arm64" else: "amd64" + + # The output static library + let nimcacheDir = outDir & "/nimcache" + let objDir = outDir & "/obj" + let vendorObjDir = outDir & "/vendor_obj" + let aFile = outDir & "/liblogosdelivery.a" + + if not dirExists objDir: + mkDir objDir + if not dirExists vendorObjDir: + mkDir vendorObjDir + + let clangBase = "clang -arch " & iosArch & " -isysroot " & sdkPath & + " -mios-version-min=18.0 -fembed-bitcode -fPIC -O2" + + # Generate C sources from Nim (no linking) + exec "nim c" & + " --nimcache:" & nimcacheDir & + " --os:ios --cpu:" & cpu & + " --compileOnly:on" & + " --noMain --mm:refc" & + " --threads:on --opt:size --header" & + " -d:metrics -d:discv5_protocol_id=d5waku" & + " --nimMainPrefix:liblogosdelivery --skipParentCfg:on" & + " --cc:clang" & + " " & extra_params & + " " & srcDir & "/liblogosdelivery.nim" + + # Compile vendor C libraries for iOS + + # --- BearSSL --- + echo "Compiling BearSSL for iOS..." + let bearSslSrcDir = bearsslPath / "bearssl/csources/src" + let bearSslIncDir = bearsslPath / "bearssl/csources/inc" + for path in walkDirRec(bearSslSrcDir): + if path.endsWith(".c"): + let relPath = path.replace(bearSslSrcDir & "/", "").replace("/", "_") + let baseName = relPath.changeFileExt("o") + let oFile = vendorObjDir / ("bearssl_" & baseName) + if not fileExists(oFile): + exec clangBase & " -I" & bearSslIncDir & " -I" & bearSslSrcDir & " -c " & path & " -o " & oFile + + # --- secp256k1 --- + echo "Compiling secp256k1 for iOS..." + let secp256k1Dir = secp256k1Path / "vendor/secp256k1" + let secp256k1Flags = " -I" & secp256k1Dir & "/include" & + " -I" & secp256k1Dir & "/src" & + " -I" & secp256k1Dir & + " -DENABLE_MODULE_RECOVERY=1" & + " -DENABLE_MODULE_ECDH=1" & + " -DECMULT_WINDOW_SIZE=15" & + " -DECMULT_GEN_PREC_BITS=4" + + # Main secp256k1 source + let secp256k1Obj = vendorObjDir / "secp256k1.o" + if not fileExists(secp256k1Obj): + exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/secp256k1.c -o " & secp256k1Obj + + # Precomputed tables (required for ecmult operations) + let secp256k1PreEcmultObj = vendorObjDir / "secp256k1_precomputed_ecmult.o" + if not fileExists(secp256k1PreEcmultObj): + exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult.c -o " & secp256k1PreEcmultObj + + let secp256k1PreEcmultGenObj = vendorObjDir / "secp256k1_precomputed_ecmult_gen.o" + if not fileExists(secp256k1PreEcmultGenObj): + exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult_gen.c -o " & secp256k1PreEcmultGenObj + + # --- miniupnpc --- + echo "Compiling miniupnpc for iOS..." + let miniupnpcSrcDir = natTraversalPath / "vendor/miniupnp/miniupnpc/src" + let miniupnpcIncDir = natTraversalPath / "vendor/miniupnp/miniupnpc/include" + let miniupnpcBuildDir = natTraversalPath / "vendor/miniupnp/miniupnpc/build" + let miniupnpcFiles = @[ + "addr_is_reserved.c", "connecthostport.c", "igd_desc_parse.c", + "minisoap.c", "minissdpc.c", "miniupnpc.c", "miniwget.c", + "minixml.c", "portlistingparse.c", "receivedata.c", "upnpcommands.c", + "upnpdev.c", "upnperrors.c", "upnpreplyparse.c" + ] + for fileName in miniupnpcFiles: + let srcPath = miniupnpcSrcDir / fileName + let oFile = vendorObjDir / ("miniupnpc_" & fileName.changeFileExt("o")) + if fileExists(srcPath) and not fileExists(oFile): + exec clangBase & + " -I" & miniupnpcIncDir & + " -I" & miniupnpcSrcDir & + " -I" & miniupnpcBuildDir & + " -DMINIUPNPC_SET_SOCKET_TIMEOUT" & + " -D_BSD_SOURCE -D_DEFAULT_SOURCE" & + " -c " & srcPath & " -o " & oFile + + # --- libnatpmp --- + echo "Compiling libnatpmp for iOS..." + let natpmpSrcDir = natTraversalPath / "vendor/libnatpmp-upstream" + # Only compile natpmp.c - getgateway.c uses net/route.h which is not available on iOS + let natpmpObj = vendorObjDir / "natpmp_natpmp.o" + if not fileExists(natpmpObj): + exec clangBase & + " -I" & natpmpSrcDir & + " -DENABLE_STRNATPMPERR" & + " -c " & natpmpSrcDir & "/natpmp.c -o " & natpmpObj + + # Use iOS-specific stub for getgateway + let getgatewayStubSrc = "./library/ios_natpmp_stubs.c" + let getgatewayStubObj = vendorObjDir / "natpmp_getgateway_stub.o" + if fileExists(getgatewayStubSrc) and not fileExists(getgatewayStubObj): + exec clangBase & " -c " & getgatewayStubSrc & " -o " & getgatewayStubObj + + # --- BearSSL stubs (for tools functions not in main library) --- + echo "Compiling BearSSL stubs for iOS..." + let bearSslStubsSrc = "./library/ios_bearssl_stubs.c" + let bearSslStubsObj = vendorObjDir / "bearssl_stubs.o" + if fileExists(bearSslStubsSrc) and not fileExists(bearSslStubsObj): + exec clangBase & " -c " & bearSslStubsSrc & " -o " & bearSslStubsObj + + # Compile all Nim-generated C files to object files + echo "Compiling Nim-generated C files for iOS..." + var cFiles: seq[string] = @[] + for kind, path in walkDir(nimcacheDir): + if kind == pcFile and path.endsWith(".c"): + cFiles.add(path) + + for cFile in cFiles: + let baseName = extractFilename(cFile).changeFileExt("o") + let oFile = objDir / baseName + exec clangBase & + " -DENABLE_STRNATPMPERR" & + " -I" & nimLibPath & + " -I" & bearsslPath & "/bearssl/csources/inc/" & + " -I" & bearsslPath & "/bearssl/csources/tools/" & + " -I" & bearsslPath & "/bearssl/abi/" & + " -I" & secp256k1Path & "/vendor/secp256k1/include/" & + " -I" & natTraversalPath & "/vendor/miniupnp/miniupnpc/include/" & + " -I" & natTraversalPath & "/vendor/libnatpmp-upstream/" & + " -I" & nimcacheDir & + " -c " & cFile & + " -o " & oFile + + # Create static library from all object files + echo "Creating static library..." + var objFiles: seq[string] = @[] + for kind, path in walkDir(objDir): + if kind == pcFile and path.endsWith(".o"): + objFiles.add(path) + for kind, path in walkDir(vendorObjDir): + if kind == pcFile and path.endsWith(".o"): + objFiles.add(path) + + exec "libtool -static -o " & aFile & " " & objFiles.join(" ") + + echo "iOS library created: " & aFile + +task libWakuIOS, "Build the mobile bindings for iOS": + let srcDir = "./library" + let extraParams = "-d:chronicles_log_level=ERROR" + buildMobileIOS srcDir, extraParams + +proc test(name: string, params = "-d:chronicles_log_level=DEBUG") = # XXX: When running `> NIM_PARAMS="-d:chronicles_log_level=INFO" make test2` # I expect compiler flag to be overridden, however it stays with whatever is # specified here. @@ -106,12 +376,12 @@ proc test(name: string, params = "-d:chronicles_log_level=DEBUG", lang = "c") = ### Waku common tasks task testcommon, "Build & run common tests": - test "all_tests_common", "-d:chronicles_log_level=WARN -d:chronosStrictException" + test "all_tests_common", "-d:chronicles_log_level=DEBUG -d:chronosStrictException" ### Waku tasks task wakunode2, "Build Waku v2 cli node": let name = "wakunode2" - buildBinary name, "apps/wakunode2/", " -d:chronicles_log_level='TRACE' " + buildBinary name, "apps/wakunode2/", " -d:chronicles_log_level=TRACE " task benchmarks, "Some benchmarks": let name = "benchmarks" @@ -150,7 +420,7 @@ task chat2, "Build example Waku chat usage": let name = "chat2" buildBinary name, "apps/chat2/", - "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level='TRACE' " + "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE " # -d:ssl - cause unlisted exception error in libp2p/utility... task chat2mix, "Build example Waku chat mix usage": @@ -161,7 +431,7 @@ task chat2mix, "Build example Waku chat mix usage": let name = "chat2mix" buildBinary name, "apps/chat2mix/", - "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level='TRACE' " + "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE " # -d:ssl - cause unlisted exception error in libp2p/utility... task chat2bridge, "Build chat2bridge": @@ -170,32 +440,33 @@ task chat2bridge, "Build chat2bridge": task liteprotocoltester, "Build liteprotocoltester": let name = "liteprotocoltester" - buildBinary name, "apps/liteprotocoltester/" + buildBinary name, "apps/liteprotocoltester/", "-d:chronicles_log_level=TRACE" task lightpushwithmix, "Build lightpushwithmix": let name = "lightpush_publisher_mix" buildBinary name, "examples/lightpush_mix/" -task api_example, "Build api_example": - let name = "api_example" - buildBinary name, "examples/api_example/" - -task buildone, "Build custom target": - let filepath = paramStr(paramCount()) - discard buildModule filepath - task buildTest, "Test custom target": - let filepath = paramStr(paramCount()) + let args = commandLineParams() + if args.len == 0: + quit "Missing test file" + + let filepath = args[^1] discard buildModule(filepath) import std/strutils task execTest, "Run test": - # Expects to be parameterized with test case name in quotes - # preceded with the nim source file name and path - # If no test case name is given still it requires empty quotes `""` - let filepath = paramStr(paramCount() - 1) - var testSuite = paramStr(paramCount()).strip(chars = {'\"'}) + let args = commandLineParams() + if args.len == 0: + quit "Missing arguments" + # expects: "" + let filepath = + if args.len >= 2: args[^2] + else: args[^1] + var testSuite = + if args.len >= 1: args[^1].strip(chars = {'\"'}) + else: "" if testSuite != "": testSuite = " \"" & testSuite & "\"" exec "build/" & filepath & ".bin " & testSuite @@ -208,203 +479,42 @@ let chroniclesParams = """-d:chronicles_disabled_topics="eth,dnsdisc.client" """ & "--warning:Deprecated:off " & "--warning:UnusedImport:on " & "-d:chronicles_log_level=TRACE" -task libwakuStatic, "Build the cbindings waku node library": - let lib_name = paramStr(paramCount()) - buildLibrary lib_name, "library/", chroniclesParams, "static" +## Libwaku build tasks -task libwakuDynamic, "Build the cbindings waku node library": - let lib_name = paramStr(paramCount()) - buildLibrary lib_name, "library/", chroniclesParams, "dynamic" +task libwakuDynamicWindows, "Generate bindings": + buildLibDynamicWindows("libwaku", "library") -### Mobile Android -task libWakuAndroid, "Build the mobile bindings for Android": - let srcDir = "./library" - let extraParams = "-d:chronicles_log_level=ERROR" - buildMobileAndroid srcDir, extraParams +task libwakuDynamicLinux, "Generate bindings": + buildLibDynamicLinux("libwaku", "library") -### Mobile iOS -import std/sequtils +task libwakuDynamicMac, "Generate bindings": + buildLibDynamicMac("libwaku", "library") -proc buildMobileIOS(srcDir = ".", params = "") = - echo "Building iOS libwaku library" +task libwakuStaticWindows, "Generate bindings": + buildLibStaticWindows("libwaku", "library") - let iosArch = getEnv("IOS_ARCH") - let iosSdk = getEnv("IOS_SDK") - let sdkPath = getEnv("IOS_SDK_PATH") +task libwakuStaticLinux, "Generate bindings": + buildLibStaticLinux("libwaku", "library") - if sdkPath.len == 0: - quit "Error: IOS_SDK_PATH not set. Set it to the path of the iOS SDK" +task libwakuStaticMac, "Generate bindings": + buildLibStaticMac("libwaku", "library") - # Use SDK name in path to differentiate device vs simulator - let outDir = "build/ios/" & iosSdk & "-" & iosArch - if not dirExists outDir: - mkDir outDir +## Liblogosdelivery build tasks - var extra_params = params - for i in 2 ..< paramCount(): - extra_params &= " " & paramStr(i) +task liblogosdeliveryDynamicWindows, "Generate bindings": + buildLibDynamicWindows("liblogosdelivery", "liblogosdelivery") - let cpu = if iosArch == "arm64": "arm64" else: "amd64" +task liblogosdeliveryDynamicLinux, "Generate bindings": + buildLibDynamicLinux("liblogosdelivery", "liblogosdelivery") - # The output static library - let nimcacheDir = outDir & "/nimcache" - let objDir = outDir & "/obj" - let vendorObjDir = outDir & "/vendor_obj" - let aFile = outDir & "/libwaku.a" +task liblogosdeliveryDynamicMac, "Generate bindings": + buildLibDynamicMac("liblogosdelivery", "liblogosdelivery") - if not dirExists objDir: - mkDir objDir - if not dirExists vendorObjDir: - mkDir vendorObjDir +task liblogosdeliveryStaticWindows, "Generate bindings": + buildLibStaticWindows("liblogosdelivery", "liblogosdelivery") - let clangBase = "clang -arch " & iosArch & " -isysroot " & sdkPath & - " -mios-version-min=18.0 -fembed-bitcode -fPIC -O2" +task liblogosdeliveryStaticLinux, "Generate bindings": + buildLibStaticLinux("liblogosdelivery", "liblogosdelivery") - # Generate C sources from Nim (no linking) - exec "nim c" & - " --nimcache:" & nimcacheDir & - " --os:ios --cpu:" & cpu & - " --compileOnly:on" & - " --noMain --mm:refc" & - " --threads:on --opt:speed --header" & - " -d:metrics -d:discv5_protocol_id=d5waku" & - " --nimMainPrefix:libwaku --skipParentCfg:on" & - " --cc:clang" & - " " & extra_params & - " " & srcDir & "/libwaku.nim" - - # Compile vendor C libraries for iOS - - # --- BearSSL --- - echo "Compiling BearSSL for iOS..." - let bearSslSrcDir = "./vendor/nim-bearssl/bearssl/csources/src" - let bearSslIncDir = "./vendor/nim-bearssl/bearssl/csources/inc" - for path in walkDirRec(bearSslSrcDir): - if path.endsWith(".c"): - let relPath = path.replace(bearSslSrcDir & "/", "").replace("/", "_") - let baseName = relPath.changeFileExt("o") - let oFile = vendorObjDir / ("bearssl_" & baseName) - if not fileExists(oFile): - exec clangBase & " -I" & bearSslIncDir & " -I" & bearSslSrcDir & " -c " & path & " -o " & oFile - - # --- secp256k1 --- - echo "Compiling secp256k1 for iOS..." - let secp256k1Dir = "./vendor/nim-secp256k1/vendor/secp256k1" - let secp256k1Flags = " -I" & secp256k1Dir & "/include" & - " -I" & secp256k1Dir & "/src" & - " -I" & secp256k1Dir & - " -DENABLE_MODULE_RECOVERY=1" & - " -DENABLE_MODULE_ECDH=1" & - " -DECMULT_WINDOW_SIZE=15" & - " -DECMULT_GEN_PREC_BITS=4" - - # Main secp256k1 source - let secp256k1Obj = vendorObjDir / "secp256k1.o" - if not fileExists(secp256k1Obj): - exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/secp256k1.c -o " & secp256k1Obj - - # Precomputed tables (required for ecmult operations) - let secp256k1PreEcmultObj = vendorObjDir / "secp256k1_precomputed_ecmult.o" - if not fileExists(secp256k1PreEcmultObj): - exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult.c -o " & secp256k1PreEcmultObj - - let secp256k1PreEcmultGenObj = vendorObjDir / "secp256k1_precomputed_ecmult_gen.o" - if not fileExists(secp256k1PreEcmultGenObj): - exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult_gen.c -o " & secp256k1PreEcmultGenObj - - # --- miniupnpc --- - echo "Compiling miniupnpc for iOS..." - let miniupnpcSrcDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/src" - let miniupnpcIncDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/include" - let miniupnpcBuildDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build" - let miniupnpcFiles = @[ - "addr_is_reserved.c", "connecthostport.c", "igd_desc_parse.c", - "minisoap.c", "minissdpc.c", "miniupnpc.c", "miniwget.c", - "minixml.c", "portlistingparse.c", "receivedata.c", "upnpcommands.c", - "upnpdev.c", "upnperrors.c", "upnpreplyparse.c" - ] - for fileName in miniupnpcFiles: - let srcPath = miniupnpcSrcDir / fileName - let oFile = vendorObjDir / ("miniupnpc_" & fileName.changeFileExt("o")) - if fileExists(srcPath) and not fileExists(oFile): - exec clangBase & - " -I" & miniupnpcIncDir & - " -I" & miniupnpcSrcDir & - " -I" & miniupnpcBuildDir & - " -DMINIUPNPC_SET_SOCKET_TIMEOUT" & - " -D_BSD_SOURCE -D_DEFAULT_SOURCE" & - " -c " & srcPath & " -o " & oFile - - # --- libnatpmp --- - echo "Compiling libnatpmp for iOS..." - let natpmpSrcDir = "./vendor/nim-nat-traversal/vendor/libnatpmp-upstream" - # Only compile natpmp.c - getgateway.c uses net/route.h which is not available on iOS - let natpmpObj = vendorObjDir / "natpmp_natpmp.o" - if not fileExists(natpmpObj): - exec clangBase & - " -I" & natpmpSrcDir & - " -DENABLE_STRNATPMPERR" & - " -c " & natpmpSrcDir & "/natpmp.c -o " & natpmpObj - - # Use iOS-specific stub for getgateway - let getgatewayStubSrc = "./library/ios_natpmp_stubs.c" - let getgatewayStubObj = vendorObjDir / "natpmp_getgateway_stub.o" - if fileExists(getgatewayStubSrc) and not fileExists(getgatewayStubObj): - exec clangBase & " -c " & getgatewayStubSrc & " -o " & getgatewayStubObj - - # --- BearSSL stubs (for tools functions not in main library) --- - echo "Compiling BearSSL stubs for iOS..." - let bearSslStubsSrc = "./library/ios_bearssl_stubs.c" - let bearSslStubsObj = vendorObjDir / "bearssl_stubs.o" - if fileExists(bearSslStubsSrc) and not fileExists(bearSslStubsObj): - exec clangBase & " -c " & bearSslStubsSrc & " -o " & bearSslStubsObj - - # Compile all Nim-generated C files to object files - echo "Compiling Nim-generated C files for iOS..." - var cFiles: seq[string] = @[] - for kind, path in walkDir(nimcacheDir): - if kind == pcFile and path.endsWith(".c"): - cFiles.add(path) - - for cFile in cFiles: - let baseName = extractFilename(cFile).changeFileExt("o") - let oFile = objDir / baseName - exec clangBase & - " -DENABLE_STRNATPMPERR" & - " -I./vendor/nimbus-build-system/vendor/Nim/lib/" & - " -I./vendor/nim-bearssl/bearssl/csources/inc/" & - " -I./vendor/nim-bearssl/bearssl/csources/tools/" & - " -I./vendor/nim-bearssl/bearssl/abi/" & - " -I./vendor/nim-secp256k1/vendor/secp256k1/include/" & - " -I./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/include/" & - " -I./vendor/nim-nat-traversal/vendor/libnatpmp-upstream/" & - " -I" & nimcacheDir & - " -c " & cFile & - " -o " & oFile - - # Create static library from all object files - echo "Creating static library..." - var objFiles: seq[string] = @[] - for kind, path in walkDir(objDir): - if kind == pcFile and path.endsWith(".o"): - objFiles.add(path) - for kind, path in walkDir(vendorObjDir): - if kind == pcFile and path.endsWith(".o"): - objFiles.add(path) - - exec "libtool -static -o " & aFile & " " & objFiles.join(" ") - - echo "✔ iOS library created: " & aFile - -task libWakuIOS, "Build the mobile bindings for iOS": - let srcDir = "./library" - let extraParams = "-d:chronicles_log_level=ERROR" - buildMobileIOS srcDir, extraParams - -task liblogosdeliveryStatic, "Build the liblogosdelivery (Logos Messaging Delivery API) static library": - let lib_name = paramStr(paramCount()) - buildLibrary lib_name, "liblogosdelivery/", chroniclesParams, "static", "liblogosdelivery.nim", "liblogosdelivery" - -task liblogosdeliveryDynamic, "Build the liblogosdelivery (Logos Messaging Delivery API) dynamic library": - let lib_name = paramStr(paramCount()) - buildLibrary lib_name, "liblogosdelivery/", chroniclesParams, "dynamic", "liblogosdelivery.nim", "liblogosdelivery" +task liblogosdeliveryStaticMac, "Generate bindings": + buildLibStaticMac("liblogosdelivery", "liblogosdelivery") diff --git a/waku/api/api_conf.nim b/waku/api/api_conf.nim index 30dfd1b2c..3606be596 100644 --- a/waku/api/api_conf.nim +++ b/waku/api/api_conf.nim @@ -13,24 +13,24 @@ import export json_serialization, json_options -type AutoShardingConfig* {.requiresInit.} = object +type AutoShardingConfig* = object numShardsInCluster*: uint16 -type RlnConfig* {.requiresInit.} = object +type RlnConfig* = object contractAddress*: string chainId*: uint epochSizeSec*: uint64 -type NetworkingConfig* {.requiresInit.} = object +type NetworkingConfig* = object listenIpv4*: string p2pTcpPort*: uint16 discv5UdpPort*: uint16 -type MessageValidation* {.requiresInit.} = object +type MessageValidation* = object maxMessageSize*: string # Accepts formats like "150 KiB", "1500 B" rlnConfig*: Option[RlnConfig] -type ProtocolsConfig* {.requiresInit.} = object +type ProtocolsConfig* = object entryNodes: seq[string] staticStoreNodes: seq[string] clusterId: uint16 @@ -526,7 +526,7 @@ proc decodeNodeConfigFromJson*( var val = NodeConfig.init() # default-initialized try: var stream = unsafeMemoryInput(jsonStr) - var reader = JsonReader[DefaultFlavor].init(stream) + var reader = (JsonReader[DefaultFlavor].init(stream)) reader.readValue(val) except IOError as err: raise (ref SerializationError)(msg: err.msg) From 59bd365c16fd40506c167e8dd7e439684ab4bd11 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Wed, 8 Apr 2026 15:33:16 +0200 Subject: [PATCH 02/25] setting num-shards-in-network to 0 by default (#3748) Co-authored-by: darshankabariya --- tests/wakunode2/test_cli_args.nim | 12 ++++++------ tools/confutils/cli_args.nim | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/wakunode2/test_cli_args.nim b/tests/wakunode2/test_cli_args.nim index 9197afe02..d08544c2c 100644 --- a/tests/wakunode2/test_cli_args.nim +++ b/tests/wakunode2/test_cli_args.nim @@ -23,9 +23,8 @@ import suite "Waku external config - default values": test "Default sharding value": ## Setup - let defaultShardingMode = AutoSharding - let defaultNumShardsInCluster = 1.uint16 - let defaultSubscribeShards = @[0.uint16] + let defaultShardingMode = StaticSharding + let defaultSubscribeShards: seq[uint16] = @[] ## Given let preConfig = defaultWakuNodeConf().get() @@ -37,7 +36,6 @@ suite "Waku external config - default values": ## Then let conf = res.get() check conf.shardingConf.kind == defaultShardingMode - check conf.shardingConf.numShardsInCluster == defaultNumShardsInCluster check conf.subscribeShards == defaultSubscribeShards test "Default shards value in static sharding": @@ -212,7 +210,7 @@ suite "Waku external config - Shards": let vRes = wakuConf.validate() assert vRes.isOk(), $vRes.error - test "Imvalid shard is passed without num shards": + test "Any shard is valid without num shards in static sharding mode": ## Setup ## Given @@ -222,7 +220,9 @@ suite "Waku external config - Shards": let res = wakuNodeConf.toWakuConf() ## Then - assert res.isErr(), "Invalid shard was accepted" + let wakuConf = res.get() + let vRes = wakuConf.validate() + assert vRes.isOk(), $vRes.error suite "Waku external config - store retention policy": test "Default retention policy": diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index 5c1934712..d63b5880c 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -331,7 +331,7 @@ hence would have reachability issues.""", numShardsInNetwork* {. desc: "Enables autosharding and set number of shards in the cluster, set to `0` to use static sharding", - defaultValue: 1, + defaultValue: 0, name: "num-shards-in-network" .}: uint16 From 5503529531ffc5818c8b4289ed9b55d8a95e725d Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Thu, 9 Apr 2026 07:17:17 +0200 Subject: [PATCH 03/25] chore: Add pre-check of options used in config Json for liblogosdelivery pre-createNode, treat unrecognized options as error (#3801) * Add pre-check of options used in config Json for logos-delivery-api pre-createNode, treat unrecognized options as error * Collect all unrecognized options and report them at once. * Refactor json config parsing and error detection --- .../logos_delivery_api/node_api.nim | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/liblogosdelivery/logos_delivery_api/node_api.nim b/liblogosdelivery/logos_delivery_api/node_api.nim index cd644abd7..90630717b 100644 --- a/liblogosdelivery/logos_delivery_api/node_api.nim +++ b/liblogosdelivery/logos_delivery_api/node_api.nim @@ -1,4 +1,4 @@ -import std/[json, strutils] +import std/[json, strutils, tables] import chronos, chronicles, results, confutils, confutils/std/net, ffi import waku/factory/waku, @@ -23,22 +23,50 @@ registerReqFFI(CreateNodeRequest, ctx: ptr FFIContext[Waku]): try: jsonNode = parseJson($configJson) except Exception: + let exceptionMsg = getCurrentExceptionMsg() + error "Failed to parse config JSON", + error = exceptionMsg, configJson = $configJson return err( - "Failed to parse config JSON: " & getCurrentExceptionMsg() & - " configJson string: " & $configJson + "Failed to parse config JSON: " & exceptionMsg & " configJson string: " & + $configJson ) + var jsonFields: Table[string, (string, JsonNode)] + for key, value in jsonNode: + let lowerKey = key.toLowerAscii() + + if jsonFields.hasKey(lowerKey): + error "Duplicate configuration option found when normalized to lowercase", + key = key + return err( + "Duplicate configuration option found when normalized to lowercase: '" & key & + "'" + ) + + jsonFields[lowerKey] = (key, value) + for confField, confValue in fieldPairs(conf): - if jsonNode.contains(confField): - let formattedString = ($jsonNode[confField]).strip(chars = {'\"'}) + let lowerField = confField.toLowerAscii() + if jsonFields.hasKey(lowerField): + let (jsonKey, jsonValue) = jsonFields[lowerField] + let formattedString = ($jsonValue).strip(chars = {'\"'}) try: confValue = parseCmdArg(typeof(confValue), formattedString) except Exception: return err( - "Failed to parse field '" & confField & "': " & getCurrentExceptionMsg() & - ". Value: " & formattedString + "Failed to parse field '" & confField & "' from JSON key '" & jsonKey & "': " & + getCurrentExceptionMsg() & ". Value: " & formattedString ) + jsonFields.del(lowerField) + + if jsonFields.len > 0: + var unknownKeys = newSeq[string]() + for _, (jsonKey, _) in pairs(jsonFields): + unknownKeys.add(jsonKey) + error "Unrecognized configuration option(s) found", option = unknownKeys + return err("Unrecognized configuration option(s) found: " & $unknownKeys) + # Create the node ctx.myLib[] = (await api.createNode(conf)).valueOr: let errMsg = $error From ca7ec3de056120635ad552db24b2e44a8bc99a1d Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:51:46 +0200 Subject: [PATCH 04/25] add main loop lag monitor (#3803) * add loop lagging as health status --- .../health_monitor/event_loop_monitor.nim | 58 +++++++++++++++++++ waku/node/health_monitor/health_status.nim | 1 + .../health_monitor/node_health_monitor.nim | 20 ++++++- 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 waku/node/health_monitor/event_loop_monitor.nim diff --git a/waku/node/health_monitor/event_loop_monitor.nim b/waku/node/health_monitor/event_loop_monitor.nim new file mode 100644 index 000000000..d4b8d98d2 --- /dev/null +++ b/waku/node/health_monitor/event_loop_monitor.nim @@ -0,0 +1,58 @@ +{.push raises: [].} + +import chronos, chronicles, metrics + +logScope: + topics = "waku event_loop_monitor" + +const CheckInterval = 5.seconds + +declarePublicGauge event_loop_lag_seconds, + "chronos event loop lag in seconds: difference between actual and expected wake-up interval" + +type OnLagChange* = proc(lagTooHigh: bool) {.gcsafe, raises: [].} + +proc eventLoopMonitorLoop*(onLagChange: OnLagChange = nil) {.async.} = + ## Monitors chronos event loop responsiveness. + ## + ## Schedules a task every `CheckInterval`. Because chronos is single-threaded + ## and cooperative, the task can only resume after all previously queued work + ## completes. The actual elapsed time between iterations therefore reflects + ## how saturated the event loop is: + ## + ## actual_elapsed ≈ CheckInterval → loop is healthy + ## actual_elapsed >> CheckInterval → tasks are accumulating / loop is stalling + ## + ## The lag (actual - expected) is exposed via `event_loop_lag_seconds`. + ## When lag transitions above or below `CheckInterval`, `onLagChange` is called. + + var lastWakeup = Moment.now() + var lagWasHigh = false + while true: + await sleepAsync(CheckInterval) + + let now = Moment.now() + let actualElapsed = now - lastWakeup + let lag = actualElapsed - CheckInterval + let lagSecs = lag.nanoseconds.float64 / 1_000_000_000.0 + + event_loop_lag_seconds.set(lagSecs) + + let lagIsHigh = lag > CheckInterval + + if lag > CheckInterval: + warn "chronos event loop severely lagging, many tasks may be accumulating", + expected_secs = CheckInterval.seconds, + actual_secs = actualElapsed.nanoseconds.float64 / 1_000_000_000.0, + lag_secs = lagSecs + elif lag > (CheckInterval div 2): + info "chronos event loop lag detected", + expected_secs = CheckInterval.seconds, + actual_secs = actualElapsed.nanoseconds.float64 / 1_000_000_000.0, + lag_secs = lagSecs + + if not isNil(onLagChange) and lagIsHigh != lagWasHigh: + lagWasHigh = lagIsHigh + onLagChange(lagIsHigh) + + lastWakeup = now diff --git a/waku/node/health_monitor/health_status.nim b/waku/node/health_monitor/health_status.nim index 4dd2bdd9a..91663a507 100644 --- a/waku/node/health_monitor/health_status.nim +++ b/waku/node/health_monitor/health_status.nim @@ -7,6 +7,7 @@ type HealthStatus* {.pure.} = enum NOT_READY NOT_MOUNTED SHUTTING_DOWN + EVENT_LOOP_LAGGING proc init*(t: typedesc[HealthStatus], strRep: string): Result[HealthStatus, string] = try: diff --git a/waku/node/health_monitor/node_health_monitor.nim b/waku/node/health_monitor/node_health_monitor.nim index 066e7776a..c92dc1aaf 100644 --- a/waku/node/health_monitor/node_health_monitor.nim +++ b/waku/node/health_monitor/node_health_monitor.nim @@ -21,6 +21,7 @@ import node/health_monitor/health_report, node/health_monitor/connection_status, node/health_monitor/protocol_health, + node/health_monitor/event_loop_monitor, requests/health_requests, ] @@ -36,6 +37,7 @@ type NodeHealthMonitor* = ref object onlineMonitor*: OnlineMonitor keepAliveFut: Future[void] healthLoopFut: Future[void] + eventLoopMonitorFut: Future[void] healthUpdateEvent: AsyncEvent connectionStatus: ConnectionStatus onConnectionStatusChange*: ConnectionStatusChangeHandler @@ -48,6 +50,9 @@ type NodeHealthMonitor* = ref object relayObserver: PubSubObserver peerEventListener: WakuPeerEventListener shardHealthListener: EventShardTopicHealthChangeListener + eventLoopLagExceeded: bool + ## set to true when the chronos event loop lag exceeds the severe threshold, + ## causing the node health to be reported as EVENT_LOOP_LAGGING until lag recovers. func getHealth*(report: HealthReport, kind: WakuProtocol): ProtocolHealth = for h in report.protocolsHealth: @@ -441,7 +446,8 @@ proc getNodeHealthReport*(hm: NodeHealthMonitor): Future[HealthReport] {.async.} hm.cachedProtocols = await hm.getAllProtocolHealthInfo() hm.connectionStatus = hm.calculateConnectionState() - report.nodeHealth = HealthStatus.READY + report.nodeHealth = + if hm.eventLoopLagExceeded: HealthStatus.EVENT_LOOP_LAGGING else: HealthStatus.READY report.connectionStatus = hm.connectionStatus report.protocolsHealth = hm.cachedProtocols return report @@ -461,7 +467,8 @@ proc getSyncNodeHealthReport*(hm: NodeHealthMonitor): HealthReport = hm.cachedProtocols = hm.getSyncAllProtocolHealthInfo() hm.connectionStatus = hm.calculateConnectionState() - report.nodeHealth = HealthStatus.READY + report.nodeHealth = + if hm.eventLoopLagExceeded: HealthStatus.EVENT_LOOP_LAGGING else: HealthStatus.READY report.connectionStatus = hm.connectionStatus report.protocolsHealth = hm.cachedProtocols return report @@ -694,9 +701,15 @@ proc startHealthMonitor*(hm: NodeHealthMonitor): Result[void, string] = hm.healthUpdateEvent.fire() hm.healthLoopFut = hm.healthLoop() + hm.eventLoopMonitorFut = eventLoopMonitorLoop( + proc(lagTooHigh: bool) {.gcsafe, raises: [].} = + hm.eventLoopLagExceeded = lagTooHigh + hm.healthUpdateEvent.fire() + ) hm.startKeepalive().isOkOr: return err("startHealthMonitor: failed starting keep alive: " & error) + return ok() proc stopHealthMonitor*(hm: NodeHealthMonitor) {.async.} = @@ -709,6 +722,9 @@ proc stopHealthMonitor*(hm: NodeHealthMonitor) {.async.} = if not isNil(hm.healthLoopFut): await hm.healthLoopFut.cancelAndWait() + if not isNil(hm.eventLoopMonitorFut): + await hm.eventLoopMonitorFut.cancelAndWait() + WakuPeerEvent.dropListener(hm.node.brokerCtx, hm.peerEventListener) EventShardTopicHealthChange.dropListener(hm.node.brokerCtx, hm.shardHealthListener) From 494ea94606adb7c0efc60b3ad03eab87eb3832df Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Thu, 9 Apr 2026 14:29:17 -0300 Subject: [PATCH 05/25] fix: recv_service delivers store-recovered messages (#3805) * recv_service now delivers store-recovered messages via MessageReceivedEvent * add regression test_api_receive to prove store recovery actually delivers messages * fix confusing "UNSUCCESSFUL / Missed message" log message * removed some dead/duplicated code Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Co-authored by Zoltan Nagy --- .github/workflows/ci.yml | 6 +- .github/workflows/container-image.yml | 2 +- .github/workflows/windows-build.yml | 2 +- BearSSL.mk | 9 +- Nat.mk | 11 +- tests/api/test_all.nim | 1 + tests/api/test_api_receive.nim | 193 ++++++++++++++++++ waku/events/delivery_events.nim | 16 -- .../delivery_service/delivery_service.nim | 2 +- .../recv_service/recv_service.nim | 134 ++++++------ 10 files changed, 279 insertions(+), 97 deletions(-) create mode 100644 tests/api/test_api_receive.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2de4e50e..1f6fcdfec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' @@ -136,7 +136,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' @@ -215,7 +215,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index ae132a477..c2fb9d4d2 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -69,7 +69,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: ${{ steps.secrets.outcome == 'success' && steps.cache-nimbledeps.outputs.cache-hit != 'true' }} diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 09ef05a5d..5b0894368 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -87,7 +87,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock') }} + key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' diff --git a/BearSSL.mk b/BearSSL.mk index 98e933ebd..355e46563 100644 --- a/BearSSL.mk +++ b/BearSSL.mk @@ -22,6 +22,13 @@ BEARSSL_NIMBLEDEPS_DIR := $(shell ls -dt $(CURDIR)/nimbledeps/pkgs2/bearssl-* 2>/dev/null | head -1) BEARSSL_CSOURCES_DIR := $(BEARSSL_NIMBLEDEPS_DIR)/bearssl/csources +BEARSSL_UNAME_M := $(shell uname -m) +ifeq ($(BEARSSL_UNAME_M),x86_64) + PORTABLE_BEARSSL_CFLAGS := -W -Wall -Os -fPIC -mssse3 +else + PORTABLE_BEARSSL_CFLAGS := -W -Wall -Os -fPIC +endif + .PHONY: clean-bearssl-nimbledeps rebuild-bearssl-nimbledeps clean-bearssl-nimbledeps: @@ -36,4 +43,4 @@ ifeq ($(BEARSSL_NIMBLEDEPS_DIR),) $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first) endif @echo "Rebuilding bearssl from $(BEARSSL_CSOURCES_DIR)" - + "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" lib \ No newline at end of file + + "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" CFLAGS="$(PORTABLE_BEARSSL_CFLAGS)" lib \ No newline at end of file diff --git a/Nat.mk b/Nat.mk index 31ad4e018..90d0b2ead 100644 --- a/Nat.mk +++ b/Nat.mk @@ -21,6 +21,13 @@ NAT_TRAVERSAL_NIMBLEDEPS_DIR := $(shell ls -dt $(CURDIR)/nimbledeps/pkgs2/nat_traversal-* 2>/dev/null | head -1) +NAT_UNAME_M := $(shell uname -m) +ifeq ($(NAT_UNAME_M),x86_64) + PORTABLE_NAT_MARCH := -mssse3 +else + PORTABLE_NAT_MARCH := +endif + .PHONY: clean-cross-nimbledeps rebuild-nat-libs-nimbledeps clean-cross-nimbledeps: @@ -47,8 +54,8 @@ ifeq ($(OS), Windows_NT) libnatpmp.a $(HANDLE_OUTPUT) else + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" \ - CC=$(CC) CFLAGS="-Os -fPIC" build/libminiupnpc.a $(HANDLE_OUTPUT) - + "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -fPIC -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \ + CC=$(CC) CFLAGS="-Os -fPIC $(PORTABLE_NAT_MARCH)" build/libminiupnpc.a $(HANDLE_OUTPUT) + + "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -fPIC $(PORTABLE_NAT_MARCH) -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \ -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \ CC=$(CC) libnatpmp.a $(HANDLE_OUTPUT) endif diff --git a/tests/api/test_all.nim b/tests/api/test_all.nim index 4617c8cdb..56be19c27 100644 --- a/tests/api/test_all.nim +++ b/tests/api/test_all.nim @@ -5,4 +5,5 @@ import ./test_node_conf, ./test_api_send, ./test_api_subscription, + ./test_api_receive, ./test_api_health diff --git a/tests/api/test_api_receive.nim b/tests/api/test_api_receive.nim new file mode 100644 index 000000000..52f8713f9 --- /dev/null +++ b/tests/api/test_api_receive.nim @@ -0,0 +1,193 @@ +{.used.} + +import std/[options, sequtils, net, sets] +import chronos, testutils/unittests, stew/byteutils +import libp2p/[peerid, peerinfo, crypto/crypto] +import ../testlib/[common, wakucore, wakunode, testasync] +import ../waku_archive/archive_utils + +import + waku, + waku/[ + waku_node, + waku_core, + common/broker/broker_context, + events/message_events, + waku_relay/protocol, + waku_archive, + waku_archive/common as archive_common, + node/delivery_service/delivery_service, + node/delivery_service/recv_service, + ] +import waku/factory/waku_conf +import tools/confutils/cli_args + +const TestTimeout = chronos.seconds(60) + +type ReceiveEventListenerManager = ref object + brokerCtx: BrokerContext + receivedListener: MessageReceivedEventListener + receivedEvent: AsyncEvent + receivedMessages: seq[WakuMessage] + targetCount: int + +proc newReceiveEventListenerManager( + brokerCtx: BrokerContext, expectedCount: int = 1 +): ReceiveEventListenerManager = + let manager = ReceiveEventListenerManager( + brokerCtx: brokerCtx, receivedMessages: @[], targetCount: expectedCount + ) + manager.receivedEvent = newAsyncEvent() + + manager.receivedListener = MessageReceivedEvent + .listen( + brokerCtx, + proc(event: MessageReceivedEvent) {.async: (raises: []).} = + manager.receivedMessages.add(event.message) + if manager.receivedMessages.len >= manager.targetCount: + manager.receivedEvent.fire() + , + ) + .expect("Failed to listen to MessageReceivedEvent") + + return manager + +proc teardown(manager: ReceiveEventListenerManager) = + MessageReceivedEvent.dropListener(manager.brokerCtx, manager.receivedListener) + +proc waitForEvents( + manager: ReceiveEventListenerManager, timeout: Duration +): Future[bool] {.async.} = + return await manager.receivedEvent.wait().withTimeout(timeout) + +proc createApiNodeConf(numShards: uint16 = 1): WakuNodeConf = + var conf = defaultWakuNodeConf().valueOr: + raiseAssert error + conf.mode = cli_args.WakuMode.Core + conf.listenAddress = parseIpAddress("0.0.0.0") + conf.tcpPort = Port(0) + conf.discv5UdpPort = Port(0) + conf.clusterId = 3'u16 + conf.numShardsInNetwork = numShards + conf.reliabilityEnabled = true + conf.rest = false + result = conf + +suite "Messaging API, Receive Service (store recovery)": + asyncTest "recv_service delivers store-recovered messages via MessageReceivedEvent": + ## Message gets archived before subscriber exists, checkStore() recovers it. + ## This is a regression test: it proves that messages recovered via store by + ## the RecvService (instead of receiving via a live relay sub) are actually + ## delivered via the MessageReceivedEvent API. + + let numShards: uint16 = 1 + let shards = @[PubsubTopic("/waku/2/rs/3/0")] + let shard = shards[0] + let testTopic = ContentTopic("/waku/2/recv-test/proto") + + proc dummyHandler(topic: PubsubTopic, msg: WakuMessage) {.async, gcsafe.} = + discard + + # store node has archive, store, relay + # it archives messages from relay and serves them to the + # subscriber's store client when it comes up (later) + var storeNode: WakuNode + lockNewGlobalBrokerContext: + storeNode = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + storeNode.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on storeNode" + ) + (await storeNode.mountRelay()).expect("Failed to mount relay on storeNode") + let archiveDriver = newSqliteArchiveDriver() + storeNode.mountArchive(archiveDriver).expect("Failed to mount archive") + await storeNode.mountStore() + await storeNode.mountLibp2pPing() + await storeNode.start() + + for s in shards: + storeNode.subscribe((kind: PubsubSub, topic: s), dummyHandler).expect( + "Failed to sub storeNode" + ) + + let storeNodePeerInfo = storeNode.peerInfo.toRemotePeerInfo() + + # publisher node (relay) + var publisher: WakuNode + lockNewGlobalBrokerContext: + publisher = + newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0)) + publisher.mountMetadata(3, toSeq(0'u16 ..< numShards)).expect( + "Failed to mount metadata on publisher" + ) + (await publisher.mountRelay()).expect("Failed to mount relay on publisher") + await publisher.mountLibp2pPing() + await publisher.start() + + for s in shards: + publisher.subscribe((kind: PubsubSub, topic: s), dummyHandler).expect( + "Failed to sub publisher" + ) + + # connect publisher to store so messages get archived + await publisher.connectToNodes(@[storeNodePeerInfo]) + + # wait for relay mesh + for _ in 0 ..< 50: + if publisher.wakuRelay.getNumPeersInMesh(shard).valueOr(0) > 0: + break + await sleepAsync(100.milliseconds) + + # publish before subscriber exists, gets archived + let missedPayload = "This message was missed".toBytes() + let missedMsg = WakuMessage( + payload: missedPayload, contentTopic: testTopic, version: 0, timestamp: now() + ) + discard (await publisher.publish(some(shard), missedMsg)).expect( + "Publish missed msg failed" + ) + + # wait for archive + block waitArchive: + for _ in 0 ..< 50: + let query = archive_common.ArchiveQuery( + includeData: false, contentTopics: @[testTopic], pubsubTopic: some(shard) + ) + let res = await storeNode.wakuArchive.findMessages(query) + if res.isOk() and res.get().hashes.len > 0: + break waitArchive + await sleepAsync(100.milliseconds) + raiseAssert "Message was not archived in time" + + # create subscriber + var subscriber: Waku + lockNewGlobalBrokerContext: + subscriber = (await createNode(createApiNodeConf(numShards))).expect( + "Failed to create subscriber" + ) + (await startWaku(addr subscriber)).expect("Failed to start subscriber") + + # connect subscriber to store (not publisher, so msg won't come via relay to it) + await subscriber.node.connectToNodes(@[storeNodePeerInfo]) + + # subscribe to content topic + (await subscriber.subscribe(testTopic)).expect("Failed to subscribe") + + # listen before triggering store check + let eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1) + defer: + eventManager.teardown() + + # trigger store check, should recover and deliver via MessageReceivedEvent + await subscriber.deliveryService.recvService.checkStore() + + let received = await eventManager.waitForEvents(TestTimeout) + check received + check eventManager.receivedMessages.len == 1 + if eventManager.receivedMessages.len > 0: + check eventManager.receivedMessages[0].payload == missedPayload + + # cleanup + (await subscriber.stop()).expect("Failed to stop subscriber") + await publisher.stop() + await storeNode.stop() diff --git a/waku/events/delivery_events.nim b/waku/events/delivery_events.nim index f8eb0f48d..f27f02721 100644 --- a/waku/events/delivery_events.nim +++ b/waku/events/delivery_events.nim @@ -1,21 +1,5 @@ import waku/waku_core/[message/message, message/digest], waku/common/broker/event_broker -type DeliveryDirection* {.pure.} = enum - PUBLISHING - RECEIVING - -type DeliverySuccess* {.pure.} = enum - SUCCESSFUL - UNSUCCESSFUL - -EventBroker: - type DeliveryFeedbackEvent* = ref object - success*: DeliverySuccess - dir*: DeliveryDirection - comment*: string - msgHash*: WakuMessageHash - msg*: WakuMessage - EventBroker: type OnFilterSubscribeEvent* = object pubsubTopic*: string diff --git a/waku/node/delivery_service/delivery_service.nim b/waku/node/delivery_service/delivery_service.nim index fd728d048..f3d78d98e 100644 --- a/waku/node/delivery_service/delivery_service.nim +++ b/waku/node/delivery_service/delivery_service.nim @@ -12,7 +12,7 @@ import type DeliveryService* = ref object sendService*: SendService - recvService: RecvService + recvService*: RecvService subscriptionManager*: SubscriptionManager proc new*( diff --git a/waku/node/delivery_service/recv_service/recv_service.nim b/waku/node/delivery_service/recv_service/recv_service.nim index 9a85df2f9..9f01ac267 100644 --- a/waku/node/delivery_service/recv_service/recv_service.nim +++ b/waku/node/delivery_service/recv_service/recv_service.nim @@ -12,7 +12,6 @@ import waku_store/common, waku_filter_v2/client, waku_core/topics, - events/delivery_events, events/message_events, waku_node, common/broker/broker_context, @@ -27,7 +26,8 @@ const PruneOldMsgsPeriod = chronos.minutes(1) const DelayExtra* = chronos.seconds(5) ## Additional security time to overlap the missing messages queries -type TupleHashAndMsg = tuple[hash: WakuMessageHash, msg: WakuMessage] +type TupleHashAndMsg = + tuple[hash: WakuMessageHash, msg: WakuMessage, pubsubTopic: PubsubTopic] type RecvMessage = object msgHash: WakuMessageHash @@ -59,88 +59,78 @@ proc getMissingMsgsFromStore( return err("getMissingMsgsFromStore: " & $error) let otherwiseMsg = WakuMessage() - ## message to be returned if the Option message is none + let otherwiseTopic = PubsubTopic("") return ok( - storeResp.messages.mapIt((hash: it.messageHash, msg: it.message.get(otherwiseMsg))) + storeResp.messages.mapIt( + ( + hash: it.messageHash, + msg: it.message.get(otherwiseMsg), + pubsubTopic: it.pubsubTopic.get(otherwiseTopic), + ) + ) ) -proc performDeliveryFeedback( - self: RecvService, - success: DeliverySuccess, - dir: DeliveryDirection, - comment: string, - msgHash: WakuMessageHash, - msg: WakuMessage, -) {.gcsafe, raises: [].} = - info "recv monitor performDeliveryFeedback", - success, dir, comment, msg_hash = shortLog(msgHash) - - DeliveryFeedbackEvent.emit( - brokerCtx = self.brokerCtx, - success = success, - dir = dir, - comment = comment, - msgHash = msgHash, - msg = msg, - ) - -proc msgChecker(self: RecvService) {.async.} = - ## Continuously checks if a message has been received - while true: - await sleepAsync(StoreCheckPeriod) - self.endTimeToCheck = getNowInNanosecondTime() - - var msgHashesInStore = newSeq[WakuMessageHash](0) - for pubsubTopic, contentTopics in self.subscriptionManager.subscribedTopics: - let storeResp: StoreQueryResponse = ( - await self.node.wakuStoreClient.queryToAny( - StoreQueryRequest( - includeData: false, - pubsubTopic: some(pubsubTopic), - contentTopics: toSeq(contentTopics), - startTime: some(self.startTimeToCheck - DelayExtra.nanos), - endTime: some(self.endTimeToCheck + DelayExtra.nanos), - ) - ) - ).valueOr: - error "msgChecker failed to get remote msgHashes", - pubsubTopic = pubsubTopic, cTopics = toSeq(contentTopics), error = $error - continue - - msgHashesInStore.add(storeResp.messages.mapIt(it.messageHash)) - - ## compare the msgHashes seen from the store vs the ones received directly - let rxMsgHashes = self.recentReceivedMsgs.mapIt(it.msgHash) - let missedHashes: seq[WakuMessageHash] = - msgHashesInStore.filterIt(not rxMsgHashes.contains(it)) - - ## Now retrieve the missed WakuMessages - let missingMsgsRet = await self.getMissingMsgsFromStore(missedHashes) - if missingMsgsRet.isOk(): - ## Give feedback so that the api client can perfom any action with the missed messages - for msgTuple in missingMsgsRet.get(): - self.performDeliveryFeedback( - DeliverySuccess.UNSUCCESSFUL, RECEIVING, "Missed message", msgTuple.hash, - msgTuple.msg, - ) - else: - error "failed to retrieve missing messages: ", error = $missingMsgsRet.error - - ## update next check times - self.startTimeToCheck = self.endTimeToCheck - proc processIncomingMessageOfInterest( self: RecvService, pubsubTopic: string, message: WakuMessage -) = - ## Resolve an incoming network message that was already filtered by topic. +): bool = ## Deduplicate (by hash), store (saves in recently-seen messages) and emit ## the MAPI MessageReceivedEvent for every unique incoming message. + ## Returns true if the message was new and the MessageReceivedEvent was properly emitted. let msgHash = computeMessageHash(pubsubTopic, message) if not self.recentReceivedMsgs.anyIt(it.msgHash == msgHash): let rxMsg = RecvMessage(msgHash: msgHash, rxTime: message.timestamp) self.recentReceivedMsgs.add(rxMsg) MessageReceivedEvent.emit(self.brokerCtx, msgHash.to0xHex(), message) + return true + return false + +proc checkStore*(self: RecvService) {.async.} = + ## Checks the store for messages that were not received directly and + ## delivers them via MessageReceivedEvent. + self.endTimeToCheck = getNowInNanosecondTime() + + ## query store and deliver new recovered messages per subscribed topic + for pubsubTopic, contentTopics in self.subscriptionManager.subscribedTopics: + let storeResp: StoreQueryResponse = ( + await self.node.wakuStoreClient.queryToAny( + StoreQueryRequest( + includeData: false, + pubsubTopic: some(pubsubTopic), + contentTopics: toSeq(contentTopics), + startTime: some(self.startTimeToCheck - DelayExtra.nanos), + endTime: some(self.endTimeToCheck + DelayExtra.nanos), + ) + ) + ).valueOr: + error "msgChecker failed to get remote msgHashes", + pubsubTopic = pubsubTopic, cTopics = toSeq(contentTopics), error = $error + continue + + ## compare the msgHashes seen from the store vs the ones received directly + let msgHashesInStore = storeResp.messages.mapIt(it.messageHash) + let rxMsgHashes = self.recentReceivedMsgs.mapIt(it.msgHash) + let missedHashes: seq[WakuMessageHash] = + msgHashesInStore.filterIt(not rxMsgHashes.contains(it)) + + ## Now retrieve the missing WakuMessages and deliver them + let missingMsgsRet = await self.getMissingMsgsFromStore(missedHashes) + if missingMsgsRet.isOk(): + for msgTuple in missingMsgsRet.get(): + if self.processIncomingMessageOfInterest(msgTuple.pubsubTopic, msgTuple.msg): + info "recv service store-recovered message", + msg_hash = shortLog(msgTuple.hash), pubsubTopic = msgTuple.pubsubTopic + else: + error "failed to retrieve missing messages: ", error = $missingMsgsRet.error + + ## update next check times + self.startTimeToCheck = self.endTimeToCheck + +proc msgChecker(self: RecvService) {.async.} = + ## Continuously checks if a message has been received + while true: + await sleepAsync(StoreCheckPeriod) + await self.checkStore() proc new*(T: typedesc[RecvService], node: WakuNode, s: SubscriptionManager): T = ## The storeClient will help to acquire any possible missed messages @@ -176,7 +166,7 @@ proc startRecvService*(self: RecvService) = shard = event.topic, contenttopic = event.message.contentTopic return - self.processIncomingMessageOfInterest(event.topic, event.message), + discard self.processIncomingMessageOfInterest(event.topic, event.message), ).valueOr: error "Failed to set MessageSeenEvent listener", error = error quit(QuitFailure) From c04df751db25ab5d5eb3a51f8247a9d03d8b63c7 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Fri, 10 Apr 2026 07:38:02 -0300 Subject: [PATCH 06/25] Fix BearSSL and NAT lib build reproducibility (#3806) * pass -mssse3 on x86_64 to BearSSL and NAT C lib builds * add BearSSL.mk and Nat.mk to nimbledeps cache key From 166dc69c390a95efa82f6b439d002eabe712b57f Mon Sep 17 00:00:00 2001 From: Gabriel Cruz <8129788+gmelodie@users.noreply.github.com> Date: Mon, 13 Apr 2026 16:44:30 -0300 Subject: [PATCH 07/25] chore: bump nim-jwt version (#3812) --- waku.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku.nimble b/waku.nimble index a6e824528..3a7956873 100644 --- a/waku.nimble +++ b/waku.nimble @@ -64,7 +64,7 @@ requires "nim >= 2.2.4", requires "https://github.com/logos-messaging/nim-ffi" requires "https://github.com/vacp2p/nim-lsquic" -requires "https://github.com/vacp2p/nim-jwt.git#18f8378de52b241f321c1f9ea905456e89b95c6f" +requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2" proc getMyCPU(): string = ## Need to set cpu more explicit manner to avoid arch issues between dependencies From 509c8755336948b5759310555fc0db9ffc895ada Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:12:52 +0200 Subject: [PATCH 08/25] chore: enable postgres support in nix liblogosdelivery build (#3813) Add -d:postgres and -d:nimDebugDlOpen to both the dynamic and static nim c invocations in nix/default.nix, matching the POSTGRES=1 flag already used in the Make-based build path. --- nix/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nix/default.nix b/nix/default.nix index f90b8185e..0d1de2ece 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -62,6 +62,8 @@ pkgs.stdenv.mkDerivation { --path:$NAT_TRAV/src \ --passL:"-L${zerokitRln}/lib -lrln" \ --define:disable_libbacktrace \ + --define:postgres \ + --define:nimDebugDlOpen \ --out:build/liblogosdelivery.${libExt} \ --app:lib \ --threads:on \ @@ -81,6 +83,8 @@ pkgs.stdenv.mkDerivation { --path:$NAT_TRAV/src \ --passL:"-L${zerokitRln}/lib -lrln" \ --define:disable_libbacktrace \ + --define:postgres \ + --define:nimDebugDlOpen \ --out:build/liblogosdelivery.a \ --app:staticlib \ --threads:on \ From ca4dbb19e0cb61a496921c286ffcef76e62ee9b9 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Mon, 20 Apr 2026 08:05:54 -0300 Subject: [PATCH 09/25] Improve logging of content topic on server (#3818) --- waku/waku_filter_v2/protocol.nim | 3 ++- waku/waku_lightpush/protocol.nim | 1 + waku/waku_lightpush_legacy/protocol.nim | 1 + waku/waku_relay/protocol.nim | 5 ++++- waku/waku_rln_relay/rln_relay.nim | 23 ++++++++++++++++------- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/waku/waku_filter_v2/protocol.nim b/waku/waku_filter_v2/protocol.nim index 451bf5cb2..35620b6cd 100644 --- a/waku/waku_filter_v2/protocol.nim +++ b/waku/waku_filter_v2/protocol.nim @@ -244,7 +244,8 @@ proc handleMessage*( ) {.async.} = let msgHash = computeMessageHash(pubsubTopic, message).to0xHex() - info "handling message", pubsubTopic = pubsubTopic, msg_hash = msgHash + info "handling message", + pubsubTopic = pubsubTopic, contentTopic = message.contentTopic, msg_hash = msgHash let handleMessageStartTime = Moment.now() diff --git a/waku/waku_lightpush/protocol.nim b/waku/waku_lightpush/protocol.nim index ecbff8461..8336f4dfc 100644 --- a/waku/waku_lightpush/protocol.nim +++ b/waku/waku_lightpush/protocol.nim @@ -68,6 +68,7 @@ proc handleRequest( peer_id = peerId, requestId = pushRequest.requestId, pubsubTopic = pushRequest.pubsubTopic, + contentTopic = pushRequest.message.contentTopic, msg_hash = msg_hash, receivedTime = getNowInNanosecondTime() diff --git a/waku/waku_lightpush_legacy/protocol.nim b/waku/waku_lightpush_legacy/protocol.nim index 72fc963ee..f5ed60134 100644 --- a/waku/waku_lightpush_legacy/protocol.nim +++ b/waku/waku_lightpush_legacy/protocol.nim @@ -50,6 +50,7 @@ proc handleRequest*( peer_id = peerId, requestId = requestId, pubsubTopic = pubsubTopic, + contentTopic = message.contentTopic, msg_hash = msg_hash, receivedTime = getNowInNanosecondTime() diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim index 490feae87..79d3702eb 100644 --- a/waku/waku_relay/protocol.nim +++ b/waku/waku_relay/protocol.nim @@ -223,6 +223,7 @@ proc logMessageInfo*( msg_id = msg_id_short, from_peer_id = remotePeerId, topic = topic, + contentTopic = msg.contentTopic, receivedTime = getNowInNanosecondTime(), payloadSizeBytes = payloadSize else: @@ -232,6 +233,7 @@ proc logMessageInfo*( msg_id = msg_id_short, to_peer_id = remotePeerId, topic = topic, + contentTopic = msg.contentTopic, sentTime = getNowInNanosecondTime(), payloadSizeBytes = payloadSize @@ -680,7 +682,8 @@ proc publish*( let data = message.encode().buffer let msgHash = computeMessageHash(pubsubTopic, message).to0xHex() - notice "start publish Waku message", msg_hash = msgHash, pubsubTopic = pubsubTopic + notice "start publish Waku message", + msg_hash = msgHash, pubsubTopic = pubsubTopic, contentTopic = message.contentTopic let relayedPeerCount = await procCall GossipSub(w).publish(pubsubTopic, data) diff --git a/waku/waku_rln_relay/rln_relay.nim b/waku/waku_rln_relay/rln_relay.nim index 8559dcd66..ac128b5bc 100644 --- a/waku/waku_rln_relay/rln_relay.nim +++ b/waku/waku_rln_relay/rln_relay.nim @@ -201,14 +201,18 @@ proc validateMessage*( if timeDiff > rlnPeer.rlnMaxTimestampGap: warn "invalid message: timestamp difference exceeds threshold", - timeDiff = timeDiff, maxTimestampGap = rlnPeer.rlnMaxTimestampGap + timeDiff = timeDiff, + maxTimestampGap = rlnPeer.rlnMaxTimestampGap, + contentTopic = msg.contentTopic waku_rln_invalid_messages_total.inc(labelValues = ["invalid_timestamp"]) return MessageValidationResult.Invalid let computedEpoch = rlnPeer.calcEpoch(messageTime) if proof.epoch != computedEpoch: warn "invalid message: timestamp mismatches epoch", - proofEpoch = fromEpoch(proof.epoch), computedEpoch = fromEpoch(computedEpoch) + proofEpoch = fromEpoch(proof.epoch), + computedEpoch = fromEpoch(computedEpoch), + contentTopic = msg.contentTopic waku_rln_invalid_messages_total.inc(labelValues = ["timestamp_mismatch"]) return MessageValidationResult.Invalid @@ -216,7 +220,8 @@ proc validateMessage*( if not rootValidationRes: warn "invalid message: provided root does not belong to acceptable window of roots", provided = proof.merkleRoot.inHex(), - validRoots = rlnPeer.groupManager.validRoots.mapIt(it.inHex()) + validRoots = rlnPeer.groupManager.validRoots.mapIt(it.inHex()), + contentTopic = msg.contentTopic waku_rln_invalid_messages_total.inc(labelValues = ["invalid_root"]) return MessageValidationResult.Invalid @@ -233,12 +238,14 @@ proc validateMessage*( proofVerificationRes.isOkOr: waku_rln_errors_total.inc(labelValues = ["proof_verification"]) - warn "invalid message: proof verification failed", payloadLen = msg.payload.len + warn "invalid message: proof verification failed", + payloadLen = msg.payload.len, contentTopic = msg.contentTopic return MessageValidationResult.Invalid if not proofVerificationRes.value(): # invalid proof - warn "invalid message: invalid proof", payloadLen = msg.payload.len + warn "invalid message: invalid proof", + payloadLen = msg.payload.len, contentTopic = msg.contentTopic waku_rln_invalid_messages_total.inc(labelValues = ["invalid_proof"]) return MessageValidationResult.Invalid @@ -252,11 +259,13 @@ proc validateMessage*( if hasDup.isErr(): waku_rln_errors_total.inc(labelValues = ["duplicate_check"]) elif hasDup.value == true: - trace "invalid message: message is spam", payloadLen = msg.payload.len + trace "invalid message: message is spam", + payloadLen = msg.payload.len, contentTopic = msg.contentTopic waku_rln_spam_messages_total.inc() return MessageValidationResult.Spam - trace "message is valid", payloadLen = msg.payload.len + trace "message is valid", + payloadLen = msg.payload.len, contentTopic = msg.contentTopic # Metric increment moved to validator to include shard label return MessageValidationResult.Valid From 9ae108b4a7c4f15218e40d5df04ee01fc7327c3e Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Mon, 20 Apr 2026 08:16:01 -0300 Subject: [PATCH 10/25] Fix peer stats endpoint (#3815) --- waku/rest_api/endpoint/admin/handlers.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/waku/rest_api/endpoint/admin/handlers.nim b/waku/rest_api/endpoint/admin/handlers.nim index 1904d43f9..304fdabf8 100644 --- a/waku/rest_api/endpoint/admin/handlers.nim +++ b/waku/rest_api/endpoint/admin/handlers.nim @@ -344,7 +344,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = for ps in relayPeers: totalRelayPeers += ps.peers.len stat[$ps.shard] = ps.peers.len - stat["Total relay peers"] = relayPeers.len + stat["Total relay peers"] = totalRelayPeers stat # stats of mesh peers @@ -355,7 +355,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = for ps in meshPeers: totalMeshPeers += ps.peers.len stat[$ps.shard] = ps.peers.len - stat["Total mesh peers"] = meshPeers.len + stat["Total mesh peers"] = totalMeshPeers stat var protoStats = initOrderedTable[string, int]() From 9cbb4e7338c62abe9345fe70cc81694373be87e2 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Mon, 20 Apr 2026 08:48:27 -0300 Subject: [PATCH 11/25] fix: prefer --num-shards-in-network over preset (#3816) * fill numShardsInCluster from preset when builder slot is none * add regression tests --- tests/factory/test_waku_conf.nim | 48 +++++++++++++++++++ .../conf_builder/waku_conf_builder.nim | 13 +++-- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/tests/factory/test_waku_conf.nim b/tests/factory/test_waku_conf.nim index 9d05f7fb5..eeacf791b 100644 --- a/tests/factory/test_waku_conf.nim +++ b/tests/factory/test_waku_conf.nim @@ -213,6 +213,54 @@ suite "Waku Conf - build with cluster conf": check rlnRelayConf.epochSizeSec == networkConf.rlnEpochSizeSec check rlnRelayConf.userMessageLimit == userMessageLimit.uint + test "num-shards-in-network > 0 overrides preset": + ## Setup + let networkConf = NetworkConf.LogosDevConf() + var builder = WakuConfBuilder.init() + + # Sanity check + check networkConf.shardingConf.kind == AutoSharding + check networkConf.shardingConf.numShardsInCluster > 1 + + ## Given: preset says >1 shards but user explicitly sets 1 + builder.withNetworkConf(networkConf) + builder.withNumShardsInCluster(1) + builder.withShardingConf(AutoSharding) + + ## When + let conf = builder.build().expect("build should succeed") + + ## Then: user value wins, not preset + conf.validate().expect("conf should validate") + check conf.shardingConf.kind == AutoSharding + check conf.shardingConf.numShardsInCluster == 1 + + test "num-shards-in-network == 0 does not override preset": + ## Passing an AutoSharding preset and trying to override with + ## --num-shards-in-network=0 (which is StaticSharding) doesn't work. + ## Note that --num-shards-in-network=0 and omitting the switch are + ## internally the same. Promoting the config to an Option[uint16] is + ## probably not worth it since overriding an AutoSharding preset with + ## StaticSharding shouldn't make any sense (that is, no use case). + + ## Given: emulate --preset=logos.dev --num-shards-in-network=0 + let networkConf = NetworkConf.LogosDevConf() + var builder = WakuConfBuilder.init() + builder.withNetworkConf(networkConf) + # Note: builder.withNumShardsInCluster() is not called when the + # value that comes from the CLI path is 0 (which means it was + # either set to 0 or was left unset). + builder.withShardingConf(StaticSharding) + + ## When + let conf = builder.build().expect("build should succeed") + + ## Then: preset wins and StaticSharding user intent is lost + conf.validate().expect("conf should validate") + check conf.shardingConf.kind == networkConf.shardingConf.kind + check conf.shardingConf.numShardsInCluster == + networkConf.shardingConf.numShardsInCluster + suite "Waku Conf - node key": test "Node key is generated": ## Setup diff --git a/waku/factory/conf_builder/waku_conf_builder.nim b/waku/factory/conf_builder/waku_conf_builder.nim index 956d733d3..78dbd9eb9 100644 --- a/waku/factory/conf_builder/waku_conf_builder.nim +++ b/waku/factory/conf_builder/waku_conf_builder.nim @@ -299,7 +299,6 @@ proc buildShardingConf( bNumShardsInCluster: Option[uint16], bSubscribeShards: Option[seq[uint16]], ): (ShardingConf, seq[uint16]) = - echo "bSubscribeShards: ", bSubscribeShards case bShardingConfKind.get(AutoSharding) of StaticSharding: (ShardingConf(kind: StaticSharding), bSubscribeShards.get(@[])) @@ -374,17 +373,17 @@ proc applyNetworkConf(builder: var WakuConfBuilder) = warn "Sharding Conf was provided alongside a network conf", used = networkConf.shardingConf.kind, discarded = builder.shardingConf - if builder.numShardsInCluster.isSome(): - warn "Num Shards In Cluster was provided alongside a network conf", - used = networkConf.shardingConf.numShardsInCluster, - discarded = builder.numShardsInCluster - case networkConf.shardingConf.kind of StaticSharding: builder.shardingConf = some(StaticSharding) of AutoSharding: builder.shardingConf = some(AutoSharding) - builder.numShardsInCluster = some(networkConf.shardingConf.numShardsInCluster) + if builder.numShardsInCluster.isSome(): + warn "Num Shards In Cluster overrides network conf preset", + used = builder.numShardsInCluster.get(), + ignored = networkConf.shardingConf.numShardsInCluster + else: + builder.numShardsInCluster = some(networkConf.shardingConf.numShardsInCluster) if networkConf.discv5Discovery: if builder.discv5Conf.enabled.isNone: From cda0197168bed3ebf2f3002338a9135a4dc9706c Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Mon, 20 Apr 2026 13:54:34 +0200 Subject: [PATCH 12/25] use nimble 0.22.3 and more appropriate nimble.lock (#3809) --- .github/workflows/ci.yml | 8 +- .github/workflows/container-image.yml | 4 +- .github/workflows/windows-build.yml | 19 +- Makefile | 54 +- config.nims | 3 +- flake.nix | 14 +- nimble.lock | 750 ++++++++++--------- nix/deps.nix | 314 ++++---- nix/shell.nix | 17 +- scripts/install_nim.sh | 72 ++ waku.nimble | 12 +- waku/incentivization/eligibility_manager.nim | 7 +- 12 files changed, 696 insertions(+), 578 deletions(-) create mode 100755 scripts/install_nim.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f6fcdfec..b45853e21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: MAKEFLAGS: "-j${NPROC}" NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none" NIM_VERSION: '2.2.4' - NIMBLE_VERSION: '0.18.2' + NIMBLE_VERSION: '0.22.3' jobs: changes: # changes detection @@ -83,7 +83,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' @@ -136,7 +136,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' @@ -215,7 +215,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index c2fb9d4d2..0783c1f66 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -16,7 +16,7 @@ env: MAKEFLAGS: "-j${NPROC}" NIMFLAGS: "--parallelBuild:${NPROC}" NIM_VERSION: '2.2.4' - NIMBLE_VERSION: '0.18.2' + NIMBLE_VERSION: '0.22.3' # This workflow should not run for outside contributors # If org secrets are not available, we'll avoid building and publishing the docker image and we'll pass the workflow @@ -69,7 +69,7 @@ jobs: path: | nimbledeps/ nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} - name: Install nimble deps if: ${{ steps.secrets.outcome == 'success' && steps.cache-nimbledeps.outputs.cache-hit != 'true' }} diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 5b0894368..50f1602cd 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -10,7 +10,7 @@ on: env: NPROC: 4 NIM_VERSION: '2.2.4' - NIMBLE_VERSION: '0.18.2' + NIMBLE_VERSION: '0.22.3' jobs: build: @@ -80,14 +80,13 @@ jobs: cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y echo "$HOME/.nimble/bin" >> $GITHUB_PATH - - name: Cache nimble deps - id: cache-nimbledeps - uses: actions/cache@v4 - with: - path: | - nimbledeps/ - nimble.paths - key: ${{ runner.os }}-nimbledeps-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }} + - name: Patch nimble.lock for Windows nim checksum + # nimble.exe uses Windows Git (core.autocrlf=true by default), which converts LF→CRLF + # on checkout. This changes the SHA1 of the nim package source tree relative to the + # Linux-computed checksum stored in nimble.lock. Patch the lock file with the + # Windows-computed checksum before nimble reads it. + run: | + sed -i 's/68bb85cbfb1832ce4db43943911b046c3af3caab/a092a045d3a427d127a5334a6e59c76faff54686/g' nimble.lock - name: Install nimble deps if: steps.cache-nimbledeps.outputs.cache-hit != 'true' @@ -124,4 +123,4 @@ jobs: else echo "Build failed: libwaku.dll not found" exit 1 - fi + fi \ No newline at end of file diff --git a/Makefile b/Makefile index cabeec80f..7ba417527 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,11 @@ ifneq (,$(findstring MINGW,$(detected_OS))) detected_OS := Windows endif +# Ensure the nim/nimble installed by install-nim/install-nimble are found first +export PATH := $(HOME)/.nimble/bin:$(PATH) + # NIM binary location -NIM_BINARY := $(shell which nim) +NIM_BINARY := $(shell which nim 2>/dev/null) NPH := $(HOME)/.nimble/bin/nph NIMBLEDEPS_STAMP := nimbledeps/.nimble-setup @@ -39,7 +42,7 @@ endif ########## ## Main ## ########## -.PHONY: all test update clean examples deps nimble +.PHONY: all test update clean examples deps nimble install-nim install-nimble # default target all: | wakunode2 libwaku liblogosdelivery @@ -67,7 +70,7 @@ waku.nims: ln -s waku.nimble $@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims - @if ! command -v nimble > /dev/null 2>&1; then $(MAKE) install-nimble; fi + $(MAKE) install-nimble nimble setup --localdeps $(MAKE) build-nph $(MAKE) rebuild-bearssl-nimbledeps @@ -81,59 +84,28 @@ update: clean: rm -rf build 2> /dev/null || true rm -rf nimbledeps 2> /dev/null || true - rm nimble.lock 2> /dev/null || true rm -fr nimcache 2> /dev/null || true rm nimble.paths 2> /dev/null || true nimble clean -REQUIRED_NIM_VERSION := $(shell grep -E '^const NimVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') -REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const NimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') +REQUIRED_NIM_VERSION := $(shell grep -E '^const RequiredNimVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') +REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const RequiredNimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') install-nim: - $(eval NIM_OS := $(shell uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/')) - $(eval NIM_ARCH := $(shell uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/')) - $(eval NIM_INSTALL_DIR := $(HOME)/.nim_runtime) - @nim_ver=$$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ - if [ "$$nim_ver" = "$(REQUIRED_NIM_VERSION)" ]; then \ - echo "nim $(REQUIRED_NIM_VERSION) already installed, skipping."; \ - else \ - curl -L "https://github.com/nim-lang/Nim/releases/download/v$(REQUIRED_NIM_VERSION)/nim-$(REQUIRED_NIM_VERSION)-$(NIM_OS)_$(NIM_ARCH).tar.xz" \ - -o /tmp/nim-$(REQUIRED_NIM_VERSION).tar.xz && \ - tar -xJf /tmp/nim-$(REQUIRED_NIM_VERSION).tar.xz -C /tmp && \ - mkdir -p $(NIM_INSTALL_DIR) && \ - cd /tmp/nim-$(REQUIRED_NIM_VERSION) && ./install.sh $(NIM_INSTALL_DIR); \ - fi + scripts/install_nim.sh $(REQUIRED_NIM_VERSION) install-nimble: install-nim @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \ echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \ else \ - cd /tmp && PATH="$(HOME)/.nim_runtime/bin:$$PATH" \ - nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \ + cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \ fi build: - @nim_ver=$$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ - if [ "$$nim_ver" != "$(REQUIRED_NIM_VERSION)" ]; then \ - echo "Error: Nim $(REQUIRED_NIM_VERSION) is required, but found '$$nim_ver'"; \ - exit 1; \ - fi - @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \ - if [ "$$nimble_ver" != "$(REQUIRED_NIMBLE_VERSION)" ]; then \ - echo "Error: Nimble $(REQUIRED_NIMBLE_VERSION) is required, but found '$$nimble_ver'"; \ - exit 1; \ - fi mkdir -p build -nimble: - echo "Inside nimble target, checking for nimble..." && \ - command -v nimble >/dev/null 2>&1 || { \ - mv nimbledeps nimbledeps_backup 2>/dev/null || true; \ - echo "choosenim not found, installing ..."; \ - curl -sSf https://nim-lang.org/choosenim/init.sh | sh; \ - mv nimbledeps_backup nimbledeps 2>/dev/null || true; \ - } +nimble: install-nimble ## Possible values: prod; debug TARGET ?= prod @@ -230,7 +202,7 @@ clean: | clean-librln ################# .PHONY: testcommon -testcommon: | build +testcommon: | $(NIMBLEDEPS_STAMP) build echo -e $(BUILD_MSG) "build/$@" && \ nimble testcommon @@ -239,7 +211,7 @@ testcommon: | build ########## .PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester -testwaku: | build rln-deps librln +testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln echo -e $(BUILD_MSG) "build/$@" && \ nimble test diff --git a/config.nims b/config.nims index 329384ac4..0f6052c9b 100644 --- a/config.nims +++ b/config.nims @@ -83,8 +83,9 @@ if not defined(macosx) and not defined(android): # add debugging symbols and original files and line numbers --debugger: native - if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace): + when defined(enable_libbacktrace): # light-weight stack traces using libbacktrace and libunwind + # opt-in: pass -d:enable_libbacktrace (requires libbacktrace in project deps) --define: nimStackTraceOverride switch("import", "libbacktrace") diff --git a/flake.nix b/flake.nix index 57592722b..31d5a120c 100644 --- a/flake.nix +++ b/flake.nix @@ -36,9 +36,21 @@ forAllSystems = nixpkgs.lib.genAttrs systems; + nimbleOverlay = final: prev: { + nimble = prev.nimble.overrideAttrs (_: { + version = "0.22.3"; + src = prev.fetchFromGitHub { + owner = "nim-lang"; + repo = "nimble"; + rev = "v0.22.3"; + sha256 = "sha256-f7DYpRGVUeSi6basK1lfu5AxZpMFOSJ3oYsy+urYErg="; + }; + }); + }; + pkgsFor = system: import nixpkgs { inherit system; - overlays = [ (import rust-overlay) ]; + overlays = [ (import rust-overlay) nimbleOverlay ]; }; in { packages = forAllSystems (system: diff --git a/nimble.lock b/nimble.lock index 96f64baf3..7c76f7fa9 100644 --- a/nimble.lock +++ b/nimble.lock @@ -1,103 +1,66 @@ { "version": 2, "packages": { + "nim": { + "version": "2.2.4", + "vcsRevision": "911e0dbb1f76de61fa0215ab1bb85af5334cc9a8", + "url": "https://github.com/nim-lang/Nim.git", + "downloadMethod": "git", + "dependencies": [], + "checksums": { + "sha1": "68bb85cbfb1832ce4db43943911b046c3af3caab" + } + }, "unittest2": { "version": "0.2.5", "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189", "url": "https://github.com/status-im/nim-unittest2", "downloadMethod": "git", - "dependencies": [], + "dependencies": [ + "nim" + ], "checksums": { "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9" } }, "bearssl": { - "version": "0.2.7", - "vcsRevision": "3b341f30d8c619b9a75c154243f9a55468a404e2", + "version": "0.2.8", + "vcsRevision": "22c6a76ce015bc07e011562bdcfc51d9446c1e82", "url": "https://github.com/status-im/nim-bearssl", "downloadMethod": "git", "dependencies": [ + "nim", "unittest2" ], "checksums": { - "sha1": "a85aab15b1b9a8b2438e9a128ac2eba41227da79" + "sha1": "da4dd7ae96d536bdaf42dca9c85d7aed024b6a86" } }, "bearssl_pkey_decoder": { - "version": "0.1.0", + "version": "#21dd3710df9345ed2ad8bf8f882761e07863b8e0", "vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0", "url": "https://github.com/vacp2p/bearssl_pkey_decoder", "downloadMethod": "git", "dependencies": [ + "nim", "bearssl" ], "checksums": { "sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714" } }, - "results": { - "version": "0.5.1", - "vcsRevision": "df8113dda4c2d74d460a8fa98252b0b771bf1f27", - "url": "https://github.com/arnetheduck/nim-results", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "a9c011f74bc9ed5c91103917b9f382b12e82a9e7" - } - }, - "stew": { - "version": "0.5.0", - "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa", - "url": "https://github.com/status-im/nim-stew", + "jwt": { + "version": "#18f8378de52b241f321c1f9ea905456e89b95c6f", + "vcsRevision": "18f8378de52b241f321c1f9ea905456e89b95c6f", + "url": "https://github.com/vacp2p/nim-jwt.git", "downloadMethod": "git", "dependencies": [ - "results", - "unittest2" + "nim", + "bearssl", + "bearssl_pkey_decoder" ], "checksums": { - "sha1": "db22942939773ab7d5a0f2b2668c237240c67dd6" - } - }, - "faststreams": { - "version": "0.5.0", - "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e", - "url": "https://github.com/status-im/nim-faststreams", - "downloadMethod": "git", - "dependencies": [ - "stew", - "unittest2" - ], - "checksums": { - "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383" - } - }, - "serialization": { - "version": "0.5.2", - "vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b", - "url": "https://github.com/status-im/nim-serialization", - "downloadMethod": "git", - "dependencies": [ - "faststreams", - "unittest2", - "stew" - ], - "checksums": { - "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8" - } - }, - "json_serialization": { - "version": "0.4.4", - "vcsRevision": "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44", - "url": "https://github.com/status-im/nim-json-serialization", - "downloadMethod": "git", - "dependencies": [ - "faststreams", - "serialization", - "stew", - "results" - ], - "checksums": { - "sha1": "8b3115354104858a0ac9019356fb29720529c2bd" + "sha1": "bcfd6fc9c5e10a52b87117219b7ab5c98136bc8e" } }, "testutils": { @@ -106,25 +69,76 @@ "url": "https://github.com/status-im/nim-testutils", "downloadMethod": "git", "dependencies": [ + "nim", "unittest2" ], "checksums": { "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3" } }, - "chronicles": { - "version": "0.12.2", - "vcsRevision": "27ec507429a4eb81edc20f28292ee8ec420be05b", - "url": "https://github.com/status-im/nim-chronicles", + "db_connector": { + "version": "0.1.0", + "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", + "url": "https://github.com/nim-lang/db_connector", "downloadMethod": "git", "dependencies": [ - "faststreams", - "serialization", - "json_serialization", - "testutils" + "nim" ], "checksums": { - "sha1": "02febb20d088120b2836d3306cfa21f434f88f65" + "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" + } + }, + "results": { + "version": "0.5.1", + "vcsRevision": "df8113dda4c2d74d460a8fa98252b0b771bf1f27", + "url": "https://github.com/arnetheduck/nim-results", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "a9c011f74bc9ed5c91103917b9f382b12e82a9e7" + } + }, + "nat_traversal": { + "version": "0.0.1", + "vcsRevision": "860e18c37667b5dd005b94c63264560c35d88004", + "url": "https://github.com/status-im/nim-nat-traversal", + "downloadMethod": "git", + "dependencies": [ + "nim", + "results" + ], + "checksums": { + "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97" + } + }, + "stew": { + "version": "0.5.0", + "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa", + "url": "https://github.com/status-im/nim-stew", + "downloadMethod": "git", + "dependencies": [ + "nim", + "results", + "unittest2" + ], + "checksums": { + "sha1": "db22942939773ab7d5a0f2b2668c237240c67dd6" + } + }, + "zlib": { + "version": "0.1.0", + "vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe", + "url": "https://github.com/status-im/nim-zlib", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "results" + ], + "checksums": { + "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f" } }, "httputils": { @@ -133,6 +147,7 @@ "url": "https://github.com/status-im/nim-http-utils", "downloadMethod": "git", "dependencies": [ + "nim", "stew", "results", "unittest2" @@ -147,6 +162,7 @@ "url": "https://github.com/status-im/nim-chronos", "downloadMethod": "git", "dependencies": [ + "nim", "results", "stew", "bearssl", @@ -157,95 +173,13 @@ "sha1": "3a4c9477df8cef20a04e4f1b54a2d74fdfc2a3d0" } }, - "confutils": { - "version": "0.1.0", - "vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a", - "url": "https://github.com/status-im/nim-confutils", - "downloadMethod": "git", - "dependencies": [ - "stew", - "serialization", - "results" - ], - "checksums": { - "sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e" - } - }, - "db_connector": { - "version": "0.1.0", - "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6", - "url": "https://github.com/nim-lang/db_connector", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4" - } - }, - "dnsclient": { - "version": "0.3.4", - "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", - "url": "https://github.com/ba0f3/dnsclient.nim", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" - } - }, - "nimcrypto": { - "version": "0.6.4", - "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", - "url": "https://github.com/cheatfate/nimcrypto", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" - } - }, - "stint": { - "version": "0.8.2", - "vcsRevision": "470b7892561b5179ab20bd389a69217d6213fe58", - "url": "https://github.com/status-im/nim-stint", - "downloadMethod": "git", - "dependencies": [ - "stew", - "unittest2" - ], - "checksums": { - "sha1": "d8f871fd617e7857192d4609fe003b48942a8ae5" - } - }, - "secp256k1": { - "version": "0.6.0.3.2", - "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", - "url": "https://github.com/status-im/nim-secp256k1", - "downloadMethod": "git", - "dependencies": [ - "stew", - "results", - "nimcrypto" - ], - "checksums": { - "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" - } - }, - "nat_traversal": { - "version": "0.0.1", - "vcsRevision": "860e18c37667b5dd005b94c63264560c35d88004", - "url": "https://github.com/status-im/nim-nat-traversal", - "downloadMethod": "git", - "dependencies": [ - "results" - ], - "checksums": { - "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97" - } - }, "metrics": { "version": "0.2.1", "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2", "url": "https://github.com/status-im/nim-metrics", "downloadMethod": "git", "dependencies": [ + "nim", "chronos", "results", "stew" @@ -254,27 +188,18 @@ "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2" } }, - "sqlite3_abi": { - "version": "3.52.0.0", - "vcsRevision": "4b79c5e1882b7fc6c00aec311daf1ed50ad653d5", - "url": "https://github.com/arnetheduck/nim-sqlite3-abi", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "b56b489a7cb01eef8821d66d38d411923a14316d" - } - }, - "minilru": { - "version": "0.1.0", - "vcsRevision": "6dd93feb60f4cded3c05e7af7209cf63fb677893", - "url": "https://github.com/status-im/nim-minilru", + "faststreams": { + "version": "0.5.0", + "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e", + "url": "https://github.com/status-im/nim-faststreams", "downloadMethod": "git", "dependencies": [ - "results", + "nim", + "stew", "unittest2" ], "checksums": { - "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" + "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383" } }, "snappy": { @@ -283,6 +208,7 @@ "url": "https://github.com/status-im/nim-snappy", "downloadMethod": "git", "dependencies": [ + "nim", "faststreams", "unittest2", "results", @@ -292,12 +218,271 @@ "sha1": "e572d60d6a3178c5b1cde2400c51ad771812cd3d" } }, + "serialization": { + "version": "0.5.2", + "vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b", + "url": "https://github.com/status-im/nim-serialization", + "downloadMethod": "git", + "dependencies": [ + "nim", + "faststreams", + "unittest2", + "stew" + ], + "checksums": { + "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8" + } + }, + "toml_serialization": { + "version": "0.2.18", + "vcsRevision": "b5b387e6fb2a7cc75d54a269b07cc6218361bd46", + "url": "https://github.com/status-im/nim-toml-serialization", + "downloadMethod": "git", + "dependencies": [ + "nim", + "faststreams", + "serialization", + "stew" + ], + "checksums": { + "sha1": "76ae1c2af5dd092849b41750ff29217980dc9ca3" + } + }, + "confutils": { + "version": "0.1.0", + "vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a", + "url": "https://github.com/status-im/nim-confutils", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "serialization", + "results" + ], + "checksums": { + "sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e" + } + }, + "json_serialization": { + "version": "0.4.4", + "vcsRevision": "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44", + "url": "https://github.com/status-im/nim-json-serialization", + "downloadMethod": "git", + "dependencies": [ + "nim", + "faststreams", + "serialization", + "stew", + "results" + ], + "checksums": { + "sha1": "8b3115354104858a0ac9019356fb29720529c2bd" + } + }, + "chronicles": { + "version": "0.12.2", + "vcsRevision": "27ec507429a4eb81edc20f28292ee8ec420be05b", + "url": "https://github.com/status-im/nim-chronicles", + "downloadMethod": "git", + "dependencies": [ + "nim", + "faststreams", + "serialization", + "json_serialization", + "testutils" + ], + "checksums": { + "sha1": "02febb20d088120b2836d3306cfa21f434f88f65" + } + }, + "presto": { + "version": "0.1.1", + "vcsRevision": "d66043dd7ede146442e6c39720c76a20bde5225f", + "url": "https://github.com/status-im/nim-presto", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronos", + "chronicles", + "metrics", + "results", + "stew" + ], + "checksums": { + "sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2" + } + }, + "stint": { + "version": "0.8.2", + "vcsRevision": "470b7892561b5179ab20bd389a69217d6213fe58", + "url": "https://github.com/status-im/nim-stint", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "unittest2" + ], + "checksums": { + "sha1": "d8f871fd617e7857192d4609fe003b48942a8ae5" + } + }, + "minilru": { + "version": "0.1.0", + "vcsRevision": "6dd93feb60f4cded3c05e7af7209cf63fb677893", + "url": "https://github.com/status-im/nim-minilru", + "downloadMethod": "git", + "dependencies": [ + "nim", + "results", + "unittest2" + ], + "checksums": { + "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4" + } + }, + "sqlite3_abi": { + "version": "3.53.0.0", + "vcsRevision": "8240e8e2819dfce1b67fa2733135d01b5cc80ae0", + "url": "https://github.com/arnetheduck/nim-sqlite3-abi", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "fb7a6e6f36fc4eb4dfa6634dbcbf5cd0dfd0ebf0" + } + }, + "dnsclient": { + "version": "0.3.4", + "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8", + "url": "https://github.com/ba0f3/dnsclient.nim", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40" + } + }, + "unicodedb": { + "version": "0.13.2", + "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", + "url": "https://github.com/nitely/nim-unicodedb", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" + } + }, + "regex": { + "version": "0.26.3", + "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", + "url": "https://github.com/nitely/nim-regex", + "downloadMethod": "git", + "dependencies": [ + "nim", + "unicodedb" + ], + "checksums": { + "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" + } + }, + "nimcrypto": { + "version": "0.6.4", + "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774", + "url": "https://github.com/cheatfate/nimcrypto", + "downloadMethod": "git", + "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec" + } + }, + "websock": { + "version": "0.3.0", + "vcsRevision": "c105d98e6522e0e2cbe3dfa11b07a273e9fd0e7b", + "url": "https://github.com/status-im/nim-websock", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronos", + "httputils", + "chronicles", + "stew", + "nimcrypto", + "bearssl", + "results", + "zlib" + ], + "checksums": { + "sha1": "1294a66520fa4541e261dec8a6a84f774fb8c0ac" + } + }, + "json_rpc": { + "version": "#43bbf499143eb45046c83ac9794c9e3280a2b8e7", + "vcsRevision": "43bbf499143eb45046c83ac9794c9e3280a2b8e7", + "url": "https://github.com/status-im/nim-json-rpc.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "nimcrypto", + "stint", + "chronos", + "httputils", + "chronicles", + "websock", + "serialization", + "json_serialization", + "unittest2" + ], + "checksums": { + "sha1": "30ff6ead115b88c79862c5c7e37b1c9852eea59f" + } + }, + "lsquic": { + "version": "0.0.1", + "vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f", + "url": "https://github.com/vacp2p/nim-lsquic", + "downloadMethod": "git", + "dependencies": [ + "nim", + "zlib", + "stew", + "chronos", + "nimcrypto", + "unittest2", + "chronicles" + ], + "checksums": { + "sha1": "f465fa994346490d0924d162f53d9b5aec62f948" + } + }, + "secp256k1": { + "version": "0.6.0.3.2", + "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15", + "url": "https://github.com/status-im/nim-secp256k1", + "downloadMethod": "git", + "dependencies": [ + "nim", + "stew", + "results", + "nimcrypto" + ], + "checksums": { + "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10" + } + }, "eth": { "version": "0.9.0", "vcsRevision": "d9135e6c3c5d6d819afdfb566aa8d958756b73a8", "url": "https://github.com/status-im/nim-eth", "downloadMethod": "git", "dependencies": [ + "nim", "nimcrypto", "stint", "secp256k1", @@ -318,12 +503,37 @@ "sha1": "2e01b0cfff9523d110562af70d19948280f8013e" } }, + "web3": { + "version": "0.8.0", + "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", + "url": "https://github.com/status-im/nim-web3", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronicles", + "chronos", + "bearssl", + "eth", + "faststreams", + "json_rpc", + "serialization", + "json_serialization", + "nimcrypto", + "stew", + "stint", + "results" + ], + "checksums": { + "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" + } + }, "dnsdisc": { "version": "0.1.0", "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e", "url": "https://github.com/status-im/nim-dnsdisc", "downloadMethod": "git", "dependencies": [ + "nim", "bearssl", "chronicles", "chronos", @@ -339,119 +549,13 @@ "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154" } }, - "taskpools": { - "version": "0.1.0", - "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", - "url": "https://github.com/status-im/nim-taskpools", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" - } - }, - "ffi": { - "version": "0.1.3", - "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", - "url": "https://github.com/logos-messaging/nim-ffi", - "downloadMethod": "git", - "dependencies": [ - "chronos", - "chronicles", - "taskpools" - ], - "checksums": { - "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" - } - }, - "zlib": { - "version": "0.1.0", - "vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe", - "url": "https://github.com/status-im/nim-zlib", - "downloadMethod": "git", - "dependencies": [ - "stew", - "results" - ], - "checksums": { - "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f" - } - }, - "websock": { - "version": "0.2.2", - "vcsRevision": "3918ce3900c83e1cc7496232a307709f195f7acd", - "url": "https://github.com/status-im/nim-websock", - "downloadMethod": "git", - "dependencies": [ - "chronos", - "httputils", - "chronicles", - "stew", - "nimcrypto", - "bearssl", - "results", - "zlib" - ], - "checksums": { - "sha1": "3c424661eff56c925b01e1cd1a911ff744e72962" - } - }, - "json_rpc": { - "version": "0.5.4", - "vcsRevision": "b6e40a776fa2d00b97a9366761fb7da18f31ae5c", - "url": "https://github.com/status-im/nim-json-rpc", - "downloadMethod": "git", - "dependencies": [ - "stew", - "nimcrypto", - "stint", - "chronos", - "httputils", - "chronicles", - "websock", - "serialization", - "json_serialization", - "unittest2" - ], - "checksums": { - "sha1": "d8e8be795fcf098f4ce03b5826f6b3153f6a6e07" - } - }, - "jwt": { - "version": "0.2", - "vcsRevision": "18f8378de52b241f321c1f9ea905456e89b95c6f", - "url": "https://github.com/vacp2p/nim-jwt.git", - "downloadMethod": "git", - "dependencies": [ - "bearssl", - "bearssl_pkey_decoder" - ], - "checksums": { - "sha1": "bcfd6fc9c5e10a52b87117219b7ab5c98136bc8e" - } - }, - "lsquic": { - "version": "0.0.1", - "vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f", - "url": "https://github.com/vacp2p/nim-lsquic", - "downloadMethod": "git", - "dependencies": [ - "zlib", - "stew", - "chronos", - "nimcrypto", - "unittest2", - "chronicles" - ], - "checksums": { - "sha1": "f465fa994346490d0924d162f53d9b5aec62f948" - } - }, "libp2p": { - "version": "1.15.2", + "version": "#ff8d51857b4b79a68468e7bcc27b2026cca02996", "vcsRevision": "ff8d51857b4b79a68468e7bcc27b2026cca02996", "url": "https://github.com/vacp2p/nim-libp2p.git", "downloadMethod": "git", "dependencies": [ + "nim", "nimcrypto", "dnsclient", "bearssl", @@ -471,79 +575,31 @@ "sha1": "fa2a7552c6ec860717b77ce34cf0b7afe4570234" } }, - "presto": { - "version": "0.1.1", - "vcsRevision": "d66043dd7ede146442e6c39720c76a20bde5225f", - "url": "https://github.com/status-im/nim-presto", + "taskpools": { + "version": "0.1.0", + "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb", + "url": "https://github.com/status-im/nim-taskpools", "downloadMethod": "git", "dependencies": [ + "nim" + ], + "checksums": { + "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" + } + }, + "ffi": { + "version": "0.1.3", + "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", + "url": "https://github.com/logos-messaging/nim-ffi", + "downloadMethod": "git", + "dependencies": [ + "nim", "chronos", "chronicles", - "metrics", - "results", - "stew" + "taskpools" ], "checksums": { - "sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2" - } - }, - "unicodedb": { - "version": "0.13.2", - "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561", - "url": "https://github.com/nitely/nim-unicodedb", - "downloadMethod": "git", - "dependencies": [], - "checksums": { - "sha1": "739102d885d99bb4571b1955f5f12aee423c935b" - } - }, - "regex": { - "version": "0.26.3", - "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c", - "url": "https://github.com/nitely/nim-regex", - "downloadMethod": "git", - "dependencies": [ - "unicodedb" - ], - "checksums": { - "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f" - } - }, - "toml_serialization": { - "version": "0.2.18", - "vcsRevision": "b5b387e6fb2a7cc75d54a269b07cc6218361bd46", - "url": "https://github.com/status-im/nim-toml-serialization", - "downloadMethod": "git", - "dependencies": [ - "faststreams", - "serialization", - "stew" - ], - "checksums": { - "sha1": "76ae1c2af5dd092849b41750ff29217980dc9ca3" - } - }, - "web3": { - "version": "0.8.0", - "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918", - "url": "https://github.com/status-im/nim-web3", - "downloadMethod": "git", - "dependencies": [ - "chronicles", - "chronos", - "bearssl", - "eth", - "faststreams", - "json_rpc", - "serialization", - "json_serialization", - "nimcrypto", - "stew", - "stint", - "results" - ], - "checksums": { - "sha1": "26a112af032ef1536f97da2ca7364af618a11b80" + "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0" } } }, diff --git a/nix/deps.nix b/nix/deps.nix index 2f30a572c..0d9986528 100644 --- a/nix/deps.nix +++ b/nix/deps.nix @@ -12,8 +12,8 @@ bearssl = pkgs.fetchgit { url = "https://github.com/status-im/nim-bearssl"; - rev = "3b341f30d8c619b9a75c154243f9a55468a404e2"; - sha256 = "059avc2dh39vv9c3a1qayah98fjm5pw04r3dn2bqrgs6vf7licmv"; + rev = "22c6a76ce015bc07e011562bdcfc51d9446c1e82"; + sha256 = "1cvdd7lfrpa6asmc39al3g4py5nqhpqmvypc36r5qyv7p5arc8a3"; fetchSubmodules = true; }; @@ -24,38 +24,10 @@ fetchSubmodules = true; }; - results = pkgs.fetchgit { - url = "https://github.com/arnetheduck/nim-results"; - rev = "df8113dda4c2d74d460a8fa98252b0b771bf1f27"; - sha256 = "1h7amas16sbhlr7zb7n3jb5434k98ji375vzw72k1fsc86vnmcr9"; - fetchSubmodules = true; - }; - - stew = pkgs.fetchgit { - url = "https://github.com/status-im/nim-stew"; - rev = "4382b18f04b3c43c8409bfcd6b62063773b2bbaa"; - sha256 = "0mx9g5m636h3sk5pllcpylk51brf7lx91izx3gc23k3ih3hrxyk2"; - fetchSubmodules = true; - }; - - faststreams = pkgs.fetchgit { - url = "https://github.com/status-im/nim-faststreams"; - rev = "ce27581a3e881f782f482cb66dc5b07a02bd615e"; - sha256 = "0y6bw2scnmr8cxj4fg18w7f34l2bh9qwg5nhlgd84m9fpr5bqarn"; - fetchSubmodules = true; - }; - - serialization = pkgs.fetchgit { - url = "https://github.com/status-im/nim-serialization"; - rev = "b0f2fa32960ea532a184394b0f27be37bd80248b"; - sha256 = "0wip1fjx7ka39ck1g1xvmyarzq1p5dlngpqil6zff8k8z5skiz27"; - fetchSubmodules = true; - }; - - json_serialization = pkgs.fetchgit { - url = "https://github.com/status-im/nim-json-serialization"; - rev = "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44"; - sha256 = "0i8sq51nqj8lshf6bfixaz9a7sq0ahsbvq3chkxdvv4khsqvam91"; + jwt = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-jwt.git"; + rev = "18f8378de52b241f321c1f9ea905456e89b95c6f"; + sha256 = "1986czmszdxj6g9yr7xn1fx8y2y9mwpb3f1bn9nc6973qawsdm0p"; fetchSubmodules = true; }; @@ -66,10 +38,38 @@ fetchSubmodules = true; }; - chronicles = pkgs.fetchgit { - url = "https://github.com/status-im/nim-chronicles"; - rev = "27ec507429a4eb81edc20f28292ee8ec420be05b"; - sha256 = "1xx9fcfwgcaizq3s7i3s03mclz253r5j8va38l9ycl19fcbc96z9"; + db_connector = pkgs.fetchgit { + url = "https://github.com/nim-lang/db_connector"; + rev = "29450a2063970712422e1ab857695c12d80112a6"; + sha256 = "11dna09ccdhj3pzpqa04j7a95ibx907z6n1ff33yf0n92qa4x59z"; + fetchSubmodules = true; + }; + + results = pkgs.fetchgit { + url = "https://github.com/arnetheduck/nim-results"; + rev = "df8113dda4c2d74d460a8fa98252b0b771bf1f27"; + sha256 = "1h7amas16sbhlr7zb7n3jb5434k98ji375vzw72k1fsc86vnmcr9"; + fetchSubmodules = true; + }; + + nat_traversal = pkgs.fetchgit { + url = "https://github.com/status-im/nim-nat-traversal"; + rev = "860e18c37667b5dd005b94c63264560c35d88004"; + sha256 = "0319k5bbl468phwfnvlrh7725sc80rnf7m9gyj0i3cb5hb9q78bs"; + fetchSubmodules = true; + }; + + stew = pkgs.fetchgit { + url = "https://github.com/status-im/nim-stew"; + rev = "4382b18f04b3c43c8409bfcd6b62063773b2bbaa"; + sha256 = "0mx9g5m636h3sk5pllcpylk51brf7lx91izx3gc23k3ih3hrxyk2"; + fetchSubmodules = true; + }; + + zlib = pkgs.fetchgit { + url = "https://github.com/status-im/nim-zlib"; + rev = "e680f269fb01af2c34a2ba879ff281795a5258fe"; + sha256 = "1xw9f1gjsgqihdg7kdkbaq1wankgnx2vn9l3ihc6nqk2jzv5bvk5"; fetchSubmodules = true; }; @@ -87,55 +87,6 @@ fetchSubmodules = true; }; - confutils = pkgs.fetchgit { - url = "https://github.com/status-im/nim-confutils"; - rev = "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a"; - sha256 = "18bj1ilx10jm2vmqx2wy2xl9rzy7alymi2m4n9jgpa4sbxnfh0x3"; - fetchSubmodules = true; - }; - - db_connector = pkgs.fetchgit { - url = "https://github.com/nim-lang/db_connector"; - rev = "29450a2063970712422e1ab857695c12d80112a6"; - sha256 = "11dna09ccdhj3pzpqa04j7a95ibx907z6n1ff33yf0n92qa4x59z"; - fetchSubmodules = true; - }; - - dnsclient = pkgs.fetchgit { - url = "https://github.com/ba0f3/dnsclient.nim"; - rev = "23214235d4784d24aceed99bbfe153379ea557c8"; - sha256 = "03mf3lw5c0m5nq9ppa49nylrl8ibkv2zzlc0wyhqg7w09kz6hks6"; - fetchSubmodules = true; - }; - - nimcrypto = pkgs.fetchgit { - url = "https://github.com/cheatfate/nimcrypto"; - rev = "721fb99ee099b632eb86dfad1f0d96ee87583774"; - sha256 = "178vzb3q8wzjq295ik2pd25rrqf32w381ck76hm5x2d8qnzfmkkc"; - fetchSubmodules = true; - }; - - stint = pkgs.fetchgit { - url = "https://github.com/status-im/nim-stint"; - rev = "470b7892561b5179ab20bd389a69217d6213fe58"; - sha256 = "1isfwmbj98qfi5pm9acy0yyvq0vlz38nxp30xl43jx2mmaga2w22"; - fetchSubmodules = true; - }; - - secp256k1 = pkgs.fetchgit { - url = "https://github.com/status-im/nim-secp256k1"; - rev = "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15"; - sha256 = "1qjrmwbngb73f6r1fznvig53nyal7wj41d1cmqfksrmivk2sgrn2"; - fetchSubmodules = true; - }; - - nat_traversal = pkgs.fetchgit { - url = "https://github.com/status-im/nim-nat-traversal"; - rev = "860e18c37667b5dd005b94c63264560c35d88004"; - sha256 = "0319k5bbl468phwfnvlrh7725sc80rnf7m9gyj0i3cb5hb9q78bs"; - fetchSubmodules = true; - }; - metrics = pkgs.fetchgit { url = "https://github.com/status-im/nim-metrics"; rev = "a1296caf3ebb5f30f51a5feae7749a30df2824c2"; @@ -143,17 +94,10 @@ fetchSubmodules = true; }; - sqlite3_abi = pkgs.fetchgit { - url = "https://github.com/arnetheduck/nim-sqlite3-abi"; - rev = "4b79c5e1882b7fc6c00aec311daf1ed50ad653d5"; - sha256 = "0qa6p2vnxmf6r2w19mfydr5rzv7bg1lfxccnpdhk0akzxnc7i5gy"; - fetchSubmodules = true; - }; - - minilru = pkgs.fetchgit { - url = "https://github.com/status-im/nim-minilru"; - rev = "6dd93feb60f4cded3c05e7af7209cf63fb677893"; - sha256 = "1xgx4j56ais3hk8b51zhnfs9q85g2afkp3y1j9ky5iziqvcs2sml"; + faststreams = pkgs.fetchgit { + url = "https://github.com/status-im/nim-faststreams"; + rev = "ce27581a3e881f782f482cb66dc5b07a02bd615e"; + sha256 = "0y6bw2scnmr8cxj4fg18w7f34l2bh9qwg5nhlgd84m9fpr5bqarn"; fetchSubmodules = true; }; @@ -164,73 +108,38 @@ fetchSubmodules = true; }; - eth = pkgs.fetchgit { - url = "https://github.com/status-im/nim-eth"; - rev = "d9135e6c3c5d6d819afdfb566aa8d958756b73a8"; - sha256 = "15r6aszalnbk6mkyfbv5rnz5vcf1mmgj6yg332wry53xsd2ipg7r"; + serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-serialization"; + rev = "b0f2fa32960ea532a184394b0f27be37bd80248b"; + sha256 = "0wip1fjx7ka39ck1g1xvmyarzq1p5dlngpqil6zff8k8z5skiz27"; fetchSubmodules = true; }; - dnsdisc = pkgs.fetchgit { - url = "https://github.com/status-im/nim-dnsdisc"; - rev = "38f2e0f52c0a8f032ef4530835e519d550706d9e"; - sha256 = "0dk787ny49n41bmzhlrvm87giwajr01gwdw9nlmphch89rdqpxxn"; + toml_serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-toml-serialization"; + rev = "b5b387e6fb2a7cc75d54a269b07cc6218361bd46"; + sha256 = "175swdj01rz57h1hvflkyaz4x76qbfn0174ysrk3qk385i1zlg5z"; fetchSubmodules = true; }; - taskpools = pkgs.fetchgit { - url = "https://github.com/status-im/nim-taskpools"; - rev = "9e8ccc754631ac55ac2fd495e167e74e86293edb"; - sha256 = "1y78l33vdjxmb9dkr455pbphxa73rgdsh8m9gpkf4d9b1wm1yivy"; + confutils = pkgs.fetchgit { + url = "https://github.com/status-im/nim-confutils"; + rev = "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a"; + sha256 = "18bj1ilx10jm2vmqx2wy2xl9rzy7alymi2m4n9jgpa4sbxnfh0x3"; fetchSubmodules = true; }; - ffi = pkgs.fetchgit { - url = "https://github.com/logos-messaging/nim-ffi"; - rev = "06111de155253b34e47ed2aaed1d61d08d62cc1b"; - sha256 = "0rb0d2i519amgsp7q0bn6m5465z1vwj4rab89529pyiivh3fgh8j"; + json_serialization = pkgs.fetchgit { + url = "https://github.com/status-im/nim-json-serialization"; + rev = "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44"; + sha256 = "0i8sq51nqj8lshf6bfixaz9a7sq0ahsbvq3chkxdvv4khsqvam91"; fetchSubmodules = true; }; - zlib = pkgs.fetchgit { - url = "https://github.com/status-im/nim-zlib"; - rev = "e680f269fb01af2c34a2ba879ff281795a5258fe"; - sha256 = "1xw9f1gjsgqihdg7kdkbaq1wankgnx2vn9l3ihc6nqk2jzv5bvk5"; - fetchSubmodules = true; - }; - - websock = pkgs.fetchgit { - url = "https://github.com/status-im/nim-websock"; - rev = "3918ce3900c83e1cc7496232a307709f195f7acd"; - sha256 = "16zvdjyasfpb04708d072rpvg12pyz3gmszi3md5brmlhbc3x8jp"; - fetchSubmodules = true; - }; - - json_rpc = pkgs.fetchgit { - url = "https://github.com/status-im/nim-json-rpc"; - rev = "b6e40a776fa2d00b97a9366761fb7da18f31ae5c"; - sha256 = "0c86glijpzcxdb5fagdk98hm9dmsrgw179nn3ixbapl48pvly9nr"; - fetchSubmodules = true; - }; - - jwt = pkgs.fetchgit { - url = "https://github.com/vacp2p/nim-jwt.git"; - rev = "18f8378de52b241f321c1f9ea905456e89b95c6f"; - sha256 = "1986czmszdxj6g9yr7xn1fx8y2y9mwpb3f1bn9nc6973qawsdm0p"; - fetchSubmodules = true; - }; - - lsquic = pkgs.fetchgit { - url = "https://github.com/vacp2p/nim-lsquic"; - rev = "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f"; - sha256 = "0qdhcd4hyp185szc9sv3jvwdwc9zp3j0syy7glxv13k9bchfmkfg"; - fetchSubmodules = true; - }; - - libp2p = pkgs.fetchgit { - url = "https://github.com/vacp2p/nim-libp2p.git"; - rev = "ff8d51857b4b79a68468e7bcc27b2026cca02996"; - sha256 = "08y4s0zhqzsd780bwaixfqbi79km0mcq5g8nyw7awfvcbjqsa53l"; + chronicles = pkgs.fetchgit { + url = "https://github.com/status-im/nim-chronicles"; + rev = "27ec507429a4eb81edc20f28292ee8ec420be05b"; + sha256 = "1xx9fcfwgcaizq3s7i3s03mclz253r5j8va38l9ycl19fcbc96z9"; fetchSubmodules = true; }; @@ -241,6 +150,34 @@ fetchSubmodules = true; }; + stint = pkgs.fetchgit { + url = "https://github.com/status-im/nim-stint"; + rev = "470b7892561b5179ab20bd389a69217d6213fe58"; + sha256 = "1isfwmbj98qfi5pm9acy0yyvq0vlz38nxp30xl43jx2mmaga2w22"; + fetchSubmodules = true; + }; + + minilru = pkgs.fetchgit { + url = "https://github.com/status-im/nim-minilru"; + rev = "6dd93feb60f4cded3c05e7af7209cf63fb677893"; + sha256 = "1xgx4j56ais3hk8b51zhnfs9q85g2afkp3y1j9ky5iziqvcs2sml"; + fetchSubmodules = true; + }; + + sqlite3_abi = pkgs.fetchgit { + url = "https://github.com/arnetheduck/nim-sqlite3-abi"; + rev = "8240e8e2819dfce1b67fa2733135d01b5cc80ae0"; + sha256 = "0g8bc0kiwxxh3h5w06ksa23cw81hnx87rdn93v64m2f053nb6bcm"; + fetchSubmodules = true; + }; + + dnsclient = pkgs.fetchgit { + url = "https://github.com/ba0f3/dnsclient.nim"; + rev = "23214235d4784d24aceed99bbfe153379ea557c8"; + sha256 = "03mf3lw5c0m5nq9ppa49nylrl8ibkv2zzlc0wyhqg7w09kz6hks6"; + fetchSubmodules = true; + }; + unicodedb = pkgs.fetchgit { url = "https://github.com/nitely/nim-unicodedb"; rev = "66f2458710dc641dd4640368f9483c8a0ec70561"; @@ -255,10 +192,45 @@ fetchSubmodules = true; }; - toml_serialization = pkgs.fetchgit { - url = "https://github.com/status-im/nim-toml-serialization"; - rev = "b5b387e6fb2a7cc75d54a269b07cc6218361bd46"; - sha256 = "175swdj01rz57h1hvflkyaz4x76qbfn0174ysrk3qk385i1zlg5z"; + nimcrypto = pkgs.fetchgit { + url = "https://github.com/cheatfate/nimcrypto"; + rev = "721fb99ee099b632eb86dfad1f0d96ee87583774"; + sha256 = "178vzb3q8wzjq295ik2pd25rrqf32w381ck76hm5x2d8qnzfmkkc"; + fetchSubmodules = true; + }; + + websock = pkgs.fetchgit { + url = "https://github.com/status-im/nim-websock"; + rev = "c105d98e6522e0e2cbe3dfa11b07a273e9fd0e7b"; + sha256 = "1zrigw27nwcmg7mw9867581ipcp3ckrqq3cwl2snabcjhkp5dm2c"; + fetchSubmodules = true; + }; + + json_rpc = pkgs.fetchgit { + url = "https://github.com/status-im/nim-json-rpc.git"; + rev = "43bbf499143eb45046c83ac9794c9e3280a2b8e7"; + sha256 = "1c1msxg958jm2ggvs875b6wh6n829d3lh7x4ch6dcxawda16qf95"; + fetchSubmodules = true; + }; + + lsquic = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-lsquic"; + rev = "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f"; + sha256 = "0qdhcd4hyp185szc9sv3jvwdwc9zp3j0syy7glxv13k9bchfmkfg"; + fetchSubmodules = true; + }; + + secp256k1 = pkgs.fetchgit { + url = "https://github.com/status-im/nim-secp256k1"; + rev = "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15"; + sha256 = "1qjrmwbngb73f6r1fznvig53nyal7wj41d1cmqfksrmivk2sgrn2"; + fetchSubmodules = true; + }; + + eth = pkgs.fetchgit { + url = "https://github.com/status-im/nim-eth"; + rev = "d9135e6c3c5d6d819afdfb566aa8d958756b73a8"; + sha256 = "15r6aszalnbk6mkyfbv5rnz5vcf1mmgj6yg332wry53xsd2ipg7r"; fetchSubmodules = true; }; @@ -269,4 +241,32 @@ fetchSubmodules = true; }; + dnsdisc = pkgs.fetchgit { + url = "https://github.com/status-im/nim-dnsdisc"; + rev = "38f2e0f52c0a8f032ef4530835e519d550706d9e"; + sha256 = "0dk787ny49n41bmzhlrvm87giwajr01gwdw9nlmphch89rdqpxxn"; + fetchSubmodules = true; + }; + + libp2p = pkgs.fetchgit { + url = "https://github.com/vacp2p/nim-libp2p.git"; + rev = "ff8d51857b4b79a68468e7bcc27b2026cca02996"; + sha256 = "08y4s0zhqzsd780bwaixfqbi79km0mcq5g8nyw7awfvcbjqsa53l"; + fetchSubmodules = true; + }; + + taskpools = pkgs.fetchgit { + url = "https://github.com/status-im/nim-taskpools"; + rev = "9e8ccc754631ac55ac2fd495e167e74e86293edb"; + sha256 = "1y78l33vdjxmb9dkr455pbphxa73rgdsh8m9gpkf4d9b1wm1yivy"; + fetchSubmodules = true; + }; + + ffi = pkgs.fetchgit { + url = "https://github.com/logos-messaging/nim-ffi"; + rev = "06111de155253b34e47ed2aaed1d61d08d62cc1b"; + sha256 = "0rb0d2i519amgsp7q0bn6m5465z1vwj4rab89529pyiivh3fgh8j"; + fetchSubmodules = true; + }; + } diff --git a/nix/shell.nix b/nix/shell.nix index 80e3b7930..edff468ae 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1,5 +1,17 @@ { pkgs }: +let + nimble = pkgs.nimble.overrideAttrs (_: { + version = "0.22.3"; + src = pkgs.fetchFromGitHub { + owner = "nim-lang"; + repo = "nimble"; + rev = "v0.22.3"; + sha256 = "sha256-f7DYpRGVUeSi6basK1lfu5AxZpMFOSJ3oYsy+urYErg="; + }; + }); +in + pkgs.mkShell { inputsFrom = [ pkgs.androidShell @@ -8,13 +20,12 @@ pkgs.mkShell { pkgs.darwin.apple_sdk.frameworks.Security ]; - buildInputs = with pkgs; [ + buildInputs = (with pkgs; [ git cargo rustup rustc cmake nim-2_2 - nimble - ]; + ]) ++ [ nimble ]; # nimble pinned to 0.22.3 via let binding above } diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh new file mode 100755 index 000000000..c8d0f439d --- /dev/null +++ b/scripts/install_nim.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Installs a specific Nim version. +# Usage: install_nim.sh +# +# Installs to ~/.nim/nim-/ and symlinks binaries into ~/.nimble/bin/, +# which is the idiomatic Nim location already on PATH. +# +# Pre-built binaries are downloaded from nim-lang.org when available. +# Falls back to building from source otherwise (e.g. macOS on older releases). + +set -e + +NIM_VERSION="${1:-}" + +if [ -z "${NIM_VERSION}" ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +# Check if the right version is already installed +nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) +if [ "${nim_ver}" = "${NIM_VERSION}" ]; then + echo "Nim ${NIM_VERSION} already installed, skipping." + exit 0 +fi + +if [ -n "${nim_ver}" ]; then + newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1) + if [ "${newer}" = "${nim_ver}" ]; then + echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2 + echo "WARNING: The build will proceed but may behave differently." >&2 + exit 0 + fi +fi + +OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/') +ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/') + +NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}" +BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz" +WORK_DIR=$(mktemp -d) +trap 'rm -rf "${WORK_DIR}"' EXIT + +echo "Checking for pre-built Nim ${NIM_VERSION} (${OS}_${ARCH})..." +HTTP_STATUS=$(curl -sI "${BINARY_URL}" | head -1 | grep -oE '[0-9]{3}' || true) + +if [ "${HTTP_STATUS}" = "200" ]; then + echo "Downloading pre-built binary from ${BINARY_URL}..." + curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz" + tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}" + rm -rf "${NIM_DEST}" + mkdir -p "${HOME}/.nim" + cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}" +else + echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..." + SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz" + curl -fL "${SRC_URL}" -o "${WORK_DIR}/nim-src.tar.gz" + tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}" + cd "${WORK_DIR}/Nim-${NIM_VERSION}" + sh build_all.sh + rm -rf "${NIM_DEST}" + mkdir -p "${HOME}/.nim" + cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}" +fi + +mkdir -p "${HOME}/.nimble/bin" +for bin_path in "${NIM_DEST}/bin/"*; do + ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")" +done + +echo "Nim ${NIM_VERSION} installed to ${NIM_DEST}" +echo "Binaries symlinked in ~/.nimble/bin — ensure it is in your PATH." diff --git a/waku.nimble b/waku.nimble index 3a7956873..d99f05e84 100644 --- a/waku.nimble +++ b/waku.nimble @@ -8,13 +8,11 @@ version = "0.37.4" author = "Status Research & Development GmbH" description = "Waku, Private P2P Messaging for Resource-Restricted Devices" license = "MIT or Apache License 2.0" -#bin = @["build/waku"] -## This indicates the nim compiler version we are currently working on. It may compile with others -## but we haven't tested. -const NimVersion = "2.2.4" -## This is the underlying nimble version that gets installed after doing `choosenim 2.2.4`. -const NimbleVersion = "0.18.2" +const RequiredNimVersion = "2.2.4" + ## This is the nim compiler version that we are working on. Other versions may behave differently. +const RequiredNimbleVersion = "0.22.3" + ## Enforced nimble version to ensure a reproducible flow ### Dependencies requires "nim >= 2.2.4", @@ -41,7 +39,7 @@ requires "nim >= 2.2.4", "secp256k1", "bearssl", # RPC & APIs - "json_rpc", + "https://github.com/status-im/nim-json-rpc.git#43bbf499143eb45046c83ac9794c9e3280a2b8e7", "presto", "web3", # Database diff --git a/waku/incentivization/eligibility_manager.nim b/waku/incentivization/eligibility_manager.nim index 29443536a..cbbf4774c 100644 --- a/waku/incentivization/eligibility_manager.nim +++ b/waku/incentivization/eligibility_manager.nim @@ -38,11 +38,8 @@ proc getMinedTransactionReceipt( proc getTxAndTxReceipt( eligibilityManager: EligibilityManager, txHash: TxHash ): Future[Result[(TransactionObject, ReceiptObject), string]] {.async.} = - let txFuture = eligibilityManager.getTransactionByHash(txHash) - let receiptFuture = eligibilityManager.getMinedTransactionReceipt(txHash) - await allFutures(txFuture, receiptFuture) - let tx = txFuture.read() - let txReceipt = receiptFuture.read().valueOr: + let tx = await eligibilityManager.getTransactionByHash(txHash) + let txReceipt = (await eligibilityManager.getMinedTransactionReceipt(txHash)).valueOr: return err("Cannot get tx receipt: " & error) return ok((tx, txReceipt)) From 260def68add27840b36bf80bc622ce16aff6c1c0 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Mon, 20 Apr 2026 18:05:44 +0200 Subject: [PATCH 13/25] use EWMA to show main loop lag information (#3808) --- .../health_monitor/event_loop_monitor.nim | 76 ++++++++++++------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/waku/node/health_monitor/event_loop_monitor.nim b/waku/node/health_monitor/event_loop_monitor.nim index d4b8d98d2..49a2bcc6e 100644 --- a/waku/node/health_monitor/event_loop_monitor.nim +++ b/waku/node/health_monitor/event_loop_monitor.nim @@ -1,58 +1,78 @@ {.push raises: [].} +import std/math import chronos, chronicles, metrics logScope: topics = "waku event_loop_monitor" -const CheckInterval = 5.seconds - -declarePublicGauge event_loop_lag_seconds, - "chronos event loop lag in seconds: difference between actual and expected wake-up interval" +declarePublicGauge event_loop_load, + "chronos event loop load EWMA by window (1.0 = sustained lag at MaxAcceptedLag)", + labels = ["window"] type OnLagChange* = proc(lagTooHigh: bool) {.gcsafe, raises: [].} proc eventLoopMonitorLoop*(onLagChange: OnLagChange = nil) {.async.} = - ## Monitors chronos event loop responsiveness. + ## Monitors chronos event loop responsiveness by measuring how much each + ## iteration oversleeps its `CheckInterval`. ## - ## Schedules a task every `CheckInterval`. Because chronos is single-threaded - ## and cooperative, the task can only resume after all previously queued work - ## completes. The actual elapsed time between iterations therefore reflects - ## how saturated the event loop is: + ## The lag is normalised against `MaxAcceptedLag` and tracked as an EWMA + ## over 1, 5, and 15-minute windows (Unix load-average decay model), + ## exposed via the `event_loop_load` gauge (labelled by window: 1m/5m/15m): ## - ## actual_elapsed ≈ CheckInterval → loop is healthy - ## actual_elapsed >> CheckInterval → tasks are accumulating / loop is stalling + ## load < 1.0 → within budget + ## load = 1.0 → sustained lag at MaxAcceptedLag (fully loaded) + ## load > 1.0 → over budget; e.g. 2.0 means twice the accepted lag ## - ## The lag (actual - expected) is exposed via `event_loop_lag_seconds`. - ## When lag transitions above or below `CheckInterval`, `onLagChange` is called. + ## `onLagChange` is called when instantaneous lag crosses `MaxAcceptedLag`. - var lastWakeup = Moment.now() + const CheckInterval = 5.seconds + const MaxAcceptedLag = 50.milliseconds + + # Decay factors: α = 1 − e^(−CheckInterval_secs / window_secs) + # Mirrors the Unix load-average convention so each EWMA has a half-life equal + # to its named window. + const alpha1m = 1.0 - exp(-5.0 / 60.0) # ≈ 0.0821 + const alpha5m = 1.0 - exp(-5.0 / 300.0) # ≈ 0.0165 + const alpha15m = 1.0 - exp(-5.0 / 900.0) # ≈ 0.0055 + + var ewma1m = 0.0 + var ewma5m = 0.0 + var ewma15m = 0.0 + + var now = Moment.now() var lagWasHigh = false + while true: + let lastWakeup = now await sleepAsync(CheckInterval) + now = Moment.now() - let now = Moment.now() let actualElapsed = now - lastWakeup - let lag = actualElapsed - CheckInterval + let lag = max(ZeroDuration, actualElapsed - CheckInterval) + const maxAcceptedLagSecs = MaxAcceptedLag.nanoseconds.float64 / 1_000_000_000.0 + let lagSecs = lag.nanoseconds.float64 / 1_000_000_000.0 + let load = lagSecs / maxAcceptedLagSecs - event_loop_lag_seconds.set(lagSecs) + ewma1m = alpha1m * load + (1.0 - alpha1m) * ewma1m + ewma5m = alpha5m * load + (1.0 - alpha5m) * ewma5m + ewma15m = alpha15m * load + (1.0 - alpha15m) * ewma15m - let lagIsHigh = lag > CheckInterval + event_loop_load.set(round(ewma1m, 4), labelValues = ["1m"]) + event_loop_load.set(round(ewma5m, 4), labelValues = ["5m"]) + event_loop_load.set(round(ewma15m, 4), labelValues = ["15m"]) + + let lagIsHigh = lag > MaxAcceptedLag if lag > CheckInterval: warn "chronos event loop severely lagging, many tasks may be accumulating", expected_secs = CheckInterval.seconds, - actual_secs = actualElapsed.nanoseconds.float64 / 1_000_000_000.0, - lag_secs = lagSecs - elif lag > (CheckInterval div 2): - info "chronos event loop lag detected", - expected_secs = CheckInterval.seconds, - actual_secs = actualElapsed.nanoseconds.float64 / 1_000_000_000.0, - lag_secs = lagSecs + lag_secs = round(lagSecs, 4), + load_1m = round(ewma1m, 4), + load_5m = round(ewma5m, 4), + load_15m = round(ewma15m, 4) - if not isNil(onLagChange) and lagIsHigh != lagWasHigh: + if not onLagChange.isNil() and lagIsHigh != lagWasHigh: lagWasHigh = lagIsHigh onLagChange(lagIsHigh) - - lastWakeup = now From 43948432998064342e9d691381291224724782d6 Mon Sep 17 00:00:00 2001 From: Igor Sirotin Date: Tue, 21 Apr 2026 22:20:53 +0100 Subject: [PATCH 14/25] fix: make update and wakunode2 build on arm64 after Nimble migration (#3814) Rebuild nat libs (miniupnpc, libnatpmp) for the host architecture during nimble deps setup. The prebuilt libs from the nimble cache are x86_64 and fail to link on arm64 (Apple Silicon). Co-authored-by: Claude Sonnet 4.6 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 7ba417527..be9e14027 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,7 @@ $(NIMBLEDEPS_STAMP): nimble.lock | waku.nims nimble setup --localdeps $(MAKE) build-nph $(MAKE) rebuild-bearssl-nimbledeps + $(MAKE) rebuild-nat-libs-nimbledeps touch $@ update: From bb8a7e878233e8f51fd67cf266ba4df4b7bae08c Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Wed, 22 Apr 2026 09:52:57 -0300 Subject: [PATCH 15/25] Fix redundant start/stop calls (#3817) * remove redundant proto start/stop calls from node start/stop * fix WakuRelay start/stop not overriding GossipSub start/stop * replace startRelay with reconnectRelayPeers --- tests/waku_store_sync/sync_utils.nim | 6 +-- tests/waku_store_sync/test_protocol.nim | 36 ++++++------- waku/node/kernel_api/relay.nim | 3 +- waku/node/waku_node.nim | 68 +++++++------------------ waku/waku_metadata/protocol.nim | 6 --- waku/waku_mix/protocol.nim | 6 --- waku/waku_relay/protocol.nim | 4 +- waku/waku_rendezvous/protocol.nim | 15 ++---- waku/waku_store_sync/reconciliation.nim | 15 +++--- waku/waku_store_sync/transfer.nim | 11 ++-- 10 files changed, 61 insertions(+), 109 deletions(-) diff --git a/tests/waku_store_sync/sync_utils.nim b/tests/waku_store_sync/sync_utils.nim index fe62e02a1..888b10a83 100644 --- a/tests/waku_store_sync/sync_utils.nim +++ b/tests/waku_store_sync/sync_utils.nim @@ -45,7 +45,7 @@ proc newTestWakuRecon*( let proto = res.get() - proto.start() + await proto.start() switch.mount(proto) return proto @@ -55,7 +55,7 @@ proc newTestWakuTransfer*( idsTx: AsyncQueue[(SyncID, PubsubTopic, ContentTopic)], wantsRx: AsyncQueue[PeerId], needsRx: AsyncQueue[(PeerId, WakuMessageHash)], -): SyncTransfer = +): Future[SyncTransfer] {.async.} = let peerManager = PeerManager.new(switch) let proto = SyncTransfer.new( @@ -66,7 +66,7 @@ proc newTestWakuTransfer*( remoteNeedsRx = needsRx, ) - proto.start() + await proto.start() switch.mount(proto) return proto diff --git a/tests/waku_store_sync/test_protocol.nim b/tests/waku_store_sync/test_protocol.nim index d051eebd7..d1f5a102a 100644 --- a/tests/waku_store_sync/test_protocol.nim +++ b/tests/waku_store_sync/test_protocol.nim @@ -63,8 +63,8 @@ suite "Waku Sync: reconciliation": clientPeerInfo = clientSwitch.peerInfo.toRemotePeerInfo() asyncTeardown: - server.stop() - client.stop() + await server.stop() + await client.stop() await allFutures(serverSwitch.stop(), clientSwitch.stop()) @@ -561,8 +561,8 @@ suite "Waku Sync: reconciliation": ) defer: - server.stop() - client.stop() + await server.stop() + await client.stop() let res = await client.storeSynchronization(some(serverPeerInfo)) assert res.isOk(), $res.error @@ -610,8 +610,8 @@ suite "Waku Sync: reconciliation": ) defer: - server.stop() - client.stop() + await server.stop() + await client.stop() let res = await client.storeSynchronization(some(serverPeerInfo)) assert res.isOk(), $res.error @@ -657,8 +657,8 @@ suite "Waku Sync: reconciliation": ) defer: - server.stop() - client.stop() + await server.stop() + await client.stop() let res = await client.storeSynchronization(some(serverPeerInfo)) assert res.isOk(), $res.error @@ -701,8 +701,8 @@ suite "Waku Sync: reconciliation": ) defer: - server.stop() - client.stop() + await server.stop() + await client.stop() let res = await client.storeSynchronization(some(serverPeerInfo)) assert res.isOk(), $res.error @@ -736,8 +736,8 @@ suite "Waku Sync: reconciliation": ) defer: - server.stop() - client.stop() + await server.stop() + await client.stop() let res = await client.storeSynchronization(some(serverPeerInfo)) assert res.isOk(), $res.error @@ -773,8 +773,8 @@ suite "Waku Sync: reconciliation": ) defer: - server.stop() - client.stop() + await server.stop() + await client.stop() let res = await client.storeSynchronization(some(serverPeerInfo)) assert res.isOk(), $res.error @@ -848,8 +848,8 @@ suite "Waku Sync: transfer": remoteNeedsRx = clientRemoteNeeds, ) - server.start() - client.start() + await server.start() + await client.start() serverSwitch.mount(server) clientSwitch.mount(client) @@ -861,8 +861,8 @@ suite "Waku Sync: transfer": clientPeermanager.addPeer(serverPeerInfo) asyncTeardown: - server.stop() - client.stop() + await server.stop() + await client.stop() await allFutures(serverSwitch.stop(), clientSwitch.stop()) diff --git a/waku/node/kernel_api/relay.nim b/waku/node/kernel_api/relay.nim index c5a11ff02..fe46f5bd2 100644 --- a/waku/node/kernel_api/relay.nim +++ b/waku/node/kernel_api/relay.nim @@ -263,7 +263,8 @@ proc mountRelay*( node.wakuRelay.routingRecordsHandler.add(peerExchangeHandler.get()) if node.started: - await node.startRelay() + await node.wakuRelay.start() + await node.reconnectRelayPeers() node.switch.mount(node.wakuRelay, protocolMatcher(WakuRelayCodec)) diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 506a3e592..45080d9d0 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -369,30 +369,16 @@ proc mountStoreSync*( return ok() -proc startRelay*(node: WakuNode) {.async.} = - ## Setup and start relay protocol - info "starting relay protocol" - +proc reconnectRelayPeers*(node: WakuNode) {.async.} = + ## Reconnect to previously-seen WakuRelay peers. if node.wakuRelay.isNil(): - error "Failed to start relay. Not mounted." return - - ## Setup relay protocol - - # Resume previous relay connections - if node.peerManager.switch.peerStore.hasPeers(protocolMatcher(WakuRelayCodec)): - info "Found previous WakuRelay peers. Reconnecting." - - # Reconnect to previous relay peers. This will respect a backoff period, if necessary - let backoffPeriod = - node.wakuRelay.parameters.pruneBackoff + chronos.seconds(BackoffSlackTime) - - await node.peerManager.reconnectPeers(WakuRelayCodec, backoffPeriod) - - # Start the WakuRelay protocol - await node.wakuRelay.start() - - info "relay started successfully" + if not node.peerManager.switch.peerStore.hasPeers(protocolMatcher(WakuRelayCodec)): + return + info "Found previous WakuRelay peers. Reconnecting." + let backoffPeriod = + node.wakuRelay.parameters.pruneBackoff + chronos.seconds(BackoffSlackTime) + await node.peerManager.reconnectPeers(WakuRelayCodec, backoffPeriod) proc selectRandomPeers*(peers: seq[PeerId], numRandomPeers: int): seq[PeerId] = var randomPeers = peers @@ -430,7 +416,10 @@ proc mountRendezvous*( return if node.started: - await node.wakuRendezvous.start() + try: + await node.wakuRendezvous.start() + except CancelledError as exc: + error "failed to start wakuRendezvous", error = exc.msg try: node.switch.mount(node.wakuRendezvous, protocolMatcher(WakuRendezVousCodec)) @@ -578,31 +567,12 @@ proc start*(node: WakuNode) {.async.} = if isBindIpWithZeroPort(address): zeroPortPresent = true - # Perform relay-specific startup tasks TODO: this should be rethought - if not node.wakuRelay.isNil(): - await node.startRelay() - - if not node.wakuMix.isNil(): - node.wakuMix.start() - - if not node.wakuMetadata.isNil(): - node.wakuMetadata.start() - if not node.wakuStoreResume.isNil(): await node.wakuStoreResume.start() - if not node.wakuRendezvous.isNil(): - await node.wakuRendezvous.start() - if not node.wakuRendezvousClient.isNil(): await node.wakuRendezvousClient.start() - if not node.wakuStoreReconciliation.isNil(): - node.wakuStoreReconciliation.start() - - if not node.wakuStoreTransfer.isNil(): - node.wakuStoreTransfer.start() - ## The switch uses this mapper to update peer info addrs ## with announced addrs after start let addressMapper = proc( @@ -612,8 +582,12 @@ proc start*(node: WakuNode) {.async.} = node.switch.peerInfo.addressMappers.add(addressMapper) ## The switch will update addresses after start using the addressMapper + ## NOTE: This will dispatch gossipsub start to the WakuRelay.start method override await node.switch.start() + # After switch.start, run custom Logos Delivery relay start logic + await node.reconnectRelayPeers() + node.started = true if not node.wakuFilterClient.isNil(): @@ -637,6 +611,7 @@ proc stop*(node: WakuNode) {.async.} = node.stopProvidersAndListeners() + ## NOTE: This will dispatch gossipsub stop to the WakuRelay.stop method override await node.switch.stop() node.peerManager.stop() @@ -653,12 +628,6 @@ proc stop*(node: WakuNode) {.async.} = if not node.wakuStoreResume.isNil(): await node.wakuStoreResume.stopWait() - if not node.wakuStoreReconciliation.isNil(): - node.wakuStoreReconciliation.stop() - - if not node.wakuStoreTransfer.isNil(): - node.wakuStoreTransfer.stop() - if not node.wakuPeerExchangeClient.isNil() and not node.wakuPeerExchangeClient.pxLoopHandle.isNil(): await node.wakuPeerExchangeClient.pxLoopHandle.cancelAndWait() @@ -666,9 +635,6 @@ proc stop*(node: WakuNode) {.async.} = if not node.wakuKademlia.isNil(): await node.wakuKademlia.stop() - if not node.wakuRendezvous.isNil(): - await node.wakuRendezvous.stopWait() - if not node.wakuRendezvousClient.isNil(): await node.wakuRendezvousClient.stopWait() diff --git a/waku/waku_metadata/protocol.nim b/waku/waku_metadata/protocol.nim index 623cbb6c3..7c72a6934 100644 --- a/waku/waku_metadata/protocol.nim +++ b/waku/waku_metadata/protocol.nim @@ -108,9 +108,3 @@ proc new*(T: type WakuMetadata, clusterId: uint32, getShards: GetShards): T = clusterId = wm.clusterId, shards = wm.getShards() return wm - -proc start*(wm: WakuMetadata) = - wm.started = true - -proc stop*(wm: WakuMetadata) = - wm.started = false diff --git a/waku/waku_mix/protocol.nim b/waku/waku_mix/protocol.nim index e31929b71..ac8b69eaf 100644 --- a/waku/waku_mix/protocol.nim +++ b/waku/waku_mix/protocol.nim @@ -104,10 +104,4 @@ proc new*( proc poolSize*(mix: WakuMix): int = mix.nodePool.len -method start*(mix: WakuMix) = - info "starting waku mix protocol" - -method stop*(mix: WakuMix) {.async.} = - discard - # Mix Protocol diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim index 79d3702eb..b19173d36 100644 --- a/waku/waku_relay/protocol.nim +++ b/waku/waku_relay/protocol.nim @@ -517,12 +517,12 @@ proc topicsHealthLoop(w: WakuRelay) {.async.} = # safety cooldown to protect from edge cases await sleepAsync(100.milliseconds) -method start*(w: WakuRelay) {.async, base.} = +method start*(w: WakuRelay) {.async: (raises: [CancelledError]).} = info "start" await procCall GossipSub(w).start() w.topicHealthLoopHandle = w.topicsHealthLoop() -method stop*(w: WakuRelay) {.async, base.} = +method stop*(w: WakuRelay) {.async: (raises: []).} = info "stop" await procCall GossipSub(w).stop() diff --git a/waku/waku_rendezvous/protocol.nim b/waku/waku_rendezvous/protocol.nim index 00b5f1a5c..89433f533 100644 --- a/waku/waku_rendezvous/protocol.nim +++ b/waku/waku_rendezvous/protocol.nim @@ -211,29 +211,22 @@ proc new*( return ok(wrv) -proc start*(self: WakuRendezVous) {.async: (raises: []).} = +method start*(self: WakuRendezVous) {.async: (raises: [CancelledError]).} = # Start the parent GenericRendezVous (starts the register deletion loop) if self.started: warn "waku rendezvous already started" return - try: - await procCall GenericRendezVous[WakuPeerRecord](self).start() - except CancelledError as exc: - error "failed to start GenericRendezVous", cause = exc.msg - return + await procCall GenericRendezVous[WakuPeerRecord](self).start() # start registering forever self.periodicRegistrationFut = self.periodicRegistration() info "waku rendezvous discovery started" -proc stopWait*(self: WakuRendezVous) {.async: (raises: []).} = +method stop*(self: WakuRendezVous) {.async: (raises: []).} = if not self.periodicRegistrationFut.isNil(): await self.periodicRegistrationFut.cancelAndWait() # Stop the parent GenericRendezVous (stops the register deletion loop) - await GenericRendezVous[WakuPeerRecord](self).stop() - - # Stop the parent GenericRendezVous (stops the register deletion loop) - await GenericRendezVous[WakuPeerRecord](self).stop() + await procCall GenericRendezVous[WakuPeerRecord](self).stop() info "waku rendezvous discovery stopped" diff --git a/waku/waku_store_sync/reconciliation.nim b/waku/waku_store_sync/reconciliation.nim index 23f513322..9dd308255 100644 --- a/waku/waku_store_sync/reconciliation.nim +++ b/waku/waku_store_sync/reconciliation.nim @@ -468,7 +468,7 @@ proc idsReceiverLoop(self: SyncReconciliation) {.async.} = self.messageIngress(id, pubsub, content) -proc start*(self: SyncReconciliation) = +method start*(self: SyncReconciliation) {.async: (raises: [CancelledError]).} = if self.started: return @@ -484,13 +484,16 @@ proc start*(self: SyncReconciliation) = info "Store Sync Reconciliation protocol started" -proc stop*(self: SyncReconciliation) = - if self.syncInterval > ZeroDuration: - self.periodicSyncFut.cancelSoon() +method stop*(self: SyncReconciliation) {.async: (raises: []).} = + defer: + self.started = false if self.syncInterval > ZeroDuration: - self.periodicPruneFut.cancelSoon() + await self.periodicSyncFut.cancelAndWait() - self.idsReceiverFut.cancelSoon() + if self.syncInterval > ZeroDuration: + await self.periodicPruneFut.cancelAndWait() + + await self.idsReceiverFut.cancelAndWait() info "Store Sync Reconciliation protocol stopped" diff --git a/waku/waku_store_sync/transfer.nim b/waku/waku_store_sync/transfer.nim index 6a600b4e3..5d20afb18 100644 --- a/waku/waku_store_sync/transfer.nim +++ b/waku/waku_store_sync/transfer.nim @@ -217,7 +217,7 @@ proc new*( return transfer -proc start*(self: SyncTransfer) = +method start*(self: SyncTransfer) {.async: (raises: [CancelledError]).} = if self.started: return @@ -228,10 +228,11 @@ proc start*(self: SyncTransfer) = info "Store Sync Transfer protocol started" -proc stop*(self: SyncTransfer) = - self.started = false +method stop*(self: SyncTransfer) {.async: (raises: []).} = + defer: + self.started = false - self.localWantsRxFut.cancelSoon() - self.remoteNeedsRxFut.cancelSoon() + await self.localWantsRxFut.cancelAndWait() + await self.remoteNeedsRxFut.cancelAndWait() info "Store Sync Transfer protocol stopped" From 820ccc6e1012ba6d5dfeef27f8bf4d126c2e7940 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Thu, 23 Apr 2026 18:24:55 +0200 Subject: [PATCH 16/25] Add ci support for liblogosdeliery, build and artifacts (#3746) --- .github/workflows/ci.yml | 1 + .github/workflows/release-assets.yml | 48 +++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b45853e21..52d20157a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: - 'waku.nimble' - 'Makefile' - 'library/**' + - 'liblogosdelivery/**' v2: - 'waku/**' - 'apps/**' diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 50e3c4c3d..274eb564c 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -4,7 +4,7 @@ on: push: tags: - 'v*' # "e.g. v0.4" - + workflow_dispatch: env: @@ -65,6 +65,16 @@ jobs: echo "libwaku=${LIBWAKU_ARTIFACT_NAME}" >> $GITHUB_OUTPUT + if [[ "${{ runner.os }}" == "Linux" ]]; then + LIBLOGOSDELIVERY_ARTIFACT_NAME=$(echo "liblogosdelivery-${VERSION}-${{matrix.arch}}-${{runner.os}}-linux.deb" | tr "[:upper:]" "[:lower:]") + fi + + if [[ "${{ runner.os }}" == "macOS" ]]; then + LIBLOGOSDELIVERY_ARTIFACT_NAME=$(echo "liblogosdelivery-${VERSION}-${{matrix.arch}}-macos.tar.gz" | tr "[:upper:]" "[:lower:]") + fi + + echo "liblogosdelivery=${LIBLOGOSDELIVERY_ARTIFACT_NAME}" >> $GITHUB_OUTPUT + - name: Install build dependencies run: | if [[ "${{ runner.os }}" == "Linux" ]]; then @@ -83,6 +93,9 @@ jobs: make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false libwaku make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false STATIC=1 libwaku + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false liblogosdelivery + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false STATIC=1 liblogosdelivery + - name: Create distributable libwaku package run: | VERSION=${{ steps.version.outputs.version }} @@ -109,6 +122,32 @@ jobs: tar -cvzf ${{steps.vars.outputs.libwaku}} ./build/libwaku.dylib ./build/libwaku.a ./library/libwaku.h fi + - name: Create distributable liblogosdelivery package + run: | + VERSION=${{ steps.version.outputs.version }} + + if [[ "${{ runner.os }}" == "Linux" ]]; then + rm -rf pkg + mkdir -p pkg/DEBIAN pkg/usr/local/lib pkg/usr/local/include + cp build/liblogosdelivery.so pkg/usr/local/lib/ + cp build/liblogosdelivery.a pkg/usr/local/lib/ + cp liblogosdelivery/liblogosdelivery.h pkg/usr/local/include/ + + echo "Package: logosdelivery" >> pkg/DEBIAN/control + echo "Version: ${VERSION}" >> pkg/DEBIAN/control + echo "Priority: optional" >> pkg/DEBIAN/control + echo "Section: libs" >> pkg/DEBIAN/control + echo "Architecture: ${{matrix.arch}}" >> pkg/DEBIAN/control + echo "Maintainer: Logos Messaging Team" >> pkg/DEBIAN/control + echo "Description: Logos Delivery library" >> pkg/DEBIAN/control + + dpkg-deb --build pkg ${{steps.vars.outputs.liblogosdelivery}} + fi + + if [[ "${{ runner.os }}" == "macOS" ]]; then + tar -cvzf ${{steps.vars.outputs.liblogosdelivery}} ./build/liblogosdelivery.dylib ./build/liblogosdelivery.a ./liblogosdelivery/liblogosdelivery.h + fi + - name: Upload waku artifact uses: actions/upload-artifact@v4.4.0 with: @@ -122,3 +161,10 @@ jobs: name: libwaku-${{ steps.version.outputs.version }}-${{ matrix.arch }}-${{ runner.os }} path: ${{ steps.vars.outputs.libwaku }} if-no-files-found: error + + - name: Upload liblogosdelivery artifact + uses: actions/upload-artifact@v4.4.0 + with: + name: liblogosdelivery-${{ steps.version.outputs.version }}-${{ matrix.arch }}-${{ runner.os }} + path: ${{ steps.vars.outputs.liblogosdelivery }} + if-no-files-found: error From ff98d853138c3f07c8c5074aa7a9d5ab6cf316b8 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Thu, 23 Apr 2026 16:02:34 -0300 Subject: [PATCH 17/25] fix: relay validator registration and sync filter (#3823) * reuse stored validator in relay * fix skip check in store sync * increase sync tolerance in test (matches similar test) --- tests/waku_store_sync/test_protocol.nim | 2 +- waku/waku_relay/protocol.nim | 2 +- waku/waku_store_sync/reconciliation.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/waku_store_sync/test_protocol.nim b/tests/waku_store_sync/test_protocol.nim index d1f5a102a..3ffa7ad4a 100644 --- a/tests/waku_store_sync/test_protocol.nim +++ b/tests/waku_store_sync/test_protocol.nim @@ -372,7 +372,7 @@ suite "Waku Sync: reconciliation": const msgCount = 400_000 diffCount = 100_000 - tol = 1000 + tol = 10_000 var diffMsgHashes: HashSet[WakuMessageHash] var missingIdx: HashSet[int] diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim index b19173d36..e7b2c99cb 100644 --- a/waku/waku_relay/protocol.nim +++ b/waku/waku_relay/protocol.nim @@ -618,7 +618,7 @@ proc subscribe*(w: WakuRelay, pubsubTopic: PubsubTopic, handler: WakuRelayHandle # Otherwise this might lead to unintended behaviour. if not w.topicValidator.hasKey(pubSubTopic): let newValidator = w.generateOrderedValidator() - procCall GossipSub(w).addValidator(pubSubTopic, w.generateOrderedValidator()) + procCall GossipSub(w).addValidator(pubSubTopic, newValidator) w.topicValidator[pubSubTopic] = newValidator # set this topic parameters for scoring diff --git a/waku/waku_store_sync/reconciliation.nim b/waku/waku_store_sync/reconciliation.nim index 9dd308255..b18251fff 100644 --- a/waku/waku_store_sync/reconciliation.nim +++ b/waku/waku_store_sync/reconciliation.nim @@ -145,7 +145,7 @@ proc preProcessPayload( # convert to skip range before processing for i in 0 ..< payload.ranges.len: let rangeType = payload.ranges[i][1] - if rangeType != RangeType.Skip: + if rangeType == RangeType.Skip: continue let upperBound = payload.ranges[i][0].b.time From 324048430bf1b84ffa0376041b79ce46f3df96de Mon Sep 17 00:00:00 2001 From: Darshan <35736874+darshankabariya@users.noreply.github.com> Date: Sat, 25 Apr 2026 00:03:46 +0530 Subject: [PATCH 18/25] fix: restore -d:postgres in nimble task and propagate NIMFLAGS (#3830) --- .github/workflows/container-image.yml | 2 +- .github/workflows/pre-release.yml | 4 ++-- .github/workflows/release-assets.yml | 10 +++++----- Dockerfile | 3 ++- ci/Jenkinsfile.release | 6 ++++-- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 0783c1f66..0ff427d87 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -83,7 +83,7 @@ jobs: id: build if: ${{ steps.secrets.outcome == 'success' }} run: | - make -j${NPROC} V=1 NIMFLAGS="-d:disableMarchNative -d:postgres -d:chronicles_colors:none" wakunode2 + make -j${NPROC} V=1 POSTGRES=1 NIMFLAGS="-d:disableMarchNative -d:chronicles_colors:none" wakunode2 SHORT_REF=$(git rev-parse --short HEAD) diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index e3c8bb575..52a50adc8 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -66,8 +66,8 @@ jobs: make V=1 CI=false NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \ update - make V=1 CI=false\ - NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" \ + make V=1 CI=false POSTGRES=1\ + NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \ wakunode2\ chat2\ tools diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 274eb564c..fc1f819d9 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -86,15 +86,15 @@ jobs: OS=$([[ "${{runner.os}}" == "macOS" ]] && echo "macosx" || echo "linux") make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" V=1 update - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false wakunode2 + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false wakunode2 make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" CI=false chat2 tar -cvzf ${{steps.vars.outputs.waku}} ./build/ - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false libwaku - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false STATIC=1 libwaku + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false libwaku + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false STATIC=1 libwaku - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false liblogosdelivery - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false STATIC=1 liblogosdelivery + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false liblogosdelivery + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false STATIC=1 liblogosdelivery - name: Create distributable libwaku package run: | diff --git a/Dockerfile b/Dockerfile index 412d0977a..05525774b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ ARG NIMFLAGS ARG MAKE_TARGET=wakunode2 ARG NIM_COMMIT ARG HEAPTRACK_BUILD=0 +ARG POSTGRES=0 # Get build tools and required header files RUN apk add --no-cache bash git build-base openssl-dev linux-headers curl jq libbsd-dev @@ -26,7 +27,7 @@ RUN if [ "$HEAPTRACK_BUILD" = "1" ]; then \ RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT} # Build the final node binary -RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET NIMFLAGS="${NIMFLAGS}" +RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET NIMFLAGS="${NIMFLAGS}" POSTGRES=${POSTGRES} # PRODUCTION IMAGE ------------------------------------------------------------- diff --git a/ci/Jenkinsfile.release b/ci/Jenkinsfile.release index 570a37d5f..d8237f009 100644 --- a/ci/Jenkinsfile.release +++ b/ci/Jenkinsfile.release @@ -85,7 +85,8 @@ pipeline { "--label=commit='${git.commit()}' " + "--label=version='${git.describe('--tags')}' " + "--build-arg=MAKE_TARGET='${params.MAKE_TARGET}' " + - "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:postgres -d:heaptracker ' " + + "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:heaptracker ' " + + "--build-arg=POSTGRES='1' " + "--build-arg=LOG_LEVEL='${params.LOWEST_LOG_LEVEL_ALLOWED}' " + "--build-arg=DEBUG='${params.DEBUG ? "1" : "0"} ' " + "--build-arg=NIM_COMMIT='NIM_COMMIT=heaptrack_support_v2.0.12' " + @@ -98,7 +99,8 @@ pipeline { "--label=commit='${git.commit()}' " + "--label=version='${git.describe('--tags')}' " + "--build-arg=MAKE_TARGET='${params.MAKE_TARGET}' " + - "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:postgres ' " + + "--build-arg=NIMFLAGS='${params.NIMFLAGS}' " + + "--build-arg=POSTGRES='1' " + "--build-arg=LOG_LEVEL='${params.LOWEST_LOG_LEVEL_ALLOWED}' " + "--build-arg=DEBUG='${params.DEBUG ? "1" : "0"} ' " + "--target='prod' ." From 5034086fefe2f32bf95319cdd39aa62fc622e4bc Mon Sep 17 00:00:00 2001 From: osmaczko <33099791+osmaczko@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:51:39 +0200 Subject: [PATCH 19/25] Chore/make nix build phase configurable (#3826) * nix: parameterize build flags with named args Expose `enablePostgres`, `enableNimDebugDlOpen`, and `chroniclesLogLevel` as arguments on `nix/default.nix`. Defaults preserve today's hardcoded behavior, so `nix build .#liblogosdelivery` with no overrides is a no-op change. Consume the package via `callPackage` in `flake.nix` so consumers can use `.override { ... }` without extra wrapping. * nix: link libstdc++ on Linux so consumers don't need patchelf Append `stdenv.cc.cc.lib` to `buildInputs` on Linux and add `-lstdc++` to the Nim `--passL` flags. Nix stdenv's fixupPhase will auto-inject `${stdenv.cc.cc.lib}/lib` into the output's RUNPATH, so downstream consumers can drop their patchelf step. macOS resolves the C++ stdlib via dyld/libc++ and is unaffected. * nix: bundle librln into the output for a self-contained package Copy the librln shared library (`librln.so` / `librln.dylib`) from the zerokit input into `$out/lib` and rewrite the internal reference in `liblogosdelivery`: - Darwin: set librln's install name to `@rpath/librln.dylib`, change the consumer's reference to match, and add `@loader_path` as an rpath. - Linux: add `$ORIGIN` to the rpath so `librln.so` resolves from the sibling directory, preserving the gcc-lib entry injected by the stdenv fixupPhase for libstdc++. The installed `liblogosdelivery` no longer carries a `/nix/store/...` absolute path to zerokit, so downstream consumers can ship the bundle as-is. --- flake.nix | 11 ++++++----- nix/default.nix | 51 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/flake.nix b/flake.nix index 31d5a120c..50b6dc0b5 100644 --- a/flake.nix +++ b/flake.nix @@ -56,13 +56,14 @@ packages = forAllSystems (system: let pkgs = pkgsFor system; - mkPkg = zerokitRln: import ./nix/default.nix { - inherit pkgs zerokitRln; + liblogosdelivery = pkgs.callPackage ./nix/default.nix { + inherit pkgs; src = ./.; + zerokitRln = zerokit.packages.${system}.rln; }; - in rec { - liblogosdelivery = mkPkg zerokit.packages.${system}.rln; - default = liblogosdelivery; + in { + inherit liblogosdelivery; + default = liblogosdelivery; } ); diff --git a/nix/default.nix b/nix/default.nix index 0d1de2ece..a9ea0f598 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,8 +1,22 @@ -{ pkgs, src, zerokitRln }: +{ pkgs +, src +, zerokitRln +, enablePostgres ? true +, enableNimDebugDlOpen ? true +, chroniclesLogLevel ? null +}: let deps = import ./deps.nix { inherit pkgs; }; + nimDefineArgs = pkgs.lib.concatStringsSep " \\\n " ( + [ "--define:disable_libbacktrace" ] + ++ pkgs.lib.optional enablePostgres "--define:postgres" + ++ pkgs.lib.optional enableNimDebugDlOpen "--define:nimDebugDlOpen" + ++ pkgs.lib.optional (chroniclesLogLevel != null) + "--define:chronicles_log_level=${toString chroniclesLogLevel}" + ); + # nat_traversal is excluded from the static pathArgs; it is handled # separately in buildPhase (its bundled C libs must be compiled first). otherDeps = builtins.removeAttrs deps [ "nat_traversal" ]; @@ -32,7 +46,8 @@ pkgs.stdenv.mkDerivation { which ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.cctools ]; - buildInputs = [ zerokitRln ]; + buildInputs = [ zerokitRln ] + ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.stdenv.cc.cc.lib ]; buildPhase = '' export HOME=$TMPDIR @@ -60,10 +75,8 @@ pkgs.stdenv.mkDerivation { ${pathArgs} \ --path:$NAT_TRAV \ --path:$NAT_TRAV/src \ - --passL:"-L${zerokitRln}/lib -lrln" \ - --define:disable_libbacktrace \ - --define:postgres \ - --define:nimDebugDlOpen \ + --passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \ + ${nimDefineArgs} \ --out:build/liblogosdelivery.${libExt} \ --app:lib \ --threads:on \ @@ -81,10 +94,8 @@ pkgs.stdenv.mkDerivation { ${pathArgs} \ --path:$NAT_TRAV \ --path:$NAT_TRAV/src \ - --passL:"-L${zerokitRln}/lib -lrln" \ - --define:disable_libbacktrace \ - --define:postgres \ - --define:nimDebugDlOpen \ + --passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \ + ${nimDefineArgs} \ --out:build/liblogosdelivery.a \ --app:staticlib \ --threads:on \ @@ -97,9 +108,29 @@ pkgs.stdenv.mkDerivation { ''; installPhase = '' + runHook preInstall mkdir -p $out/lib $out/include cp build/liblogosdelivery.${libExt} $out/lib/ 2>/dev/null || true cp build/liblogosdelivery.a $out/lib/ 2>/dev/null || true cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true + runHook postInstall ''; + + # Bundle librln alongside liblogosdelivery so the output is self-contained. + # Use --add-rpath (not --set-rpath) so fixupPhase's stdenv RUNPATH injection + # for libstdc++ is preserved. + postInstall = + pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + cp ${zerokitRln}/lib/librln.dylib $out/lib/ + chmod +w $out/lib/librln.dylib $out/lib/liblogosdelivery.dylib + install_name_tool -id @rpath/liblogosdelivery.dylib $out/lib/liblogosdelivery.dylib + install_name_tool -id @rpath/librln.dylib $out/lib/librln.dylib + old=$(otool -L $out/lib/liblogosdelivery.dylib | awk 'NR>1{print $1}' | grep librln) + install_name_tool -change "$old" @rpath/librln.dylib $out/lib/liblogosdelivery.dylib + install_name_tool -add_rpath @loader_path $out/lib/liblogosdelivery.dylib + '' + + pkgs.lib.optionalString pkgs.stdenv.isLinux '' + cp ${zerokitRln}/lib/librln.so $out/lib/ + patchelf --add-rpath '$ORIGIN' $out/lib/liblogosdelivery.so + ''; } From 300f584efcb69c3d562762ec27515bb94a4d06b2 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Wed, 29 Apr 2026 15:10:21 +0200 Subject: [PATCH 20/25] Removed duplicates of announcedAddresses, extMultiaddresses (#3831) Removing duplicates of multiaddresses for Enr. Safe building Enr Record. Co-authored-by: Copilot --- tests/factory/test_node_factory.nim | 49 ++++++++++++++++++++++++++++- tests/test_waku_enr.nim | 38 ++++++++++++++++++++++ tests/test_waku_netconfig.nim | 25 +++++++++++++++ waku/factory/internal_config.nim | 33 ++++++++++++++++--- waku/node/net_config.nim | 10 ++++-- waku/waku_enr/multiaddr.nim | 2 +- 6 files changed, 148 insertions(+), 9 deletions(-) diff --git a/tests/factory/test_node_factory.nim b/tests/factory/test_node_factory.nim index f30e079b5..4b2085e82 100644 --- a/tests/factory/test_node_factory.nim +++ b/tests/factory/test_node_factory.nim @@ -1,11 +1,19 @@ {.used.} -import testutils/unittests, chronos, libp2p/protocols/connectivity/relay/relay +import + std/[options, sequtils], + testutils/unittests, + chronos, + libp2p/multiaddress, + libp2p/protocols/connectivity/relay/relay +import eth/p2p/discoveryv5/enr import ../testlib/wakunode, waku/waku_node, + waku/waku_enr, waku/factory/node_factory, + waku/factory/internal_config, waku/factory/conf_builder/conf_builder, waku/factory/conf_builder/web_socket_conf_builder @@ -38,6 +46,45 @@ suite "Node Factory": not node.wakuStore.isNil() not node.wakuArchive.isNil() + test "ENR configuration trims multiaddrs until record fits": + var conf = defaultTestWakuConf() + let bindIp = conf.endpointConf.p2pListenAddress + let bindPort = Port(30303) + + let oversizedMultiaddrs = (0 .. 11).mapIt( + MultiAddress + .init( + "/dns4/very-long-logical-hostname-" & $it & + ".example.logos.dev.status.im/tcp/30303/wss" + ) + .get() + ) + + let netConfig = NetConfig.init( + clusterId = conf.clusterId, + bindIp = bindIp, + bindPort = bindPort, + extMultiAddrs = oversizedMultiaddrs, + extMultiAddrsOnly = true, + wakuFlags = some(conf.wakuFlags), + ).valueOr: + raiseAssert error + + let record = enrConfiguration(conf, netConfig).valueOr: + raiseAssert error + + let typedRecord = record.toTyped() + require typedRecord.isOk() + + let multiaddrsOpt = typedRecord.value.multiaddrs + require multiaddrsOpt.isSome() + + let retainedMultiaddrs = multiaddrsOpt.get() + check: + retainedMultiaddrs.len < oversizedMultiaddrs.len + retainedMultiaddrs.len > 0 + retainedMultiaddrs == oversizedMultiaddrs[0 ..< retainedMultiaddrs.len] + asynctest "Set up a node with Filter enabled": var confBuilder = defaultTestWakuConfBuilder() confBuilder.filterServiceConf.withEnabled(true) diff --git a/tests/test_waku_enr.nim b/tests/test_waku_enr.nim index 2ffff5e57..10183adf5 100644 --- a/tests/test_waku_enr.nim +++ b/tests/test_waku_enr.nim @@ -271,6 +271,44 @@ suite "Waku ENR - Multiaddresses": multiaddrs.contains(expectedAddr1) multiaddrs.contains(addr2) + test "encode and decode record with multiaddrs field deduplicates duplicate entries": + ## Given + let + enrSeqNum = 1u64 + enrPrivKey = generatesecp256k1key() + + let + addr1 = MultiAddress + .init( + "/ip4/127.0.0.1/tcp/80/ws/p2p/16Uiu2HAm4v86W3bmT1BiH6oSPzcsSr31iDQpSN5Qa882BCjjwgrD" + ) + .get() + addr1NoPeerId = MultiAddress.init("/ip4/127.0.0.1/tcp/80/ws").get() + addr2 = MultiAddress.init("/ip4/127.0.0.1/tcp/443/wss").get() + + ## When + var builder = EnrBuilder.init(enrPrivKey, seqNum = enrSeqNum) + builder.withMultiaddrs(@[addr1, addr1NoPeerId, addr2, addr2]) + + let recordRes = builder.build() + + require recordRes.isOk() + let record = recordRes.tryGet() + + let typedRecord = record.toTyped() + require typedRecord.isOk() + + let multiaddrsOpt = typedRecord.value.multiaddrs + + ## Then + check multiaddrsOpt.isSome() + + let multiaddrs = multiaddrsOpt.get() + check: + multiaddrs.len == 2 + multiaddrs.contains(addr1NoPeerId) + multiaddrs.contains(addr2) + suite "Waku ENR - Relay static sharding": test "new relay shards object with single invalid shard id": ## Given diff --git a/tests/test_waku_netconfig.nim b/tests/test_waku_netconfig.nim index 5f9ff4b46..20d95d59b 100644 --- a/tests/test_waku_netconfig.nim +++ b/tests/test_waku_netconfig.nim @@ -152,6 +152,31 @@ suite "Waku NetConfig": netConfig.announcedAddresses.len == 1 # DNS address netConfig.announcedAddresses[0] == dns4TcpEndPoint(dns4DomainName, extPort) + asyncTest "AnnouncedAddresses and enrMultiaddrs deduplicate dns4DomainName and extMultiAddrs overlap": + let + conf = defaultTestWakuConf() + dns4DomainName = "example.com" + extPort = Port(1234) + dns4Address = dns4TcpEndPoint(dns4DomainName, extPort) + + let netConfigRes = NetConfig.init( + bindIp = conf.endpointConf.p2pListenAddress, + bindPort = conf.endpointConf.p2pTcpPort, + dns4DomainName = some(dns4DomainName), + extPort = some(extPort), + extMultiAddrs = @[dns4Address], + ) + + assert netConfigRes.isOk(), $netConfigRes.error + + let netConfig = netConfigRes.get() + + check: + netConfig.announcedAddresses.len == 1 + netConfig.announcedAddresses[0] == dns4Address + netConfig.enrMultiAddrs.len == 1 + netConfig.enrMultiAddrs[0] == dns4Address + asyncTest "AnnouncedAddresses includes WebSocket addresses when enabled": var confBuilder = defaultTestWakuConfBuilder() diff --git a/waku/factory/internal_config.nim b/waku/factory/internal_config.nim index 7aad6e615..fd85c26a5 100644 --- a/waku/factory/internal_config.nim +++ b/waku/factory/internal_config.nim @@ -10,8 +10,8 @@ import import ../common/utils/nat, ../node/net_config, ../waku_enr, ../waku_core, ./waku_conf -proc enrConfiguration*( - conf: WakuConf, netConfig: NetConfig +proc tryBuildEnrRecord( + conf: WakuConf, netConfig: NetConfig, multiaddrs: seq[MultiAddress] ): Result[enr.Record, string] = var enrBuilder = EnrBuilder.init(conf.nodeKey) @@ -22,7 +22,8 @@ proc enrConfiguration*( if netConfig.wakuFlags.isSome(): enrBuilder.withWakuCapabilities(netConfig.wakuFlags.get()) - enrBuilder.withMultiaddrs(netConfig.enrMultiaddrs) + if multiaddrs.len > 0: + enrBuilder.withMultiaddrs(multiaddrs) enrBuilder.withWakuRelaySharding( RelayShards(clusterId: conf.clusterId, shardIds: conf.subscribeShards) @@ -30,11 +31,35 @@ proc enrConfiguration*( return err("could not initialize ENR with shards") let record = enrBuilder.build().valueOr: - error "failed to create enr record", error = error return err($error) return ok(record) +proc enrConfiguration*( + conf: WakuConf, netConfig: NetConfig +): Result[enr.Record, string] = + for retained in countdown(netConfig.enrMultiaddrs.len, 0): + let multiaddrs = netConfig.enrMultiaddrs[0 ..< retained] + let record = tryBuildEnrRecord(conf, netConfig, multiaddrs).valueOr: + if retained > 0: + warn "failed to create enr record, retrying with fewer multiaddrs", + error = error, + totalMultiaddrs = netConfig.enrMultiaddrs.len, + retainedMultiaddrs = retained - 1, + removedMultiaddr = multiaddrs[^1] + continue + + error "failed to create enr record", error = error + return err($error) + + if retained < netConfig.enrMultiaddrs.len: + warn "created enr record after trimming multiaddrs", + totalMultiaddrs = netConfig.enrMultiaddrs.len, retainedMultiaddrs = retained + + return ok(record) + + return err("failed to create enr record") + proc dnsResolve*( domain: string, dnsAddrsNameServers: seq[IpAddress] ): Future[Result[string, string]] {.async.} = diff --git a/waku/node/net_config.nim b/waku/node/net_config.nim index 4802694c4..fc4b42fe6 100644 --- a/waku/node/net_config.nim +++ b/waku/node/net_config.nim @@ -156,12 +156,16 @@ proc init*( if extMultiAddrs.len > 0: announcedAddresses.add(extMultiAddrs) + announcedAddresses = announcedAddresses.deduplicate() + let # enrMultiaddrs are just addresses which cannot be represented in ENR, as described in # https://rfc.vac.dev/spec/31/#many-connection-types - enrMultiaddrs = announcedAddresses.filterIt( - it.hasProtocol("dns4") or it.hasProtocol("dns6") or it.hasProtocol("ws") or - it.hasProtocol("wss") + enrMultiaddrs = deduplicate( + announcedAddresses.filterIt( + it.hasProtocol("dns4") or it.hasProtocol("dns6") or it.hasProtocol("ws") or + it.hasProtocol("wss") + ) ) ok( diff --git a/waku/waku_enr/multiaddr.nim b/waku/waku_enr/multiaddr.nim index c343fff51..4d6e9baa7 100644 --- a/waku/waku_enr/multiaddr.nim +++ b/waku/waku_enr/multiaddr.nim @@ -74,7 +74,7 @@ func stripPeerId(multiaddr: MultiAddress): MultiAddress = return cleanAddr func withMultiaddrs*(builder: var EnrBuilder, multiaddrs: seq[MultiAddress]) = - let multiaddrs = multiaddrs.map(stripPeerId) + let multiaddrs = deduplicate(multiaddrs.map(stripPeerId)) let value = encodeMultiaddrs(multiaddrs) builder.addFieldPair(MultiaddrEnrField, value) From 587014e34fbfaf15fa151616bf636aeede58d30a Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Thu, 30 Apr 2026 00:27:38 +0200 Subject: [PATCH 21/25] add event_loop_accumulates_lag_secs (#3833) --- waku/node/health_monitor/event_loop_monitor.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/waku/node/health_monitor/event_loop_monitor.nim b/waku/node/health_monitor/event_loop_monitor.nim index 49a2bcc6e..bd1a33e4e 100644 --- a/waku/node/health_monitor/event_loop_monitor.nim +++ b/waku/node/health_monitor/event_loop_monitor.nim @@ -10,6 +10,9 @@ declarePublicGauge event_loop_load, "chronos event loop load EWMA by window (1.0 = sustained lag at MaxAcceptedLag)", labels = ["window"] +declarePublicCounter event_loop_accumulated_lag_secs, + "chronos event loop total accumulated lag in seconds since node start" + type OnLagChange* = proc(lagTooHigh: bool) {.gcsafe, raises: [].} proc eventLoopMonitorLoop*(onLagChange: OnLagChange = nil) {.async.} = @@ -55,6 +58,8 @@ proc eventLoopMonitorLoop*(onLagChange: OnLagChange = nil) {.async.} = let lagSecs = lag.nanoseconds.float64 / 1_000_000_000.0 let load = lagSecs / maxAcceptedLagSecs + event_loop_accumulated_lag_secs.inc(lagSecs) + ewma1m = alpha1m * load + (1.0 - alpha1m) * ewma1m ewma5m = alpha5m * load + (1.0 - alpha5m) * ewma5m ewma15m = alpha15m * load + (1.0 - alpha15m) * ewma15m From 75864a705ea0b913d517a5f3640747f8709e9e53 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Thu, 30 Apr 2026 14:20:11 +0200 Subject: [PATCH 22/25] Fix websock nimble dependency version restriction to match lock file. (#3829) --- waku.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku.nimble b/waku.nimble index d99f05e84..591307c23 100644 --- a/waku.nimble +++ b/waku.nimble @@ -33,7 +33,7 @@ requires "nim >= 2.2.4", "dnsdisc", "dnsclient", "httputils >= 0.4.1", - "websock >= 0.2.1", + "websock >= 0.3.0", # Cryptography "nimcrypto == 0.6.4", # 0.6.4 used in libp2p. Version 0.7.3 makes test to crash on Ubuntu. "secp256k1", From 27ae07adaaea7beeae02cea9f8647b18cd9fb482 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Wed, 6 May 2026 19:58:19 +0200 Subject: [PATCH 23/25] receive_service: ensure fetch msgs query is performed when missing msg (#3849) --- .../recv_service/recv_service.nim | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/waku/node/delivery_service/recv_service/recv_service.nim b/waku/node/delivery_service/recv_service/recv_service.nim index 9f01ac267..64f4d683d 100644 --- a/waku/node/delivery_service/recv_service/recv_service.nim +++ b/waku/node/delivery_service/recv_service/recv_service.nim @@ -113,15 +113,19 @@ proc checkStore*(self: RecvService) {.async.} = let missedHashes: seq[WakuMessageHash] = msgHashesInStore.filterIt(not rxMsgHashes.contains(it)) - ## Now retrieve the missing WakuMessages and deliver them - let missingMsgsRet = await self.getMissingMsgsFromStore(missedHashes) - if missingMsgsRet.isOk(): - for msgTuple in missingMsgsRet.get(): - if self.processIncomingMessageOfInterest(msgTuple.pubsubTopic, msgTuple.msg): - info "recv service store-recovered message", - msg_hash = shortLog(msgTuple.hash), pubsubTopic = msgTuple.pubsubTopic - else: - error "failed to retrieve missing messages: ", error = $missingMsgsRet.error + if missedHashes.len > 0: + info "missed messages detected, checking store for missed messages", + pubsubTopic = pubsubTopic, missedCount = missedHashes.len + + ## Now retrieve the missing WakuMessages and deliver them + let missingMsgsRet = await self.getMissingMsgsFromStore(missedHashes) + if missingMsgsRet.isOk(): + for msgTuple in missingMsgsRet.get(): + if self.processIncomingMessageOfInterest(msgTuple.pubsubTopic, msgTuple.msg): + info "recv service store-recovered message", + msg_hash = shortLog(msgTuple.hash), pubsubTopic = msgTuple.pubsubTopic + else: + error "failed to retrieve missing messages: ", error = $missingMsgsRet.error ## update next check times self.startTimeToCheck = self.endTimeToCheck From a62ab1e7b13ee5dcd52708952b910b61345396fc Mon Sep 17 00:00:00 2001 From: Darshan <35736874+darshankabariya@users.noreply.github.com> Date: Mon, 11 May 2026 19:02:25 +0530 Subject: [PATCH 24/25] chore: add nim-sds (no runtime integration yet) (#3820) --- nimble.lock | 20 ++++++++++++++++++++ waku.nimble | 2 ++ 2 files changed, 22 insertions(+) diff --git a/nimble.lock b/nimble.lock index 7c76f7fa9..0a76565c4 100644 --- a/nimble.lock +++ b/nimble.lock @@ -587,6 +587,26 @@ "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81" } }, + "sds": { + "version": "#2e9a7683f0e180bf112135fae3a3803eed8490d4", + "vcsRevision": "2e9a7683f0e180bf112135fae3a3803eed8490d4", + "url": "https://github.com/logos-messaging/nim-sds.git", + "downloadMethod": "git", + "dependencies": [ + "nim", + "chronos", + "libp2p", + "chronicles", + "stew", + "stint", + "metrics", + "results", + "taskpools" + ], + "checksums": { + "sha1": "d13f1bf8d1b90b27e9edfc063b043831242cda19" + } + }, "ffi": { "version": "0.1.3", "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b", diff --git a/waku.nimble b/waku.nimble index 591307c23..f944aaae1 100644 --- a/waku.nimble +++ b/waku.nimble @@ -61,6 +61,8 @@ requires "nim >= 2.2.4", # Packages not on nimble (use git URLs) requires "https://github.com/logos-messaging/nim-ffi" +requires "https://github.com/logos-messaging/nim-sds.git#2e9a7683f0e180bf112135fae3a3803eed8490d4" + requires "https://github.com/vacp2p/nim-lsquic" requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2" From 71a369ffad2ed15a3d18c9662c0da0db53e7f5d4 Mon Sep 17 00:00:00 2001 From: Fabiana Cecin Date: Mon, 11 May 2026 15:22:22 -0300 Subject: [PATCH 25/25] feat: allow a port value of zero for service ports (auto-assign port) (#3828) * any port set to 0 on conf results in a random port bound * Debug API MyBoundPorts reports actually bound ports for all services, reports 0 if disabled * write back bound values to both WakuConf and WakuNode.ports * setupDiscoveryV5 returns Result and errors out on port 0 * rename setupAndStartDiscv5WithAutoPort to setupAndStartDiscv5 * updateWaku ENR rebuild now runs after discv5 startup * Add DefaultP2pTcpPort, DefaultDiscv5UdpPort, DefaultWebSocketPort, DefaultRestPort, DefaultMetricsHttpPort * add tests --- tests/api/test_node_conf.nim | 2 + tests/factory/test_node_factory.nim | 109 ++++++++++++++++-- tests/factory/test_waku_conf.nim | 2 +- tests/test_waku_netconfig.nim | 2 +- tests/testlib/wakunode.nim | 1 - tests/waku_discv5/test_waku_discv5.nim | 6 +- tests/wakunode2/test_app.nim | 52 ++++++++- tests/wakunode_rest/test_rest_debug.nim | 1 + waku/discovery/waku_discv5.nim | 53 ++++++++- .../conf_builder/discv5_conf_builder.nim | 8 +- .../metrics_server_conf_builder.nim | 4 +- .../conf_builder/rest_server_conf_builder.nim | 6 +- .../conf_builder/waku_conf_builder.nim | 23 ++-- .../conf_builder/web_socket_conf_builder.nim | 10 +- waku/factory/internal_config.nim | 2 +- waku/factory/waku.nim | 75 +++++++----- waku/factory/waku_state_info.nim | 5 +- waku/net/auto_port.nim | 48 ++++++++ waku/net/bound_ports.nim | 20 ++++ waku/{node => net}/net_config.nim | 0 waku/node/waku_metrics.nim | 40 ++++--- waku/node/waku_node.nim | 4 +- waku/waku_node.nim | 2 +- 23 files changed, 376 insertions(+), 99 deletions(-) create mode 100644 waku/net/auto_port.nim create mode 100644 waku/net/bound_ports.nim rename waku/{node => net}/net_config.nim (100%) diff --git a/tests/api/test_node_conf.nim b/tests/api/test_node_conf.nim index b19739393..e171c5207 100644 --- a/tests/api/test_node_conf.nim +++ b/tests/api/test_node_conf.nim @@ -376,6 +376,7 @@ suite "WakuConfBuilder - store retention policies": test "Multiple retention policies": ## Given var b = WakuConfBuilder.init() + b.withP2pTcpPort(0'u16) b.storeServiceConf.withEnabled(true) b.storeServiceConf.withDbUrl("sqlite://test.db") b.storeServiceConf.withRetentionPolicies( @@ -420,6 +421,7 @@ suite "WakuConfBuilder - store retention policies": test "Store disabled - no retention policy applied": ## Given var b = WakuConfBuilder.init() + b.withP2pTcpPort(0'u16) # storeServiceConf not enabled ## When diff --git a/tests/factory/test_node_factory.nim b/tests/factory/test_node_factory.nim index 4b2085e82..1fe242532 100644 --- a/tests/factory/test_node_factory.nim +++ b/tests/factory/test_node_factory.nim @@ -1,21 +1,23 @@ {.used.} import - std/[options, sequtils], + std/[net, options, sequtils, strutils], testutils/unittests, chronos, - libp2p/multiaddress, - libp2p/protocols/connectivity/relay/relay -import eth/p2p/discoveryv5/enr + chronos/transports/[stream, datagram, common], + metrics/chronos_httpserver, + libp2p/[crypto/crypto, multiaddress, protocols/connectivity/relay/relay], + eth/p2p/discoveryv5/enr import - ../testlib/wakunode, - waku/waku_node, - waku/waku_enr, - waku/factory/node_factory, - waku/factory/internal_config, - waku/factory/conf_builder/conf_builder, - waku/factory/conf_builder/web_socket_conf_builder + tests/testlib/[wakunode, wakucore], + waku/[waku_node, waku_enr, net/auto_port, discovery/waku_discv5, node/waku_metrics], + waku/factory/[ + node_factory, + internal_config, + conf_builder/conf_builder, + conf_builder/web_socket_conf_builder, + ] suite "Node Factory": asynctest "Set up a node based on default configurations": @@ -115,5 +117,90 @@ asynctest "Start a node based on default test configuration": check: node.started == true + # Default conf has p2pTcpPort=0, so the OS must have assigned a real port. + var hasNonZeroTcp = false + for a in node.switch.peerInfo.listenAddrs: + let s = $a + if ("/tcp/" in s) and not ("/tcp/0" in s): + hasNonZeroTcp = true + check hasNonZeroTcp + ## Cleanup await node.stop() + +suite "Auto-port retry": + asynctest "metrics binds on free TCP port, fails on taken": + let takenPort = Port(55100) + let freePort = Port(55101) + let taken = createStreamServer(initTAddress("127.0.0.1", takenPort)) + defer: + taken.stop() + await taken.closeWait() + + proc buildMetricsConf(port: Port): MetricsServerConf = + var b = MetricsServerConfBuilder.init() + b.withEnabled(true) + b.withHttpPort(port) + b.build().value.get() + + let failRes = await startMetricsServerAndLogging(buildMetricsConf(takenPort), 0'u16) + check failRes.isErr() + + let okRes = await startMetricsServerAndLogging(buildMetricsConf(freePort), 0'u16) + check okRes.isOk() + if okRes.isOk(): + await okRes.get().server.close() + + asynctest "discv5 binds on free UDP port, fails on taken": + let takenPort = Port(55200) + let freePort = Port(55201) + + proc dummyCb( + transp: DatagramTransport, raddr: TransportAddress + ): Future[void] {.async: (raises: []).} = + discard + + let takenUdp = + newDatagramTransport(dummyCb, local = initTAddress("0.0.0.0", takenPort)) + defer: + await takenUdp.closeWait() + + let nodeKey = generateSecp256k1Key() + let node = newTestWakuNode(nodeKey, parseIpAddress("0.0.0.0"), Port(0)) + await node.start() + defer: + await node.stop() + + proc buildDiscv5Conf(port: Port): Discv5Conf = + var b = Discv5ConfBuilder.init() + b.withEnabled(true) + b.withUdpPort(port) + b.build().value.get() + + let failRes = await setupAndStartDiscv5( + node.enr, + node.peerManager, + node.topicSubscriptionQueue, + buildDiscv5Conf(takenPort), + @[], + node.rng, + nodeKey, + parseIpAddress("0.0.0.0"), + 0'u16, + ) + check failRes.isErr() + + let okRes = await setupAndStartDiscv5( + node.enr, + node.peerManager, + node.topicSubscriptionQueue, + buildDiscv5Conf(freePort), + @[], + node.rng, + nodeKey, + parseIpAddress("0.0.0.0"), + 0'u16, + ) + check okRes.isOk() + if okRes.isOk(): + await okRes.get().stop() diff --git a/tests/factory/test_waku_conf.nim b/tests/factory/test_waku_conf.nim index eeacf791b..885e22867 100644 --- a/tests/factory/test_waku_conf.nim +++ b/tests/factory/test_waku_conf.nim @@ -4,7 +4,7 @@ import libp2p/crypto/[crypto, secp], libp2p/multiaddress, nimcrypto/utils, - std/[options, random, sequtils], + std/[net, options, random, sequtils], results, testutils/unittests import diff --git a/tests/test_waku_netconfig.nim b/tests/test_waku_netconfig.nim index 20d95d59b..0aff64121 100644 --- a/tests/test_waku_netconfig.nim +++ b/tests/test_waku_netconfig.nim @@ -5,7 +5,7 @@ import chronos, confutils/toml/std/net, libp2p/multiaddress, testutils/unittests import ./testlib/wakunode, waku/waku_enr/capabilities include - waku/node/net_config, + waku/net/net_config, waku/factory/conf_builder/web_socket_conf_builder, waku/factory/conf_builder/conf_builder diff --git a/tests/testlib/wakunode.nim b/tests/testlib/wakunode.nim index e904604ab..77c017d96 100644 --- a/tests/testlib/wakunode.nim +++ b/tests/testlib/wakunode.nim @@ -27,7 +27,6 @@ import # TODO: migrate to usage of a test cluster conf proc defaultTestWakuConfBuilder*(): WakuConfBuilder = var builder = WakuConfBuilder.init() - builder.withP2pTcpPort(Port(0)) builder.withP2pListenAddress(parseIpAddress("0.0.0.0")) builder.restServerConf.withListenAddress(parseIpAddress("127.0.0.1")) builder.withDnsAddrsNameServers( diff --git a/tests/waku_discv5/test_waku_discv5.nim b/tests/waku_discv5/test_waku_discv5.nim index 20a0c6965..936c01826 100644 --- a/tests/waku_discv5/test_waku_discv5.nim +++ b/tests/waku_discv5/test_waku_discv5.nim @@ -506,7 +506,8 @@ suite "Waku Discovery v5": waku.conf.nodeKey, waku.conf.endpointConf.p2pListenAddress, waku.conf.portsShift, - ) + ).valueOr: + raiseAssert "failed setup discv5 in test: " & $error check: waku.node.peerManager.switch.peerStore.peers().anyIt( @@ -537,7 +538,8 @@ suite "Waku Discovery v5": waku.conf.nodeKey, waku.conf.endpointConf.p2pListenAddress, waku.conf.portsShift, - ) + ).valueOr: + raiseAssert "failed setup discv5 in test: " & $error check: not waku.node.peerManager.switch.peerStore.peers().anyIt( diff --git a/tests/wakunode2/test_app.nim b/tests/wakunode2/test_app.nim index 6ec6043fe..7621ab1e7 100644 --- a/tests/wakunode2/test_app.nim +++ b/tests/wakunode2/test_app.nim @@ -1,14 +1,13 @@ {.used.} import + std/json, testutils/unittests, chronicles, chronos, - libp2p/crypto/crypto, - libp2p/crypto/secp, - libp2p/multiaddress, - libp2p/switch -import ../testlib/wakucore, ../testlib/wakunode + libp2p/[crypto/crypto, crypto/secp, multiaddress, switch], + tests/testlib/[wakucore, wakunode], + waku/factory/conf_builder/conf_builder include waku/factory/waku, waku/common/enr/typed_record @@ -99,3 +98,46 @@ suite "Wakunode2 - Waku initialization": ## Cleanup (waitFor waku.stop()).isOkOr: raiseAssert error + + test "explicit port=0 triggers auto-bind across all services": + var builder = defaultTestWakuConfBuilder() + builder.withP2pTcpPort(Port(0)) + builder.discv5Conf.withEnabled(true) + builder.discv5Conf.withUdpPort(Port(0)) + builder.restServerConf.withEnabled(true) + builder.restServerConf.withRelayCacheCapacity(50'u32) + builder.restServerConf.withPort(Port(0)) + builder.metricsServerConf.withEnabled(true) + builder.metricsServerConf.withHttpPort(Port(0)) + builder.webSocketConf.withEnabled(true) + builder.webSocketConf.withWebSocketPort(Port(0)) + + let conf = builder.build().valueOr: + raiseAssert error + + check: + conf.endpointConf.p2pTcpPort == Port(0) + conf.discv5Conf.get().udpPort == Port(0) + conf.restServerConf.get().port == Port(0) + conf.metricsServerConf.get().httpPort == Port(0) + conf.webSocketConf.get().port == Port(0) + + var waku = (waitFor Waku.new(conf)).valueOr: + raiseAssert error + defer: + (waitFor waku.stop()).isOkOr: + raiseAssert error + + (waitFor startWaku(addr waku)).isOkOr: + raiseAssert error + + let portsJson = waku.stateInfo.getNodeInfoItem(NodeInfoId.MyBoundPorts) + let parsed = parseJson(portsJson) + + check: + parsed.kind == JObject + parsed["tcp"].getInt() != 0 + parsed["webSocket"].getInt() != 0 + parsed["rest"].getInt() != 0 + parsed["discv5Udp"].getInt() != 0 + parsed["metrics"].getInt() != 0 diff --git a/tests/wakunode_rest/test_rest_debug.nim b/tests/wakunode_rest/test_rest_debug.nim index 4bd2e8c02..1171f5878 100644 --- a/tests/wakunode_rest/test_rest_debug.nim +++ b/tests/wakunode_rest/test_rest_debug.nim @@ -1,6 +1,7 @@ {.used.} import + std/options, testutils/unittests, presto, presto/client as presto_client, diff --git a/waku/discovery/waku_discv5.nim b/waku/discovery/waku_discv5.nim index 0eb329fa4..c1b253c8c 100644 --- a/waku/discovery/waku_discv5.nim +++ b/waku/discovery/waku_discv5.nim @@ -10,7 +10,7 @@ import eth/keys as eth_keys, eth/p2p/discoveryv5/node, eth/p2p/discoveryv5/protocol -import ../node/peer_manager/peer_manager, ../waku_core, ../waku_enr +import waku/[net/auto_port, node/peer_manager/peer_manager, waku_core, waku_enr] export protocol, waku_enr @@ -409,7 +409,15 @@ proc setupDiscoveryV5*( key: crypto.PrivateKey, p2pListenAddress: IpAddress, portsShift: uint16, -): WakuDiscoveryV5 = +): Result[WakuDiscoveryV5, string] = + ## Public only for testing. Callers should use `setupAndStartDiscv5`, which + ## additionally handles `udpPort == 0` via auto-port retry. + if conf.udpPort == Port(0): + return err( + "setupDiscoveryV5: udpPort must be non-zero; " & + "use setupAndStartDiscv5 for port=0 auto-port retry" + ) + let dynamicBootstrapEnrs = dynamicBootstrapNodes.filterIt(it.hasUdpPort()).mapIt(it.enr.get()) @@ -441,10 +449,47 @@ proc setupDiscoveryV5*( autoupdateRecord: conf.enrAutoUpdate, ) - WakuDiscoveryV5.new( - rng, discv5Conf, some(myENR), some(nodePeerManager), nodeTopicSubscriptionQueue + return ok( + WakuDiscoveryV5.new( + rng, discv5Conf, some(myENR), some(nodePeerManager), nodeTopicSubscriptionQueue + ) ) +proc setupAndStartDiscv5*( + myENR: enr.Record, + nodePeerManager: PeerManager, + nodeTopicSubscriptionQueue: AsyncEventQueue[SubscriptionEvent], + conf: Discv5Conf, + dynamicBootstrapNodes: seq[RemotePeerInfo], + rng: ref HmacDrbgContext, + key: crypto.PrivateKey, + p2pListenAddress: IpAddress, + portsShift: uint16, +): Future[Result[WakuDiscoveryV5, string]] {.async: (raises: []).} = + ## Construct and start a `WakuDiscoveryV5` instance, handling auto-port + ## retry when the caller asks for `udpPort == 0`. + proc attempt( + port: Port + ): Future[Result[WakuDiscoveryV5, string]] {.async: (raises: []).} = + var c = conf + c.udpPort = port + let wd = setupDiscoveryV5( + myENR, nodePeerManager, nodeTopicSubscriptionQueue, c, dynamicBootstrapNodes, rng, + key, p2pListenAddress, portsShift, + ).valueOr: + return err(error) + let startRes = await wd.start() + if startRes.isErr(): + return err("failed to start discovery, attempt: " & startRes.error) + return ok(wd) + + let wd = (await tryWithAutoPort[WakuDiscoveryV5](conf.udpPort, attempt)).valueOr: + return err("setupAndStartDiscv5: " & error) + return ok(wd) + +proc udpPort*(wd: WakuDiscoveryV5): Port = + wd.conf.port + proc updateBootstrapRecords*( self: var WakuDiscoveryV5, newRecordsString: string ): Result[void, string] = diff --git a/waku/factory/conf_builder/discv5_conf_builder.nim b/waku/factory/conf_builder/discv5_conf_builder.nim index e2729021e..5dd269d23 100644 --- a/waku/factory/conf_builder/discv5_conf_builder.nim +++ b/waku/factory/conf_builder/discv5_conf_builder.nim @@ -4,6 +4,8 @@ import ../waku_conf logScope: topics = "waku conf builder discv5" +const DefaultDiscv5UdpPort*: Port = Port(9000) + ########################### ## Discv5 Config Builder ## ########################### @@ -38,8 +40,8 @@ proc withTableIpLimit*(b: var Discv5ConfBuilder, tableIpLimit: uint) = proc withUdpPort*(b: var Discv5ConfBuilder, udpPort: Port) = b.udpPort = some(udpPort) -proc withUdpPort*(b: var Discv5ConfBuilder, udpPort: uint) = - b.udpPort = some(Port(udpPort.uint16)) +proc withUdpPort*(b: var Discv5ConfBuilder, udpPort: uint16) = + b.udpPort = some(Port(udpPort)) proc withBootstrapNodes*(b: var Discv5ConfBuilder, bootstrapNodes: seq[string]) = # TODO: validate ENRs? @@ -57,7 +59,7 @@ proc build*(b: Discv5ConfBuilder): Result[Option[Discv5Conf], string] = bucketIpLimit: b.bucketIpLimit.get(2), enrAutoUpdate: b.enrAutoUpdate.get(true), tableIpLimit: b.tableIpLimit.get(10), - udpPort: b.udpPort.get(9000.Port), + udpPort: b.udpPort.get(DefaultDiscv5UdpPort), ) ) ) diff --git a/waku/factory/conf_builder/metrics_server_conf_builder.nim b/waku/factory/conf_builder/metrics_server_conf_builder.nim index 0f0d18564..8b2ea4eb8 100644 --- a/waku/factory/conf_builder/metrics_server_conf_builder.nim +++ b/waku/factory/conf_builder/metrics_server_conf_builder.nim @@ -4,6 +4,8 @@ import ../waku_conf logScope: topics = "waku conf builder metrics server" +const DefaultMetricsHttpPort*: Port = Port(8008) + ################################### ## Metrics Server Config Builder ## ################################### @@ -40,7 +42,7 @@ proc build*(b: MetricsServerConfBuilder): Result[Option[MetricsServerConf], stri some( MetricsServerConf( httpAddress: b.httpAddress.get(static parseIpAddress("127.0.0.1")), - httpPort: b.httpPort.get(8008.Port), + httpPort: b.httpPort.get(DefaultMetricsHttpPort), logging: b.logging.get(false), ) ) diff --git a/waku/factory/conf_builder/rest_server_conf_builder.nim b/waku/factory/conf_builder/rest_server_conf_builder.nim index 2efd91f02..dcafbb56a 100644 --- a/waku/factory/conf_builder/rest_server_conf_builder.nim +++ b/waku/factory/conf_builder/rest_server_conf_builder.nim @@ -4,6 +4,8 @@ import ../waku_conf logScope: topics = "waku conf builder rest server" +const DefaultRestPort*: Port = Port(8645) + ################################ ## REST Server Config Builder ## ################################ @@ -46,8 +48,6 @@ proc build*(b: RestServerConfBuilder): Result[Option[RestServerConf], string] = if b.listenAddress.isNone(): return err("restServer.listenAddress is not specified") - if b.port.isNone(): - return err("restServer.port is not specified") if b.relayCacheCapacity.isNone(): return err("restServer.relayCacheCapacity is not specified") @@ -56,7 +56,7 @@ proc build*(b: RestServerConfBuilder): Result[Option[RestServerConf], string] = RestServerConf( allowOrigin: b.allowOrigin, listenAddress: b.listenAddress.get(), - port: b.port.get(), + port: b.port.get(DefaultRestPort), admin: b.admin.get(false), relayCacheCapacity: b.relayCacheCapacity.get(), ) diff --git a/waku/factory/conf_builder/waku_conf_builder.nim b/waku/factory/conf_builder/waku_conf_builder.nim index 78dbd9eb9..5954bbe58 100644 --- a/waku/factory/conf_builder/waku_conf_builder.nim +++ b/waku/factory/conf_builder/waku_conf_builder.nim @@ -8,11 +8,13 @@ import results import - ../waku_conf, - ../networks_config, - ../../common/logging, - ../../common/utils/parse_size_units, - ../../waku_enr/capabilities, + waku/[ + factory/waku_conf, + factory/networks_config, + common/logging, + common/utils/parse_size_units, + waku_enr/capabilities, + ], tools/confutils/entry_nodes import @@ -32,7 +34,9 @@ import logScope: topics = "waku conf builder" -const DefaultMaxConnections* = 150 +const + DefaultMaxConnections* = 150 + DefaultP2pTcpPort*: Port = Port(60000) type MaxMessageSizeKind* = enum mmskNone @@ -574,12 +578,7 @@ proc build*( warn "Nat Strategy is not specified, defaulting to none" "none" - let p2pTcpPort = - if builder.p2pTcpPort.isSome(): - builder.p2pTcpPort.get() - else: - warn "P2P Listening TCP Port is not specified, listening on 60000" - 60000.Port + let p2pTcpPort = builder.p2pTcpPort.get(DefaultP2pTcpPort) let p2pListenAddress = if builder.p2pListenAddress.isSome(): diff --git a/waku/factory/conf_builder/web_socket_conf_builder.nim b/waku/factory/conf_builder/web_socket_conf_builder.nim index 88edc0941..61334d958 100644 --- a/waku/factory/conf_builder/web_socket_conf_builder.nim +++ b/waku/factory/conf_builder/web_socket_conf_builder.nim @@ -4,6 +4,8 @@ import waku/factory/waku_conf logScope: topics = "waku conf builder websocket" +const DefaultWebSocketPort*: Port = Port(8000) + ############################## ## WebSocket Config Builder ## ############################## @@ -41,14 +43,12 @@ proc build*(b: WebSocketConfBuilder): Result[Option[WebSocketConf], string] = if not b.enabled.get(false): return ok(none(WebSocketConf)) - if b.webSocketPort.isNone(): - return err("websocket.port is not specified") - if not b.secureEnabled.get(false): return ok( some( WebSocketConf( - port: b.websocketPort.get(), secureConf: none(WebSocketSecureConf) + port: b.webSocketPort.get(DefaultWebSocketPort), + secureConf: none(WebSocketSecureConf), ) ) ) @@ -61,7 +61,7 @@ proc build*(b: WebSocketConfBuilder): Result[Option[WebSocketConf], string] = return ok( some( WebSocketConf( - port: b.webSocketPort.get(), + port: b.webSocketPort.get(DefaultWebSocketPort), secureConf: some( WebSocketSecureConf(keyPath: b.keyPath.get(), certPath: b.certPath.get()) ), diff --git a/waku/factory/internal_config.nim b/waku/factory/internal_config.nim index fd85c26a5..fa36aff57 100644 --- a/waku/factory/internal_config.nim +++ b/waku/factory/internal_config.nim @@ -8,7 +8,7 @@ import std/[options, sequtils, net], results -import ../common/utils/nat, ../node/net_config, ../waku_enr, ../waku_core, ./waku_conf +import waku/[common/utils/nat, net/net_config, waku_enr, waku_core], ./waku_conf proc tryBuildEnrRecord( conf: WakuConf, netConfig: NetConfig, multiaddrs: seq[MultiAddress] diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index 45e0edee0..395841130 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -202,6 +202,11 @@ proc new*( else: nil + if not restServer.isNil(): + let boundRestPort = restServer.httpServer.address.port + node.ports.rest = boundRestPort.uint16 + wakuConf.restServerConf.get().port = boundRestPort + # Set the extMultiAddrsOnly flag so the node knows not to replace explicit addresses node.extMultiAddrsOnly = wakuConf.endpointConf.extMultiAddrsOnly @@ -249,7 +254,7 @@ proc getPorts( return ok((tcpPort: tcpPort, websocketPort: websocketPort)) proc getRunningNetConfig(waku: ptr Waku): Future[Result[NetConfig, string]] {.async.} = - var conf = waku[].conf + let conf = waku[].conf let (tcpPort, websocketPort) = getPorts(waku[].node.switch.peerInfo.listenAddrs).valueOr: return err("Could not retrieve ports: " & error) @@ -281,6 +286,10 @@ proc updateEnr(waku: ptr Waku): Future[Result[void, string]] {.async.} = waku[].node.enr = record + # If TCP/WS was configured with port 0, node.announcedAddresses was built + # pre-bind with a port value of 0. In any case, the resync is harmless. + waku[].node.announcedAddresses = netConf.announcedAddresses + return ok() proc updateAddressInENR(waku: ptr Waku): Result[void, string] = @@ -312,11 +321,8 @@ proc updateAddressInENR(waku: ptr Waku): Result[void, string] = return ok() proc updateWaku(waku: ptr Waku): Future[Result[void, string]] {.async.} = - let conf = waku[].conf - if conf.endpointConf.p2pTcpPort == Port(0) or - (conf.websocketConf.isSome() and conf.websocketConf.get.port == Port(0)): - (await updateEnr(waku)).isOkOr: - return err("error calling updateEnr: " & $error) + (await updateEnr(waku)).isOkOr: + return err("error calling updateEnr: " & $error) ?updateAnnouncedAddrWithPrimaryIpAddr(waku[].node) @@ -390,29 +396,37 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: (await startNode(waku.node, waku.conf, waku.dynamicBootstrapNodes)).isOkOr: return err("error while calling startNode: " & $error) - ## Update waku data that is set dynamically on node start - try: - (await updateWaku(waku)).isOkOr: - return err("Error in updateApp: " & $error) - except CatchableError: - return err("Caught exception in updateApp: " & getCurrentExceptionMsg()) + let bound = getPorts(waku.node.switch.peerInfo.listenAddrs).valueOr: + return err("failed to read bound ports from switch: " & $error) + waku[].node.ports.tcp = bound.tcpPort.get(Port(0)).uint16 + waku[].node.ports.webSocket = bound.websocketPort.get(Port(0)).uint16 ## Discv5 if conf.discv5Conf.isSome(): - waku[].wakuDiscV5 = waku_discv5.setupDiscoveryV5( - waku.node.enr, - waku.node.peerManager, - waku.node.topicSubscriptionQueue, - conf.discv5Conf.get(), - waku.dynamicBootstrapNodes, - waku.rng, - conf.nodeKey, - conf.endpointConf.p2pListenAddress, - conf.portsShift, - ) + waku[].wakuDiscV5 = ( + await waku_discv5.setupAndStartDiscv5( + waku.node.enr, + waku.node.peerManager, + waku.node.topicSubscriptionQueue, + conf.discv5Conf.get(), + waku.dynamicBootstrapNodes, + waku.rng, + conf.nodeKey, + conf.endpointConf.p2pListenAddress, + conf.portsShift, + ) + ).valueOr: + return err("failed to start waku discovery v5: " & error) - (await waku.wakuDiscV5.start()).isOkOr: - return err("failed to start waku discovery v5: " & $error) + waku[].node.ports.discv5Udp = waku[].wakuDiscV5.udpPort.uint16 + waku[].conf.discv5Conf.get().udpPort = waku[].wakuDiscV5.udpPort + + ## Update waku data that is set dynamically on node start + try: + (await updateWaku(waku)).isOkOr: + return err("Error in startWaku: " & $error) + except CatchableError: + return err("Caught exception in startWaku: " & getCurrentExceptionMsg()) ## Reliability if not waku[].deliveryService.isNil(): @@ -482,14 +496,15 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises: if conf.metricsServerConf.isSome(): try: - waku[].metricsServer = ( - await ( - waku_metrics.startMetricsServerAndLogging( - conf.metricsServerConf.get(), conf.portsShift - ) + let (server, port) = ( + await waku_metrics.startMetricsServerAndLogging( + conf.metricsServerConf.get(), conf.portsShift ) ).valueOr: return err("Starting monitoring and external interfaces failed: " & error) + waku[].metricsServer = server + waku[].node.ports.metrics = port.uint16 + waku[].conf.metricsServerConf.get().httpPort = port except CatchableError: return err( "Caught exception starting monitoring and external interfaces failed: " & diff --git a/waku/factory/waku_state_info.nim b/waku/factory/waku_state_info.nim index 5dc72a693..397b90d6d 100644 --- a/waku/factory/waku_state_info.nim +++ b/waku/factory/waku_state_info.nim @@ -6,7 +6,7 @@ import std/[tables, sequtils, strutils] import metrics, eth/p2p/discoveryv5/enr, libp2p/peerid -import waku/waku_node +import waku/[waku_node, net/bound_ports] type NodeInfoId* {.pure.} = enum @@ -15,6 +15,7 @@ type MyMultiaddresses MyENR MyPeerId + MyBoundPorts WakuStateInfo* {.requiresInit.} = object node: WakuNode @@ -43,6 +44,8 @@ proc getNodeInfoItem*(self: WakuStateInfo, infoItemId: NodeInfoId): string = return self.node.enr.toURI() of NodeInfoId.MyPeerId: return $PeerId(self.node.peerId()) + of NodeInfoId.MyBoundPorts: + return $self.node.ports else: return "unknown info item id" diff --git a/waku/net/auto_port.nim b/waku/net/auto_port.nim new file mode 100644 index 000000000..38176d27d --- /dev/null +++ b/waku/net/auto_port.nim @@ -0,0 +1,48 @@ +{.push raises: [].} + +import std/[net, random] +import chronos, results + +const + AutoPortRetryCount* = 20 + AutoPortMin = 50000'u16 + AutoPortMax = 59000'u16 + AutoPortAttemptTimeout = chronos.seconds(30) + +proc getAutoPort*(): uint16 = + var rng = initRand() + uint16(rng.rand(AutoPortMin.int .. AutoPortMax.int)) + +proc tryWithAutoPort*[T]( + startingPort: Port, + attempt: proc(p: Port): Future[Result[T, string]] {.async: (raises: []).}, +): Future[Result[T, string]] {.async: (raises: []).} = + ## If `startingPort == Port(0)`, call `attempt` up to `AutoPortRetryCount` + ## times with random ports. Otherwise call it once with `startingPort`. + ## Returns the first ok or the last err. + let autoMode = startingPort == Port(0) + let attempts = if autoMode: AutoPortRetryCount else: 1 + var lastErr = "" + for i in 1 .. attempts: + let port = + if autoMode: + Port(getAutoPort()) + else: + startingPort + let fut = attempt(port) + let res = + try: + if await fut.withTimeout(AutoPortAttemptTimeout): + await fut + else: + fut.cancelSoon() + Result[T, string].err("bind attempt timed out") + except CancelledError: + fut.cancelSoon() + Result[T, string].err("bind attempt cancelled") + if res.isOk(): + return ok(res.get()) + lastErr = res.error + if autoMode: + return err("auto-port exhausted; last error: " & lastErr) + return err("port bind failed: " & lastErr) diff --git a/waku/net/bound_ports.nim b/waku/net/bound_ports.nim new file mode 100644 index 000000000..f8f561940 --- /dev/null +++ b/waku/net/bound_ports.nim @@ -0,0 +1,20 @@ +{.push raises: [].} + +import std/json + +type BoundPorts* {.requiresInit.} = object + ## Set by the factory once each service has bound to a port. + ## A value of 0 means the service was not enabled or did not bind. + tcp*: uint16 + webSocket*: uint16 + rest*: uint16 + discv5Udp*: uint16 + metrics*: uint16 + +proc init*(T: type BoundPorts): BoundPorts = + return BoundPorts( + tcp: 0'u16, webSocket: 0'u16, rest: 0'u16, discv5Udp: 0'u16, metrics: 0'u16 + ) + +proc `$`*(p: BoundPorts): string = + return $(%*p) diff --git a/waku/node/net_config.nim b/waku/net/net_config.nim similarity index 100% rename from waku/node/net_config.nim rename to waku/net/net_config.nim diff --git a/waku/node/waku_metrics.nim b/waku/node/waku_metrics.nim index 8d38624c1..af74b1532 100644 --- a/waku/node/waku_metrics.nim +++ b/waku/node/waku_metrics.nim @@ -2,8 +2,7 @@ import chronicles, chronos, metrics, metrics/chronos_httpserver import - ../waku_rln_relay/protocol_metrics as rln_metrics, - ../utils/collector, + waku/[net/auto_port, waku_rln_relay/protocol_metrics as rln_metrics, utils/collector], ./peer_manager, ./waku_node @@ -57,27 +56,36 @@ proc startMetricsLog*() = discard setTimer(Moment.fromNow(LogInterval), logMetrics) +type StartedMetricsServer* = tuple[server: MetricsHttpServerRef, port: Port] + proc startMetricsServer( serverIp: IpAddress, serverPort: Port -): Future[Result[MetricsHttpServerRef, string]] {.async.} = - info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $serverPort +): Future[Result[StartedMetricsServer, string]] {.async.} = + proc attempt( + port: Port + ): Future[Result[StartedMetricsServer, string]] {.async: (raises: []).} = + info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $port - let server = MetricsHttpServerRef.new($serverIp, serverPort).valueOr: - return err("metrics HTTP server start failed: " & $error) + let server = MetricsHttpServerRef.new($serverIp, port).valueOr: + return err("fail to start service metrics server, attempt:" & $error) - try: - await server.start() - except CatchableError: - return err("metrics HTTP server start failed: " & getCurrentExceptionMsg()) + try: + await server.start() + except CatchableError: + return + err("exception while startMetricsServer, attempt: " & getCurrentExceptionMsg()) - info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $serverPort - return ok(server) + info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $port + return ok((server: server, port: port)) + + let started = (await tryWithAutoPort[StartedMetricsServer](serverPort, attempt)).valueOr: + return err("metrics HTTP server start failed: " & error) + return ok(started) proc startMetricsServerAndLogging*( conf: MetricsServerConf, portsShift: uint16 -): Future[Result[MetricsHttpServerRef, string]] {.async.} = - var metricsServer: MetricsHttpServerRef - metricsServer = ( +): Future[Result[StartedMetricsServer, string]] {.async.} = + let started = ( await ( startMetricsServer(conf.httpAddress, Port(conf.httpPort.uint16 + portsShift)) ) @@ -87,4 +95,4 @@ proc startMetricsServerAndLogging*( if conf.logging: startMetricsLog() - return ok(metricsServer) + return ok(started) diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 45080d9d0..7cd334b53 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -62,7 +62,7 @@ import events/message_events, ], waku/discovery/waku_kademlia, - ./net_config, + waku/net/[bound_ports, net_config], ./peer_manager, ./health_monitor/health_status, ./health_monitor/topic_health @@ -140,6 +140,7 @@ type wakuMix*: WakuMix kademliaDiscoveryLoop*: Future[void] wakuKademlia*: WakuKademlia + ports*: BoundPorts proc deduceRelayShard( node: WakuNode, @@ -224,6 +225,7 @@ proc new*( announcedAddresses: netConfig.announcedAddresses, topicSubscriptionQueue: queue, rateLimitSettings: rateLimitSettings, + ports: BoundPorts.init(), ) peerManager.setShardGetter(node.getShardsGetter(@[])) diff --git a/waku/waku_node.nim b/waku/waku_node.nim index e782e616b..c8b13d4ea 100644 --- a/waku/waku_node.nim +++ b/waku/waku_node.nim @@ -1,5 +1,5 @@ import - ./node/net_config, + ./net/net_config, ./node/waku_switch as switch, ./node/waku_node as node, ./node/health_monitor as health_monitor,