diff --git a/.github/ISSUE_TEMPLATE/bump_dependencies.md b/.github/ISSUE_TEMPLATE/bump_dependencies.md
index 0413cbfd2..59f46f08b 100644
--- a/.github/ISSUE_TEMPLATE/bump_dependencies.md
+++ b/.github/ISSUE_TEMPLATE/bump_dependencies.md
@@ -1,7 +1,7 @@
---
name: Bump dependencies
-about: Bump vendor dependencies for release
-title: 'Bump vendor dependencies for release 0.0.0'
+about: Bump dependencies for release
+title: 'Bump dependencies for release 0.X.0'
labels: dependencies
assignees: ''
@@ -9,40 +9,10 @@ assignees: ''
-Update `nwaku` "vendor" dependencies.
+### Bumped items
+- [ ] Update nimble dependencies
+ 1. Edit manually waku.nimble. For some dependencies, we want to bump versions manually and use a pinned version, f.e., nim-libp2p and all its dependencies.
+ 2. Run `nimble lock` (make sure `nimble --version` shows the Nimble version pinned in waku.nimble)
+ 3. Run `./tools/gen-nix-deps.sh nimble.lock nix/deps.nix` to update nix deps
-### Items to bump
-- [ ] dnsclient.nim ( update to the latest tag version )
-- [ ] nim-bearssl
-- [ ] nimbus-build-system
-- [ ] nim-chronicles
-- [ ] nim-chronos
-- [ ] nim-confutils
-- [ ] nimcrypto
-- [ ] nim-dnsdisc
-- [ ] nim-eth
-- [ ] nim-faststreams
-- [ ] nim-http-utils
-- [ ] nim-json-rpc
-- [ ] nim-json-serialization
-- [ ] nim-libbacktrace
-- [ ] nim-libp2p ( update to the latest tag version )
-- [ ] nim-metrics
-- [ ] nim-nat-traversal
-- [ ] nim-presto
-- [ ] nim-regex ( update to the latest tag version )
-- [ ] nim-results
-- [ ] nim-secp256k1
-- [ ] nim-serialization
-- [ ] nim-sqlite3-abi ( update to the latest tag version )
-- [ ] nim-stew
-- [ ] nim-stint
-- [ ] nim-taskpools ( update to the latest tag version )
-- [ ] nim-testutils ( update to the latest tag version )
-- [ ] nim-toml-serialization
-- [ ] nim-unicodedb
-- [ ] nim-unittest2 ( update to the latest tag version )
-- [ ] nim-web3 ( update to the latest tag version )
-- [ ] nim-websock ( update to the latest tag version )
-- [ ] nim-zlib
-- [ ] zerokit ( this should be kept in version `v0.7.0` )
+- [ ] Update vendor/zerokit dependency.
diff --git a/.github/ISSUE_TEMPLATE/prepare_release.md b/.github/ISSUE_TEMPLATE/prepare_release.md
index 83456e79a..de67b3eaf 100644
--- a/.github/ISSUE_TEMPLATE/prepare_release.md
+++ b/.github/ISSUE_TEMPLATE/prepare_release.md
@@ -18,6 +18,7 @@ For detailed info on the release process refer to https://github.com/logos-messa
All items below are to be completed by the owner of the given release.
- [ ] Create release branch with major and minor only ( e.g. release/v0.X ) if it doesn't exist.
+- [ ] Update the `version` field in `waku.nimble` to match the release version (e.g. `version = "0.X.0"`).
- [ ] Assign release candidate tag to the release branch HEAD (e.g. `v0.X.0-rc.0`, `v0.X.0-rc.1`, ... `v0.X.0-rc.N`).
- [ ] Generate and edit release notes in CHANGELOG.md.
diff --git a/.github/workflows/ci-daily.yml b/.github/workflows/ci-daily.yml
index b442014a6..a4cf39340 100644
--- a/.github/workflows/ci-daily.yml
+++ b/.github/workflows/ci-daily.yml
@@ -40,7 +40,7 @@ jobs:
run: make update
- name: Build binaries
- run: make V=1 QUICK_AND_DIRTY_COMPILER=1 examples tools
+ run: make V=1 examples tools
- name: Notify Discord
if: always()
diff --git a/.github/workflows/ci-nix.yml b/.github/workflows/ci-nix.yml
index 8fc7ac985..7319f64aa 100644
--- a/.github/workflows/ci-nix.yml
+++ b/.github/workflows/ci-nix.yml
@@ -16,14 +16,7 @@ jobs:
- aarch64-darwin
- x86_64-linux
nixpkg:
- - libwaku
- - libwaku-android-arm64
- - wakucanary
-
- exclude:
- # Android SDK limitation
- - system: aarch64-darwin
- nixpkg: libwaku-android-arm64
+ - liblogosdelivery
include:
- system: aarch64-darwin
@@ -36,12 +29,10 @@ jobs:
runs-on: ${{ matrix.runs_on }}
steps:
- uses: actions/checkout@v4
- with:
- submodules: recursive
- - name: 'Run Nix build for {{ matrix.nixpkg }}'
+ - name: 'Run Nix build for ${{ matrix.nixpkg }}'
shell: bash
- run: nix build -L '.?submodules=1#${{ matrix.nixpkg }}'
+ run: nix build -L '.#${{ matrix.nixpkg }}'
- name: 'Show result contents'
shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3c84f5c6f..52d20157a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,6 +14,8 @@ env:
NPROC: 2
MAKEFLAGS: "-j${NPROC}"
NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none"
+ NIM_VERSION: '2.2.4'
+ NIMBLE_VERSION: '0.22.3'
jobs:
changes: # changes detection
@@ -30,10 +32,11 @@ jobs:
filters: |
common:
- '.github/workflows/**'
- - 'vendor/**'
- - 'Makefile'
+ - 'nimble.lock'
- 'waku.nimble'
+ - 'Makefile'
- 'library/**'
+ - 'liblogosdelivery/**'
v2:
- 'waku/**'
- 'apps/**'
@@ -63,24 +66,36 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
- - name: Get submodules hash
- id: submodules
- run: |
- echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
+ - name: Install Nim ${{ env.NIM_VERSION }}
+ uses: jiro4989/setup-nim-action@v2
+ with:
+ nim-version: ${{ env.NIM_VERSION }}
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache submodules
+ - name: Install Nimble ${{ env.NIMBLE_VERSION }}
+ run: |
+ cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y
+ echo "$HOME/.nimble/bin" >> $GITHUB_PATH
+
+ - name: Cache nimble deps
+ id: cache-nimbledeps
uses: actions/cache@v3
with:
path: |
- vendor/
- .git/modules
- key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
+ nimbledeps/
+ nimble.paths
+ key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }}
- - name: Make update
- run: make update
+ - name: Install nimble deps
+ if: steps.cache-nimbledeps.outputs.cache-hit != 'true'
+ run: |
+ nimble setup --localdeps -y
+ make rebuild-nat-libs-nimbledeps
+ make rebuild-bearssl-nimbledeps
+ touch nimbledeps/.nimble-setup
- name: Build binaries
- run: make V=1 QUICK_AND_DIRTY_COMPILER=1 all
+ run: make V=1 all
build-windows:
needs: changes
@@ -104,21 +119,33 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
- - name: Get submodules hash
- id: submodules
- run: |
- echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
+ - name: Install Nim ${{ env.NIM_VERSION }}
+ uses: jiro4989/setup-nim-action@v2
+ with:
+ nim-version: ${{ env.NIM_VERSION }}
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache submodules
+ - name: Install Nimble ${{ env.NIMBLE_VERSION }}
+ run: |
+ cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y
+ echo "$HOME/.nimble/bin" >> $GITHUB_PATH
+
+ - name: Cache nimble deps
+ id: cache-nimbledeps
uses: actions/cache@v3
with:
path: |
- vendor/
- .git/modules
- key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
+ nimbledeps/
+ nimble.paths
+ key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }}
- - name: Make update
- run: make update
+ - name: Install nimble deps
+ if: steps.cache-nimbledeps.outputs.cache-hit != 'true'
+ run: |
+ nimble setup --localdeps -y
+ make rebuild-nat-libs-nimbledeps
+ make rebuild-bearssl-nimbledeps
+ touch nimbledeps/.nimble-setup
- name: Run tests
run: |
@@ -132,13 +159,13 @@ jobs:
export NIMFLAGS="--colors:off -d:chronicles_colors:none"
export USE_LIBBACKTRACE=0
- make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 POSTGRES=$postgres_enabled test
- make V=1 LOG_LEVEL=DEBUG QUICK_AND_DIRTY_COMPILER=1 POSTGRES=$postgres_enabled testwakunode2
+ make V=1 POSTGRES=$postgres_enabled test
+ make V=1 POSTGRES=$postgres_enabled testwakunode2
build-docker-image:
needs: changes
if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' || needs.changes.outputs.docker == 'true' }}
- uses: logos-messaging/logos-delivery/.github/workflows/container-image.yml@10dc3d3eb4b6a3d4313f7b2cc4a85a925e9ce039
+ uses: ./.github/workflows/container-image.yml
secrets: inherit
nwaku-nwaku-interop-tests:
@@ -171,18 +198,33 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
- - name: Get submodules hash
- id: submodules
- run: |
- echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
+ - name: Install Nim ${{ env.NIM_VERSION }}
+ uses: jiro4989/setup-nim-action@v2
+ with:
+ nim-version: ${{ env.NIM_VERSION }}
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache submodules
+ - name: Install Nimble ${{ env.NIMBLE_VERSION }}
+ run: |
+ cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y
+ echo "$HOME/.nimble/bin" >> $GITHUB_PATH
+
+ - name: Cache nimble deps
+ id: cache-nimbledeps
uses: actions/cache@v3
with:
path: |
- vendor/
- .git/modules
- key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
+ nimbledeps/
+ nimble.paths
+ key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }}
+
+ - name: Install nimble deps
+ if: steps.cache-nimbledeps.outputs.cache-hit != 'true'
+ run: |
+ nimble setup --localdeps -y
+ make rebuild-nat-libs-nimbledeps
+ make rebuild-bearssl-nimbledeps
+ touch nimbledeps/.nimble-setup
- name: Build nph
run: |
diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml
index 2bc08be2f..0ff427d87 100644
--- a/.github/workflows/container-image.yml
+++ b/.github/workflows/container-image.yml
@@ -15,6 +15,8 @@ env:
NPROC: 2
MAKEFLAGS: "-j${NPROC}"
NIMFLAGS: "--parallelBuild:${NPROC}"
+ NIM_VERSION: '2.2.4'
+ NIMBLE_VERSION: '0.22.3'
# This workflow should not run for outside contributors
# If org secrets are not available, we'll avoid building and publishing the docker image and we'll pass the workflow
@@ -46,28 +48,42 @@ jobs:
if: ${{ steps.secrets.outcome == 'success' }}
uses: actions/checkout@v4
- - name: Get submodules hash
- id: submodules
+ - name: Install Nim ${{ env.NIM_VERSION }}
+ if: ${{ steps.secrets.outcome == 'success' }}
+ uses: jiro4989/setup-nim-action@v2
+ with:
+ nim-version: ${{ env.NIM_VERSION }}
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Install Nimble ${{ env.NIMBLE_VERSION }}
if: ${{ steps.secrets.outcome == 'success' }}
run: |
- echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
+ cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y
+ echo "$HOME/.nimble/bin" >> $GITHUB_PATH
- - name: Cache submodules
+ - name: Cache nimble deps
if: ${{ steps.secrets.outcome == 'success' }}
+ id: cache-nimbledeps
uses: actions/cache@v3
with:
path: |
- vendor/
- .git/modules
- key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
+ nimbledeps/
+ nimble.paths
+ key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }}
+
+ - name: Install nimble deps
+ if: ${{ steps.secrets.outcome == 'success' && steps.cache-nimbledeps.outputs.cache-hit != 'true' }}
+ run: |
+ nimble setup --localdeps -y
+ make rebuild-nat-libs-nimbledeps
+ make rebuild-bearssl-nimbledeps
+ touch nimbledeps/.nimble-setup
- name: Build binaries
id: build
if: ${{ steps.secrets.outcome == 'success' }}
run: |
- make update
-
- make -j${NPROC} V=1 QUICK_AND_DIRTY_COMPILER=1 NIMFLAGS="-d:disableMarchNative -d:postgres -d:chronicles_colors:none" wakunode2
+ make -j${NPROC} V=1 POSTGRES=1 NIMFLAGS="-d:disableMarchNative -d:chronicles_colors:none" wakunode2
SHORT_REF=$(git rev-parse --short HEAD)
diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
index e145e28ae..52a50adc8 100644
--- a/.github/workflows/pre-release.yml
+++ b/.github/workflows/pre-release.yml
@@ -63,11 +63,11 @@ jobs:
run: |
OS=$([[ "${{runner.os}}" == "macOS" ]] && echo "macosx" || echo "linux")
- make QUICK_AND_DIRTY_COMPILER=1 V=1 CI=false NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \
+ make V=1 CI=false NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \
update
- make QUICK_AND_DIRTY_COMPILER=1 V=1 CI=false\
- NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" \
+ make V=1 CI=false POSTGRES=1\
+ NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" \
wakunode2\
chat2\
tools
diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml
index 50e3c4c3d..fc1f819d9 100644
--- a/.github/workflows/release-assets.yml
+++ b/.github/workflows/release-assets.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- 'v*' # "e.g. v0.4"
-
+
workflow_dispatch:
env:
@@ -65,6 +65,16 @@ jobs:
echo "libwaku=${LIBWAKU_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
+ if [[ "${{ runner.os }}" == "Linux" ]]; then
+ LIBLOGOSDELIVERY_ARTIFACT_NAME=$(echo "liblogosdelivery-${VERSION}-${{matrix.arch}}-${{runner.os}}-linux.deb" | tr "[:upper:]" "[:lower:]")
+ fi
+
+ if [[ "${{ runner.os }}" == "macOS" ]]; then
+ LIBLOGOSDELIVERY_ARTIFACT_NAME=$(echo "liblogosdelivery-${VERSION}-${{matrix.arch}}-macos.tar.gz" | tr "[:upper:]" "[:lower:]")
+ fi
+
+ echo "liblogosdelivery=${LIBLOGOSDELIVERY_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
+
- name: Install build dependencies
run: |
if [[ "${{ runner.os }}" == "Linux" ]]; then
@@ -76,12 +86,15 @@ jobs:
OS=$([[ "${{runner.os}}" == "macOS" ]] && echo "macosx" || echo "linux")
make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" V=1 update
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false wakunode2
+ make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false wakunode2
make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" CI=false chat2
tar -cvzf ${{steps.vars.outputs.waku}} ./build/
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false libwaku
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}} -d:postgres" CI=false STATIC=1 libwaku
+ make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false libwaku
+ make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false STATIC=1 libwaku
+
+ make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false liblogosdelivery
+ make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" POSTGRES=1 CI=false STATIC=1 liblogosdelivery
- name: Create distributable libwaku package
run: |
@@ -109,6 +122,32 @@ jobs:
tar -cvzf ${{steps.vars.outputs.libwaku}} ./build/libwaku.dylib ./build/libwaku.a ./library/libwaku.h
fi
+ - name: Create distributable liblogosdelivery package
+ run: |
+ VERSION=${{ steps.version.outputs.version }}
+
+ if [[ "${{ runner.os }}" == "Linux" ]]; then
+ rm -rf pkg
+ mkdir -p pkg/DEBIAN pkg/usr/local/lib pkg/usr/local/include
+ cp build/liblogosdelivery.so pkg/usr/local/lib/
+ cp build/liblogosdelivery.a pkg/usr/local/lib/
+ cp liblogosdelivery/liblogosdelivery.h pkg/usr/local/include/
+
+ echo "Package: logosdelivery" >> pkg/DEBIAN/control
+ echo "Version: ${VERSION}" >> pkg/DEBIAN/control
+ echo "Priority: optional" >> pkg/DEBIAN/control
+ echo "Section: libs" >> pkg/DEBIAN/control
+ echo "Architecture: ${{matrix.arch}}" >> pkg/DEBIAN/control
+ echo "Maintainer: Logos Messaging Team" >> pkg/DEBIAN/control
+ echo "Description: Logos Delivery library" >> pkg/DEBIAN/control
+
+ dpkg-deb --build pkg ${{steps.vars.outputs.liblogosdelivery}}
+ fi
+
+ if [[ "${{ runner.os }}" == "macOS" ]]; then
+ tar -cvzf ${{steps.vars.outputs.liblogosdelivery}} ./build/liblogosdelivery.dylib ./build/liblogosdelivery.a ./liblogosdelivery/liblogosdelivery.h
+ fi
+
- name: Upload waku artifact
uses: actions/upload-artifact@v4.4.0
with:
@@ -122,3 +161,10 @@ jobs:
name: libwaku-${{ steps.version.outputs.version }}-${{ matrix.arch }}-${{ runner.os }}
path: ${{ steps.vars.outputs.libwaku }}
if-no-files-found: error
+
+ - name: Upload liblogosdelivery artifact
+ uses: actions/upload-artifact@v4.4.0
+ with:
+ name: liblogosdelivery-${{ steps.version.outputs.version }}-${{ matrix.arch }}-${{ runner.os }}
+ path: ${{ steps.vars.outputs.liblogosdelivery }}
+ if-no-files-found: error
diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index 9c1b1eab0..50f1602cd 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -7,20 +7,25 @@ on:
required: true
type: string
+env:
+ NPROC: 4
+ NIM_VERSION: '2.2.4'
+ NIMBLE_VERSION: '0.22.3'
+
jobs:
build:
runs-on: windows-latest
defaults:
run:
- shell: msys2 {0}
+ shell: msys2 {0}
env:
MSYSTEM: MINGW64
steps:
- name: Checkout code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup MSYS2
uses: msys2/setup-msys2@v2
@@ -51,50 +56,60 @@ jobs:
run: |
bash scripts/install_nasm_in_windows.sh
source $HOME/.bashrc
-
+
- name: Add UPX to PATH
run: |
echo "/usr/bin:$PATH" >> $GITHUB_PATH
echo "/mingw64/bin:$PATH" >> $GITHUB_PATH
echo "/usr/lib:$PATH" >> $GITHUB_PATH
- echo "/mingw64/lib:$PATH" >> $GITHUB_PATH
+ echo "/mingw64/lib:$PATH" >> $GITHUB_PATH
- name: Verify dependencies
run: |
which upx gcc g++ make cmake cargo rustc python nasm
- - name: Updating submodules
- run: git submodule update --init --recursive
+ - name: Install Nim ${{ env.NIM_VERSION }}
+ uses: jiro4989/setup-nim-action@v2
+ with:
+ nim-version: ${{ env.NIM_VERSION }}
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Install Nimble ${{ env.NIMBLE_VERSION }}
+ run: |
+ export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$PATH"
+ cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y
+ echo "$HOME/.nimble/bin" >> $GITHUB_PATH
+
+ - name: Patch nimble.lock for Windows nim checksum
+ # nimble.exe uses Windows Git (core.autocrlf=true by default), which converts LF→CRLF
+ # on checkout. This changes the SHA1 of the nim package source tree relative to the
+ # Linux-computed checksum stored in nimble.lock. Patch the lock file with the
+ # Windows-computed checksum before nimble reads it.
+ run: |
+ sed -i 's/68bb85cbfb1832ce4db43943911b046c3af3caab/a092a045d3a427d127a5334a6e59c76faff54686/g' nimble.lock
+
+ - name: Install nimble deps
+ if: steps.cache-nimbledeps.outputs.cache-hit != 'true'
+ run: |
+ export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH"
+ nimble setup --localdeps -y
+ make rebuild-nat-libs-nimbledeps CC=gcc
+ make rebuild-bearssl-nimbledeps CC=gcc
+ touch nimbledeps/.nimble-setup
- name: Creating tmp directory
run: mkdir -p tmp
- - name: Building Nim
- run: |
- cd vendor/nimbus-build-system/vendor/Nim
- ./build_all.bat
- cd ../../../..
-
- - name: Building miniupnpc
- run: |
- cd vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc
- make -f Makefile.mingw CC=gcc CXX=g++ libminiupnpc.a V=1
- cd ../../../../..
-
- - name: Building libnatpmp
- run: |
- cd ./vendor/nim-nat-traversal/vendor/libnatpmp-upstream
- make CC="gcc -fPIC -D_WIN32_WINNT=0x0600 -DNATPMP_STATICLIB" libnatpmp.a V=1
- cd ../../../../
-
- name: Building wakunode2.exe
run: |
- make wakunode2 LOG_LEVEL=DEBUG V=3 -j8
+ export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH"
+ make wakunode2 V=3 -j${{ env.NPROC }}
- name: Building libwaku.dll
run: |
- make libwaku STATIC=0 LOG_LEVEL=DEBUG V=1 -j
-
+ export PATH="$GITHUB_WORKSPACE/.nim_runtime/bin:$HOME/.nimble/bin:$PATH"
+ make libwaku STATIC=0 V=1 -j
+
- name: Check Executable
run: |
if [ -f "./build/wakunode2.exe" ]; then
@@ -108,4 +123,4 @@ jobs:
else
echo "Build failed: libwaku.dll not found"
exit 1
- fi
+ fi
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 5222a0d5e..188090b19 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,9 +3,6 @@
# Executables shall be put in an ignored build/ directory
/build
-# Nimble packages
-/vendor/.nimble
-
# Generated Files
*.generated.nim
@@ -45,9 +42,6 @@ node_modules/
rlnKeystore.json
*.tar.gz
-# Nimbus Build System
-nimbus-build-system.paths
-
# sqlite db
*.db
*.db-shm
diff --git a/.gitmodules b/.gitmodules
index 6a63491e3..ac07235b8 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,197 +1,10 @@
-[submodule "vendor/nim-eth"]
- path = vendor/nim-eth
- url = https://github.com/status-im/nim-eth.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-secp256k1"]
- path = vendor/nim-secp256k1
- url = https://github.com/status-im/nim-secp256k1.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-libp2p"]
- path = vendor/nim-libp2p
- url = https://github.com/vacp2p/nim-libp2p.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-stew"]
- path = vendor/nim-stew
- url = https://github.com/status-im/nim-stew.git
- ignore = dirty
- branch = master
-[submodule "vendor/nimbus-build-system"]
- path = vendor/nimbus-build-system
- url = https://github.com/status-im/nimbus-build-system.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-nat-traversal"]
- path = vendor/nim-nat-traversal
- url = https://github.com/status-im/nim-nat-traversal.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-libbacktrace"]
- path = vendor/nim-libbacktrace
- url = https://github.com/status-im/nim-libbacktrace.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-confutils"]
- path = vendor/nim-confutils
- url = https://github.com/status-im/nim-confutils.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-chronicles"]
- path = vendor/nim-chronicles
- url = https://github.com/status-im/nim-chronicles.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-faststreams"]
- path = vendor/nim-faststreams
- url = https://github.com/status-im/nim-faststreams.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-chronos"]
- path = vendor/nim-chronos
- url = https://github.com/status-im/nim-chronos.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-json-serialization"]
- path = vendor/nim-json-serialization
- url = https://github.com/status-im/nim-json-serialization.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-serialization"]
- path = vendor/nim-serialization
- url = https://github.com/status-im/nim-serialization.git
- ignore = dirty
- branch = master
-[submodule "vendor/nimcrypto"]
- path = vendor/nimcrypto
- url = https://github.com/cheatfate/nimcrypto.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-metrics"]
- path = vendor/nim-metrics
- url = https://github.com/status-im/nim-metrics.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-stint"]
- path = vendor/nim-stint
- url = https://github.com/status-im/nim-stint.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-json-rpc"]
- path = vendor/nim-json-rpc
- url = https://github.com/status-im/nim-json-rpc.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-http-utils"]
- path = vendor/nim-http-utils
- url = https://github.com/status-im/nim-http-utils.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-bearssl"]
- path = vendor/nim-bearssl
- url = https://github.com/status-im/nim-bearssl.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-sqlite3-abi"]
- path = vendor/nim-sqlite3-abi
- url = https://github.com/arnetheduck/nim-sqlite3-abi.git
- ignore = dirty
- branch = master
-[submodule "vendor/nim-web3"]
- path = vendor/nim-web3
- url = https://github.com/status-im/nim-web3.git
-[submodule "vendor/nim-testutils"]
- path = vendor/nim-testutils
- url = https://github.com/status-im/nim-testutils.git
- ignore = untracked
- branch = master
-[submodule "vendor/nim-unittest2"]
- path = vendor/nim-unittest2
- url = https://github.com/status-im/nim-unittest2.git
- ignore = untracked
- branch = master
-[submodule "vendor/nim-websock"]
- path = vendor/nim-websock
- url = https://github.com/status-im/nim-websock.git
- ignore = untracked
- branch = main
-[submodule "vendor/nim-zlib"]
- path = vendor/nim-zlib
- url = https://github.com/status-im/nim-zlib.git
- ignore = untracked
- branch = master
-[submodule "vendor/nim-dnsdisc"]
- path = vendor/nim-dnsdisc
- url = https://github.com/status-im/nim-dnsdisc.git
- ignore = untracked
- branch = main
-[submodule "vendor/dnsclient.nim"]
- path = vendor/dnsclient.nim
- url = https://github.com/ba0f3/dnsclient.nim.git
- ignore = untracked
- branch = master
-[submodule "vendor/nim-toml-serialization"]
- path = vendor/nim-toml-serialization
- url = https://github.com/status-im/nim-toml-serialization.git
-[submodule "vendor/nim-presto"]
- path = vendor/nim-presto
- url = https://github.com/status-im/nim-presto.git
- ignore = untracked
- branch = master
[submodule "vendor/zerokit"]
path = vendor/zerokit
url = https://github.com/vacp2p/zerokit.git
ignore = dirty
branch = v0.5.1
-[submodule "vendor/nim-regex"]
- path = vendor/nim-regex
- url = https://github.com/nitely/nim-regex.git
- ignore = untracked
- branch = master
-[submodule "vendor/nim-unicodedb"]
- path = vendor/nim-unicodedb
- url = https://github.com/nitely/nim-unicodedb.git
- ignore = untracked
- branch = master
-[submodule "vendor/nim-taskpools"]
- path = vendor/nim-taskpools
- url = https://github.com/status-im/nim-taskpools.git
- ignore = untracked
- branch = stable
-[submodule "vendor/nim-results"]
- ignore = untracked
- branch = master
- path = vendor/nim-results
- url = https://github.com/arnetheduck/nim-results.git
-[submodule "vendor/db_connector"]
- path = vendor/db_connector
- url = https://github.com/nim-lang/db_connector.git
- ignore = untracked
- branch = devel
-[submodule "vendor/nph"]
- ignore = untracked
- branch = master
- path = vendor/nph
- url = https://github.com/arnetheduck/nph.git
-[submodule "vendor/nim-minilru"]
- path = vendor/nim-minilru
- url = https://github.com/status-im/nim-minilru.git
- ignore = untracked
- branch = master
[submodule "vendor/waku-rlnv2-contract"]
path = vendor/waku-rlnv2-contract
url = https://github.com/logos-messaging/waku-rlnv2-contract.git
ignore = untracked
branch = master
-[submodule "vendor/nim-lsquic"]
- path = vendor/nim-lsquic
- url = https://github.com/vacp2p/nim-lsquic
-[submodule "vendor/nim-jwt"]
- path = vendor/nim-jwt
- url = https://github.com/vacp2p/nim-jwt.git
-[submodule "vendor/nim-ffi"]
- path = vendor/nim-ffi
- url = https://github.com/logos-messaging/nim-ffi/
- ignore = untracked
- branch = master
diff --git a/BearSSL.mk b/BearSSL.mk
new file mode 100644
index 000000000..355e46563
--- /dev/null
+++ b/BearSSL.mk
@@ -0,0 +1,46 @@
+# Copyright (c) 2022 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+###########################
+## bearssl (nimbledeps) ##
+###########################
+# Rebuilds libbearssl.a from the package installed by nimble under
+# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP).
+#
+# BEARSSL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that
+# depend on it must be invoked via a recursive $(MAKE) call so the sub-make
+# re-evaluates the variable after nimble setup has populated nimbledeps/.
+#
+# `ls -dt` (sort by modification time, newest first) is used to pick the
+# latest installed version and is portable across Linux, macOS, and
+# Windows (MSYS/MinGW).
+
+BEARSSL_NIMBLEDEPS_DIR := $(shell ls -dt $(CURDIR)/nimbledeps/pkgs2/bearssl-* 2>/dev/null | head -1)
+BEARSSL_CSOURCES_DIR := $(BEARSSL_NIMBLEDEPS_DIR)/bearssl/csources
+
+BEARSSL_UNAME_M := $(shell uname -m)
+ifeq ($(BEARSSL_UNAME_M),x86_64)
+ PORTABLE_BEARSSL_CFLAGS := -W -Wall -Os -fPIC -mssse3
+else
+ PORTABLE_BEARSSL_CFLAGS := -W -Wall -Os -fPIC
+endif
+
+.PHONY: clean-bearssl-nimbledeps rebuild-bearssl-nimbledeps
+
+clean-bearssl-nimbledeps:
+ifeq ($(BEARSSL_NIMBLEDEPS_DIR),)
+ $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first)
+endif
+ + [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \
+ "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true
+
+rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps
+ifeq ($(BEARSSL_NIMBLEDEPS_DIR),)
+ $(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first)
+endif
+ @echo "Rebuilding bearssl from $(BEARSSL_CSOURCES_DIR)"
+ + "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" CFLAGS="$(PORTABLE_BEARSSL_CFLAGS)" lib
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 5b16b9eee..05525774b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,8 +4,8 @@ FROM rustlang/rust:nightly-alpine3.19 AS nim-build
ARG NIMFLAGS
ARG MAKE_TARGET=wakunode2
ARG NIM_COMMIT
-ARG LOG_LEVEL=TRACE
ARG HEAPTRACK_BUILD=0
+ARG POSTGRES=0
# Get build tools and required header files
RUN apk add --no-cache bash git build-base openssl-dev linux-headers curl jq libbsd-dev
@@ -27,7 +27,7 @@ RUN if [ "$HEAPTRACK_BUILD" = "1" ]; then \
RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT}
# Build the final node binary
-RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}"
+RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET NIMFLAGS="${NIMFLAGS}" POSTGRES=${POSTGRES}
# PRODUCTION IMAGE -------------------------------------------------------------
diff --git a/Makefile b/Makefile
index afd2389d2..be9e14027 100644
--- a/Makefile
+++ b/Makefile
@@ -4,28 +4,13 @@
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
-export BUILD_SYSTEM_DIR := vendor/nimbus-build-system
-export EXCLUDED_NIM_PACKAGES := vendor/nim-dnsdisc/vendor
+
+include Nat.mk
+include BearSSL.mk
+
LINK_PCRE := 0
FORMAT_MSG := "\\x1B[95mFormatting:\\x1B[39m"
-# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
--include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
-
-
-ifeq ($(NIM_PARAMS),)
-# "variables.mk" was not included, so we update the submodules.
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
-.DEFAULT:
- +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
- $(GIT_SUBMODULE_UPDATE); \
- echo
-# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
-# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
-#
-# After restarting, it will execute its original goal, so we don't have to start a child Make here
-# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
-
-else # "variables.mk" was included. Business as usual until the end of this file.
+BUILD_MSG := "Building:"
# Determine the OS
detected_OS := $(shell uname -s)
@@ -33,29 +18,34 @@ ifneq (,$(findstring MINGW,$(detected_OS)))
detected_OS := Windows
endif
+# Ensure the nim/nimble installed by install-nim/install-nimble are found first
+export PATH := $(HOME)/.nimble/bin:$(PATH)
+
+# NIM binary location
+NIM_BINARY := $(shell which nim 2>/dev/null)
+NPH := $(HOME)/.nimble/bin/nph
+NIMBLEDEPS_STAMP := nimbledeps/.nimble-setup
+
+# Compilation parameters
+NIM_PARAMS ?=
+
ifeq ($(detected_OS),Windows)
- # Update MINGW_PATH to standard MinGW location
MINGW_PATH = /mingw64
NIM_PARAMS += --passC:"-I$(MINGW_PATH)/include"
NIM_PARAMS += --passL:"-L$(MINGW_PATH)/lib"
- NIM_PARAMS += --passL:"-Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc"
- NIM_PARAMS += --passL:"-Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream"
-
- LIBS = -lws2_32 -lbcrypt -liphlpapi -luserenv -lntdll -lminiupnpc -lnatpmp -lpq
+ LIBS = -lws2_32 -lbcrypt -liphlpapi -luserenv -lntdll -lpq
NIM_PARAMS += $(foreach lib,$(LIBS),--passL:"$(lib)")
NIM_PARAMS += --passL:"-Wl,--allow-multiple-definition"
-
export PATH := /c/msys64/usr/bin:/c/msys64/mingw64/bin:/c/msys64/usr/lib:/c/msys64/mingw64/lib:$(PATH)
-
endif
##########
## Main ##
##########
-.PHONY: all test update clean examples
+.PHONY: all test update clean examples deps nimble install-nim install-nimble
-# default target, because it's the first one that doesn't start with '.'
-all: | wakunode2 libwaku
+# default target
+all: | wakunode2 libwaku liblogosdelivery
examples: | example2 chat2 chat2bridge
@@ -71,102 +61,86 @@ ifeq ($(strip $(test_file)),)
else
$(MAKE) compile-test TEST_FILE="$(test_file)" TEST_NAME="$(call test_name)"
endif
-# this prevents make from erroring on unknown targets like "Index"
+
+# this prevents make from erroring on unknown targets
%:
@true
waku.nims:
ln -s waku.nimble $@
-update: | update-common
- rm -rf waku.nims && \
- $(MAKE) waku.nims $(HANDLE_OUTPUT)
+$(NIMBLEDEPS_STAMP): nimble.lock | waku.nims
+ $(MAKE) install-nimble
+ nimble setup --localdeps
$(MAKE) build-nph
+ $(MAKE) rebuild-bearssl-nimbledeps
+ $(MAKE) rebuild-nat-libs-nimbledeps
+ touch $@
+
+update:
+ rm -f $(NIMBLEDEPS_STAMP)
+ $(MAKE) $(NIMBLEDEPS_STAMP)
+ nimble lock
clean:
- rm -rf build
+ rm -rf build 2> /dev/null || true
+ rm -rf nimbledeps 2> /dev/null || true
+ rm -fr nimcache 2> /dev/null || true
+ rm nimble.paths 2> /dev/null || true
+ nimble clean
-# must be included after the default target
--include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
+REQUIRED_NIM_VERSION := $(shell grep -E '^const RequiredNimVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
+REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const RequiredNimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
+
+install-nim:
+ scripts/install_nim.sh $(REQUIRED_NIM_VERSION)
+
+install-nimble: install-nim
+ @nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \
+ if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \
+ echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \
+ else \
+ cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \
+ fi
+
+build:
+ mkdir -p build
+
+nimble: install-nimble
## Possible values: prod; debug
TARGET ?= prod
## Git version
GIT_VERSION ?= $(shell git describe --abbrev=6 --always --tags)
-## Compilation parameters. If defined in the CLI the assignments won't be executed
NIM_PARAMS := $(NIM_PARAMS) -d:git_version=\"$(GIT_VERSION)\"
## Heaptracker options
HEAPTRACKER ?= 0
HEAPTRACKER_INJECT ?= 0
ifeq ($(HEAPTRACKER), 1)
-# Assumes Nim's lib/system/alloc.nim is patched!
TARGET := debug-with-heaptrack
-
ifeq ($(HEAPTRACKER_INJECT), 1)
-# the Nim compiler will load 'libheaptrack_inject.so'
HEAPTRACK_PARAMS := -d:heaptracker -d:heaptracker_inject
NIM_PARAMS := $(NIM_PARAMS) -d:heaptracker -d:heaptracker_inject
else
-# the Nim compiler will load 'libheaptrack_preload.so'
HEAPTRACK_PARAMS := -d:heaptracker
NIM_PARAMS := $(NIM_PARAMS) -d:heaptracker
endif
-
-endif
-## end of Heaptracker options
-
-##################
-## Dependencies ##
-##################
-.PHONY: deps libbacktrace
-
-FOUNDRY_VERSION := 1.5.0
-PNPM_VERSION := 10.23.0
-
-
-rustup:
-ifeq (, $(shell which cargo))
-# Install Rustup if it's not installed
-# -y: Assume "yes" for all prompts
-# --default-toolchain stable: Install the stable toolchain
- curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
endif
-rln-deps: rustup
- ./scripts/install_rln_tests_dependencies.sh $(FOUNDRY_VERSION) $(PNPM_VERSION)
-
-deps: | deps-common nat-libs waku.nims
-
-
-### nim-libbacktrace
-
-# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
+# Debug/Release mode
ifeq ($(DEBUG), 0)
-NIM_PARAMS := $(NIM_PARAMS) -d:release -d:lto_incremental -d:strip
+NIM_PARAMS := $(NIM_PARAMS) -d:release
else
NIM_PARAMS := $(NIM_PARAMS) -d:debug
endif
-ifeq ($(USE_LIBBACKTRACE), 0)
NIM_PARAMS := $(NIM_PARAMS) -d:disable_libbacktrace
-endif
# enable experimental exit is dest feature in libp2p mix
NIM_PARAMS := $(NIM_PARAMS) -d:libp2p_mix_experimental_exit_is_dest
-libbacktrace:
- + $(MAKE) -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
-
-clean-libbacktrace:
- + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
-
-# Extend deps and clean targets
-ifneq ($(USE_LIBBACKTRACE), 0)
-deps: | libbacktrace
-endif
-
ifeq ($(POSTGRES), 1)
NIM_PARAMS := $(NIM_PARAMS) -d:postgres -d:nimDebugDlOpen
endif
@@ -175,14 +149,26 @@ ifeq ($(DEBUG_DISCV5), 1)
NIM_PARAMS := $(NIM_PARAMS) -d:debugDiscv5
endif
-clean: | clean-libbacktrace
+# Export NIM_PARAMS so nimble can access it
+export NIM_PARAMS
-### Create nimble links (used when building with Nix)
+##################
+## Dependencies ##
+##################
+.PHONY: deps
-nimbus-build-system-nimble-dir:
- NIMBLE_DIR="$(CURDIR)/$(NIMBLE_DIR)" \
- PWD_CMD="$(PWD)" \
- $(CURDIR)/scripts/generate_nimble_links.sh
+FOUNDRY_VERSION := 1.5.0
+PNPM_VERSION := 10.23.0
+
+rustup:
+ifeq (, $(shell which cargo))
+ curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
+endif
+
+rln-deps: rustup
+ ./scripts/install_rln_tests_dependencies.sh $(FOUNDRY_VERSION) $(PNPM_VERSION)
+
+deps: | nimble
##################
## RLN ##
@@ -199,8 +185,9 @@ LIBRLN_FILE ?= librln_$(LIBRLN_VERSION).a
endif
$(LIBRLN_FILE):
+ git submodule update --init vendor/zerokit
echo -e $(BUILD_MSG) "$@" && \
- ./scripts/build_rln.sh $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(LIBRLN_FILE)
+ bash scripts/build_rln.sh $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(LIBRLN_FILE)
librln: | $(LIBRLN_FILE)
$(eval NIM_PARAMS += --passL:$(LIBRLN_FILE) --passL:-lm)
@@ -209,7 +196,6 @@ clean-librln:
cargo clean --manifest-path vendor/zerokit/rln/Cargo.toml
rm -f $(LIBRLN_FILE)
-# Extend clean target
clean: | clean-librln
#################
@@ -217,74 +203,71 @@ clean: | clean-librln
#################
.PHONY: testcommon
-testcommon: | build deps
+testcommon: | $(NIMBLEDEPS_STAMP) build
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim testcommon $(NIM_PARAMS) waku.nims
-
+ nimble testcommon
##########
## Waku ##
##########
.PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester
-# install rln-deps only for the testwaku target
-testwaku: | build deps rln-deps librln
+testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim test -d:os=$(shell uname) $(NIM_PARAMS) waku.nims
+ nimble test
-wakunode2: | build deps librln
+wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- \
- $(ENV_SCRIPT) nim wakunode2 $(NIM_PARAMS) waku.nims
+ nimble wakunode2
-benchmarks: | build deps librln
+benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim benchmarks $(NIM_PARAMS) waku.nims
+ nimble benchmarks
-testwakunode2: | build deps librln
+testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim testwakunode2 $(NIM_PARAMS) waku.nims
+ nimble testwakunode2
-example2: | build deps librln
+example2: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim example2 $(NIM_PARAMS) waku.nims
+ nimble example2
-chat2: | build deps librln
+chat2: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim chat2 $(NIM_PARAMS) waku.nims
+ nimble chat2
-chat2mix: | build deps librln
+chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim chat2mix $(NIM_PARAMS) waku.nims
+ nimble chat2mix
-rln-db-inspector: | build deps librln
+rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim rln_db_inspector $(NIM_PARAMS) waku.nims
+ nimble rln_db_inspector
-chat2bridge: | build deps librln
+chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim chat2bridge $(NIM_PARAMS) waku.nims
+ nimble chat2bridge
-liteprotocoltester: | build deps librln
+liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim liteprotocoltester $(NIM_PARAMS) waku.nims
+ nimble liteprotocoltester
-lightpushwithmix: | build deps librln
+lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim lightpushwithmix $(NIM_PARAMS) waku.nims
+ nimble lightpushwithmix
-api_example: | build deps librln
+api_example: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims
-build/%: | build deps librln
+build/%: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$*" && \
- $(ENV_SCRIPT) nim buildone $(NIM_PARAMS) waku.nims $*
+ nimble buildone $*
-compile-test: | build deps librln
+compile-test: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \
- $(ENV_SCRIPT) nim buildTest $(NIM_PARAMS) waku.nims $(TEST_FILE) && \
- $(ENV_SCRIPT) nim execTest $(NIM_PARAMS) waku.nims $(TEST_FILE) "\"$(TEST_NAME)\""; \
+ nimble buildTest $(TEST_FILE) && \
+ nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\""
################
## Waku tools ##
@@ -293,29 +276,30 @@ compile-test: | build deps librln
tools: networkmonitor wakucanary
-wakucanary: | build deps librln
+wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim wakucanary $(NIM_PARAMS) waku.nims
+ nimble wakucanary
-networkmonitor: | build deps librln
+networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim networkmonitor $(NIM_PARAMS) waku.nims
+ nimble networkmonitor
############
## Format ##
############
-.PHONY: build-nph install-nph clean-nph print-nph-path
-
-# Default location for nph binary shall be next to nim binary to make it available on the path.
-NPH:=$(shell dirname $(NIM_BINARY))/nph
+.PHONY: build-nph install-nph print-nph-path
build-nph: | build deps
-ifeq ("$(wildcard $(NPH))","")
- $(ENV_SCRIPT) nim c --skipParentCfg:on vendor/nph/src/nph.nim && \
- mv vendor/nph/src/nph $(shell dirname $(NPH))
- echo "nph utility is available at " $(NPH)
+ifneq ($(detected_OS),Windows)
+ if command -v nph > /dev/null 2>&1; then \
+ echo "nph already installed, skipping"; \
+ else \
+ echo "Installing nph globally"; \
+ (cd /tmp && nimble install nph@0.7.0 --accept -g); \
+ fi
+ command -v nph
else
- echo "nph utility already exists at " $(NPH)
+ echo "Skipping nph build on Windows (nph is only used on Unix-like systems)"
endif
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
@@ -332,39 +316,30 @@ nph/%: | build-nph
echo -e $(FORMAT_MSG) "nph/$*" && \
$(NPH) $*
-clean-nph:
- rm -f $(NPH)
-
-# To avoid hardcoding nph binary location in several places
print-nph-path:
- echo "$(NPH)"
+ @echo "$(NPH)"
-clean: | clean-nph
+clean:
###################
## Documentation ##
###################
.PHONY: docs coverage
-# TODO: Remove unused target
docs: | build deps
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) nim doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims
+ nimble doc --run --index:on --project --out:.gh-pages waku/waku.nim waku.nims
coverage:
echo -e $(BUILD_MSG) "build/$@" && \
- $(ENV_SCRIPT) ./scripts/run_cov.sh -y
-
+ ./scripts/run_cov.sh -y
#####################
## Container image ##
#####################
-# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
-# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
DOCKER_IMAGE_NIMFLAGS ?= -d:chronicles_colors:none -d:insecure -d:postgres
DOCKER_IMAGE_NIMFLAGS := $(DOCKER_IMAGE_NIMFLAGS) $(HEAPTRACK_PARAMS)
-# build a docker image for the fleet
docker-image: MAKE_TARGET ?= wakunode2
docker-image: DOCKER_IMAGE_TAG ?= $(MAKE_TARGET)-$(GIT_VERSION)
docker-image: DOCKER_IMAGE_NAME ?= wakuorg/nwaku:$(DOCKER_IMAGE_TAG)
@@ -372,8 +347,6 @@ docker-image:
docker build \
--build-arg="MAKE_TARGET=$(MAKE_TARGET)" \
--build-arg="NIMFLAGS=$(DOCKER_IMAGE_NIMFLAGS)" \
- --build-arg="NIM_COMMIT=$(DOCKER_NIM_COMMIT)" \
- --build-arg="LOG_LEVEL=$(LOG_LEVEL)" \
--build-arg="HEAPTRACK_BUILD=$(HEAPTRACKER)" \
--label="commit=$(shell git rev-parse HEAD)" \
--label="version=$(GIT_VERSION)" \
@@ -384,7 +357,7 @@ docker-quick-image: MAKE_TARGET ?= wakunode2
docker-quick-image: DOCKER_IMAGE_TAG ?= $(MAKE_TARGET)-$(GIT_VERSION)
docker-quick-image: DOCKER_IMAGE_NAME ?= wakuorg/nwaku:$(DOCKER_IMAGE_TAG)
docker-quick-image: NIM_PARAMS := $(NIM_PARAMS) -d:chronicles_colors:none -d:insecure -d:postgres --passL:$(LIBRLN_FILE) --passL:-lm
-docker-quick-image: | build deps librln wakunode2
+docker-quick-image: | build librln wakunode2
docker build \
--build-arg="MAKE_TARGET=$(MAKE_TARGET)" \
--tag $(DOCKER_IMAGE_NAME) \
@@ -398,20 +371,14 @@ docker-push:
####################################
## Container lite-protocol-tester ##
####################################
-# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
-# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
DOCKER_LPT_NIMFLAGS ?= -d:chronicles_colors:none -d:insecure
-# build a docker image for the fleet
docker-liteprotocoltester: DOCKER_LPT_TAG ?= latest
docker-liteprotocoltester: DOCKER_LPT_NAME ?= wakuorg/liteprotocoltester:$(DOCKER_LPT_TAG)
-# --no-cache
docker-liteprotocoltester:
docker build \
--build-arg="MAKE_TARGET=liteprotocoltester" \
--build-arg="NIMFLAGS=$(DOCKER_LPT_NIMFLAGS)" \
- --build-arg="NIM_COMMIT=$(DOCKER_NIM_COMMIT)" \
- --build-arg="LOG_LEVEL=TRACE" \
--label="commit=$(shell git rev-parse HEAD)" \
--label="version=$(GIT_VERSION)" \
--target $(if $(filter deploy,$(DOCKER_LPT_TAG)),deployment_lpt,standalone_lpt) \
@@ -430,39 +397,38 @@ docker-quick-liteprotocoltester: | liteprotocoltester
docker-liteprotocoltester-push:
docker push $(DOCKER_LPT_NAME)
-
################
## C Bindings ##
################
.PHONY: cbindings cwaku_example libwaku liblogosdelivery liblogosdelivery_example
+detected_OS ?= Linux
+ifeq ($(OS),Windows_NT)
+detected_OS := Windows
+else
+detected_OS := $(shell uname -s)
+endif
+
+BUILD_COMMAND ?= Dynamic
STATIC ?= 0
-LIBWAKU_BUILD_COMMAND ?= libwakuDynamic
-LIBLOGOSDELIVERY_BUILD_COMMAND ?= liblogosdeliveryDynamic
+ifeq ($(STATIC), 1)
+ BUILD_COMMAND = Static
+endif
ifeq ($(detected_OS),Windows)
- LIB_EXT_DYNAMIC = dll
- LIB_EXT_STATIC = lib
+ BUILD_COMMAND := $(BUILD_COMMAND)Windows
else ifeq ($(detected_OS),Darwin)
- LIB_EXT_DYNAMIC = dylib
- LIB_EXT_STATIC = a
+ BUILD_COMMAND := $(BUILD_COMMAND)Mac
+ export IOS_SDK_PATH := $(shell xcrun --sdk iphoneos --show-sdk-path)
else ifeq ($(detected_OS),Linux)
- LIB_EXT_DYNAMIC = so
- LIB_EXT_STATIC = a
+ BUILD_COMMAND := $(BUILD_COMMAND)Linux
endif
-LIB_EXT := $(LIB_EXT_DYNAMIC)
-ifeq ($(STATIC), 1)
- LIB_EXT = $(LIB_EXT_STATIC)
- LIBWAKU_BUILD_COMMAND = libwakuStatic
- LIBLOGOSDELIVERY_BUILD_COMMAND = liblogosdeliveryStatic
-endif
+libwaku: | $(NIMBLEDEPS_STAMP) librln
+ nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble
-libwaku: | build deps librln
- echo -e $(BUILD_MSG) "build/$@.$(LIB_EXT)" && $(ENV_SCRIPT) nim $(LIBWAKU_BUILD_COMMAND) $(NIM_PARAMS) waku.nims $@.$(LIB_EXT)
-
-liblogosdelivery: | build deps librln
- echo -e $(BUILD_MSG) "build/$@.$(LIB_EXT)" && $(ENV_SCRIPT) nim $(LIBLOGOSDELIVERY_BUILD_COMMAND) $(NIM_PARAMS) waku.nims $@.$(LIB_EXT)
+liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln
+ nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble
logosdelivery_example: | build liblogosdelivery
@echo -e $(BUILD_MSG) "build/$@"
@@ -492,17 +458,35 @@ else ifeq ($(detected_OS),Windows)
-lws2_32
endif
+cwaku_example: | build libwaku
+ echo -e $(BUILD_MSG) "build/$@" && \
+ cc -o "build/$@" \
+ ./examples/cbindings/waku_example.c \
+ ./examples/cbindings/base64.c \
+ -lwaku -Lbuild/ \
+ -pthread -ldl -lm
+
+cppwaku_example: | build libwaku
+ echo -e $(BUILD_MSG) "build/$@" && \
+ g++ -o "build/$@" \
+ ./examples/cpp/waku.cpp \
+ ./examples/cpp/base64.cpp \
+ -lwaku -Lbuild/ \
+ -pthread -ldl -lm
+
+nodejswaku: | build deps
+ echo -e $(BUILD_MSG) "build/$@" && \
+ node-gyp build --directory=examples/nodejs/
+
#####################
## Mobile Bindings ##
#####################
.PHONY: libwaku-android \
- libwaku-android-precheck \
- libwaku-android-arm64 \
- libwaku-android-amd64 \
- libwaku-android-x86 \
- libwaku-android-arm \
- rebuild-nat-libs \
- build-libwaku-for-android-arch
+ libwaku-android-precheck \
+ libwaku-android-arm64 \
+ libwaku-android-amd64 \
+ libwaku-android-x86 \
+ libwaku-android-arm
ANDROID_TARGET ?= 30
ifeq ($(detected_OS),Darwin)
@@ -511,22 +495,19 @@ else
ANDROID_TOOLCHAIN_DIR := $(ANDROID_NDK_HOME)/toolchains/llvm/prebuilt/linux-x86_64
endif
-rebuild-nat-libs: | clean-cross nat-libs
-
libwaku-android-precheck:
ifndef ANDROID_NDK_HOME
- $(error ANDROID_NDK_HOME is not set)
+ $(error ANDROID_NDK_HOME is not set)
endif
build-libwaku-for-android-arch:
ifneq ($(findstring /nix/store,$(LIBRLN_FILE)),)
mkdir -p $(CURDIR)/build/android/$(ABIDIR)/
- cp $(LIBRLN_FILE) $(CURDIR)/build/android/$(ABIDIR)/
+ CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) nimble libWakuAndroid
else
./scripts/build_rln_android.sh $(CURDIR)/build $(LIBRLN_BUILDDIR) $(LIBRLN_VERSION) $(CROSS_TARGET) $(ABIDIR)
endif
- $(MAKE) rebuild-nat-libs CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_COMPILER)
- CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_ARCH=$(ANDROID_ARCH) ANDROID_COMPILER=$(ANDROID_COMPILER) ANDROID_TOOLCHAIN_DIR=$(ANDROID_TOOLCHAIN_DIR) $(ENV_SCRIPT) nim libWakuAndroid $(NIM_PARAMS) waku.nims
+ $(MAKE) rebuild-nat-libs-nimbledeps CC=$(ANDROID_TOOLCHAIN_DIR)/bin/$(ANDROID_COMPILER)
libwaku-android-arm64: ANDROID_ARCH=aarch64-linux-android
libwaku-android-arm64: CPU=arm64
@@ -550,29 +531,23 @@ libwaku-android-arm: ANDROID_ARCH=armv7a-linux-androideabi
libwaku-android-arm: CPU=arm
libwaku-android-arm: ABIDIR=armeabi-v7a
libwaku-android-arm: | libwaku-android-precheck build deps
-# cross-rs target architecture name does not match the one used in android
$(MAKE) build-libwaku-for-android-arch ANDROID_ARCH=$(ANDROID_ARCH) CROSS_TARGET=armv7-linux-androideabi CPU=$(CPU) ABIDIR=$(ABIDIR) ANDROID_COMPILER=$(ANDROID_ARCH)$(ANDROID_TARGET)-clang
libwaku-android:
$(MAKE) libwaku-android-amd64
$(MAKE) libwaku-android-arm64
$(MAKE) libwaku-android-x86
-# This target is disabled because on recent versions of cross-rs complain with the following error
-# relocation R_ARM_THM_ALU_PREL_11_0 cannot be used against symbol 'stack_init_trampoline_return'; recompile with -fPIC
-# It's likely this architecture is not used so we might just not support it.
-# $(MAKE) libwaku-android-arm
#################
## iOS Bindings #
#################
.PHONY: libwaku-ios-precheck \
- libwaku-ios-device \
- libwaku-ios-simulator \
- libwaku-ios
+ libwaku-ios-device \
+ libwaku-ios-simulator \
+ libwaku-ios
IOS_DEPLOYMENT_TARGET ?= 18.0
-# Get SDK paths dynamically using xcrun
define get_ios_sdk_path
$(shell xcrun --sdk $(1) --show-sdk-path 2>/dev/null)
endef
@@ -584,59 +559,25 @@ else
$(error iOS builds are only supported on macOS)
endif
-# Build for iOS architecture
build-libwaku-for-ios-arch:
- IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) $(ENV_SCRIPT) nim libWakuIOS $(NIM_PARAMS) waku.nims
+ IOS_SDK=$(IOS_SDK) IOS_ARCH=$(IOS_ARCH) IOS_SDK_PATH=$(IOS_SDK_PATH) nimble libWakuIOS
-# iOS device (arm64)
libwaku-ios-device: IOS_ARCH=arm64
libwaku-ios-device: IOS_SDK=iphoneos
libwaku-ios-device: IOS_SDK_PATH=$(call get_ios_sdk_path,iphoneos)
libwaku-ios-device: | libwaku-ios-precheck build deps
$(MAKE) build-libwaku-for-ios-arch IOS_ARCH=$(IOS_ARCH) IOS_SDK=$(IOS_SDK) IOS_SDK_PATH=$(IOS_SDK_PATH)
-# iOS simulator (arm64 - Apple Silicon Macs)
libwaku-ios-simulator: IOS_ARCH=arm64
libwaku-ios-simulator: IOS_SDK=iphonesimulator
libwaku-ios-simulator: IOS_SDK_PATH=$(call get_ios_sdk_path,iphonesimulator)
libwaku-ios-simulator: | libwaku-ios-precheck build deps
$(MAKE) build-libwaku-for-ios-arch IOS_ARCH=$(IOS_ARCH) IOS_SDK=$(IOS_SDK) IOS_SDK_PATH=$(IOS_SDK_PATH)
-# Build all iOS targets
libwaku-ios:
$(MAKE) libwaku-ios-device
$(MAKE) libwaku-ios-simulator
-cwaku_example: | build libwaku
- echo -e $(BUILD_MSG) "build/$@" && \
- cc -o "build/$@" \
- ./examples/cbindings/waku_example.c \
- ./examples/cbindings/base64.c \
- -lwaku -Lbuild/ \
- -pthread -ldl -lm \
- -lminiupnpc -Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/ \
- -lnatpmp -Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream/ \
- vendor/nim-libbacktrace/libbacktrace_wrapper.o \
- vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a
-
-cppwaku_example: | build libwaku
- echo -e $(BUILD_MSG) "build/$@" && \
- g++ -o "build/$@" \
- ./examples/cpp/waku.cpp \
- ./examples/cpp/base64.cpp \
- -lwaku -Lbuild/ \
- -pthread -ldl -lm \
- -lminiupnpc -Lvendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build/ \
- -lnatpmp -Lvendor/nim-nat-traversal/vendor/libnatpmp-upstream/ \
- vendor/nim-libbacktrace/libbacktrace_wrapper.o \
- vendor/nim-libbacktrace/install/usr/lib/libbacktrace.a
-
-nodejswaku: | build deps
- echo -e $(BUILD_MSG) "build/$@" && \
- node-gyp build --directory=examples/nodejs/
-
-endif # "variables.mk" was not included
-
###################
# Release Targets #
###################
@@ -649,6 +590,4 @@ release-notes:
-u $(shell id -u) \
docker.io/wakuorg/sv4git:latest \
release-notes |\
- sed -E 's@#([0-9]+)@[#\1](https://github.com/waku-org/nwaku/issues/\1)@g'
-# I could not get the tool to replace issue ids with links, so using sed for now,
-# asked here: https://github.com/bvieira/sv4git/discussions/101
+ sed -E 's@#([0-9]+)@[#\1](https://github.com/waku-org/nwaku/issues/\1)@g'
\ No newline at end of file
diff --git a/Nat.mk b/Nat.mk
new file mode 100644
index 000000000..90d0b2ead
--- /dev/null
+++ b/Nat.mk
@@ -0,0 +1,61 @@
+# Copyright (c) 2022 Status Research & Development GmbH. Licensed under
+# either of:
+# - Apache License, version 2.0
+# - MIT license
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+###########################
+## nat-libs (nimbledeps) ##
+###########################
+# Builds miniupnpc and libnatpmp from the package installed by nimble under
+# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP).
+#
+# NAT_TRAVERSAL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that
+# depend on it must be invoked via a recursive $(MAKE) call so the sub-make
+# re-evaluates the variable after nimble setup has populated nimbledeps/.
+#
+# `ls -dt` (sort by modification time, newest first) is used to pick the
+# latest installed version and is portable across Linux, macOS, and
+# Windows (MSYS/MinGW).
+
+NAT_TRAVERSAL_NIMBLEDEPS_DIR := $(shell ls -dt $(CURDIR)/nimbledeps/pkgs2/nat_traversal-* 2>/dev/null | head -1)
+
+NAT_UNAME_M := $(shell uname -m)
+ifeq ($(NAT_UNAME_M),x86_64)
+ PORTABLE_NAT_MARCH := -mssse3
+else
+ PORTABLE_NAT_MARCH :=
+endif
+
+.PHONY: clean-cross-nimbledeps rebuild-nat-libs-nimbledeps
+
+clean-cross-nimbledeps:
+ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),)
+ $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first)
+endif
+ + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" ] && \
+ "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" CC=$(CC) clean $(HANDLE_OUTPUT) || true
+ + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \
+ "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true
+
+rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps
+ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),)
+ $(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first)
+endif
+ @echo "Rebuilding nat-libs from $(NAT_TRAVERSAL_NIMBLEDEPS_DIR)"
+ifeq ($(OS), Windows_NT)
+ + [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc/libminiupnpc.a" ] || \
+ PATH=".;$${PATH}" "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" \
+ -f Makefile.mingw CC=$(CC) CFLAGS="-Os -fPIC" libminiupnpc.a $(HANDLE_OUTPUT)
+ + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \
+ OS=mingw CC=$(CC) \
+ CFLAGS="-Wall -Wno-cpp -Os -fPIC -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \
+ libnatpmp.a $(HANDLE_OUTPUT)
+else
+ + "$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" \
+ CC=$(CC) CFLAGS="-Os -fPIC $(PORTABLE_NAT_MARCH)" build/libminiupnpc.a $(HANDLE_OUTPUT)
+ + "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -fPIC $(PORTABLE_NAT_MARCH) -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \
+ -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \
+ CC=$(CC) libnatpmp.a $(HANDLE_OUTPUT)
+endif
diff --git a/apps/wakunode2/wakunode2.nim b/apps/wakunode2/wakunode2.nim
index c8132ff4e..484adf68f 100644
--- a/apps/wakunode2/wakunode2.nim
+++ b/apps/wakunode2/wakunode2.nim
@@ -5,7 +5,6 @@ import
chronicles,
chronos,
metrics,
- libbacktrace,
system/ansi_c,
libp2p/crypto/crypto
import
@@ -88,7 +87,7 @@ when isMainModule:
when defined(posix):
proc handleSigsegv(signal: cint) {.noconv.} =
# Require --debugger:native
- fatal "Shutting down after receiving SIGSEGV", stacktrace = getBacktrace()
+ fatal "Shutting down after receiving SIGSEGV"
# Not available in -d:release mode
writeStackTrace()
diff --git a/ci/Jenkinsfile.release b/ci/Jenkinsfile.release
index 570a37d5f..d8237f009 100644
--- a/ci/Jenkinsfile.release
+++ b/ci/Jenkinsfile.release
@@ -85,7 +85,8 @@ pipeline {
"--label=commit='${git.commit()}' " +
"--label=version='${git.describe('--tags')}' " +
"--build-arg=MAKE_TARGET='${params.MAKE_TARGET}' " +
- "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:postgres -d:heaptracker ' " +
+ "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:heaptracker ' " +
+ "--build-arg=POSTGRES='1' " +
"--build-arg=LOG_LEVEL='${params.LOWEST_LOG_LEVEL_ALLOWED}' " +
"--build-arg=DEBUG='${params.DEBUG ? "1" : "0"} ' " +
"--build-arg=NIM_COMMIT='NIM_COMMIT=heaptrack_support_v2.0.12' " +
@@ -98,7 +99,8 @@ pipeline {
"--label=commit='${git.commit()}' " +
"--label=version='${git.describe('--tags')}' " +
"--build-arg=MAKE_TARGET='${params.MAKE_TARGET}' " +
- "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:postgres ' " +
+ "--build-arg=NIMFLAGS='${params.NIMFLAGS}' " +
+ "--build-arg=POSTGRES='1' " +
"--build-arg=LOG_LEVEL='${params.LOWEST_LOG_LEVEL_ALLOWED}' " +
"--build-arg=DEBUG='${params.DEBUG ? "1" : "0"} ' " +
"--target='prod' ."
diff --git a/config.nims b/config.nims
index 0655bf092..0f6052c9b 100644
--- a/config.nims
+++ b/config.nims
@@ -9,12 +9,6 @@ if defined(windows):
switch("passL", "rln.lib")
switch("define", "postgres=false")
- # Automatically add all vendor subdirectories
- for dir in walkDir("./vendor"):
- if dir.kind == pcDir:
- switch("path", dir.path)
- switch("path", dir.path / "src")
-
# disable timestamps in Windows PE headers - https://wiki.debian.org/ReproducibleBuilds/TimestampsInPEBinaries
switch("passL", "-Wl,--no-insert-timestamp")
# increase stack size
@@ -89,8 +83,9 @@ if not defined(macosx) and not defined(android):
# add debugging symbols and original files and line numbers
--debugger:
native
- if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace):
+ when defined(enable_libbacktrace):
# light-weight stack traces using libbacktrace and libunwind
+ # opt-in: pass -d:enable_libbacktrace (requires libbacktrace in project deps)
--define:
nimStackTraceOverride
switch("import", "libbacktrace")
@@ -121,3 +116,8 @@ if defined(android):
switch("passC", "--sysroot=" & sysRoot)
switch("passL", "--sysroot=" & sysRoot)
switch("cincludes", sysRoot & "/usr/include/")
+# begin Nimble config (version 2)
+when withDir(thisDir(), system.fileExists("nimble.paths")):
+ --noNimblePath
+ include "nimble.paths"
+# end Nimble config
diff --git a/env.sh b/env.sh
deleted file mode 100755
index f90ba9a74..000000000
--- a/env.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
-# and we fall back to a Zsh-specific special var to also support Zsh.
-REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
-ABS_PATH="$(cd ${REL_PATH}; pwd)"
-source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
-
diff --git a/flake.lock b/flake.lock
index b927e8807..9b5db728d 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,27 +2,48 @@
"nodes": {
"nixpkgs": {
"locked": {
- "lastModified": 1757590060,
- "narHash": "sha256-EWwwdKLMZALkgHFyKW7rmyhxECO74+N+ZO5xTDnY/5c=",
+ "lastModified": 1770464364,
+ "narHash": "sha256-z5NJPSBwsLf/OfD8WTmh79tlSU8XgIbwmk6qB1/TFzY=",
"owner": "NixOS",
"repo": "nixpkgs",
- "rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
+ "rev": "23d72dabcb3b12469f57b37170fcbc1789bd7457",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
- "rev": "0ef228213045d2cdb5a169a95d63ded38670b293",
+ "rev": "23d72dabcb3b12469f57b37170fcbc1789bd7457",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
+ "rust-overlay": "rust-overlay",
"zerokit": "zerokit"
}
},
"rust-overlay": {
+ "inputs": {
+ "nixpkgs": [
+ "nixpkgs"
+ ]
+ },
+ "locked": {
+ "lastModified": 1775099554,
+ "narHash": "sha256-3xBsGnGDLOFtnPZ1D3j2LU19wpAlYefRKTlkv648rU0=",
+ "owner": "oxalica",
+ "repo": "rust-overlay",
+ "rev": "8d6387ed6d8e6e6672fd3ed4b61b59d44b124d99",
+ "type": "github"
+ },
+ "original": {
+ "owner": "oxalica",
+ "repo": "rust-overlay",
+ "type": "github"
+ }
+ },
+ "rust-overlay_2": {
"inputs": {
"nixpkgs": [
"zerokit",
@@ -30,11 +51,11 @@
]
},
"locked": {
- "lastModified": 1748399823,
- "narHash": "sha256-kahD8D5hOXOsGbNdoLLnqCL887cjHkx98Izc37nDjlA=",
+ "lastModified": 1771211437,
+ "narHash": "sha256-lcNK438i4DGtyA+bPXXyVLHVmJjYpVKmpux9WASa3ro=",
"owner": "oxalica",
"repo": "rust-overlay",
- "rev": "d68a69dc71bc19beb3479800392112c2f6218159",
+ "rev": "c62195b3d6e1bb11e0c2fb2a494117d3b55d410f",
"type": "github"
},
"original": {
@@ -48,21 +69,21 @@
"nixpkgs": [
"nixpkgs"
],
- "rust-overlay": "rust-overlay"
+ "rust-overlay": "rust-overlay_2"
},
"locked": {
- "lastModified": 1762211504,
- "narHash": "sha256-SbDoBElFYJ4cYebltxlO2lYnz6qOaDAVY6aNJ5bqHDE=",
- "ref": "refs/heads/master",
- "rev": "3160d9504d07791f2fc9b610948a6cf9a58ed488",
- "revCount": 342,
- "type": "git",
- "url": "https://github.com/vacp2p/zerokit"
+ "lastModified": 1771279884,
+ "narHash": "sha256-tzkQPwSl4vPTUo1ixHh6NCENjsBDroMKTjifg2q8QX8=",
+ "owner": "vacp2p",
+ "repo": "zerokit",
+ "rev": "53b18098e6d5d046e3eb1ac338a8f4f651432477",
+ "type": "github"
},
"original": {
- "rev": "3160d9504d07791f2fc9b610948a6cf9a58ed488",
- "type": "git",
- "url": "https://github.com/vacp2p/zerokit"
+ "owner": "vacp2p",
+ "repo": "zerokit",
+ "rev": "53b18098e6d5d046e3eb1ac338a8f4f651432477",
+ "type": "github"
}
}
},
diff --git a/flake.nix b/flake.nix
index 13ca5e618..50b6dc0b5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,92 +1,83 @@
{
- description = "Logos Messaging Nim build flake";
+ description = "logos-delivery nim build flake";
nixConfig = {
extra-substituters = [ "https://nix-cache.status.im/" ];
- extra-trusted-public-keys = [ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY=" ];
+ extra-trusted-public-keys = [
+ "nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY="
+ ];
};
inputs = {
- # Ensure Nix fetches git submodules (vendor/*) when evaluating this flake.
- # Requires Nix >= 2.27. Consumers no longer need '?submodules=1' in the URL.
- self.submodules = true;
+ # Pinning the commit to use same commit across different projects.
+ # A commit from nixpkgs 25.11 release: https://github.com/NixOS/nixpkgs/tree/release-25.11
+ nixpkgs.url = "github:NixOS/nixpkgs?rev=23d72dabcb3b12469f57b37170fcbc1789bd7457";
- # We are pinning the commit because ultimately we want to use same commit across different projects.
- # A commit from nixpkgs 24.11 release : https://github.com/NixOS/nixpkgs/tree/release-24.11
- nixpkgs.url = "github:NixOS/nixpkgs/0ef228213045d2cdb5a169a95d63ded38670b293";
- # WARNING: Remember to update commit and use 'nix flake update' to update flake.lock.
+ rust-overlay = {
+ url = "github:oxalica/rust-overlay";
+ inputs.nixpkgs.follows = "nixpkgs";
+ };
+
+ # External flake input: Zerokit pinned to a specific commit.
+ # Update the rev here when a new zerokit version is needed.
zerokit = {
- url = "git+https://github.com/vacp2p/zerokit?rev=3160d9504d07791f2fc9b610948a6cf9a58ed488";
+ url = "github:vacp2p/zerokit/53b18098e6d5d046e3eb1ac338a8f4f651432477";
inputs.nixpkgs.follows = "nixpkgs";
};
};
- outputs = { self, nixpkgs, zerokit }:
+ outputs = { self, nixpkgs, rust-overlay, zerokit }:
let
- stableSystems = [
+ systems = [
"x86_64-linux" "aarch64-linux"
"x86_64-darwin" "aarch64-darwin"
- "x86_64-windows" "i686-linux"
- "i686-windows"
+ "x86_64-windows"
];
- forAllSystems = f: nixpkgs.lib.genAttrs stableSystems (system: f system);
+ forAllSystems = nixpkgs.lib.genAttrs systems;
- pkgsFor = forAllSystems (
- system: import nixpkgs {
- inherit system;
- config = {
- android_sdk.accept_license = true;
- allowUnfree = true;
+ nimbleOverlay = final: prev: {
+ nimble = prev.nimble.overrideAttrs (_: {
+ version = "0.22.3";
+ src = prev.fetchFromGitHub {
+ owner = "nim-lang";
+ repo = "nimble";
+ rev = "v0.22.3";
+ sha256 = "sha256-f7DYpRGVUeSi6basK1lfu5AxZpMFOSJ3oYsy+urYErg=";
};
- overlays = [
- (final: prev: {
- androidEnvCustom = prev.callPackage ./nix/pkgs/android-sdk { };
- androidPkgs = final.androidEnvCustom.pkgs;
- androidShell = final.androidEnvCustom.shell;
- })
- ];
+ });
+ };
+
+ pkgsFor = system: import nixpkgs {
+ inherit system;
+ overlays = [ (import rust-overlay) nimbleOverlay ];
+ };
+ in {
+ packages = forAllSystems (system:
+ let
+ pkgs = pkgsFor system;
+ liblogosdelivery = pkgs.callPackage ./nix/default.nix {
+ inherit pkgs;
+ src = ./.;
+ zerokitRln = zerokit.packages.${system}.rln;
+ };
+ in {
+ inherit liblogosdelivery;
+ default = liblogosdelivery;
}
);
- in rec {
- packages = forAllSystems (system: let
- pkgs = pkgsFor.${system};
- in rec {
- libwaku-android-arm64 = pkgs.callPackage ./nix/default.nix {
- inherit stableSystems;
- src = self;
- targets = ["libwaku-android-arm64"];
- abidir = "arm64-v8a";
- zerokitRln = zerokit.packages.${system}.rln-android-arm64;
- };
-
- libwaku = pkgs.callPackage ./nix/default.nix {
- inherit stableSystems;
- src = self;
- targets = ["libwaku"];
- zerokitRln = zerokit.packages.${system}.rln;
- };
-
- wakucanary = pkgs.callPackage ./nix/default.nix {
- inherit stableSystems;
- src = self;
- targets = ["wakucanary"];
- zerokitRln = zerokit.packages.${system}.rln;
- };
-
- liblogosdelivery = pkgs.callPackage ./nix/default.nix {
- inherit stableSystems;
- src = self;
- targets = ["liblogosdelivery"];
- zerokitRln = zerokit.packages.${system}.rln;
- };
-
- default = libwaku;
- });
-
- devShells = forAllSystems (system: {
- default = pkgsFor.${system}.callPackage ./nix/shell.nix {};
- });
+ devShells = forAllSystems (system:
+ let
+ pkgs = pkgsFor system;
+ in {
+ default = pkgs.mkShell {
+ nativeBuildInputs = with pkgs; [
+ nim-2_2
+ nimble
+ ];
+ };
+ }
+ );
};
}
diff --git a/liblogosdelivery/logos_delivery_api/node_api.nim b/liblogosdelivery/logos_delivery_api/node_api.nim
index cd644abd7..90630717b 100644
--- a/liblogosdelivery/logos_delivery_api/node_api.nim
+++ b/liblogosdelivery/logos_delivery_api/node_api.nim
@@ -1,4 +1,4 @@
-import std/[json, strutils]
+import std/[json, strutils, tables]
import chronos, chronicles, results, confutils, confutils/std/net, ffi
import
waku/factory/waku,
@@ -23,22 +23,50 @@ registerReqFFI(CreateNodeRequest, ctx: ptr FFIContext[Waku]):
try:
jsonNode = parseJson($configJson)
except Exception:
+ let exceptionMsg = getCurrentExceptionMsg()
+ error "Failed to parse config JSON",
+ error = exceptionMsg, configJson = $configJson
return err(
- "Failed to parse config JSON: " & getCurrentExceptionMsg() &
- " configJson string: " & $configJson
+ "Failed to parse config JSON: " & exceptionMsg & " configJson string: " &
+ $configJson
)
+ var jsonFields: Table[string, (string, JsonNode)]
+ for key, value in jsonNode:
+ let lowerKey = key.toLowerAscii()
+
+ if jsonFields.hasKey(lowerKey):
+ error "Duplicate configuration option found when normalized to lowercase",
+ key = key
+ return err(
+ "Duplicate configuration option found when normalized to lowercase: '" & key &
+ "'"
+ )
+
+ jsonFields[lowerKey] = (key, value)
+
for confField, confValue in fieldPairs(conf):
- if jsonNode.contains(confField):
- let formattedString = ($jsonNode[confField]).strip(chars = {'\"'})
+ let lowerField = confField.toLowerAscii()
+ if jsonFields.hasKey(lowerField):
+ let (jsonKey, jsonValue) = jsonFields[lowerField]
+ let formattedString = ($jsonValue).strip(chars = {'\"'})
try:
confValue = parseCmdArg(typeof(confValue), formattedString)
except Exception:
return err(
- "Failed to parse field '" & confField & "': " & getCurrentExceptionMsg() &
- ". Value: " & formattedString
+ "Failed to parse field '" & confField & "' from JSON key '" & jsonKey & "': " &
+ getCurrentExceptionMsg() & ". Value: " & formattedString
)
+ jsonFields.del(lowerField)
+
+ if jsonFields.len > 0:
+ var unknownKeys = newSeq[string]()
+ for _, (jsonKey, _) in pairs(jsonFields):
+ unknownKeys.add(jsonKey)
+ error "Unrecognized configuration option(s) found", option = unknownKeys
+ return err("Unrecognized configuration option(s) found: " & $unknownKeys)
+
# Create the node
ctx.myLib[] = (await api.createNode(conf)).valueOr:
let errMsg = $error
diff --git a/nimble.lock b/nimble.lock
new file mode 100644
index 000000000..0a76565c4
--- /dev/null
+++ b/nimble.lock
@@ -0,0 +1,627 @@
+{
+ "version": 2,
+ "packages": {
+ "nim": {
+ "version": "2.2.4",
+ "vcsRevision": "911e0dbb1f76de61fa0215ab1bb85af5334cc9a8",
+ "url": "https://github.com/nim-lang/Nim.git",
+ "downloadMethod": "git",
+ "dependencies": [],
+ "checksums": {
+ "sha1": "68bb85cbfb1832ce4db43943911b046c3af3caab"
+ }
+ },
+ "unittest2": {
+ "version": "0.2.5",
+ "vcsRevision": "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189",
+ "url": "https://github.com/status-im/nim-unittest2",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "02bb3751ba9ddc3c17bfd89f2e41cb6bfb8fc0c9"
+ }
+ },
+ "bearssl": {
+ "version": "0.2.8",
+ "vcsRevision": "22c6a76ce015bc07e011562bdcfc51d9446c1e82",
+ "url": "https://github.com/status-im/nim-bearssl",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "da4dd7ae96d536bdaf42dca9c85d7aed024b6a86"
+ }
+ },
+ "bearssl_pkey_decoder": {
+ "version": "#21dd3710df9345ed2ad8bf8f882761e07863b8e0",
+ "vcsRevision": "21dd3710df9345ed2ad8bf8f882761e07863b8e0",
+ "url": "https://github.com/vacp2p/bearssl_pkey_decoder",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "bearssl"
+ ],
+ "checksums": {
+ "sha1": "21b42e2e6ddca6c875d3fc50f36a5115abf51714"
+ }
+ },
+ "jwt": {
+ "version": "#18f8378de52b241f321c1f9ea905456e89b95c6f",
+ "vcsRevision": "18f8378de52b241f321c1f9ea905456e89b95c6f",
+ "url": "https://github.com/vacp2p/nim-jwt.git",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "bearssl",
+ "bearssl_pkey_decoder"
+ ],
+ "checksums": {
+ "sha1": "bcfd6fc9c5e10a52b87117219b7ab5c98136bc8e"
+ }
+ },
+ "testutils": {
+ "version": "0.8.1",
+ "vcsRevision": "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d",
+ "url": "https://github.com/status-im/nim-testutils",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "96a11cf8b84fa9bd12d4a553afa1cc4b7f9df4e3"
+ }
+ },
+ "db_connector": {
+ "version": "0.1.0",
+ "vcsRevision": "29450a2063970712422e1ab857695c12d80112a6",
+ "url": "https://github.com/nim-lang/db_connector",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "4f2e67d0e4b61af9ac5575509305660b473f01a4"
+ }
+ },
+ "results": {
+ "version": "0.5.1",
+ "vcsRevision": "df8113dda4c2d74d460a8fa98252b0b771bf1f27",
+ "url": "https://github.com/arnetheduck/nim-results",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "a9c011f74bc9ed5c91103917b9f382b12e82a9e7"
+ }
+ },
+ "nat_traversal": {
+ "version": "0.0.1",
+ "vcsRevision": "860e18c37667b5dd005b94c63264560c35d88004",
+ "url": "https://github.com/status-im/nim-nat-traversal",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "results"
+ ],
+ "checksums": {
+ "sha1": "1a376d3e710590ef2c48748a546369755f0a7c97"
+ }
+ },
+ "stew": {
+ "version": "0.5.0",
+ "vcsRevision": "4382b18f04b3c43c8409bfcd6b62063773b2bbaa",
+ "url": "https://github.com/status-im/nim-stew",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "results",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "db22942939773ab7d5a0f2b2668c237240c67dd6"
+ }
+ },
+ "zlib": {
+ "version": "0.1.0",
+ "vcsRevision": "e680f269fb01af2c34a2ba879ff281795a5258fe",
+ "url": "https://github.com/status-im/nim-zlib",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "results"
+ ],
+ "checksums": {
+ "sha1": "bbde4f5a97a84b450fef7d107461e5f35cf2b47f"
+ }
+ },
+ "httputils": {
+ "version": "0.4.1",
+ "vcsRevision": "f142cb2e8bd812dd002a6493b6082827bb248592",
+ "url": "https://github.com/status-im/nim-http-utils",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "results",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "016774ab31c3afff9a423f7d80584905ee59c570"
+ }
+ },
+ "chronos": {
+ "version": "4.2.2",
+ "vcsRevision": "45f43a9ad8bd8bcf5903b42f365c1c879bd54240",
+ "url": "https://github.com/status-im/nim-chronos",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "results",
+ "stew",
+ "bearssl",
+ "httputils",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "3a4c9477df8cef20a04e4f1b54a2d74fdfc2a3d0"
+ }
+ },
+ "metrics": {
+ "version": "0.2.1",
+ "vcsRevision": "a1296caf3ebb5f30f51a5feae7749a30df2824c2",
+ "url": "https://github.com/status-im/nim-metrics",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "chronos",
+ "results",
+ "stew"
+ ],
+ "checksums": {
+ "sha1": "84bb09873d7677c06046f391c7b473cd2fcff8a2"
+ }
+ },
+ "faststreams": {
+ "version": "0.5.0",
+ "vcsRevision": "ce27581a3e881f782f482cb66dc5b07a02bd615e",
+ "url": "https://github.com/status-im/nim-faststreams",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "ee61e507b805ae1df7ec936f03f2d101b0d72383"
+ }
+ },
+ "snappy": {
+ "version": "0.1.0",
+ "vcsRevision": "00bfcef94f8ef6981df5d5b994897f6695badfb2",
+ "url": "https://github.com/status-im/nim-snappy",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "faststreams",
+ "unittest2",
+ "results",
+ "stew"
+ ],
+ "checksums": {
+ "sha1": "e572d60d6a3178c5b1cde2400c51ad771812cd3d"
+ }
+ },
+ "serialization": {
+ "version": "0.5.2",
+ "vcsRevision": "b0f2fa32960ea532a184394b0f27be37bd80248b",
+ "url": "https://github.com/status-im/nim-serialization",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "faststreams",
+ "unittest2",
+ "stew"
+ ],
+ "checksums": {
+ "sha1": "fa35c1bb76a0a02a2379fe86eaae0957c7527cb8"
+ }
+ },
+ "toml_serialization": {
+ "version": "0.2.18",
+ "vcsRevision": "b5b387e6fb2a7cc75d54a269b07cc6218361bd46",
+ "url": "https://github.com/status-im/nim-toml-serialization",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "faststreams",
+ "serialization",
+ "stew"
+ ],
+ "checksums": {
+ "sha1": "76ae1c2af5dd092849b41750ff29217980dc9ca3"
+ }
+ },
+ "confutils": {
+ "version": "0.1.0",
+ "vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a",
+ "url": "https://github.com/status-im/nim-confutils",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "serialization",
+ "results"
+ ],
+ "checksums": {
+ "sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e"
+ }
+ },
+ "json_serialization": {
+ "version": "0.4.4",
+ "vcsRevision": "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44",
+ "url": "https://github.com/status-im/nim-json-serialization",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "faststreams",
+ "serialization",
+ "stew",
+ "results"
+ ],
+ "checksums": {
+ "sha1": "8b3115354104858a0ac9019356fb29720529c2bd"
+ }
+ },
+ "chronicles": {
+ "version": "0.12.2",
+ "vcsRevision": "27ec507429a4eb81edc20f28292ee8ec420be05b",
+ "url": "https://github.com/status-im/nim-chronicles",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "faststreams",
+ "serialization",
+ "json_serialization",
+ "testutils"
+ ],
+ "checksums": {
+ "sha1": "02febb20d088120b2836d3306cfa21f434f88f65"
+ }
+ },
+ "presto": {
+ "version": "0.1.1",
+ "vcsRevision": "d66043dd7ede146442e6c39720c76a20bde5225f",
+ "url": "https://github.com/status-im/nim-presto",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "chronos",
+ "chronicles",
+ "metrics",
+ "results",
+ "stew"
+ ],
+ "checksums": {
+ "sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2"
+ }
+ },
+ "stint": {
+ "version": "0.8.2",
+ "vcsRevision": "470b7892561b5179ab20bd389a69217d6213fe58",
+ "url": "https://github.com/status-im/nim-stint",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "d8f871fd617e7857192d4609fe003b48942a8ae5"
+ }
+ },
+ "minilru": {
+ "version": "0.1.0",
+ "vcsRevision": "6dd93feb60f4cded3c05e7af7209cf63fb677893",
+ "url": "https://github.com/status-im/nim-minilru",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "results",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "0be03a5da29fdd4409ea74a60fd0ccce882601b4"
+ }
+ },
+ "sqlite3_abi": {
+ "version": "3.53.0.0",
+ "vcsRevision": "8240e8e2819dfce1b67fa2733135d01b5cc80ae0",
+ "url": "https://github.com/arnetheduck/nim-sqlite3-abi",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "fb7a6e6f36fc4eb4dfa6634dbcbf5cd0dfd0ebf0"
+ }
+ },
+ "dnsclient": {
+ "version": "0.3.4",
+ "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8",
+ "url": "https://github.com/ba0f3/dnsclient.nim",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40"
+ }
+ },
+ "unicodedb": {
+ "version": "0.13.2",
+ "vcsRevision": "66f2458710dc641dd4640368f9483c8a0ec70561",
+ "url": "https://github.com/nitely/nim-unicodedb",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "739102d885d99bb4571b1955f5f12aee423c935b"
+ }
+ },
+ "regex": {
+ "version": "0.26.3",
+ "vcsRevision": "4593305ed1e49731fc75af1dc572dd2559aad19c",
+ "url": "https://github.com/nitely/nim-regex",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "unicodedb"
+ ],
+ "checksums": {
+ "sha1": "4d24e7d7441137cd202e16f2359a5807ddbdc31f"
+ }
+ },
+ "nimcrypto": {
+ "version": "0.6.4",
+ "vcsRevision": "721fb99ee099b632eb86dfad1f0d96ee87583774",
+ "url": "https://github.com/cheatfate/nimcrypto",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "f9ab24fa940ed03d0fb09729a7303feb50b7eaec"
+ }
+ },
+ "websock": {
+ "version": "0.3.0",
+ "vcsRevision": "c105d98e6522e0e2cbe3dfa11b07a273e9fd0e7b",
+ "url": "https://github.com/status-im/nim-websock",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "chronos",
+ "httputils",
+ "chronicles",
+ "stew",
+ "nimcrypto",
+ "bearssl",
+ "results",
+ "zlib"
+ ],
+ "checksums": {
+ "sha1": "1294a66520fa4541e261dec8a6a84f774fb8c0ac"
+ }
+ },
+ "json_rpc": {
+ "version": "#43bbf499143eb45046c83ac9794c9e3280a2b8e7",
+ "vcsRevision": "43bbf499143eb45046c83ac9794c9e3280a2b8e7",
+ "url": "https://github.com/status-im/nim-json-rpc.git",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "nimcrypto",
+ "stint",
+ "chronos",
+ "httputils",
+ "chronicles",
+ "websock",
+ "serialization",
+ "json_serialization",
+ "unittest2"
+ ],
+ "checksums": {
+ "sha1": "30ff6ead115b88c79862c5c7e37b1c9852eea59f"
+ }
+ },
+ "lsquic": {
+ "version": "0.0.1",
+ "vcsRevision": "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f",
+ "url": "https://github.com/vacp2p/nim-lsquic",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "zlib",
+ "stew",
+ "chronos",
+ "nimcrypto",
+ "unittest2",
+ "chronicles"
+ ],
+ "checksums": {
+ "sha1": "f465fa994346490d0924d162f53d9b5aec62f948"
+ }
+ },
+ "secp256k1": {
+ "version": "0.6.0.3.2",
+ "vcsRevision": "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15",
+ "url": "https://github.com/status-im/nim-secp256k1",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "stew",
+ "results",
+ "nimcrypto"
+ ],
+ "checksums": {
+ "sha1": "6618ef9de17121846a8c1d0317026b0ce8584e10"
+ }
+ },
+ "eth": {
+ "version": "0.9.0",
+ "vcsRevision": "d9135e6c3c5d6d819afdfb566aa8d958756b73a8",
+ "url": "https://github.com/status-im/nim-eth",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "nimcrypto",
+ "stint",
+ "secp256k1",
+ "chronos",
+ "chronicles",
+ "stew",
+ "nat_traversal",
+ "metrics",
+ "sqlite3_abi",
+ "confutils",
+ "testutils",
+ "unittest2",
+ "results",
+ "minilru",
+ "snappy"
+ ],
+ "checksums": {
+ "sha1": "2e01b0cfff9523d110562af70d19948280f8013e"
+ }
+ },
+ "web3": {
+ "version": "0.8.0",
+ "vcsRevision": "cdfe5601d2812a58e54faf53ee634452d01e5918",
+ "url": "https://github.com/status-im/nim-web3",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "chronicles",
+ "chronos",
+ "bearssl",
+ "eth",
+ "faststreams",
+ "json_rpc",
+ "serialization",
+ "json_serialization",
+ "nimcrypto",
+ "stew",
+ "stint",
+ "results"
+ ],
+ "checksums": {
+ "sha1": "26a112af032ef1536f97da2ca7364af618a11b80"
+ }
+ },
+ "dnsdisc": {
+ "version": "0.1.0",
+ "vcsRevision": "38f2e0f52c0a8f032ef4530835e519d550706d9e",
+ "url": "https://github.com/status-im/nim-dnsdisc",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "bearssl",
+ "chronicles",
+ "chronos",
+ "eth",
+ "secp256k1",
+ "stew",
+ "testutils",
+ "unittest2",
+ "nimcrypto",
+ "results"
+ ],
+ "checksums": {
+ "sha1": "055b882a0f6b1d1e57a25a7af99d2e5ac6268154"
+ }
+ },
+ "libp2p": {
+ "version": "#ff8d51857b4b79a68468e7bcc27b2026cca02996",
+ "vcsRevision": "ff8d51857b4b79a68468e7bcc27b2026cca02996",
+ "url": "https://github.com/vacp2p/nim-libp2p.git",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "nimcrypto",
+ "dnsclient",
+ "bearssl",
+ "chronicles",
+ "chronos",
+ "metrics",
+ "secp256k1",
+ "stew",
+ "websock",
+ "unittest2",
+ "results",
+ "serialization",
+ "lsquic",
+ "jwt"
+ ],
+ "checksums": {
+ "sha1": "fa2a7552c6ec860717b77ce34cf0b7afe4570234"
+ }
+ },
+ "taskpools": {
+ "version": "0.1.0",
+ "vcsRevision": "9e8ccc754631ac55ac2fd495e167e74e86293edb",
+ "url": "https://github.com/status-im/nim-taskpools",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim"
+ ],
+ "checksums": {
+ "sha1": "09e1b2fdad55b973724d61227971afc0df0b7a81"
+ }
+ },
+ "sds": {
+ "version": "#2e9a7683f0e180bf112135fae3a3803eed8490d4",
+ "vcsRevision": "2e9a7683f0e180bf112135fae3a3803eed8490d4",
+ "url": "https://github.com/logos-messaging/nim-sds.git",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "chronos",
+ "libp2p",
+ "chronicles",
+ "stew",
+ "stint",
+ "metrics",
+ "results",
+ "taskpools"
+ ],
+ "checksums": {
+ "sha1": "d13f1bf8d1b90b27e9edfc063b043831242cda19"
+ }
+ },
+ "ffi": {
+ "version": "0.1.3",
+ "vcsRevision": "06111de155253b34e47ed2aaed1d61d08d62cc1b",
+ "url": "https://github.com/logos-messaging/nim-ffi",
+ "downloadMethod": "git",
+ "dependencies": [
+ "nim",
+ "chronos",
+ "chronicles",
+ "taskpools"
+ ],
+ "checksums": {
+ "sha1": "6f9d49375ea1dc71add55c72ac80a808f238e5b0"
+ }
+ }
+ },
+ "tasks": {}
+}
diff --git a/nix/default.nix b/nix/default.nix
index 816d0aed8..a9ea0f598 100644
--- a/nix/default.nix
+++ b/nix/default.nix
@@ -1,149 +1,136 @@
-{
- pkgs,
- src ? ../.,
- targets ? ["libwaku-android-arm64"],
- verbosity ? 1,
- useSystemNim ? true,
- quickAndDirty ? true,
- stableSystems ? [
- "x86_64-linux" "aarch64-linux"
- ],
- abidir ? null,
- zerokitRln,
+{ pkgs
+, src
+, zerokitRln
+, enablePostgres ? true
+, enableNimDebugDlOpen ? true
+, chroniclesLogLevel ? null
}:
-assert pkgs.lib.assertMsg (builtins.pathExists "${src}/vendor/nimbus-build-system/scripts")
- "Unable to build without submodules. Append '?submodules=1#' to the URI.";
-
let
- inherit (pkgs) stdenv lib writeScriptBin callPackage;
+ deps = import ./deps.nix { inherit pkgs; };
- androidManifest = "";
+ nimDefineArgs = pkgs.lib.concatStringsSep " \\\n " (
+ [ "--define:disable_libbacktrace" ]
+ ++ pkgs.lib.optional enablePostgres "--define:postgres"
+ ++ pkgs.lib.optional enableNimDebugDlOpen "--define:nimDebugDlOpen"
+ ++ pkgs.lib.optional (chroniclesLogLevel != null)
+ "--define:chronicles_log_level=${toString chroniclesLogLevel}"
+ );
- tools = pkgs.callPackage ./tools.nix {};
- version = tools.findKeyValue "^version = \"([a-f0-9.-]+)\"$" ../waku.nimble;
- revision = lib.substring 0 8 (src.rev or src.dirtyRev or "00000000");
- copyLibwaku = lib.elem "libwaku" targets;
- copyLiblogosdelivery = lib.elem "liblogosdelivery" targets;
- copyWakunode2 = lib.elem "wakunode2" targets;
- hasKnownInstallTarget = copyLibwaku || copyLiblogosdelivery || copyWakunode2;
+ # nat_traversal is excluded from the static pathArgs; it is handled
+ # separately in buildPhase (its bundled C libs must be compiled first).
+ otherDeps = builtins.removeAttrs deps [ "nat_traversal" ];
-in stdenv.mkDerivation {
- pname = "logos-messaging-nim";
- version = "${version}-${revision}";
+ # Some packages (e.g. regex, unicodedb) put their .nim files under src/
+ # while others use the repo root. Pass both so the compiler finds either layout.
+ pathArgs =
+ builtins.concatStringsSep " "
+ (builtins.concatMap (p: [ "--path:${p}" "--path:${p}/src" ])
+ (builtins.attrValues otherDeps));
+
+ libExt =
+ if pkgs.stdenv.hostPlatform.isWindows then "dll"
+ else if pkgs.stdenv.hostPlatform.isDarwin then "dylib"
+ else "so";
+in
+pkgs.stdenv.mkDerivation {
+ pname = "liblogosdelivery";
+ version = "dev";
inherit src;
- # Runtime dependencies
- buildInputs = with pkgs; [
- openssl gmp zip
- ];
+ nativeBuildInputs = with pkgs; [
+ nim-2_2
+ git
+ gnumake
+ which
+ ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.cctools ];
- # Dependencies that should only exist in the build environment.
- nativeBuildInputs = let
- # Fix for Nim compiler calling 'git rev-parse' and 'lsb_release'.
- fakeGit = writeScriptBin "git" "echo ${version}";
- in with pkgs; [
- cmake which zerokitRln nim-unwrapped-2_2 fakeGit
- ] ++ lib.optionals stdenv.isDarwin [
- pkgs.darwin.cctools gcc # Necessary for libbacktrace
- ];
+ buildInputs = [ zerokitRln ]
+ ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.stdenv.cc.cc.lib ];
- # Environment variables required for Android builds
- ANDROID_SDK_ROOT = "${pkgs.androidPkgs.sdk}";
- ANDROID_NDK_HOME = "${pkgs.androidPkgs.ndk}";
- NIMFLAGS = "-d:disableMarchNative -d:git_revision_override=${revision}";
- XDG_CACHE_HOME = "/tmp";
+ buildPhase = ''
+ export HOME=$TMPDIR
+ export XDG_CACHE_HOME=$TMPDIR/.cache
+ export NIMBLE_DIR=$TMPDIR/.nimble
+ export NIMCACHE=$TMPDIR/nimcache
- makeFlags = targets ++ [
- "V=${toString verbosity}"
- "QUICK_AND_DIRTY_COMPILER=${if quickAndDirty then "1" else "0"}"
- "QUICK_AND_DIRTY_NIMBLE=${if quickAndDirty then "1" else "0"}"
- "USE_SYSTEM_NIM=${if useSystemNim then "1" else "0"}"
- "LIBRLN_FILE=${zerokitRln}/lib/librln.${if abidir != null then "so" else "a"}"
- "POSTGRES=1"
- ];
+ mkdir -p build $NIMCACHE
- configurePhase = ''
- patchShebangs . vendor/nimbus-build-system > /dev/null
+ # nat_traversal bundles C sub-libraries that must be compiled before linking.
+ # Copy the fetchgit store path to a writable tmpdir, build, then pass to nim.
+ NAT_TRAV=$TMPDIR/nat_traversal
+ cp -r ${deps.nat_traversal} $NAT_TRAV
+ chmod -R +w $NAT_TRAV
- # build_nim.sh guards "rm -rf dist/checksums" with NIX_BUILD_TOP != "/build",
- # but on macOS the nix sandbox uses /private/tmp/... so the check fails and
- # dist/checksums (provided via preBuild) gets deleted. Fix the check to skip
- # the removal whenever NIX_BUILD_TOP is set (i.e. any nix build).
- substituteInPlace vendor/nimbus-build-system/scripts/build_nim.sh \
- --replace 'if [[ "''${NIX_BUILD_TOP}" != "/build" ]]; then' \
- 'if [[ -z "''${NIX_BUILD_TOP}" ]]; then'
+ make -C $NAT_TRAV/vendor/miniupnp/miniupnpc \
+ CFLAGS="-Os -fPIC" build/libminiupnpc.a
- make nimbus-build-system-paths
- make nimbus-build-system-nimble-dir
+ make -C $NAT_TRAV/vendor/libnatpmp-upstream \
+ CFLAGS="-Wall -Os -fPIC -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4" libnatpmp.a
+
+ echo "== Building liblogosdelivery (dynamic) =="
+ nim c \
+ --noNimblePath \
+ ${pathArgs} \
+ --path:$NAT_TRAV \
+ --path:$NAT_TRAV/src \
+ --passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \
+ ${nimDefineArgs} \
+ --out:build/liblogosdelivery.${libExt} \
+ --app:lib \
+ --threads:on \
+ --opt:size \
+ --noMain \
+ --mm:refc \
+ --header \
+ --nimMainPrefix:liblogosdelivery \
+ --nimcache:$NIMCACHE \
+ liblogosdelivery/liblogosdelivery.nim
+
+ echo "== Building liblogosdelivery (static) =="
+ nim c \
+ --noNimblePath \
+ ${pathArgs} \
+ --path:$NAT_TRAV \
+ --path:$NAT_TRAV/src \
+ --passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \
+ ${nimDefineArgs} \
+ --out:build/liblogosdelivery.a \
+ --app:staticlib \
+ --threads:on \
+ --opt:size \
+ --noMain \
+ --mm:refc \
+ --nimMainPrefix:liblogosdelivery \
+ --nimcache:$NIMCACHE \
+ liblogosdelivery/liblogosdelivery.nim
'';
- # For the Nim v2.2.4 built with NBS we added sat and zippy
- preBuild = lib.optionalString (!useSystemNim) ''
- pushd vendor/nimbus-build-system/vendor/Nim
- mkdir dist
- mkdir -p dist/nimble/vendor/sat
- mkdir -p dist/nimble/vendor/checksums
- mkdir -p dist/nimble/vendor/zippy
-
- cp -r ${callPackage ./nimble.nix {}}/. dist/nimble
- cp -r ${callPackage ./checksums.nix {}}/. dist/checksums
- cp -r ${callPackage ./csources.nix {}}/. csources_v2
- cp -r ${callPackage ./sat.nix {}}/. dist/nimble/vendor/sat
- cp -r ${callPackage ./checksums.nix {}}/. dist/nimble/vendor/checksums
- cp -r ${callPackage ./zippy.nix {}}/. dist/nimble/vendor/zippy
- chmod 777 -R dist/nimble csources_v2
- popd
- '';
-
- installPhase = if abidir != null then ''
- mkdir -p $out/jni
- cp -r ./build/android/${abidir}/* $out/jni/
- echo '${androidManifest}' > $out/jni/AndroidManifest.xml
- cd $out && zip -r libwaku.aar *
- '' else ''
- mkdir -p $out/bin $out/include
-
- # Copy artifacts from build directory (created by Make during buildPhase)
- # Note: build/ is in the source tree, not result/ (which is a post-build symlink)
- if [ -d build ]; then
- ${lib.optionalString copyLibwaku ''
- cp build/libwaku.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true
- ''}
-
- ${lib.optionalString copyLiblogosdelivery ''
- cp build/liblogosdelivery.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true
- ''}
-
- ${lib.optionalString copyWakunode2 ''
- cp build/wakunode2 $out/bin/ 2>/dev/null || true
- ''}
-
- ${lib.optionalString (!hasKnownInstallTarget) ''
- cp build/lib*.{so,dylib,dll,a,lib} $out/bin/ 2>/dev/null || true
- ''}
- fi
-
- # Copy header files
- ${lib.optionalString copyLibwaku ''
- cp library/libwaku.h $out/include/ 2>/dev/null || true
- ''}
-
- ${lib.optionalString copyLiblogosdelivery ''
+ installPhase = ''
+ runHook preInstall
+ mkdir -p $out/lib $out/include
+ cp build/liblogosdelivery.${libExt} $out/lib/ 2>/dev/null || true
+ cp build/liblogosdelivery.a $out/lib/ 2>/dev/null || true
cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true
- ''}
-
- ${lib.optionalString (!hasKnownInstallTarget) ''
- cp library/libwaku.h $out/include/ 2>/dev/null || true
- cp liblogosdelivery/liblogosdelivery.h $out/include/ 2>/dev/null || true
- ''}
+ runHook postInstall
'';
- meta = with pkgs.lib; {
- description = "NWaku derivation to build libwaku for mobile targets using Android NDK and Rust.";
- homepage = "https://github.com/status-im/nwaku";
- license = licenses.mit;
- platforms = stableSystems;
- };
+ # Bundle librln alongside liblogosdelivery so the output is self-contained.
+ # Use --add-rpath (not --set-rpath) so fixupPhase's stdenv RUNPATH injection
+ # for libstdc++ is preserved.
+ postInstall =
+ pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
+ cp ${zerokitRln}/lib/librln.dylib $out/lib/
+ chmod +w $out/lib/librln.dylib $out/lib/liblogosdelivery.dylib
+ install_name_tool -id @rpath/liblogosdelivery.dylib $out/lib/liblogosdelivery.dylib
+ install_name_tool -id @rpath/librln.dylib $out/lib/librln.dylib
+ old=$(otool -L $out/lib/liblogosdelivery.dylib | awk 'NR>1{print $1}' | grep librln)
+ install_name_tool -change "$old" @rpath/librln.dylib $out/lib/liblogosdelivery.dylib
+ install_name_tool -add_rpath @loader_path $out/lib/liblogosdelivery.dylib
+ ''
+ + pkgs.lib.optionalString pkgs.stdenv.isLinux ''
+ cp ${zerokitRln}/lib/librln.so $out/lib/
+ patchelf --add-rpath '$ORIGIN' $out/lib/liblogosdelivery.so
+ '';
}
diff --git a/nix/deps.nix b/nix/deps.nix
new file mode 100644
index 000000000..0d9986528
--- /dev/null
+++ b/nix/deps.nix
@@ -0,0 +1,272 @@
+# AUTOGENERATED from nimble.lock — do not edit manually.
+# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix
+{ pkgs }:
+
+{
+ unittest2 = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-unittest2";
+ rev = "26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189";
+ sha256 = "1n8n36kad50m97b64y7bzzknz9n7szffxhp0bqpk3g2v7zpda8sw";
+ fetchSubmodules = true;
+ };
+
+ bearssl = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-bearssl";
+ rev = "22c6a76ce015bc07e011562bdcfc51d9446c1e82";
+ sha256 = "1cvdd7lfrpa6asmc39al3g4py5nqhpqmvypc36r5qyv7p5arc8a3";
+ fetchSubmodules = true;
+ };
+
+ bearssl_pkey_decoder = pkgs.fetchgit {
+ url = "https://github.com/vacp2p/bearssl_pkey_decoder";
+ rev = "21dd3710df9345ed2ad8bf8f882761e07863b8e0";
+ sha256 = "0bl3f147zmkazbhdkr4cj1nipf9rqiw3g4hh1j424k9hpl55zdpg";
+ fetchSubmodules = true;
+ };
+
+ jwt = pkgs.fetchgit {
+ url = "https://github.com/vacp2p/nim-jwt.git";
+ rev = "18f8378de52b241f321c1f9ea905456e89b95c6f";
+ sha256 = "1986czmszdxj6g9yr7xn1fx8y2y9mwpb3f1bn9nc6973qawsdm0p";
+ fetchSubmodules = true;
+ };
+
+ testutils = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-testutils";
+ rev = "6ce5e5e2301ccbc04b09d27ff78741ff4d352b4d";
+ sha256 = "1vbkr6i5yxhc2ai3b7rbglhmyc98f99x874fqdp6a152a6kqgwxy";
+ fetchSubmodules = true;
+ };
+
+ db_connector = pkgs.fetchgit {
+ url = "https://github.com/nim-lang/db_connector";
+ rev = "29450a2063970712422e1ab857695c12d80112a6";
+ sha256 = "11dna09ccdhj3pzpqa04j7a95ibx907z6n1ff33yf0n92qa4x59z";
+ fetchSubmodules = true;
+ };
+
+ results = pkgs.fetchgit {
+ url = "https://github.com/arnetheduck/nim-results";
+ rev = "df8113dda4c2d74d460a8fa98252b0b771bf1f27";
+ sha256 = "1h7amas16sbhlr7zb7n3jb5434k98ji375vzw72k1fsc86vnmcr9";
+ fetchSubmodules = true;
+ };
+
+ nat_traversal = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-nat-traversal";
+ rev = "860e18c37667b5dd005b94c63264560c35d88004";
+ sha256 = "0319k5bbl468phwfnvlrh7725sc80rnf7m9gyj0i3cb5hb9q78bs";
+ fetchSubmodules = true;
+ };
+
+ stew = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-stew";
+ rev = "4382b18f04b3c43c8409bfcd6b62063773b2bbaa";
+ sha256 = "0mx9g5m636h3sk5pllcpylk51brf7lx91izx3gc23k3ih3hrxyk2";
+ fetchSubmodules = true;
+ };
+
+ zlib = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-zlib";
+ rev = "e680f269fb01af2c34a2ba879ff281795a5258fe";
+ sha256 = "1xw9f1gjsgqihdg7kdkbaq1wankgnx2vn9l3ihc6nqk2jzv5bvk5";
+ fetchSubmodules = true;
+ };
+
+ httputils = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-http-utils";
+ rev = "f142cb2e8bd812dd002a6493b6082827bb248592";
+ sha256 = "03msj4zdxraz4qx9cidb17g7v0asazxv91nng6xxbzjxz0qaqxw6";
+ fetchSubmodules = true;
+ };
+
+ chronos = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-chronos";
+ rev = "45f43a9ad8bd8bcf5903b42f365c1c879bd54240";
+ sha256 = "1v1n59zfzznp97pvwgs9kf136bqmv4x2s2y9f24msspa7qv27w39";
+ fetchSubmodules = true;
+ };
+
+ metrics = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-metrics";
+ rev = "a1296caf3ebb5f30f51a5feae7749a30df2824c2";
+ sha256 = "02vxqy20g8012ks939ac25ksc25k727q84si0p2cmihy5bw1a3qm";
+ fetchSubmodules = true;
+ };
+
+ faststreams = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-faststreams";
+ rev = "ce27581a3e881f782f482cb66dc5b07a02bd615e";
+ sha256 = "0y6bw2scnmr8cxj4fg18w7f34l2bh9qwg5nhlgd84m9fpr5bqarn";
+ fetchSubmodules = true;
+ };
+
+ snappy = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-snappy";
+ rev = "00bfcef94f8ef6981df5d5b994897f6695badfb2";
+ sha256 = "117mam97mkjjj1hs8svc07679k5ayww9yigi74yq8dyqm6fpbl6l";
+ fetchSubmodules = true;
+ };
+
+ serialization = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-serialization";
+ rev = "b0f2fa32960ea532a184394b0f27be37bd80248b";
+ sha256 = "0wip1fjx7ka39ck1g1xvmyarzq1p5dlngpqil6zff8k8z5skiz27";
+ fetchSubmodules = true;
+ };
+
+ toml_serialization = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-toml-serialization";
+ rev = "b5b387e6fb2a7cc75d54a269b07cc6218361bd46";
+ sha256 = "175swdj01rz57h1hvflkyaz4x76qbfn0174ysrk3qk385i1zlg5z";
+ fetchSubmodules = true;
+ };
+
+ confutils = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-confutils";
+ rev = "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a";
+ sha256 = "18bj1ilx10jm2vmqx2wy2xl9rzy7alymi2m4n9jgpa4sbxnfh0x3";
+ fetchSubmodules = true;
+ };
+
+ json_serialization = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-json-serialization";
+ rev = "c343b0e243d9e17e2c40f3a8a24340f7c4a71d44";
+ sha256 = "0i8sq51nqj8lshf6bfixaz9a7sq0ahsbvq3chkxdvv4khsqvam91";
+ fetchSubmodules = true;
+ };
+
+ chronicles = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-chronicles";
+ rev = "27ec507429a4eb81edc20f28292ee8ec420be05b";
+ sha256 = "1xx9fcfwgcaizq3s7i3s03mclz253r5j8va38l9ycl19fcbc96z9";
+ fetchSubmodules = true;
+ };
+
+ presto = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-presto";
+ rev = "d66043dd7ede146442e6c39720c76a20bde5225f";
+ sha256 = "1hrppcak32aigrdv3mqk124w81yy9jv1prs57vqqhfj83gl930vi";
+ fetchSubmodules = true;
+ };
+
+ stint = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-stint";
+ rev = "470b7892561b5179ab20bd389a69217d6213fe58";
+ sha256 = "1isfwmbj98qfi5pm9acy0yyvq0vlz38nxp30xl43jx2mmaga2w22";
+ fetchSubmodules = true;
+ };
+
+ minilru = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-minilru";
+ rev = "6dd93feb60f4cded3c05e7af7209cf63fb677893";
+ sha256 = "1xgx4j56ais3hk8b51zhnfs9q85g2afkp3y1j9ky5iziqvcs2sml";
+ fetchSubmodules = true;
+ };
+
+ sqlite3_abi = pkgs.fetchgit {
+ url = "https://github.com/arnetheduck/nim-sqlite3-abi";
+ rev = "8240e8e2819dfce1b67fa2733135d01b5cc80ae0";
+ sha256 = "0g8bc0kiwxxh3h5w06ksa23cw81hnx87rdn93v64m2f053nb6bcm";
+ fetchSubmodules = true;
+ };
+
+ dnsclient = pkgs.fetchgit {
+ url = "https://github.com/ba0f3/dnsclient.nim";
+ rev = "23214235d4784d24aceed99bbfe153379ea557c8";
+ sha256 = "03mf3lw5c0m5nq9ppa49nylrl8ibkv2zzlc0wyhqg7w09kz6hks6";
+ fetchSubmodules = true;
+ };
+
+ unicodedb = pkgs.fetchgit {
+ url = "https://github.com/nitely/nim-unicodedb";
+ rev = "66f2458710dc641dd4640368f9483c8a0ec70561";
+ sha256 = "092z3glgdb7rmwajm7dmqzvralkm7ixighixk8ycf8sf17zm72ck";
+ fetchSubmodules = true;
+ };
+
+ regex = pkgs.fetchgit {
+ url = "https://github.com/nitely/nim-regex";
+ rev = "4593305ed1e49731fc75af1dc572dd2559aad19c";
+ sha256 = "1b666qws5sva3n5allin0ycvnqlzdjd7xzprpdvv632ccqddzcl9";
+ fetchSubmodules = true;
+ };
+
+ nimcrypto = pkgs.fetchgit {
+ url = "https://github.com/cheatfate/nimcrypto";
+ rev = "721fb99ee099b632eb86dfad1f0d96ee87583774";
+ sha256 = "178vzb3q8wzjq295ik2pd25rrqf32w381ck76hm5x2d8qnzfmkkc";
+ fetchSubmodules = true;
+ };
+
+ websock = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-websock";
+ rev = "c105d98e6522e0e2cbe3dfa11b07a273e9fd0e7b";
+ sha256 = "1zrigw27nwcmg7mw9867581ipcp3ckrqq3cwl2snabcjhkp5dm2c";
+ fetchSubmodules = true;
+ };
+
+ json_rpc = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-json-rpc.git";
+ rev = "43bbf499143eb45046c83ac9794c9e3280a2b8e7";
+ sha256 = "1c1msxg958jm2ggvs875b6wh6n829d3lh7x4ch6dcxawda16qf95";
+ fetchSubmodules = true;
+ };
+
+ lsquic = pkgs.fetchgit {
+ url = "https://github.com/vacp2p/nim-lsquic";
+ rev = "4fb03ee7bfb39aecb3316889fdcb60bec3d0936f";
+ sha256 = "0qdhcd4hyp185szc9sv3jvwdwc9zp3j0syy7glxv13k9bchfmkfg";
+ fetchSubmodules = true;
+ };
+
+ secp256k1 = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-secp256k1";
+ rev = "d8f1288b7c72f00be5fc2c5ea72bf5cae1eafb15";
+ sha256 = "1qjrmwbngb73f6r1fznvig53nyal7wj41d1cmqfksrmivk2sgrn2";
+ fetchSubmodules = true;
+ };
+
+ eth = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-eth";
+ rev = "d9135e6c3c5d6d819afdfb566aa8d958756b73a8";
+ sha256 = "15r6aszalnbk6mkyfbv5rnz5vcf1mmgj6yg332wry53xsd2ipg7r";
+ fetchSubmodules = true;
+ };
+
+ web3 = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-web3";
+ rev = "cdfe5601d2812a58e54faf53ee634452d01e5918";
+ sha256 = "1j52vcqw868qs40bh4wzfw5cvvnywp2q0dnzhfajh31xws98jc27";
+ fetchSubmodules = true;
+ };
+
+ dnsdisc = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-dnsdisc";
+ rev = "38f2e0f52c0a8f032ef4530835e519d550706d9e";
+ sha256 = "0dk787ny49n41bmzhlrvm87giwajr01gwdw9nlmphch89rdqpxxn";
+ fetchSubmodules = true;
+ };
+
+ libp2p = pkgs.fetchgit {
+ url = "https://github.com/vacp2p/nim-libp2p.git";
+ rev = "ff8d51857b4b79a68468e7bcc27b2026cca02996";
+ sha256 = "08y4s0zhqzsd780bwaixfqbi79km0mcq5g8nyw7awfvcbjqsa53l";
+ fetchSubmodules = true;
+ };
+
+ taskpools = pkgs.fetchgit {
+ url = "https://github.com/status-im/nim-taskpools";
+ rev = "9e8ccc754631ac55ac2fd495e167e74e86293edb";
+ sha256 = "1y78l33vdjxmb9dkr455pbphxa73rgdsh8m9gpkf4d9b1wm1yivy";
+ fetchSubmodules = true;
+ };
+
+ ffi = pkgs.fetchgit {
+ url = "https://github.com/logos-messaging/nim-ffi";
+ rev = "06111de155253b34e47ed2aaed1d61d08d62cc1b";
+ sha256 = "0rb0d2i519amgsp7q0bn6m5465z1vwj4rab89529pyiivh3fgh8j";
+ fetchSubmodules = true;
+ };
+
+}
diff --git a/nix/nimble.nix b/nix/nimble.nix
deleted file mode 100644
index 337ecd672..000000000
--- a/nix/nimble.nix
+++ /dev/null
@@ -1,12 +0,0 @@
-{ pkgs ? import { } }:
-
-let
- tools = pkgs.callPackage ./tools.nix {};
- sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim;
-in pkgs.fetchFromGitHub {
- owner = "nim-lang";
- repo = "nimble";
- rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".*$" sourceFile;
- # WARNING: Requires manual updates when Nim compiler version changes.
- hash = "sha256-8iutVgNzDtttZ7V+7S11KfLEuwhKA9TsgS51mlUI08k=";
-}
diff --git a/nix/shell.nix b/nix/shell.nix
index 3b83ac93d..edff468ae 100644
--- a/nix/shell.nix
+++ b/nix/shell.nix
@@ -1,5 +1,17 @@
{ pkgs }:
+let
+ nimble = pkgs.nimble.overrideAttrs (_: {
+ version = "0.22.3";
+ src = pkgs.fetchFromGitHub {
+ owner = "nim-lang";
+ repo = "nimble";
+ rev = "v0.22.3";
+ sha256 = "sha256-f7DYpRGVUeSi6basK1lfu5AxZpMFOSJ3oYsy+urYErg=";
+ };
+ });
+in
+
pkgs.mkShell {
inputsFrom = [
pkgs.androidShell
@@ -8,12 +20,12 @@ pkgs.mkShell {
pkgs.darwin.apple_sdk.frameworks.Security
];
- buildInputs = with pkgs; [
+ buildInputs = (with pkgs; [
git
cargo
rustup
rustc
cmake
- nim-unwrapped-2_2
- ];
+ nim-2_2
+ ]) ++ [ nimble ]; # nimble pinned to 0.22.3 via let binding above
}
diff --git a/scripts/install_nim.sh b/scripts/install_nim.sh
new file mode 100755
index 000000000..c8d0f439d
--- /dev/null
+++ b/scripts/install_nim.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Installs a specific Nim version.
+# Usage: install_nim.sh
+#
+# Installs to ~/.nim/nim-/ and symlinks binaries into ~/.nimble/bin/,
+# which is the idiomatic Nim location already on PATH.
+#
+# Pre-built binaries are downloaded from nim-lang.org when available.
+# Falls back to building from source otherwise (e.g. macOS on older releases).
+
+set -e
+
+NIM_VERSION="${1:-}"
+
+if [ -z "${NIM_VERSION}" ]; then
+ echo "Usage: $0 " >&2
+ exit 1
+fi
+
+# Check if the right version is already installed
+nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
+if [ "${nim_ver}" = "${NIM_VERSION}" ]; then
+ echo "Nim ${NIM_VERSION} already installed, skipping."
+ exit 0
+fi
+
+if [ -n "${nim_ver}" ]; then
+ newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1)
+ if [ "${newer}" = "${nim_ver}" ]; then
+ echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2
+ echo "WARNING: The build will proceed but may behave differently." >&2
+ exit 0
+ fi
+fi
+
+OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/')
+ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/')
+
+NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}"
+BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz"
+WORK_DIR=$(mktemp -d)
+trap 'rm -rf "${WORK_DIR}"' EXIT
+
+echo "Checking for pre-built Nim ${NIM_VERSION} (${OS}_${ARCH})..."
+HTTP_STATUS=$(curl -sI "${BINARY_URL}" | head -1 | grep -oE '[0-9]{3}' || true)
+
+if [ "${HTTP_STATUS}" = "200" ]; then
+ echo "Downloading pre-built binary from ${BINARY_URL}..."
+ curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz"
+ tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}"
+ rm -rf "${NIM_DEST}"
+ mkdir -p "${HOME}/.nim"
+ cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}"
+else
+ echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..."
+ SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz"
+ curl -fL "${SRC_URL}" -o "${WORK_DIR}/nim-src.tar.gz"
+ tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}"
+ cd "${WORK_DIR}/Nim-${NIM_VERSION}"
+ sh build_all.sh
+ rm -rf "${NIM_DEST}"
+ mkdir -p "${HOME}/.nim"
+ cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}"
+fi
+
+mkdir -p "${HOME}/.nimble/bin"
+for bin_path in "${NIM_DEST}/bin/"*; do
+ ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")"
+done
+
+echo "Nim ${NIM_VERSION} installed to ${NIM_DEST}"
+echo "Binaries symlinked in ~/.nimble/bin — ensure it is in your PATH."
diff --git a/tests/api/test_node_conf.nim b/tests/api/test_node_conf.nim
index b19739393..e171c5207 100644
--- a/tests/api/test_node_conf.nim
+++ b/tests/api/test_node_conf.nim
@@ -376,6 +376,7 @@ suite "WakuConfBuilder - store retention policies":
test "Multiple retention policies":
## Given
var b = WakuConfBuilder.init()
+ b.withP2pTcpPort(0'u16)
b.storeServiceConf.withEnabled(true)
b.storeServiceConf.withDbUrl("sqlite://test.db")
b.storeServiceConf.withRetentionPolicies(
@@ -420,6 +421,7 @@ suite "WakuConfBuilder - store retention policies":
test "Store disabled - no retention policy applied":
## Given
var b = WakuConfBuilder.init()
+ b.withP2pTcpPort(0'u16)
# storeServiceConf not enabled
## When
diff --git a/tests/factory/test_node_factory.nim b/tests/factory/test_node_factory.nim
index f30e079b5..1fe242532 100644
--- a/tests/factory/test_node_factory.nim
+++ b/tests/factory/test_node_factory.nim
@@ -1,13 +1,23 @@
{.used.}
-import testutils/unittests, chronos, libp2p/protocols/connectivity/relay/relay
+import
+ std/[net, options, sequtils, strutils],
+ testutils/unittests,
+ chronos,
+ chronos/transports/[stream, datagram, common],
+ metrics/chronos_httpserver,
+ libp2p/[crypto/crypto, multiaddress, protocols/connectivity/relay/relay],
+ eth/p2p/discoveryv5/enr
import
- ../testlib/wakunode,
- waku/waku_node,
- waku/factory/node_factory,
- waku/factory/conf_builder/conf_builder,
- waku/factory/conf_builder/web_socket_conf_builder
+ tests/testlib/[wakunode, wakucore],
+ waku/[waku_node, waku_enr, net/auto_port, discovery/waku_discv5, node/waku_metrics],
+ waku/factory/[
+ node_factory,
+ internal_config,
+ conf_builder/conf_builder,
+ conf_builder/web_socket_conf_builder,
+ ]
suite "Node Factory":
asynctest "Set up a node based on default configurations":
@@ -38,6 +48,45 @@ suite "Node Factory":
not node.wakuStore.isNil()
not node.wakuArchive.isNil()
+ test "ENR configuration trims multiaddrs until record fits":
+ var conf = defaultTestWakuConf()
+ let bindIp = conf.endpointConf.p2pListenAddress
+ let bindPort = Port(30303)
+
+ let oversizedMultiaddrs = (0 .. 11).mapIt(
+ MultiAddress
+ .init(
+ "/dns4/very-long-logical-hostname-" & $it &
+ ".example.logos.dev.status.im/tcp/30303/wss"
+ )
+ .get()
+ )
+
+ let netConfig = NetConfig.init(
+ clusterId = conf.clusterId,
+ bindIp = bindIp,
+ bindPort = bindPort,
+ extMultiAddrs = oversizedMultiaddrs,
+ extMultiAddrsOnly = true,
+ wakuFlags = some(conf.wakuFlags),
+ ).valueOr:
+ raiseAssert error
+
+ let record = enrConfiguration(conf, netConfig).valueOr:
+ raiseAssert error
+
+ let typedRecord = record.toTyped()
+ require typedRecord.isOk()
+
+ let multiaddrsOpt = typedRecord.value.multiaddrs
+ require multiaddrsOpt.isSome()
+
+ let retainedMultiaddrs = multiaddrsOpt.get()
+ check:
+ retainedMultiaddrs.len < oversizedMultiaddrs.len
+ retainedMultiaddrs.len > 0
+ retainedMultiaddrs == oversizedMultiaddrs[0 ..< retainedMultiaddrs.len]
+
asynctest "Set up a node with Filter enabled":
var confBuilder = defaultTestWakuConfBuilder()
confBuilder.filterServiceConf.withEnabled(true)
@@ -68,5 +117,90 @@ asynctest "Start a node based on default test configuration":
check:
node.started == true
+ # Default conf has p2pTcpPort=0, so the OS must have assigned a real port.
+ var hasNonZeroTcp = false
+ for a in node.switch.peerInfo.listenAddrs:
+ let s = $a
+ if ("/tcp/" in s) and not ("/tcp/0" in s):
+ hasNonZeroTcp = true
+ check hasNonZeroTcp
+
## Cleanup
await node.stop()
+
+suite "Auto-port retry":
+ asynctest "metrics binds on free TCP port, fails on taken":
+ let takenPort = Port(55100)
+ let freePort = Port(55101)
+ let taken = createStreamServer(initTAddress("127.0.0.1", takenPort))
+ defer:
+ taken.stop()
+ await taken.closeWait()
+
+ proc buildMetricsConf(port: Port): MetricsServerConf =
+ var b = MetricsServerConfBuilder.init()
+ b.withEnabled(true)
+ b.withHttpPort(port)
+ b.build().value.get()
+
+ let failRes = await startMetricsServerAndLogging(buildMetricsConf(takenPort), 0'u16)
+ check failRes.isErr()
+
+ let okRes = await startMetricsServerAndLogging(buildMetricsConf(freePort), 0'u16)
+ check okRes.isOk()
+ if okRes.isOk():
+ await okRes.get().server.close()
+
+ asynctest "discv5 binds on free UDP port, fails on taken":
+ let takenPort = Port(55200)
+ let freePort = Port(55201)
+
+ proc dummyCb(
+ transp: DatagramTransport, raddr: TransportAddress
+ ): Future[void] {.async: (raises: []).} =
+ discard
+
+ let takenUdp =
+ newDatagramTransport(dummyCb, local = initTAddress("0.0.0.0", takenPort))
+ defer:
+ await takenUdp.closeWait()
+
+ let nodeKey = generateSecp256k1Key()
+ let node = newTestWakuNode(nodeKey, parseIpAddress("0.0.0.0"), Port(0))
+ await node.start()
+ defer:
+ await node.stop()
+
+ proc buildDiscv5Conf(port: Port): Discv5Conf =
+ var b = Discv5ConfBuilder.init()
+ b.withEnabled(true)
+ b.withUdpPort(port)
+ b.build().value.get()
+
+ let failRes = await setupAndStartDiscv5(
+ node.enr,
+ node.peerManager,
+ node.topicSubscriptionQueue,
+ buildDiscv5Conf(takenPort),
+ @[],
+ node.rng,
+ nodeKey,
+ parseIpAddress("0.0.0.0"),
+ 0'u16,
+ )
+ check failRes.isErr()
+
+ let okRes = await setupAndStartDiscv5(
+ node.enr,
+ node.peerManager,
+ node.topicSubscriptionQueue,
+ buildDiscv5Conf(freePort),
+ @[],
+ node.rng,
+ nodeKey,
+ parseIpAddress("0.0.0.0"),
+ 0'u16,
+ )
+ check okRes.isOk()
+ if okRes.isOk():
+ await okRes.get().stop()
diff --git a/tests/factory/test_waku_conf.nim b/tests/factory/test_waku_conf.nim
index 9d05f7fb5..885e22867 100644
--- a/tests/factory/test_waku_conf.nim
+++ b/tests/factory/test_waku_conf.nim
@@ -4,7 +4,7 @@ import
libp2p/crypto/[crypto, secp],
libp2p/multiaddress,
nimcrypto/utils,
- std/[options, random, sequtils],
+ std/[net, options, random, sequtils],
results,
testutils/unittests
import
@@ -213,6 +213,54 @@ suite "Waku Conf - build with cluster conf":
check rlnRelayConf.epochSizeSec == networkConf.rlnEpochSizeSec
check rlnRelayConf.userMessageLimit == userMessageLimit.uint
+ test "num-shards-in-network > 0 overrides preset":
+ ## Setup
+ let networkConf = NetworkConf.LogosDevConf()
+ var builder = WakuConfBuilder.init()
+
+ # Sanity check
+ check networkConf.shardingConf.kind == AutoSharding
+ check networkConf.shardingConf.numShardsInCluster > 1
+
+ ## Given: preset says >1 shards but user explicitly sets 1
+ builder.withNetworkConf(networkConf)
+ builder.withNumShardsInCluster(1)
+ builder.withShardingConf(AutoSharding)
+
+ ## When
+ let conf = builder.build().expect("build should succeed")
+
+ ## Then: user value wins, not preset
+ conf.validate().expect("conf should validate")
+ check conf.shardingConf.kind == AutoSharding
+ check conf.shardingConf.numShardsInCluster == 1
+
+ test "num-shards-in-network == 0 does not override preset":
+ ## Passing an AutoSharding preset and trying to override with
+ ## --num-shards-in-network=0 (which is StaticSharding) doesn't work.
+ ## Note that --num-shards-in-network=0 and omitting the switch are
+ ## internally the same. Promoting the config to an Option[uint16] is
+ ## probably not worth it since overriding an AutoSharding preset with
+ ## StaticSharding shouldn't make any sense (that is, no use case).
+
+ ## Given: emulate --preset=logos.dev --num-shards-in-network=0
+ let networkConf = NetworkConf.LogosDevConf()
+ var builder = WakuConfBuilder.init()
+ builder.withNetworkConf(networkConf)
+ # Note: builder.withNumShardsInCluster() is not called when the
+ # value that comes from the CLI path is 0 (which means it was
+ # either set to 0 or was left unset).
+ builder.withShardingConf(StaticSharding)
+
+ ## When
+ let conf = builder.build().expect("build should succeed")
+
+ ## Then: preset wins and StaticSharding user intent is lost
+ conf.validate().expect("conf should validate")
+ check conf.shardingConf.kind == networkConf.shardingConf.kind
+ check conf.shardingConf.numShardsInCluster ==
+ networkConf.shardingConf.numShardsInCluster
+
suite "Waku Conf - node key":
test "Node key is generated":
## Setup
diff --git a/tests/test_waku_enr.nim b/tests/test_waku_enr.nim
index 2ffff5e57..10183adf5 100644
--- a/tests/test_waku_enr.nim
+++ b/tests/test_waku_enr.nim
@@ -271,6 +271,44 @@ suite "Waku ENR - Multiaddresses":
multiaddrs.contains(expectedAddr1)
multiaddrs.contains(addr2)
+ test "encode and decode record with multiaddrs field deduplicates duplicate entries":
+ ## Given
+ let
+ enrSeqNum = 1u64
+ enrPrivKey = generatesecp256k1key()
+
+ let
+ addr1 = MultiAddress
+ .init(
+ "/ip4/127.0.0.1/tcp/80/ws/p2p/16Uiu2HAm4v86W3bmT1BiH6oSPzcsSr31iDQpSN5Qa882BCjjwgrD"
+ )
+ .get()
+ addr1NoPeerId = MultiAddress.init("/ip4/127.0.0.1/tcp/80/ws").get()
+ addr2 = MultiAddress.init("/ip4/127.0.0.1/tcp/443/wss").get()
+
+ ## When
+ var builder = EnrBuilder.init(enrPrivKey, seqNum = enrSeqNum)
+ builder.withMultiaddrs(@[addr1, addr1NoPeerId, addr2, addr2])
+
+ let recordRes = builder.build()
+
+ require recordRes.isOk()
+ let record = recordRes.tryGet()
+
+ let typedRecord = record.toTyped()
+ require typedRecord.isOk()
+
+ let multiaddrsOpt = typedRecord.value.multiaddrs
+
+ ## Then
+ check multiaddrsOpt.isSome()
+
+ let multiaddrs = multiaddrsOpt.get()
+ check:
+ multiaddrs.len == 2
+ multiaddrs.contains(addr1NoPeerId)
+ multiaddrs.contains(addr2)
+
suite "Waku ENR - Relay static sharding":
test "new relay shards object with single invalid shard id":
## Given
diff --git a/tests/test_waku_netconfig.nim b/tests/test_waku_netconfig.nim
index 5f9ff4b46..0aff64121 100644
--- a/tests/test_waku_netconfig.nim
+++ b/tests/test_waku_netconfig.nim
@@ -5,7 +5,7 @@ import chronos, confutils/toml/std/net, libp2p/multiaddress, testutils/unittests
import ./testlib/wakunode, waku/waku_enr/capabilities
include
- waku/node/net_config,
+ waku/net/net_config,
waku/factory/conf_builder/web_socket_conf_builder,
waku/factory/conf_builder/conf_builder
@@ -152,6 +152,31 @@ suite "Waku NetConfig":
netConfig.announcedAddresses.len == 1 # DNS address
netConfig.announcedAddresses[0] == dns4TcpEndPoint(dns4DomainName, extPort)
+ asyncTest "AnnouncedAddresses and enrMultiaddrs deduplicate dns4DomainName and extMultiAddrs overlap":
+ let
+ conf = defaultTestWakuConf()
+ dns4DomainName = "example.com"
+ extPort = Port(1234)
+ dns4Address = dns4TcpEndPoint(dns4DomainName, extPort)
+
+ let netConfigRes = NetConfig.init(
+ bindIp = conf.endpointConf.p2pListenAddress,
+ bindPort = conf.endpointConf.p2pTcpPort,
+ dns4DomainName = some(dns4DomainName),
+ extPort = some(extPort),
+ extMultiAddrs = @[dns4Address],
+ )
+
+ assert netConfigRes.isOk(), $netConfigRes.error
+
+ let netConfig = netConfigRes.get()
+
+ check:
+ netConfig.announcedAddresses.len == 1
+ netConfig.announcedAddresses[0] == dns4Address
+ netConfig.enrMultiAddrs.len == 1
+ netConfig.enrMultiAddrs[0] == dns4Address
+
asyncTest "AnnouncedAddresses includes WebSocket addresses when enabled":
var confBuilder = defaultTestWakuConfBuilder()
diff --git a/tests/testlib/wakunode.nim b/tests/testlib/wakunode.nim
index e904604ab..77c017d96 100644
--- a/tests/testlib/wakunode.nim
+++ b/tests/testlib/wakunode.nim
@@ -27,7 +27,6 @@ import
# TODO: migrate to usage of a test cluster conf
proc defaultTestWakuConfBuilder*(): WakuConfBuilder =
var builder = WakuConfBuilder.init()
- builder.withP2pTcpPort(Port(0))
builder.withP2pListenAddress(parseIpAddress("0.0.0.0"))
builder.restServerConf.withListenAddress(parseIpAddress("127.0.0.1"))
builder.withDnsAddrsNameServers(
diff --git a/tests/tools/test_confutils_envvar.nim b/tests/tools/test_confutils_envvar.nim
index ed559ad0b..76d9ddd31 100644
--- a/tests/tools/test_confutils_envvar.nim
+++ b/tests/tools/test_confutils_envvar.nim
@@ -19,7 +19,7 @@ type TestConf = object
Option[InputFile]
listenAddress* {.
- defaultValue: parseIpAddress("127.0.0.1"),
+ defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127u8, 0, 0, 1]),
desc: "Listening address",
name: "listen-address"
.}: IpAddress
@@ -62,9 +62,15 @@ suite "nim-confutils - envvar":
## Then
check confLoadRes.isOk()
+ let parsedIpAddress =
+ try:
+ parseIpAddress(listenAddress)
+ except ValueError:
+ IpAddress(family: IpAddressFamily.IPv4, address_v4: [0u8, 0, 0, 0])
+
let conf = confLoadRes.get()
check:
- conf.listenAddress == parseIpAddress(listenAddress)
+ conf.listenAddress == parsedIpAddress
conf.tcpPort == Port(8080)
conf.configFile.isSome()
diff --git a/tests/waku_core/test_peers.nim b/tests/waku_core/test_peers.nim
index 59ae2e2f3..0ba3e5b04 100644
--- a/tests/waku_core/test_peers.nim
+++ b/tests/waku_core/test_peers.nim
@@ -1,5 +1,6 @@
{.used.}
+import std/options
import
results,
testutils/unittests,
diff --git a/tests/waku_discv5/test_waku_discv5.nim b/tests/waku_discv5/test_waku_discv5.nim
index 20a0c6965..936c01826 100644
--- a/tests/waku_discv5/test_waku_discv5.nim
+++ b/tests/waku_discv5/test_waku_discv5.nim
@@ -506,7 +506,8 @@ suite "Waku Discovery v5":
waku.conf.nodeKey,
waku.conf.endpointConf.p2pListenAddress,
waku.conf.portsShift,
- )
+ ).valueOr:
+ raiseAssert "failed setup discv5 in test: " & $error
check:
waku.node.peerManager.switch.peerStore.peers().anyIt(
@@ -537,7 +538,8 @@ suite "Waku Discovery v5":
waku.conf.nodeKey,
waku.conf.endpointConf.p2pListenAddress,
waku.conf.portsShift,
- )
+ ).valueOr:
+ raiseAssert "failed setup discv5 in test: " & $error
check:
not waku.node.peerManager.switch.peerStore.peers().anyIt(
diff --git a/tests/waku_store_sync/sync_utils.nim b/tests/waku_store_sync/sync_utils.nim
index fe62e02a1..888b10a83 100644
--- a/tests/waku_store_sync/sync_utils.nim
+++ b/tests/waku_store_sync/sync_utils.nim
@@ -45,7 +45,7 @@ proc newTestWakuRecon*(
let proto = res.get()
- proto.start()
+ await proto.start()
switch.mount(proto)
return proto
@@ -55,7 +55,7 @@ proc newTestWakuTransfer*(
idsTx: AsyncQueue[(SyncID, PubsubTopic, ContentTopic)],
wantsRx: AsyncQueue[PeerId],
needsRx: AsyncQueue[(PeerId, WakuMessageHash)],
-): SyncTransfer =
+): Future[SyncTransfer] {.async.} =
let peerManager = PeerManager.new(switch)
let proto = SyncTransfer.new(
@@ -66,7 +66,7 @@ proc newTestWakuTransfer*(
remoteNeedsRx = needsRx,
)
- proto.start()
+ await proto.start()
switch.mount(proto)
return proto
diff --git a/tests/waku_store_sync/test_protocol.nim b/tests/waku_store_sync/test_protocol.nim
index d051eebd7..3ffa7ad4a 100644
--- a/tests/waku_store_sync/test_protocol.nim
+++ b/tests/waku_store_sync/test_protocol.nim
@@ -63,8 +63,8 @@ suite "Waku Sync: reconciliation":
clientPeerInfo = clientSwitch.peerInfo.toRemotePeerInfo()
asyncTeardown:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
await allFutures(serverSwitch.stop(), clientSwitch.stop())
@@ -372,7 +372,7 @@ suite "Waku Sync: reconciliation":
const
msgCount = 400_000
diffCount = 100_000
- tol = 1000
+ tol = 10_000
var diffMsgHashes: HashSet[WakuMessageHash]
var missingIdx: HashSet[int]
@@ -561,8 +561,8 @@ suite "Waku Sync: reconciliation":
)
defer:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
let res = await client.storeSynchronization(some(serverPeerInfo))
assert res.isOk(), $res.error
@@ -610,8 +610,8 @@ suite "Waku Sync: reconciliation":
)
defer:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
let res = await client.storeSynchronization(some(serverPeerInfo))
assert res.isOk(), $res.error
@@ -657,8 +657,8 @@ suite "Waku Sync: reconciliation":
)
defer:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
let res = await client.storeSynchronization(some(serverPeerInfo))
assert res.isOk(), $res.error
@@ -701,8 +701,8 @@ suite "Waku Sync: reconciliation":
)
defer:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
let res = await client.storeSynchronization(some(serverPeerInfo))
assert res.isOk(), $res.error
@@ -736,8 +736,8 @@ suite "Waku Sync: reconciliation":
)
defer:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
let res = await client.storeSynchronization(some(serverPeerInfo))
assert res.isOk(), $res.error
@@ -773,8 +773,8 @@ suite "Waku Sync: reconciliation":
)
defer:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
let res = await client.storeSynchronization(some(serverPeerInfo))
assert res.isOk(), $res.error
@@ -848,8 +848,8 @@ suite "Waku Sync: transfer":
remoteNeedsRx = clientRemoteNeeds,
)
- server.start()
- client.start()
+ await server.start()
+ await client.start()
serverSwitch.mount(server)
clientSwitch.mount(client)
@@ -861,8 +861,8 @@ suite "Waku Sync: transfer":
clientPeermanager.addPeer(serverPeerInfo)
asyncTeardown:
- server.stop()
- client.stop()
+ await server.stop()
+ await client.stop()
await allFutures(serverSwitch.stop(), clientSwitch.stop())
diff --git a/tests/wakunode2/test_app.nim b/tests/wakunode2/test_app.nim
index 6ec6043fe..7621ab1e7 100644
--- a/tests/wakunode2/test_app.nim
+++ b/tests/wakunode2/test_app.nim
@@ -1,14 +1,13 @@
{.used.}
import
+ std/json,
testutils/unittests,
chronicles,
chronos,
- libp2p/crypto/crypto,
- libp2p/crypto/secp,
- libp2p/multiaddress,
- libp2p/switch
-import ../testlib/wakucore, ../testlib/wakunode
+ libp2p/[crypto/crypto, crypto/secp, multiaddress, switch],
+ tests/testlib/[wakucore, wakunode],
+ waku/factory/conf_builder/conf_builder
include waku/factory/waku, waku/common/enr/typed_record
@@ -99,3 +98,46 @@ suite "Wakunode2 - Waku initialization":
## Cleanup
(waitFor waku.stop()).isOkOr:
raiseAssert error
+
+ test "explicit port=0 triggers auto-bind across all services":
+ var builder = defaultTestWakuConfBuilder()
+ builder.withP2pTcpPort(Port(0))
+ builder.discv5Conf.withEnabled(true)
+ builder.discv5Conf.withUdpPort(Port(0))
+ builder.restServerConf.withEnabled(true)
+ builder.restServerConf.withRelayCacheCapacity(50'u32)
+ builder.restServerConf.withPort(Port(0))
+ builder.metricsServerConf.withEnabled(true)
+ builder.metricsServerConf.withHttpPort(Port(0))
+ builder.webSocketConf.withEnabled(true)
+ builder.webSocketConf.withWebSocketPort(Port(0))
+
+ let conf = builder.build().valueOr:
+ raiseAssert error
+
+ check:
+ conf.endpointConf.p2pTcpPort == Port(0)
+ conf.discv5Conf.get().udpPort == Port(0)
+ conf.restServerConf.get().port == Port(0)
+ conf.metricsServerConf.get().httpPort == Port(0)
+ conf.webSocketConf.get().port == Port(0)
+
+ var waku = (waitFor Waku.new(conf)).valueOr:
+ raiseAssert error
+ defer:
+ (waitFor waku.stop()).isOkOr:
+ raiseAssert error
+
+ (waitFor startWaku(addr waku)).isOkOr:
+ raiseAssert error
+
+ let portsJson = waku.stateInfo.getNodeInfoItem(NodeInfoId.MyBoundPorts)
+ let parsed = parseJson(portsJson)
+
+ check:
+ parsed.kind == JObject
+ parsed["tcp"].getInt() != 0
+ parsed["webSocket"].getInt() != 0
+ parsed["rest"].getInt() != 0
+ parsed["discv5Udp"].getInt() != 0
+ parsed["metrics"].getInt() != 0
diff --git a/tests/wakunode_rest/test_rest_debug.nim b/tests/wakunode_rest/test_rest_debug.nim
index 4bd2e8c02..1171f5878 100644
--- a/tests/wakunode_rest/test_rest_debug.nim
+++ b/tests/wakunode_rest/test_rest_debug.nim
@@ -1,6 +1,7 @@
{.used.}
import
+ std/options,
testutils/unittests,
presto,
presto/client as presto_client,
diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim
index 7d531159b..d63b5880c 100644
--- a/tools/confutils/cli_args.nim
+++ b/tools/confutils/cli_args.nim
@@ -251,7 +251,10 @@ type WakuNodeConf* = object
dnsAddrsNameServers* {.
desc:
"DNS name server IPs to query for DNS multiaddrs resolution. Argument may be repeated.",
- defaultValue: @[parseIpAddress("1.1.1.1"), parseIpAddress("1.0.0.1")],
+ defaultValue: @[
+ IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1]),
+ IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 0, 0, 1]),
+ ],
name: "dns-addrs-name-server"
.}: seq[IpAddress]
@@ -480,7 +483,8 @@ with the drawback of consuming some more bandwidth.""",
restAddress* {.
desc: "Listening address of the REST HTTP server.",
- defaultValue: parseIpAddress("127.0.0.1"),
+ defaultValue:
+ IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "rest-address"
.}: IpAddress
@@ -520,7 +524,8 @@ with the drawback of consuming some more bandwidth.""",
metricsServerAddress* {.
desc: "Listening address of the metrics server.",
- defaultValue: parseIpAddress("127.0.0.1"),
+ defaultValue:
+ IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "metrics-server-address"
.}: IpAddress
@@ -774,7 +779,7 @@ proc completeCmdArg*(T: type IpAddress, val: string): seq[string] =
proc defaultListenAddress*(): IpAddress =
# TODO: Should probably listen on both ipv4 and ipv6 by default.
- (static parseIpAddress("0.0.0.0"))
+ (static IpAddress(family: IpAddressFamily.IPv4, address_v4: [0'u8, 0, 0, 0]))
proc defaultColocationLimit*(): int =
return DefaultColocationLimit
diff --git a/tools/gen-nix-deps.sh b/tools/gen-nix-deps.sh
new file mode 100755
index 000000000..9bb43e638
--- /dev/null
+++ b/tools/gen-nix-deps.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Generates nix/deps.nix from nimble.lock using nix-prefetch-git.
+# Usage: ./tools/gen-nix-deps.sh [nimble.lock] [nix/deps.nix]
+set -euo pipefail
+
+usage() {
+ cat <
+
+Example:
+ $0 nimble.lock nix/deps.nix
+EOF
+}
+
+if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
+ usage; exit 0
+fi
+
+if [[ $# -ne 2 ]]; then
+ usage; exit 1
+fi
+
+LOCKFILE="$1"
+OUTFILE="$2"
+
+command -v jq >/dev/null || { echo "error: jq required"; exit 1; }
+command -v nix-prefetch-git >/dev/null || { echo "error: nix-prefetch-git required"; exit 1; }
+
+if [[ ! -f "$LOCKFILE" ]]; then
+ echo "[!] $LOCKFILE not found"
+ echo "[*] Generating $LOCKFILE via 'nimble lock'"
+ nimble lock
+fi
+
+echo "[*] Generating $OUTFILE from $LOCKFILE"
+mkdir -p "$(dirname "$OUTFILE")"
+
+cat > "$OUTFILE" <<'EOF'
+# AUTOGENERATED from nimble.lock — do not edit manually.
+# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix
+{ pkgs }:
+
+{
+EOF
+
+jq -c '
+ .packages
+ | to_entries[]
+ | select(.value.downloadMethod == "git")
+ | select(.key != "nim" and .key != "nimble")
+' "$LOCKFILE" | while read -r entry; do
+ name=$(jq -r '.key' <<<"$entry")
+ url=$(jq -r '.value.url' <<<"$entry")
+ rev=$(jq -r '.value.vcsRevision' <<<"$entry")
+
+ echo " [*] Prefetching $name @ $rev"
+
+ sha=$(nix-prefetch-git \
+ --url "$url" \
+ --rev "$rev" \
+ --fetch-submodules \
+ | jq -r '.sha256')
+
+ cat >> "$OUTFILE" <> "$OUTFILE" <<'EOF'
+}
+EOF
+
+echo "[✓] Wrote $OUTFILE"
diff --git a/vendor/db_connector b/vendor/db_connector
deleted file mode 160000
index 74aef399e..000000000
--- a/vendor/db_connector
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 74aef399e5c232f95c9fc5c987cebac846f09d62
diff --git a/vendor/dnsclient.nim b/vendor/dnsclient.nim
deleted file mode 160000
index 23214235d..000000000
--- a/vendor/dnsclient.nim
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 23214235d4784d24aceed99bbfe153379ea557c8
diff --git a/vendor/nim-bearssl b/vendor/nim-bearssl
deleted file mode 160000
index 11e798b62..000000000
--- a/vendor/nim-bearssl
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 11e798b62b8e6beabe958e048e9e24c7e0f9ee63
diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles
deleted file mode 160000
index 54f5b7260..000000000
--- a/vendor/nim-chronicles
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 54f5b726025e8c7385e3a6529d3aa27454c6e6ff
diff --git a/vendor/nim-chronos b/vendor/nim-chronos
deleted file mode 160000
index 85af4db76..000000000
--- a/vendor/nim-chronos
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 85af4db764ecd3573c4704139560df3943216cf1
diff --git a/vendor/nim-confutils b/vendor/nim-confutils
deleted file mode 160000
index e214b3992..000000000
--- a/vendor/nim-confutils
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit e214b3992a31acece6a9aada7d0a1ad37c928f3b
diff --git a/vendor/nim-dnsdisc b/vendor/nim-dnsdisc
deleted file mode 160000
index 203abd2b3..000000000
--- a/vendor/nim-dnsdisc
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 203abd2b3e758e0ea3ae325769b20a7e1bcd1010
diff --git a/vendor/nim-eth b/vendor/nim-eth
deleted file mode 160000
index d9135e6c3..000000000
--- a/vendor/nim-eth
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d9135e6c3c5d6d819afdfb566aa8d958756b73a8
diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams
deleted file mode 160000
index ce27581a3..000000000
--- a/vendor/nim-faststreams
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit ce27581a3e881f782f482cb66dc5b07a02bd615e
diff --git a/vendor/nim-ffi b/vendor/nim-ffi
deleted file mode 160000
index 06111de15..000000000
--- a/vendor/nim-ffi
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 06111de155253b34e47ed2aaed1d61d08d62cc1b
diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils
deleted file mode 160000
index f142cb2e8..000000000
--- a/vendor/nim-http-utils
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f142cb2e8bd812dd002a6493b6082827bb248592
diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc
deleted file mode 160000
index 9665c2650..000000000
--- a/vendor/nim-json-rpc
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 9665c265035f49f5ff94bbffdeadde68e19d6221
diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization
deleted file mode 160000
index c343b0e24..000000000
--- a/vendor/nim-json-serialization
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c343b0e243d9e17e2c40f3a8a24340f7c4a71d44
diff --git a/vendor/nim-jwt b/vendor/nim-jwt
deleted file mode 160000
index 18f8378de..000000000
--- a/vendor/nim-jwt
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 18f8378de52b241f321c1f9ea905456e89b95c6f
diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace
deleted file mode 160000
index d8bd4ce5c..000000000
--- a/vendor/nim-libbacktrace
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d8bd4ce5c46bb6d2f984f6b3f3d7380897d95ecb
diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
deleted file mode 160000
index ff8d51857..000000000
--- a/vendor/nim-libp2p
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit ff8d51857b4b79a68468e7bcc27b2026cca02996
diff --git a/vendor/nim-lsquic b/vendor/nim-lsquic
deleted file mode 160000
index 4fb03ee7b..000000000
--- a/vendor/nim-lsquic
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 4fb03ee7bfb39aecb3316889fdcb60bec3d0936f
diff --git a/vendor/nim-metrics b/vendor/nim-metrics
deleted file mode 160000
index a1296caf3..000000000
--- a/vendor/nim-metrics
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a1296caf3ebb5f30f51a5feae7749a30df2824c2
diff --git a/vendor/nim-minilru b/vendor/nim-minilru
deleted file mode 160000
index 0c4b2bce9..000000000
--- a/vendor/nim-minilru
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0c4b2bce959591f0a862e9b541ba43c6d0cf3476
diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal
deleted file mode 160000
index 860e18c37..000000000
--- a/vendor/nim-nat-traversal
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 860e18c37667b5dd005b94c63264560c35d88004
diff --git a/vendor/nim-presto b/vendor/nim-presto
deleted file mode 160000
index d66043dd7..000000000
--- a/vendor/nim-presto
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d66043dd7ede146442e6c39720c76a20bde5225f
diff --git a/vendor/nim-regex b/vendor/nim-regex
deleted file mode 160000
index 4593305ed..000000000
--- a/vendor/nim-regex
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 4593305ed1e49731fc75af1dc572dd2559aad19c
diff --git a/vendor/nim-results b/vendor/nim-results
deleted file mode 160000
index df8113dda..000000000
--- a/vendor/nim-results
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27
diff --git a/vendor/nim-secp256k1 b/vendor/nim-secp256k1
deleted file mode 160000
index 9dd3df621..000000000
--- a/vendor/nim-secp256k1
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 9dd3df62124aae79d564da636bb22627c53c7676
diff --git a/vendor/nim-serialization b/vendor/nim-serialization
deleted file mode 160000
index b0f2fa329..000000000
--- a/vendor/nim-serialization
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b0f2fa32960ea532a184394b0f27be37bd80248b
diff --git a/vendor/nim-sqlite3-abi b/vendor/nim-sqlite3-abi
deleted file mode 160000
index 89ba51f55..000000000
--- a/vendor/nim-sqlite3-abi
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 89ba51f557414d3a3e17ab3df8270e1bdaa3ca2a
diff --git a/vendor/nim-stew b/vendor/nim-stew
deleted file mode 160000
index b66168735..000000000
--- a/vendor/nim-stew
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b66168735d6f3841c5239c3169d3fe5fe98b1257
diff --git a/vendor/nim-stint b/vendor/nim-stint
deleted file mode 160000
index 470b78925..000000000
--- a/vendor/nim-stint
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 470b7892561b5179ab20bd389a69217d6213fe58
diff --git a/vendor/nim-taskpools b/vendor/nim-taskpools
deleted file mode 160000
index 9e8ccc754..000000000
--- a/vendor/nim-taskpools
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 9e8ccc754631ac55ac2fd495e167e74e86293edb
diff --git a/vendor/nim-testutils b/vendor/nim-testutils
deleted file mode 160000
index e4d37dc16..000000000
--- a/vendor/nim-testutils
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit e4d37dc1652d5c63afb89907efb5a5e812261797
diff --git a/vendor/nim-toml-serialization b/vendor/nim-toml-serialization
deleted file mode 160000
index b5b387e6f..000000000
--- a/vendor/nim-toml-serialization
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b5b387e6fb2a7cc75d54a269b07cc6218361bd46
diff --git a/vendor/nim-unicodedb b/vendor/nim-unicodedb
deleted file mode 160000
index 66f245871..000000000
--- a/vendor/nim-unicodedb
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 66f2458710dc641dd4640368f9483c8a0ec70561
diff --git a/vendor/nim-unittest2 b/vendor/nim-unittest2
deleted file mode 160000
index 26f2ef3ae..000000000
--- a/vendor/nim-unittest2
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 26f2ef3ae0ec72a2a75bfe557e02e88f6a31c189
diff --git a/vendor/nim-web3 b/vendor/nim-web3
deleted file mode 160000
index 81ee8ce47..000000000
--- a/vendor/nim-web3
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 81ee8ce479d86acb73be7c4f365328e238d9b4a3
diff --git a/vendor/nim-websock b/vendor/nim-websock
deleted file mode 160000
index 35ae76f15..000000000
--- a/vendor/nim-websock
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 35ae76f1559e835c80f9c1a3943bf995d3dd9eb5
diff --git a/vendor/nim-zlib b/vendor/nim-zlib
deleted file mode 160000
index daa8723fd..000000000
--- a/vendor/nim-zlib
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit daa8723fd32299d4ca621c837430c29a5a11e19a
diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system
deleted file mode 160000
index e6c2c9da3..000000000
--- a/vendor/nimbus-build-system
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit e6c2c9da39c2d368d9cf420ac22692e99715d22c
diff --git a/vendor/nimcrypto b/vendor/nimcrypto
deleted file mode 160000
index 721fb99ee..000000000
--- a/vendor/nimcrypto
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 721fb99ee099b632eb86dfad1f0d96ee87583774
diff --git a/vendor/nph b/vendor/nph
deleted file mode 160000
index 2cacf6cc2..000000000
--- a/vendor/nph
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 2cacf6cc28116e4046e0b67a13545af5c4e756bd
diff --git a/waku.nimble b/waku.nimble
index cbbe6aa97..f944aaae1 100644
--- a/waku.nimble
+++ b/waku.nimble
@@ -4,81 +4,161 @@ import os
mode = ScriptMode.Verbose
### Package
-version = "0.36.0"
+version = "0.37.4"
author = "Status Research & Development GmbH"
description = "Waku, Private P2P Messaging for Resource-Restricted Devices"
license = "MIT or Apache License 2.0"
-#bin = @["build/waku"]
+
+const RequiredNimVersion = "2.2.4"
+ ## This is the nim compiler version that we are working on. Other versions may behave differently.
+const RequiredNimbleVersion = "0.22.3"
+ ## Enforced nimble version to ensure a reproducible flow
### Dependencies
requires "nim >= 2.2.4",
+ "chronos >= 4.2.0",
+ "taskpools",
+ # Logging & Configuration
"chronicles",
"confutils",
- "chronos",
- "dnsdisc",
- "eth",
- "json_rpc",
- "libbacktrace",
- "nimcrypto",
+ # Serialization
"serialization",
+ "json_serialization",
+ "toml_serialization",
+ "faststreams",
+ # Networking & P2P
+ "https://github.com/vacp2p/nim-libp2p.git#ff8d51857b4b79a68468e7bcc27b2026cca02996",
+ "eth",
+ "nat_traversal",
+ "dnsdisc",
+ "dnsclient",
+ "httputils >= 0.4.1",
+ "websock >= 0.3.0",
+ # Cryptography
+ "nimcrypto == 0.6.4", # 0.6.4 used in libp2p. Version 0.7.3 makes test to crash on Ubuntu.
+ "secp256k1",
+ "bearssl",
+ # RPC & APIs
+ "https://github.com/status-im/nim-json-rpc.git#43bbf499143eb45046c83ac9794c9e3280a2b8e7",
+ "presto",
+ "web3",
+ # Database
+ "db_connector",
+ "sqlite3_abi",
+ # Utilities
"stew",
"stint",
"metrics",
- "libp2p >= 1.15.0",
- "web3",
- "presto",
"regex",
+ "unicodedb",
"results",
- "db_connector",
"minilru",
- "lsquic",
- "jwt",
- "ffi"
+ "zlib",
+ # Debug & Testing
+ "testutils",
+ "unittest2"
+
+# Packages not on nimble (use git URLs)
+requires "https://github.com/logos-messaging/nim-ffi"
+
+requires "https://github.com/logos-messaging/nim-sds.git#2e9a7683f0e180bf112135fae3a3803eed8490d4"
+
+requires "https://github.com/vacp2p/nim-lsquic"
+requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2"
+
+proc getMyCPU(): string =
+ ## Need to set cpu more explicit manner to avoid arch issues between dependencies
+ when defined(macosx) and defined(arm64):
+ return " --cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" "
+ elif defined(macosx) and defined(amd64):
+ return " --cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" "
+ elif defined(arm64):
+ return " --cpu:arm64 "
+ elif defined(amd64):
+ return " --cpu:amd64 "
+
+proc getNimParams(): string =
+ return " " & getEnv("NIM_PARAMS") & " "
### Helper functions
-proc buildModule(filePath, params = "", lang = "c"): bool =
+proc buildModule(filePath, params = ""): bool =
if not dirExists "build":
mkDir "build"
- # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
- var extra_params = params
- for i in 2 ..< paramCount() - 1:
- extra_params &= " " & paramStr(i)
if not fileExists(filePath):
echo "File to build not found: " & filePath
return false
- exec "nim " & lang & " --out:build/" & filepath & ".bin --mm:refc " & extra_params &
+ exec "nim c --out:build/" & filepath & ".bin --mm:refc " & getMyCPU() & getNimParams() & " " & params &
" " & filePath
# exec will raise exception if anything goes wrong
return true
-proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
+proc buildBinary(name: string, srcDir = "./", params = "") =
if not dirExists "build":
mkDir "build"
- # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
- var extra_params = params
- for i in 2 ..< paramCount():
- extra_params &= " " & paramStr(i)
- exec "nim " & lang & " --out:build/" & name & " --mm:refc " & extra_params & " " &
+ exec "nim c --out:build/" & name & " --mm:refc " & getMyCPU() & getNimParams() & " " & params & " " &
srcDir & name & ".nim"
proc buildLibrary(lib_name: string, srcDir = "./", params = "", `type` = "static", srcFile = "libwaku.nim", mainPrefix = "libwaku") =
if not dirExists "build":
mkDir "build"
- # allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
- var extra_params = params
- for i in 2 ..< (paramCount() - 1):
- extra_params &= " " & paramStr(i)
+
if `type` == "static":
exec "nim c" & " --out:build/" & lib_name &
" --threads:on --app:staticlib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:on -d:discv5_protocol_id=d5waku " &
- extra_params & " " & srcDir & srcFile
+ getMyCPU() & getNimParams() & srcDir & "/" & srcFile
else:
exec "nim c" & " --out:build/" & lib_name &
" --threads:on --app:lib --opt:speed --noMain --mm:refc --header -d:metrics --nimMainPrefix:" & mainPrefix & " --skipParentCfg:off -d:discv5_protocol_id=d5waku " &
- extra_params & " " & srcDir & srcFile
+ getMyCPU() & getNimParams() & " " & srcDir & "/" & srcFile
+
+proc buildLibDynamicWindows(libName: string, folderName: string) =
+ buildLibrary libName & ".dll", folderName,
+ """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
+ "dynamic", libName & ".nim", libname
+
+proc buildLibDynamicLinux(libName: string, folderName: string) =
+ buildLibrary libName & ".so", folderName,
+ """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
+ "dynamic", libName & ".nim", libname
+
+proc buildLibDynamicMac(libName: string, folderName: string) =
+ let sdkPath = staticExec("xcrun --show-sdk-path").strip()
+ when defined(arm64):
+ let archFlags = "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
+ elif defined(amd64):
+ let archFlags = "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
+ else:
+ {.error: "Unsupported macOS architecture".}
+ buildLibrary libName & ".dylib", folderName,
+ archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
+ "dynamic", libName & ".nim", libname
+
+proc buildLibStaticWindows(libName: string, folderName: string) =
+ buildLibrary libName & ".lib", folderName,
+ """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
+ "static", libName & ".nim", libname
+
+proc buildLibStaticLinux(libName: string, folderName: string) =
+ buildLibrary libName & ".a", folderName,
+ """-d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE """,
+ "static", libName & ".nim", libname
+
+proc buildLibStaticMac(libName: string, folderName: string) =
+ let sdkPath = staticExec("xcrun --show-sdk-path").strip()
+ when defined(arm64):
+ let archFlags = "--cpu:arm64 --passC:\"-arch arm64\" --passL:\"-arch arm64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
+ elif defined(amd64):
+ let archFlags = "--cpu:amd64 --passC:\"-arch x86_64\" --passL:\"-arch x86_64\" --passC:\"-isysroot " & sdkPath & "\" --passL:\"-isysroot " & sdkPath & "\""
+ else:
+ {.error: "Unsupported macOS architecture".}
+ buildLibrary libName & ".a", folderName,
+ archFlags & " -d:chronicles_line_numbers --warning:Deprecated:off --warning:UnusedImport:on -d:chronicles_log_level=TRACE",
+ "static", libName & ".nim", libname
+
+### Mobile Android
proc buildMobileAndroid(srcDir = ".", params = "") =
let cpu = getEnv("CPU")
@@ -88,16 +168,206 @@ proc buildMobileAndroid(srcDir = ".", params = "") =
if not dirExists outDir:
mkDir outDir
- var extra_params = params
- for i in 2 ..< paramCount():
- extra_params &= " " & paramStr(i)
-
exec "nim c" & " --out:" & outDir &
- "/libwaku.so --threads:on --app:lib --opt:speed --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" &
- outdir & " --passL:-lrln --passL:-llog --cpu:" & cpu & " --os:android -d:androidNDK " &
- extra_params & " " & srcDir & "/libwaku.nim"
+ "/liblogosdelivery.so --threads:on --app:lib --opt:speed --noMain --mm:refc -d:chronicles_sinks=textlines[dynamic] --header -d:chronosEventEngine=epoll --passL:-L" &
+ outdir & " --passL:-lrln --passL:-llog --cpu:" & cpu & " --nimMainPrefix:liblogosdelivery --os:android -d:androidNDK " & params &
+ getNimParams() & " " & srcDir & "/liblogosdelivery.nim"
-proc test(name: string, params = "-d:chronicles_log_level=DEBUG", lang = "c") =
+task libLogosDeliveryAndroid, "Build the mobile bindings for Android":
+ let srcDir = "./library"
+ buildMobileAndroid srcDir, "-d:chronicles_log_level=ERROR"
+
+### Mobile iOS
+
+import std/sequtils
+
+proc buildMobileIOS(srcDir = ".", params = "") =
+ echo "Building iOS liblogosdelivery library"
+
+ let iosArch = getEnv("IOS_ARCH")
+ let iosSdk = getEnv("IOS_SDK")
+ let sdkPath = getEnv("IOS_SDK_PATH")
+
+ if sdkPath.len == 0:
+ quit "Error: IOS_SDK_PATH not set. Set it to the path of the iOS SDK"
+
+ # Get nimble package paths
+ let bearsslPath = gorge("nimble path bearssl").strip()
+ let secp256k1Path = gorge("nimble path secp256k1").strip()
+ let natTraversalPath = gorge("nimble path nat_traversal").strip()
+
+ # Get Nim standard library path
+ let nimPath = gorge("nim --fullhelp 2>&1 | head -1 | sed 's/.*\\[//' | sed 's/\\].*//'").strip()
+ let nimLibPath = nimPath.parentDir.parentDir / "lib"
+
+ # Use SDK name in path to differentiate device vs simulator
+ let outDir = "build/ios/" & iosSdk & "-" & iosArch
+ if not dirExists outDir:
+ mkDir outDir
+
+ var extra_params = params
+ let args = commandLineParams()
+ for arg in args:
+ extra_params &= " " & arg
+
+ let cpu = if iosArch == "arm64": "arm64" else: "amd64"
+
+ # The output static library
+ let nimcacheDir = outDir & "/nimcache"
+ let objDir = outDir & "/obj"
+ let vendorObjDir = outDir & "/vendor_obj"
+ let aFile = outDir & "/liblogosdelivery.a"
+
+ if not dirExists objDir:
+ mkDir objDir
+ if not dirExists vendorObjDir:
+ mkDir vendorObjDir
+
+ let clangBase = "clang -arch " & iosArch & " -isysroot " & sdkPath &
+ " -mios-version-min=18.0 -fembed-bitcode -fPIC -O2"
+
+ # Generate C sources from Nim (no linking)
+ exec "nim c" &
+ " --nimcache:" & nimcacheDir &
+ " --os:ios --cpu:" & cpu &
+ " --compileOnly:on" &
+ " --noMain --mm:refc" &
+ " --threads:on --opt:size --header" &
+ " -d:metrics -d:discv5_protocol_id=d5waku" &
+ " --nimMainPrefix:liblogosdelivery --skipParentCfg:on" &
+ " --cc:clang" &
+ " " & extra_params &
+ " " & srcDir & "/liblogosdelivery.nim"
+
+ # Compile vendor C libraries for iOS
+
+ # --- BearSSL ---
+ echo "Compiling BearSSL for iOS..."
+ let bearSslSrcDir = bearsslPath / "bearssl/csources/src"
+ let bearSslIncDir = bearsslPath / "bearssl/csources/inc"
+ for path in walkDirRec(bearSslSrcDir):
+ if path.endsWith(".c"):
+ let relPath = path.replace(bearSslSrcDir & "/", "").replace("/", "_")
+ let baseName = relPath.changeFileExt("o")
+ let oFile = vendorObjDir / ("bearssl_" & baseName)
+ if not fileExists(oFile):
+ exec clangBase & " -I" & bearSslIncDir & " -I" & bearSslSrcDir & " -c " & path & " -o " & oFile
+
+ # --- secp256k1 ---
+ echo "Compiling secp256k1 for iOS..."
+ let secp256k1Dir = secp256k1Path / "vendor/secp256k1"
+ let secp256k1Flags = " -I" & secp256k1Dir & "/include" &
+ " -I" & secp256k1Dir & "/src" &
+ " -I" & secp256k1Dir &
+ " -DENABLE_MODULE_RECOVERY=1" &
+ " -DENABLE_MODULE_ECDH=1" &
+ " -DECMULT_WINDOW_SIZE=15" &
+ " -DECMULT_GEN_PREC_BITS=4"
+
+ # Main secp256k1 source
+ let secp256k1Obj = vendorObjDir / "secp256k1.o"
+ if not fileExists(secp256k1Obj):
+ exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/secp256k1.c -o " & secp256k1Obj
+
+ # Precomputed tables (required for ecmult operations)
+ let secp256k1PreEcmultObj = vendorObjDir / "secp256k1_precomputed_ecmult.o"
+ if not fileExists(secp256k1PreEcmultObj):
+ exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult.c -o " & secp256k1PreEcmultObj
+
+ let secp256k1PreEcmultGenObj = vendorObjDir / "secp256k1_precomputed_ecmult_gen.o"
+ if not fileExists(secp256k1PreEcmultGenObj):
+ exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult_gen.c -o " & secp256k1PreEcmultGenObj
+
+ # --- miniupnpc ---
+ echo "Compiling miniupnpc for iOS..."
+ let miniupnpcSrcDir = natTraversalPath / "vendor/miniupnp/miniupnpc/src"
+ let miniupnpcIncDir = natTraversalPath / "vendor/miniupnp/miniupnpc/include"
+ let miniupnpcBuildDir = natTraversalPath / "vendor/miniupnp/miniupnpc/build"
+ let miniupnpcFiles = @[
+ "addr_is_reserved.c", "connecthostport.c", "igd_desc_parse.c",
+ "minisoap.c", "minissdpc.c", "miniupnpc.c", "miniwget.c",
+ "minixml.c", "portlistingparse.c", "receivedata.c", "upnpcommands.c",
+ "upnpdev.c", "upnperrors.c", "upnpreplyparse.c"
+ ]
+ for fileName in miniupnpcFiles:
+ let srcPath = miniupnpcSrcDir / fileName
+ let oFile = vendorObjDir / ("miniupnpc_" & fileName.changeFileExt("o"))
+ if fileExists(srcPath) and not fileExists(oFile):
+ exec clangBase &
+ " -I" & miniupnpcIncDir &
+ " -I" & miniupnpcSrcDir &
+ " -I" & miniupnpcBuildDir &
+ " -DMINIUPNPC_SET_SOCKET_TIMEOUT" &
+ " -D_BSD_SOURCE -D_DEFAULT_SOURCE" &
+ " -c " & srcPath & " -o " & oFile
+
+ # --- libnatpmp ---
+ echo "Compiling libnatpmp for iOS..."
+ let natpmpSrcDir = natTraversalPath / "vendor/libnatpmp-upstream"
+ # Only compile natpmp.c - getgateway.c uses net/route.h which is not available on iOS
+ let natpmpObj = vendorObjDir / "natpmp_natpmp.o"
+ if not fileExists(natpmpObj):
+ exec clangBase &
+ " -I" & natpmpSrcDir &
+ " -DENABLE_STRNATPMPERR" &
+ " -c " & natpmpSrcDir & "/natpmp.c -o " & natpmpObj
+
+ # Use iOS-specific stub for getgateway
+ let getgatewayStubSrc = "./library/ios_natpmp_stubs.c"
+ let getgatewayStubObj = vendorObjDir / "natpmp_getgateway_stub.o"
+ if fileExists(getgatewayStubSrc) and not fileExists(getgatewayStubObj):
+ exec clangBase & " -c " & getgatewayStubSrc & " -o " & getgatewayStubObj
+
+ # --- BearSSL stubs (for tools functions not in main library) ---
+ echo "Compiling BearSSL stubs for iOS..."
+ let bearSslStubsSrc = "./library/ios_bearssl_stubs.c"
+ let bearSslStubsObj = vendorObjDir / "bearssl_stubs.o"
+ if fileExists(bearSslStubsSrc) and not fileExists(bearSslStubsObj):
+ exec clangBase & " -c " & bearSslStubsSrc & " -o " & bearSslStubsObj
+
+ # Compile all Nim-generated C files to object files
+ echo "Compiling Nim-generated C files for iOS..."
+ var cFiles: seq[string] = @[]
+ for kind, path in walkDir(nimcacheDir):
+ if kind == pcFile and path.endsWith(".c"):
+ cFiles.add(path)
+
+ for cFile in cFiles:
+ let baseName = extractFilename(cFile).changeFileExt("o")
+ let oFile = objDir / baseName
+ exec clangBase &
+ " -DENABLE_STRNATPMPERR" &
+ " -I" & nimLibPath &
+ " -I" & bearsslPath & "/bearssl/csources/inc/" &
+ " -I" & bearsslPath & "/bearssl/csources/tools/" &
+ " -I" & bearsslPath & "/bearssl/abi/" &
+ " -I" & secp256k1Path & "/vendor/secp256k1/include/" &
+ " -I" & natTraversalPath & "/vendor/miniupnp/miniupnpc/include/" &
+ " -I" & natTraversalPath & "/vendor/libnatpmp-upstream/" &
+ " -I" & nimcacheDir &
+ " -c " & cFile &
+ " -o " & oFile
+
+ # Create static library from all object files
+ echo "Creating static library..."
+ var objFiles: seq[string] = @[]
+ for kind, path in walkDir(objDir):
+ if kind == pcFile and path.endsWith(".o"):
+ objFiles.add(path)
+ for kind, path in walkDir(vendorObjDir):
+ if kind == pcFile and path.endsWith(".o"):
+ objFiles.add(path)
+
+ exec "libtool -static -o " & aFile & " " & objFiles.join(" ")
+
+ echo "iOS library created: " & aFile
+
+task libWakuIOS, "Build the mobile bindings for iOS":
+ let srcDir = "./library"
+ let extraParams = "-d:chronicles_log_level=ERROR"
+ buildMobileIOS srcDir, extraParams
+
+proc test(name: string, params = "-d:chronicles_log_level=DEBUG") =
# XXX: When running `> NIM_PARAMS="-d:chronicles_log_level=INFO" make test2`
# I expect compiler flag to be overridden, however it stays with whatever is
# specified here.
@@ -106,12 +376,12 @@ proc test(name: string, params = "-d:chronicles_log_level=DEBUG", lang = "c") =
### Waku common tasks
task testcommon, "Build & run common tests":
- test "all_tests_common", "-d:chronicles_log_level=WARN -d:chronosStrictException"
+ test "all_tests_common", "-d:chronicles_log_level=DEBUG -d:chronosStrictException"
### Waku tasks
task wakunode2, "Build Waku v2 cli node":
let name = "wakunode2"
- buildBinary name, "apps/wakunode2/", " -d:chronicles_log_level='TRACE' "
+ buildBinary name, "apps/wakunode2/", " -d:chronicles_log_level=TRACE "
task benchmarks, "Some benchmarks":
let name = "benchmarks"
@@ -150,7 +420,7 @@ task chat2, "Build example Waku chat usage":
let name = "chat2"
buildBinary name,
"apps/chat2/",
- "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level='TRACE' "
+ "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE "
# -d:ssl - cause unlisted exception error in libp2p/utility...
task chat2mix, "Build example Waku chat mix usage":
@@ -161,7 +431,7 @@ task chat2mix, "Build example Waku chat mix usage":
let name = "chat2mix"
buildBinary name,
"apps/chat2mix/",
- "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level='TRACE' "
+ "-d:chronicles_sinks=textlines[file] -d:chronicles_log_level=TRACE "
# -d:ssl - cause unlisted exception error in libp2p/utility...
task chat2bridge, "Build chat2bridge":
@@ -170,32 +440,33 @@ task chat2bridge, "Build chat2bridge":
task liteprotocoltester, "Build liteprotocoltester":
let name = "liteprotocoltester"
- buildBinary name, "apps/liteprotocoltester/"
+ buildBinary name, "apps/liteprotocoltester/", "-d:chronicles_log_level=TRACE"
task lightpushwithmix, "Build lightpushwithmix":
let name = "lightpush_publisher_mix"
buildBinary name, "examples/lightpush_mix/"
-task api_example, "Build api_example":
- let name = "api_example"
- buildBinary name, "examples/api_example/"
-
-task buildone, "Build custom target":
- let filepath = paramStr(paramCount())
- discard buildModule filepath
-
task buildTest, "Test custom target":
- let filepath = paramStr(paramCount())
+ let args = commandLineParams()
+ if args.len == 0:
+ quit "Missing test file"
+
+ let filepath = args[^1]
discard buildModule(filepath)
import std/strutils
task execTest, "Run test":
- # Expects to be parameterized with test case name in quotes
- # preceded with the nim source file name and path
- # If no test case name is given still it requires empty quotes `""`
- let filepath = paramStr(paramCount() - 1)
- var testSuite = paramStr(paramCount()).strip(chars = {'\"'})
+ let args = commandLineParams()
+ if args.len == 0:
+ quit "Missing arguments"
+ # expects: ""
+ let filepath =
+ if args.len >= 2: args[^2]
+ else: args[^1]
+ var testSuite =
+ if args.len >= 1: args[^1].strip(chars = {'\"'})
+ else: ""
if testSuite != "":
testSuite = " \"" & testSuite & "\""
exec "build/" & filepath & ".bin " & testSuite
@@ -208,203 +479,42 @@ let chroniclesParams =
"""-d:chronicles_disabled_topics="eth,dnsdisc.client" """ & "--warning:Deprecated:off " &
"--warning:UnusedImport:on " & "-d:chronicles_log_level=TRACE"
-task libwakuStatic, "Build the cbindings waku node library":
- let lib_name = paramStr(paramCount())
- buildLibrary lib_name, "library/", chroniclesParams, "static"
+## Libwaku build tasks
-task libwakuDynamic, "Build the cbindings waku node library":
- let lib_name = paramStr(paramCount())
- buildLibrary lib_name, "library/", chroniclesParams, "dynamic"
+task libwakuDynamicWindows, "Generate bindings":
+ buildLibDynamicWindows("libwaku", "library")
-### Mobile Android
-task libWakuAndroid, "Build the mobile bindings for Android":
- let srcDir = "./library"
- let extraParams = "-d:chronicles_log_level=ERROR"
- buildMobileAndroid srcDir, extraParams
+task libwakuDynamicLinux, "Generate bindings":
+ buildLibDynamicLinux("libwaku", "library")
-### Mobile iOS
-import std/sequtils
+task libwakuDynamicMac, "Generate bindings":
+ buildLibDynamicMac("libwaku", "library")
-proc buildMobileIOS(srcDir = ".", params = "") =
- echo "Building iOS libwaku library"
+task libwakuStaticWindows, "Generate bindings":
+ buildLibStaticWindows("libwaku", "library")
- let iosArch = getEnv("IOS_ARCH")
- let iosSdk = getEnv("IOS_SDK")
- let sdkPath = getEnv("IOS_SDK_PATH")
+task libwakuStaticLinux, "Generate bindings":
+ buildLibStaticLinux("libwaku", "library")
- if sdkPath.len == 0:
- quit "Error: IOS_SDK_PATH not set. Set it to the path of the iOS SDK"
+task libwakuStaticMac, "Generate bindings":
+ buildLibStaticMac("libwaku", "library")
- # Use SDK name in path to differentiate device vs simulator
- let outDir = "build/ios/" & iosSdk & "-" & iosArch
- if not dirExists outDir:
- mkDir outDir
+## Liblogosdelivery build tasks
- var extra_params = params
- for i in 2 ..< paramCount():
- extra_params &= " " & paramStr(i)
+task liblogosdeliveryDynamicWindows, "Generate bindings":
+ buildLibDynamicWindows("liblogosdelivery", "liblogosdelivery")
- let cpu = if iosArch == "arm64": "arm64" else: "amd64"
+task liblogosdeliveryDynamicLinux, "Generate bindings":
+ buildLibDynamicLinux("liblogosdelivery", "liblogosdelivery")
- # The output static library
- let nimcacheDir = outDir & "/nimcache"
- let objDir = outDir & "/obj"
- let vendorObjDir = outDir & "/vendor_obj"
- let aFile = outDir & "/libwaku.a"
+task liblogosdeliveryDynamicMac, "Generate bindings":
+ buildLibDynamicMac("liblogosdelivery", "liblogosdelivery")
- if not dirExists objDir:
- mkDir objDir
- if not dirExists vendorObjDir:
- mkDir vendorObjDir
+task liblogosdeliveryStaticWindows, "Generate bindings":
+ buildLibStaticWindows("liblogosdelivery", "liblogosdelivery")
- let clangBase = "clang -arch " & iosArch & " -isysroot " & sdkPath &
- " -mios-version-min=18.0 -fembed-bitcode -fPIC -O2"
+task liblogosdeliveryStaticLinux, "Generate bindings":
+ buildLibStaticLinux("liblogosdelivery", "liblogosdelivery")
- # Generate C sources from Nim (no linking)
- exec "nim c" &
- " --nimcache:" & nimcacheDir &
- " --os:ios --cpu:" & cpu &
- " --compileOnly:on" &
- " --noMain --mm:refc" &
- " --threads:on --opt:speed --header" &
- " -d:metrics -d:discv5_protocol_id=d5waku" &
- " --nimMainPrefix:libwaku --skipParentCfg:on" &
- " --cc:clang" &
- " " & extra_params &
- " " & srcDir & "/libwaku.nim"
-
- # Compile vendor C libraries for iOS
-
- # --- BearSSL ---
- echo "Compiling BearSSL for iOS..."
- let bearSslSrcDir = "./vendor/nim-bearssl/bearssl/csources/src"
- let bearSslIncDir = "./vendor/nim-bearssl/bearssl/csources/inc"
- for path in walkDirRec(bearSslSrcDir):
- if path.endsWith(".c"):
- let relPath = path.replace(bearSslSrcDir & "/", "").replace("/", "_")
- let baseName = relPath.changeFileExt("o")
- let oFile = vendorObjDir / ("bearssl_" & baseName)
- if not fileExists(oFile):
- exec clangBase & " -I" & bearSslIncDir & " -I" & bearSslSrcDir & " -c " & path & " -o " & oFile
-
- # --- secp256k1 ---
- echo "Compiling secp256k1 for iOS..."
- let secp256k1Dir = "./vendor/nim-secp256k1/vendor/secp256k1"
- let secp256k1Flags = " -I" & secp256k1Dir & "/include" &
- " -I" & secp256k1Dir & "/src" &
- " -I" & secp256k1Dir &
- " -DENABLE_MODULE_RECOVERY=1" &
- " -DENABLE_MODULE_ECDH=1" &
- " -DECMULT_WINDOW_SIZE=15" &
- " -DECMULT_GEN_PREC_BITS=4"
-
- # Main secp256k1 source
- let secp256k1Obj = vendorObjDir / "secp256k1.o"
- if not fileExists(secp256k1Obj):
- exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/secp256k1.c -o " & secp256k1Obj
-
- # Precomputed tables (required for ecmult operations)
- let secp256k1PreEcmultObj = vendorObjDir / "secp256k1_precomputed_ecmult.o"
- if not fileExists(secp256k1PreEcmultObj):
- exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult.c -o " & secp256k1PreEcmultObj
-
- let secp256k1PreEcmultGenObj = vendorObjDir / "secp256k1_precomputed_ecmult_gen.o"
- if not fileExists(secp256k1PreEcmultGenObj):
- exec clangBase & secp256k1Flags & " -c " & secp256k1Dir & "/src/precomputed_ecmult_gen.c -o " & secp256k1PreEcmultGenObj
-
- # --- miniupnpc ---
- echo "Compiling miniupnpc for iOS..."
- let miniupnpcSrcDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/src"
- let miniupnpcIncDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/include"
- let miniupnpcBuildDir = "./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/build"
- let miniupnpcFiles = @[
- "addr_is_reserved.c", "connecthostport.c", "igd_desc_parse.c",
- "minisoap.c", "minissdpc.c", "miniupnpc.c", "miniwget.c",
- "minixml.c", "portlistingparse.c", "receivedata.c", "upnpcommands.c",
- "upnpdev.c", "upnperrors.c", "upnpreplyparse.c"
- ]
- for fileName in miniupnpcFiles:
- let srcPath = miniupnpcSrcDir / fileName
- let oFile = vendorObjDir / ("miniupnpc_" & fileName.changeFileExt("o"))
- if fileExists(srcPath) and not fileExists(oFile):
- exec clangBase &
- " -I" & miniupnpcIncDir &
- " -I" & miniupnpcSrcDir &
- " -I" & miniupnpcBuildDir &
- " -DMINIUPNPC_SET_SOCKET_TIMEOUT" &
- " -D_BSD_SOURCE -D_DEFAULT_SOURCE" &
- " -c " & srcPath & " -o " & oFile
-
- # --- libnatpmp ---
- echo "Compiling libnatpmp for iOS..."
- let natpmpSrcDir = "./vendor/nim-nat-traversal/vendor/libnatpmp-upstream"
- # Only compile natpmp.c - getgateway.c uses net/route.h which is not available on iOS
- let natpmpObj = vendorObjDir / "natpmp_natpmp.o"
- if not fileExists(natpmpObj):
- exec clangBase &
- " -I" & natpmpSrcDir &
- " -DENABLE_STRNATPMPERR" &
- " -c " & natpmpSrcDir & "/natpmp.c -o " & natpmpObj
-
- # Use iOS-specific stub for getgateway
- let getgatewayStubSrc = "./library/ios_natpmp_stubs.c"
- let getgatewayStubObj = vendorObjDir / "natpmp_getgateway_stub.o"
- if fileExists(getgatewayStubSrc) and not fileExists(getgatewayStubObj):
- exec clangBase & " -c " & getgatewayStubSrc & " -o " & getgatewayStubObj
-
- # --- BearSSL stubs (for tools functions not in main library) ---
- echo "Compiling BearSSL stubs for iOS..."
- let bearSslStubsSrc = "./library/ios_bearssl_stubs.c"
- let bearSslStubsObj = vendorObjDir / "bearssl_stubs.o"
- if fileExists(bearSslStubsSrc) and not fileExists(bearSslStubsObj):
- exec clangBase & " -c " & bearSslStubsSrc & " -o " & bearSslStubsObj
-
- # Compile all Nim-generated C files to object files
- echo "Compiling Nim-generated C files for iOS..."
- var cFiles: seq[string] = @[]
- for kind, path in walkDir(nimcacheDir):
- if kind == pcFile and path.endsWith(".c"):
- cFiles.add(path)
-
- for cFile in cFiles:
- let baseName = extractFilename(cFile).changeFileExt("o")
- let oFile = objDir / baseName
- exec clangBase &
- " -DENABLE_STRNATPMPERR" &
- " -I./vendor/nimbus-build-system/vendor/Nim/lib/" &
- " -I./vendor/nim-bearssl/bearssl/csources/inc/" &
- " -I./vendor/nim-bearssl/bearssl/csources/tools/" &
- " -I./vendor/nim-bearssl/bearssl/abi/" &
- " -I./vendor/nim-secp256k1/vendor/secp256k1/include/" &
- " -I./vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc/include/" &
- " -I./vendor/nim-nat-traversal/vendor/libnatpmp-upstream/" &
- " -I" & nimcacheDir &
- " -c " & cFile &
- " -o " & oFile
-
- # Create static library from all object files
- echo "Creating static library..."
- var objFiles: seq[string] = @[]
- for kind, path in walkDir(objDir):
- if kind == pcFile and path.endsWith(".o"):
- objFiles.add(path)
- for kind, path in walkDir(vendorObjDir):
- if kind == pcFile and path.endsWith(".o"):
- objFiles.add(path)
-
- exec "libtool -static -o " & aFile & " " & objFiles.join(" ")
-
- echo "✔ iOS library created: " & aFile
-
-task libWakuIOS, "Build the mobile bindings for iOS":
- let srcDir = "./library"
- let extraParams = "-d:chronicles_log_level=ERROR"
- buildMobileIOS srcDir, extraParams
-
-task liblogosdeliveryStatic, "Build the liblogosdelivery (Logos Messaging Delivery API) static library":
- let lib_name = paramStr(paramCount())
- buildLibrary lib_name, "liblogosdelivery/", chroniclesParams, "static", "liblogosdelivery.nim", "liblogosdelivery"
-
-task liblogosdeliveryDynamic, "Build the liblogosdelivery (Logos Messaging Delivery API) dynamic library":
- let lib_name = paramStr(paramCount())
- buildLibrary lib_name, "liblogosdelivery/", chroniclesParams, "dynamic", "liblogosdelivery.nim", "liblogosdelivery"
+task liblogosdeliveryStaticMac, "Generate bindings":
+ buildLibStaticMac("liblogosdelivery", "liblogosdelivery")
diff --git a/waku/api/api_conf.nim b/waku/api/api_conf.nim
index 30dfd1b2c..3606be596 100644
--- a/waku/api/api_conf.nim
+++ b/waku/api/api_conf.nim
@@ -13,24 +13,24 @@ import
export json_serialization, json_options
-type AutoShardingConfig* {.requiresInit.} = object
+type AutoShardingConfig* = object
numShardsInCluster*: uint16
-type RlnConfig* {.requiresInit.} = object
+type RlnConfig* = object
contractAddress*: string
chainId*: uint
epochSizeSec*: uint64
-type NetworkingConfig* {.requiresInit.} = object
+type NetworkingConfig* = object
listenIpv4*: string
p2pTcpPort*: uint16
discv5UdpPort*: uint16
-type MessageValidation* {.requiresInit.} = object
+type MessageValidation* = object
maxMessageSize*: string # Accepts formats like "150 KiB", "1500 B"
rlnConfig*: Option[RlnConfig]
-type ProtocolsConfig* {.requiresInit.} = object
+type ProtocolsConfig* = object
entryNodes: seq[string]
staticStoreNodes: seq[string]
clusterId: uint16
@@ -526,7 +526,7 @@ proc decodeNodeConfigFromJson*(
var val = NodeConfig.init() # default-initialized
try:
var stream = unsafeMemoryInput(jsonStr)
- var reader = JsonReader[DefaultFlavor].init(stream)
+ var reader = (JsonReader[DefaultFlavor].init(stream))
reader.readValue(val)
except IOError as err:
raise (ref SerializationError)(msg: err.msg)
diff --git a/waku/discovery/waku_discv5.nim b/waku/discovery/waku_discv5.nim
index 0eb329fa4..c1b253c8c 100644
--- a/waku/discovery/waku_discv5.nim
+++ b/waku/discovery/waku_discv5.nim
@@ -10,7 +10,7 @@ import
eth/keys as eth_keys,
eth/p2p/discoveryv5/node,
eth/p2p/discoveryv5/protocol
-import ../node/peer_manager/peer_manager, ../waku_core, ../waku_enr
+import waku/[net/auto_port, node/peer_manager/peer_manager, waku_core, waku_enr]
export protocol, waku_enr
@@ -409,7 +409,15 @@ proc setupDiscoveryV5*(
key: crypto.PrivateKey,
p2pListenAddress: IpAddress,
portsShift: uint16,
-): WakuDiscoveryV5 =
+): Result[WakuDiscoveryV5, string] =
+ ## Public only for testing. Callers should use `setupAndStartDiscv5`, which
+ ## additionally handles `udpPort == 0` via auto-port retry.
+ if conf.udpPort == Port(0):
+ return err(
+ "setupDiscoveryV5: udpPort must be non-zero; " &
+ "use setupAndStartDiscv5 for port=0 auto-port retry"
+ )
+
let dynamicBootstrapEnrs =
dynamicBootstrapNodes.filterIt(it.hasUdpPort()).mapIt(it.enr.get())
@@ -441,10 +449,47 @@ proc setupDiscoveryV5*(
autoupdateRecord: conf.enrAutoUpdate,
)
- WakuDiscoveryV5.new(
- rng, discv5Conf, some(myENR), some(nodePeerManager), nodeTopicSubscriptionQueue
+ return ok(
+ WakuDiscoveryV5.new(
+ rng, discv5Conf, some(myENR), some(nodePeerManager), nodeTopicSubscriptionQueue
+ )
)
+proc setupAndStartDiscv5*(
+ myENR: enr.Record,
+ nodePeerManager: PeerManager,
+ nodeTopicSubscriptionQueue: AsyncEventQueue[SubscriptionEvent],
+ conf: Discv5Conf,
+ dynamicBootstrapNodes: seq[RemotePeerInfo],
+ rng: ref HmacDrbgContext,
+ key: crypto.PrivateKey,
+ p2pListenAddress: IpAddress,
+ portsShift: uint16,
+): Future[Result[WakuDiscoveryV5, string]] {.async: (raises: []).} =
+ ## Construct and start a `WakuDiscoveryV5` instance, handling auto-port
+ ## retry when the caller asks for `udpPort == 0`.
+ proc attempt(
+ port: Port
+ ): Future[Result[WakuDiscoveryV5, string]] {.async: (raises: []).} =
+ var c = conf
+ c.udpPort = port
+ let wd = setupDiscoveryV5(
+ myENR, nodePeerManager, nodeTopicSubscriptionQueue, c, dynamicBootstrapNodes, rng,
+ key, p2pListenAddress, portsShift,
+ ).valueOr:
+ return err(error)
+ let startRes = await wd.start()
+ if startRes.isErr():
+ return err("failed to start discovery, attempt: " & startRes.error)
+ return ok(wd)
+
+ let wd = (await tryWithAutoPort[WakuDiscoveryV5](conf.udpPort, attempt)).valueOr:
+ return err("setupAndStartDiscv5: " & error)
+ return ok(wd)
+
+proc udpPort*(wd: WakuDiscoveryV5): Port =
+ wd.conf.port
+
proc updateBootstrapRecords*(
self: var WakuDiscoveryV5, newRecordsString: string
): Result[void, string] =
diff --git a/waku/factory/conf_builder/discv5_conf_builder.nim b/waku/factory/conf_builder/discv5_conf_builder.nim
index e2729021e..5dd269d23 100644
--- a/waku/factory/conf_builder/discv5_conf_builder.nim
+++ b/waku/factory/conf_builder/discv5_conf_builder.nim
@@ -4,6 +4,8 @@ import ../waku_conf
logScope:
topics = "waku conf builder discv5"
+const DefaultDiscv5UdpPort*: Port = Port(9000)
+
###########################
## Discv5 Config Builder ##
###########################
@@ -38,8 +40,8 @@ proc withTableIpLimit*(b: var Discv5ConfBuilder, tableIpLimit: uint) =
proc withUdpPort*(b: var Discv5ConfBuilder, udpPort: Port) =
b.udpPort = some(udpPort)
-proc withUdpPort*(b: var Discv5ConfBuilder, udpPort: uint) =
- b.udpPort = some(Port(udpPort.uint16))
+proc withUdpPort*(b: var Discv5ConfBuilder, udpPort: uint16) =
+ b.udpPort = some(Port(udpPort))
proc withBootstrapNodes*(b: var Discv5ConfBuilder, bootstrapNodes: seq[string]) =
# TODO: validate ENRs?
@@ -57,7 +59,7 @@ proc build*(b: Discv5ConfBuilder): Result[Option[Discv5Conf], string] =
bucketIpLimit: b.bucketIpLimit.get(2),
enrAutoUpdate: b.enrAutoUpdate.get(true),
tableIpLimit: b.tableIpLimit.get(10),
- udpPort: b.udpPort.get(9000.Port),
+ udpPort: b.udpPort.get(DefaultDiscv5UdpPort),
)
)
)
diff --git a/waku/factory/conf_builder/metrics_server_conf_builder.nim b/waku/factory/conf_builder/metrics_server_conf_builder.nim
index 0f0d18564..8b2ea4eb8 100644
--- a/waku/factory/conf_builder/metrics_server_conf_builder.nim
+++ b/waku/factory/conf_builder/metrics_server_conf_builder.nim
@@ -4,6 +4,8 @@ import ../waku_conf
logScope:
topics = "waku conf builder metrics server"
+const DefaultMetricsHttpPort*: Port = Port(8008)
+
###################################
## Metrics Server Config Builder ##
###################################
@@ -40,7 +42,7 @@ proc build*(b: MetricsServerConfBuilder): Result[Option[MetricsServerConf], stri
some(
MetricsServerConf(
httpAddress: b.httpAddress.get(static parseIpAddress("127.0.0.1")),
- httpPort: b.httpPort.get(8008.Port),
+ httpPort: b.httpPort.get(DefaultMetricsHttpPort),
logging: b.logging.get(false),
)
)
diff --git a/waku/factory/conf_builder/rest_server_conf_builder.nim b/waku/factory/conf_builder/rest_server_conf_builder.nim
index 2efd91f02..dcafbb56a 100644
--- a/waku/factory/conf_builder/rest_server_conf_builder.nim
+++ b/waku/factory/conf_builder/rest_server_conf_builder.nim
@@ -4,6 +4,8 @@ import ../waku_conf
logScope:
topics = "waku conf builder rest server"
+const DefaultRestPort*: Port = Port(8645)
+
################################
## REST Server Config Builder ##
################################
@@ -46,8 +48,6 @@ proc build*(b: RestServerConfBuilder): Result[Option[RestServerConf], string] =
if b.listenAddress.isNone():
return err("restServer.listenAddress is not specified")
- if b.port.isNone():
- return err("restServer.port is not specified")
if b.relayCacheCapacity.isNone():
return err("restServer.relayCacheCapacity is not specified")
@@ -56,7 +56,7 @@ proc build*(b: RestServerConfBuilder): Result[Option[RestServerConf], string] =
RestServerConf(
allowOrigin: b.allowOrigin,
listenAddress: b.listenAddress.get(),
- port: b.port.get(),
+ port: b.port.get(DefaultRestPort),
admin: b.admin.get(false),
relayCacheCapacity: b.relayCacheCapacity.get(),
)
diff --git a/waku/factory/conf_builder/waku_conf_builder.nim b/waku/factory/conf_builder/waku_conf_builder.nim
index 956d733d3..5954bbe58 100644
--- a/waku/factory/conf_builder/waku_conf_builder.nim
+++ b/waku/factory/conf_builder/waku_conf_builder.nim
@@ -8,11 +8,13 @@ import
results
import
- ../waku_conf,
- ../networks_config,
- ../../common/logging,
- ../../common/utils/parse_size_units,
- ../../waku_enr/capabilities,
+ waku/[
+ factory/waku_conf,
+ factory/networks_config,
+ common/logging,
+ common/utils/parse_size_units,
+ waku_enr/capabilities,
+ ],
tools/confutils/entry_nodes
import
@@ -32,7 +34,9 @@ import
logScope:
topics = "waku conf builder"
-const DefaultMaxConnections* = 150
+const
+ DefaultMaxConnections* = 150
+ DefaultP2pTcpPort*: Port = Port(60000)
type MaxMessageSizeKind* = enum
mmskNone
@@ -299,7 +303,6 @@ proc buildShardingConf(
bNumShardsInCluster: Option[uint16],
bSubscribeShards: Option[seq[uint16]],
): (ShardingConf, seq[uint16]) =
- echo "bSubscribeShards: ", bSubscribeShards
case bShardingConfKind.get(AutoSharding)
of StaticSharding:
(ShardingConf(kind: StaticSharding), bSubscribeShards.get(@[]))
@@ -374,17 +377,17 @@ proc applyNetworkConf(builder: var WakuConfBuilder) =
warn "Sharding Conf was provided alongside a network conf",
used = networkConf.shardingConf.kind, discarded = builder.shardingConf
- if builder.numShardsInCluster.isSome():
- warn "Num Shards In Cluster was provided alongside a network conf",
- used = networkConf.shardingConf.numShardsInCluster,
- discarded = builder.numShardsInCluster
-
case networkConf.shardingConf.kind
of StaticSharding:
builder.shardingConf = some(StaticSharding)
of AutoSharding:
builder.shardingConf = some(AutoSharding)
- builder.numShardsInCluster = some(networkConf.shardingConf.numShardsInCluster)
+ if builder.numShardsInCluster.isSome():
+ warn "Num Shards In Cluster overrides network conf preset",
+ used = builder.numShardsInCluster.get(),
+ ignored = networkConf.shardingConf.numShardsInCluster
+ else:
+ builder.numShardsInCluster = some(networkConf.shardingConf.numShardsInCluster)
if networkConf.discv5Discovery:
if builder.discv5Conf.enabled.isNone:
@@ -575,12 +578,7 @@ proc build*(
warn "Nat Strategy is not specified, defaulting to none"
"none"
- let p2pTcpPort =
- if builder.p2pTcpPort.isSome():
- builder.p2pTcpPort.get()
- else:
- warn "P2P Listening TCP Port is not specified, listening on 60000"
- 60000.Port
+ let p2pTcpPort = builder.p2pTcpPort.get(DefaultP2pTcpPort)
let p2pListenAddress =
if builder.p2pListenAddress.isSome():
diff --git a/waku/factory/conf_builder/web_socket_conf_builder.nim b/waku/factory/conf_builder/web_socket_conf_builder.nim
index 88edc0941..61334d958 100644
--- a/waku/factory/conf_builder/web_socket_conf_builder.nim
+++ b/waku/factory/conf_builder/web_socket_conf_builder.nim
@@ -4,6 +4,8 @@ import waku/factory/waku_conf
logScope:
topics = "waku conf builder websocket"
+const DefaultWebSocketPort*: Port = Port(8000)
+
##############################
## WebSocket Config Builder ##
##############################
@@ -41,14 +43,12 @@ proc build*(b: WebSocketConfBuilder): Result[Option[WebSocketConf], string] =
if not b.enabled.get(false):
return ok(none(WebSocketConf))
- if b.webSocketPort.isNone():
- return err("websocket.port is not specified")
-
if not b.secureEnabled.get(false):
return ok(
some(
WebSocketConf(
- port: b.websocketPort.get(), secureConf: none(WebSocketSecureConf)
+ port: b.webSocketPort.get(DefaultWebSocketPort),
+ secureConf: none(WebSocketSecureConf),
)
)
)
@@ -61,7 +61,7 @@ proc build*(b: WebSocketConfBuilder): Result[Option[WebSocketConf], string] =
return ok(
some(
WebSocketConf(
- port: b.webSocketPort.get(),
+ port: b.webSocketPort.get(DefaultWebSocketPort),
secureConf: some(
WebSocketSecureConf(keyPath: b.keyPath.get(), certPath: b.certPath.get())
),
diff --git a/waku/factory/internal_config.nim b/waku/factory/internal_config.nim
index 7aad6e615..fa36aff57 100644
--- a/waku/factory/internal_config.nim
+++ b/waku/factory/internal_config.nim
@@ -8,10 +8,10 @@ import
std/[options, sequtils, net],
results
-import ../common/utils/nat, ../node/net_config, ../waku_enr, ../waku_core, ./waku_conf
+import waku/[common/utils/nat, net/net_config, waku_enr, waku_core], ./waku_conf
-proc enrConfiguration*(
- conf: WakuConf, netConfig: NetConfig
+proc tryBuildEnrRecord(
+ conf: WakuConf, netConfig: NetConfig, multiaddrs: seq[MultiAddress]
): Result[enr.Record, string] =
var enrBuilder = EnrBuilder.init(conf.nodeKey)
@@ -22,7 +22,8 @@ proc enrConfiguration*(
if netConfig.wakuFlags.isSome():
enrBuilder.withWakuCapabilities(netConfig.wakuFlags.get())
- enrBuilder.withMultiaddrs(netConfig.enrMultiaddrs)
+ if multiaddrs.len > 0:
+ enrBuilder.withMultiaddrs(multiaddrs)
enrBuilder.withWakuRelaySharding(
RelayShards(clusterId: conf.clusterId, shardIds: conf.subscribeShards)
@@ -30,11 +31,35 @@ proc enrConfiguration*(
return err("could not initialize ENR with shards")
let record = enrBuilder.build().valueOr:
- error "failed to create enr record", error = error
return err($error)
return ok(record)
+proc enrConfiguration*(
+ conf: WakuConf, netConfig: NetConfig
+): Result[enr.Record, string] =
+ for retained in countdown(netConfig.enrMultiaddrs.len, 0):
+ let multiaddrs = netConfig.enrMultiaddrs[0 ..< retained]
+ let record = tryBuildEnrRecord(conf, netConfig, multiaddrs).valueOr:
+ if retained > 0:
+ warn "failed to create enr record, retrying with fewer multiaddrs",
+ error = error,
+ totalMultiaddrs = netConfig.enrMultiaddrs.len,
+ retainedMultiaddrs = retained - 1,
+ removedMultiaddr = multiaddrs[^1]
+ continue
+
+ error "failed to create enr record", error = error
+ return err($error)
+
+ if retained < netConfig.enrMultiaddrs.len:
+ warn "created enr record after trimming multiaddrs",
+ totalMultiaddrs = netConfig.enrMultiaddrs.len, retainedMultiaddrs = retained
+
+ return ok(record)
+
+ return err("failed to create enr record")
+
proc dnsResolve*(
domain: string, dnsAddrsNameServers: seq[IpAddress]
): Future[Result[string, string]] {.async.} =
diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim
index 45e0edee0..395841130 100644
--- a/waku/factory/waku.nim
+++ b/waku/factory/waku.nim
@@ -202,6 +202,11 @@ proc new*(
else:
nil
+ if not restServer.isNil():
+ let boundRestPort = restServer.httpServer.address.port
+ node.ports.rest = boundRestPort.uint16
+ wakuConf.restServerConf.get().port = boundRestPort
+
# Set the extMultiAddrsOnly flag so the node knows not to replace explicit addresses
node.extMultiAddrsOnly = wakuConf.endpointConf.extMultiAddrsOnly
@@ -249,7 +254,7 @@ proc getPorts(
return ok((tcpPort: tcpPort, websocketPort: websocketPort))
proc getRunningNetConfig(waku: ptr Waku): Future[Result[NetConfig, string]] {.async.} =
- var conf = waku[].conf
+ let conf = waku[].conf
let (tcpPort, websocketPort) = getPorts(waku[].node.switch.peerInfo.listenAddrs).valueOr:
return err("Could not retrieve ports: " & error)
@@ -281,6 +286,10 @@ proc updateEnr(waku: ptr Waku): Future[Result[void, string]] {.async.} =
waku[].node.enr = record
+ # If TCP/WS was configured with port 0, node.announcedAddresses was built
+ # pre-bind with a port value of 0. In any case, the resync is harmless.
+ waku[].node.announcedAddresses = netConf.announcedAddresses
+
return ok()
proc updateAddressInENR(waku: ptr Waku): Result[void, string] =
@@ -312,11 +321,8 @@ proc updateAddressInENR(waku: ptr Waku): Result[void, string] =
return ok()
proc updateWaku(waku: ptr Waku): Future[Result[void, string]] {.async.} =
- let conf = waku[].conf
- if conf.endpointConf.p2pTcpPort == Port(0) or
- (conf.websocketConf.isSome() and conf.websocketConf.get.port == Port(0)):
- (await updateEnr(waku)).isOkOr:
- return err("error calling updateEnr: " & $error)
+ (await updateEnr(waku)).isOkOr:
+ return err("error calling updateEnr: " & $error)
?updateAnnouncedAddrWithPrimaryIpAddr(waku[].node)
@@ -390,29 +396,37 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises:
(await startNode(waku.node, waku.conf, waku.dynamicBootstrapNodes)).isOkOr:
return err("error while calling startNode: " & $error)
- ## Update waku data that is set dynamically on node start
- try:
- (await updateWaku(waku)).isOkOr:
- return err("Error in updateApp: " & $error)
- except CatchableError:
- return err("Caught exception in updateApp: " & getCurrentExceptionMsg())
+ let bound = getPorts(waku.node.switch.peerInfo.listenAddrs).valueOr:
+ return err("failed to read bound ports from switch: " & $error)
+ waku[].node.ports.tcp = bound.tcpPort.get(Port(0)).uint16
+ waku[].node.ports.webSocket = bound.websocketPort.get(Port(0)).uint16
## Discv5
if conf.discv5Conf.isSome():
- waku[].wakuDiscV5 = waku_discv5.setupDiscoveryV5(
- waku.node.enr,
- waku.node.peerManager,
- waku.node.topicSubscriptionQueue,
- conf.discv5Conf.get(),
- waku.dynamicBootstrapNodes,
- waku.rng,
- conf.nodeKey,
- conf.endpointConf.p2pListenAddress,
- conf.portsShift,
- )
+ waku[].wakuDiscV5 = (
+ await waku_discv5.setupAndStartDiscv5(
+ waku.node.enr,
+ waku.node.peerManager,
+ waku.node.topicSubscriptionQueue,
+ conf.discv5Conf.get(),
+ waku.dynamicBootstrapNodes,
+ waku.rng,
+ conf.nodeKey,
+ conf.endpointConf.p2pListenAddress,
+ conf.portsShift,
+ )
+ ).valueOr:
+ return err("failed to start waku discovery v5: " & error)
- (await waku.wakuDiscV5.start()).isOkOr:
- return err("failed to start waku discovery v5: " & $error)
+ waku[].node.ports.discv5Udp = waku[].wakuDiscV5.udpPort.uint16
+ waku[].conf.discv5Conf.get().udpPort = waku[].wakuDiscV5.udpPort
+
+ ## Update waku data that is set dynamically on node start
+ try:
+ (await updateWaku(waku)).isOkOr:
+ return err("Error in startWaku: " & $error)
+ except CatchableError:
+ return err("Caught exception in startWaku: " & getCurrentExceptionMsg())
## Reliability
if not waku[].deliveryService.isNil():
@@ -482,14 +496,15 @@ proc startWaku*(waku: ptr Waku): Future[Result[void, string]] {.async: (raises:
if conf.metricsServerConf.isSome():
try:
- waku[].metricsServer = (
- await (
- waku_metrics.startMetricsServerAndLogging(
- conf.metricsServerConf.get(), conf.portsShift
- )
+ let (server, port) = (
+ await waku_metrics.startMetricsServerAndLogging(
+ conf.metricsServerConf.get(), conf.portsShift
)
).valueOr:
return err("Starting monitoring and external interfaces failed: " & error)
+ waku[].metricsServer = server
+ waku[].node.ports.metrics = port.uint16
+ waku[].conf.metricsServerConf.get().httpPort = port
except CatchableError:
return err(
"Caught exception starting monitoring and external interfaces failed: " &
diff --git a/waku/factory/waku_state_info.nim b/waku/factory/waku_state_info.nim
index 5dc72a693..397b90d6d 100644
--- a/waku/factory/waku_state_info.nim
+++ b/waku/factory/waku_state_info.nim
@@ -6,7 +6,7 @@
import std/[tables, sequtils, strutils]
import metrics, eth/p2p/discoveryv5/enr, libp2p/peerid
-import waku/waku_node
+import waku/[waku_node, net/bound_ports]
type
NodeInfoId* {.pure.} = enum
@@ -15,6 +15,7 @@ type
MyMultiaddresses
MyENR
MyPeerId
+ MyBoundPorts
WakuStateInfo* {.requiresInit.} = object
node: WakuNode
@@ -43,6 +44,8 @@ proc getNodeInfoItem*(self: WakuStateInfo, infoItemId: NodeInfoId): string =
return self.node.enr.toURI()
of NodeInfoId.MyPeerId:
return $PeerId(self.node.peerId())
+ of NodeInfoId.MyBoundPorts:
+ return $self.node.ports
else:
return "unknown info item id"
diff --git a/waku/incentivization/eligibility_manager.nim b/waku/incentivization/eligibility_manager.nim
index 29443536a..cbbf4774c 100644
--- a/waku/incentivization/eligibility_manager.nim
+++ b/waku/incentivization/eligibility_manager.nim
@@ -38,11 +38,8 @@ proc getMinedTransactionReceipt(
proc getTxAndTxReceipt(
eligibilityManager: EligibilityManager, txHash: TxHash
): Future[Result[(TransactionObject, ReceiptObject), string]] {.async.} =
- let txFuture = eligibilityManager.getTransactionByHash(txHash)
- let receiptFuture = eligibilityManager.getMinedTransactionReceipt(txHash)
- await allFutures(txFuture, receiptFuture)
- let tx = txFuture.read()
- let txReceipt = receiptFuture.read().valueOr:
+ let tx = await eligibilityManager.getTransactionByHash(txHash)
+ let txReceipt = (await eligibilityManager.getMinedTransactionReceipt(txHash)).valueOr:
return err("Cannot get tx receipt: " & error)
return ok((tx, txReceipt))
diff --git a/waku/net/auto_port.nim b/waku/net/auto_port.nim
new file mode 100644
index 000000000..38176d27d
--- /dev/null
+++ b/waku/net/auto_port.nim
@@ -0,0 +1,48 @@
+{.push raises: [].}
+
+import std/[net, random]
+import chronos, results
+
+const
+ AutoPortRetryCount* = 20
+ AutoPortMin = 50000'u16
+ AutoPortMax = 59000'u16
+ AutoPortAttemptTimeout = chronos.seconds(30)
+
+proc getAutoPort*(): uint16 =
+ var rng = initRand()
+ uint16(rng.rand(AutoPortMin.int .. AutoPortMax.int))
+
+proc tryWithAutoPort*[T](
+ startingPort: Port,
+ attempt: proc(p: Port): Future[Result[T, string]] {.async: (raises: []).},
+): Future[Result[T, string]] {.async: (raises: []).} =
+ ## If `startingPort == Port(0)`, call `attempt` up to `AutoPortRetryCount`
+ ## times with random ports. Otherwise call it once with `startingPort`.
+ ## Returns the first ok or the last err.
+ let autoMode = startingPort == Port(0)
+ let attempts = if autoMode: AutoPortRetryCount else: 1
+ var lastErr = ""
+ for i in 1 .. attempts:
+ let port =
+ if autoMode:
+ Port(getAutoPort())
+ else:
+ startingPort
+ let fut = attempt(port)
+ let res =
+ try:
+ if await fut.withTimeout(AutoPortAttemptTimeout):
+ await fut
+ else:
+ fut.cancelSoon()
+ Result[T, string].err("bind attempt timed out")
+ except CancelledError:
+ fut.cancelSoon()
+ Result[T, string].err("bind attempt cancelled")
+ if res.isOk():
+ return ok(res.get())
+ lastErr = res.error
+ if autoMode:
+ return err("auto-port exhausted; last error: " & lastErr)
+ return err("port bind failed: " & lastErr)
diff --git a/waku/net/bound_ports.nim b/waku/net/bound_ports.nim
new file mode 100644
index 000000000..f8f561940
--- /dev/null
+++ b/waku/net/bound_ports.nim
@@ -0,0 +1,20 @@
+{.push raises: [].}
+
+import std/json
+
+type BoundPorts* {.requiresInit.} = object
+ ## Set by the factory once each service has bound to a port.
+ ## A value of 0 means the service was not enabled or did not bind.
+ tcp*: uint16
+ webSocket*: uint16
+ rest*: uint16
+ discv5Udp*: uint16
+ metrics*: uint16
+
+proc init*(T: type BoundPorts): BoundPorts =
+ return BoundPorts(
+ tcp: 0'u16, webSocket: 0'u16, rest: 0'u16, discv5Udp: 0'u16, metrics: 0'u16
+ )
+
+proc `$`*(p: BoundPorts): string =
+ return $(%*p)
diff --git a/waku/node/net_config.nim b/waku/net/net_config.nim
similarity index 95%
rename from waku/node/net_config.nim
rename to waku/net/net_config.nim
index 4802694c4..fc4b42fe6 100644
--- a/waku/node/net_config.nim
+++ b/waku/net/net_config.nim
@@ -156,12 +156,16 @@ proc init*(
if extMultiAddrs.len > 0:
announcedAddresses.add(extMultiAddrs)
+ announcedAddresses = announcedAddresses.deduplicate()
+
let
# enrMultiaddrs are just addresses which cannot be represented in ENR, as described in
# https://rfc.vac.dev/spec/31/#many-connection-types
- enrMultiaddrs = announcedAddresses.filterIt(
- it.hasProtocol("dns4") or it.hasProtocol("dns6") or it.hasProtocol("ws") or
- it.hasProtocol("wss")
+ enrMultiaddrs = deduplicate(
+ announcedAddresses.filterIt(
+ it.hasProtocol("dns4") or it.hasProtocol("dns6") or it.hasProtocol("ws") or
+ it.hasProtocol("wss")
+ )
)
ok(
diff --git a/waku/node/health_monitor/event_loop_monitor.nim b/waku/node/health_monitor/event_loop_monitor.nim
new file mode 100644
index 000000000..bd1a33e4e
--- /dev/null
+++ b/waku/node/health_monitor/event_loop_monitor.nim
@@ -0,0 +1,83 @@
+{.push raises: [].}
+
+import std/math
+import chronos, chronicles, metrics
+
+logScope:
+ topics = "waku event_loop_monitor"
+
+declarePublicGauge event_loop_load,
+ "chronos event loop load EWMA by window (1.0 = sustained lag at MaxAcceptedLag)",
+ labels = ["window"]
+
+declarePublicCounter event_loop_accumulated_lag_secs,
+ "chronos event loop total accumulated lag in seconds since node start"
+
+type OnLagChange* = proc(lagTooHigh: bool) {.gcsafe, raises: [].}
+
+proc eventLoopMonitorLoop*(onLagChange: OnLagChange = nil) {.async.} =
+ ## Monitors chronos event loop responsiveness by measuring how much each
+ ## iteration oversleeps its `CheckInterval`.
+ ##
+ ## The lag is normalised against `MaxAcceptedLag` and tracked as an EWMA
+ ## over 1, 5, and 15-minute windows (Unix load-average decay model),
+ ## exposed via the `event_loop_load` gauge (labelled by window: 1m/5m/15m):
+ ##
+ ## load < 1.0 → within budget
+ ## load = 1.0 → sustained lag at MaxAcceptedLag (fully loaded)
+ ## load > 1.0 → over budget; e.g. 2.0 means twice the accepted lag
+ ##
+ ## `onLagChange` is called when instantaneous lag crosses `MaxAcceptedLag`.
+
+ const CheckInterval = 5.seconds
+ const MaxAcceptedLag = 50.milliseconds
+
+ # Decay factors: α = 1 − e^(−CheckInterval_secs / window_secs)
+ # Mirrors the Unix load-average convention so each EWMA has a half-life equal
+ # to its named window.
+ const alpha1m = 1.0 - exp(-5.0 / 60.0) # ≈ 0.0821
+ const alpha5m = 1.0 - exp(-5.0 / 300.0) # ≈ 0.0165
+ const alpha15m = 1.0 - exp(-5.0 / 900.0) # ≈ 0.0055
+
+ var ewma1m = 0.0
+ var ewma5m = 0.0
+ var ewma15m = 0.0
+
+ var now = Moment.now()
+ var lagWasHigh = false
+
+ while true:
+ let lastWakeup = now
+ await sleepAsync(CheckInterval)
+ now = Moment.now()
+
+ let actualElapsed = now - lastWakeup
+ let lag = max(ZeroDuration, actualElapsed - CheckInterval)
+ const maxAcceptedLagSecs = MaxAcceptedLag.nanoseconds.float64 / 1_000_000_000.0
+
+ let lagSecs = lag.nanoseconds.float64 / 1_000_000_000.0
+ let load = lagSecs / maxAcceptedLagSecs
+
+ event_loop_accumulated_lag_secs.inc(lagSecs)
+
+ ewma1m = alpha1m * load + (1.0 - alpha1m) * ewma1m
+ ewma5m = alpha5m * load + (1.0 - alpha5m) * ewma5m
+ ewma15m = alpha15m * load + (1.0 - alpha15m) * ewma15m
+
+ event_loop_load.set(round(ewma1m, 4), labelValues = ["1m"])
+ event_loop_load.set(round(ewma5m, 4), labelValues = ["5m"])
+ event_loop_load.set(round(ewma15m, 4), labelValues = ["15m"])
+
+ let lagIsHigh = lag > MaxAcceptedLag
+
+ if lag > CheckInterval:
+ warn "chronos event loop severely lagging, many tasks may be accumulating",
+ expected_secs = CheckInterval.seconds,
+ lag_secs = round(lagSecs, 4),
+ load_1m = round(ewma1m, 4),
+ load_5m = round(ewma5m, 4),
+ load_15m = round(ewma15m, 4)
+
+ if not onLagChange.isNil() and lagIsHigh != lagWasHigh:
+ lagWasHigh = lagIsHigh
+ onLagChange(lagIsHigh)
diff --git a/waku/node/health_monitor/health_status.nim b/waku/node/health_monitor/health_status.nim
index 4dd2bdd9a..91663a507 100644
--- a/waku/node/health_monitor/health_status.nim
+++ b/waku/node/health_monitor/health_status.nim
@@ -7,6 +7,7 @@ type HealthStatus* {.pure.} = enum
NOT_READY
NOT_MOUNTED
SHUTTING_DOWN
+ EVENT_LOOP_LAGGING
proc init*(t: typedesc[HealthStatus], strRep: string): Result[HealthStatus, string] =
try:
diff --git a/waku/node/health_monitor/node_health_monitor.nim b/waku/node/health_monitor/node_health_monitor.nim
index 066e7776a..c92dc1aaf 100644
--- a/waku/node/health_monitor/node_health_monitor.nim
+++ b/waku/node/health_monitor/node_health_monitor.nim
@@ -21,6 +21,7 @@ import
node/health_monitor/health_report,
node/health_monitor/connection_status,
node/health_monitor/protocol_health,
+ node/health_monitor/event_loop_monitor,
requests/health_requests,
]
@@ -36,6 +37,7 @@ type NodeHealthMonitor* = ref object
onlineMonitor*: OnlineMonitor
keepAliveFut: Future[void]
healthLoopFut: Future[void]
+ eventLoopMonitorFut: Future[void]
healthUpdateEvent: AsyncEvent
connectionStatus: ConnectionStatus
onConnectionStatusChange*: ConnectionStatusChangeHandler
@@ -48,6 +50,9 @@ type NodeHealthMonitor* = ref object
relayObserver: PubSubObserver
peerEventListener: WakuPeerEventListener
shardHealthListener: EventShardTopicHealthChangeListener
+ eventLoopLagExceeded: bool
+ ## set to true when the chronos event loop lag exceeds the severe threshold,
+ ## causing the node health to be reported as EVENT_LOOP_LAGGING until lag recovers.
func getHealth*(report: HealthReport, kind: WakuProtocol): ProtocolHealth =
for h in report.protocolsHealth:
@@ -441,7 +446,8 @@ proc getNodeHealthReport*(hm: NodeHealthMonitor): Future[HealthReport] {.async.}
hm.cachedProtocols = await hm.getAllProtocolHealthInfo()
hm.connectionStatus = hm.calculateConnectionState()
- report.nodeHealth = HealthStatus.READY
+ report.nodeHealth =
+ if hm.eventLoopLagExceeded: HealthStatus.EVENT_LOOP_LAGGING else: HealthStatus.READY
report.connectionStatus = hm.connectionStatus
report.protocolsHealth = hm.cachedProtocols
return report
@@ -461,7 +467,8 @@ proc getSyncNodeHealthReport*(hm: NodeHealthMonitor): HealthReport =
hm.cachedProtocols = hm.getSyncAllProtocolHealthInfo()
hm.connectionStatus = hm.calculateConnectionState()
- report.nodeHealth = HealthStatus.READY
+ report.nodeHealth =
+ if hm.eventLoopLagExceeded: HealthStatus.EVENT_LOOP_LAGGING else: HealthStatus.READY
report.connectionStatus = hm.connectionStatus
report.protocolsHealth = hm.cachedProtocols
return report
@@ -694,9 +701,15 @@ proc startHealthMonitor*(hm: NodeHealthMonitor): Result[void, string] =
hm.healthUpdateEvent.fire()
hm.healthLoopFut = hm.healthLoop()
+ hm.eventLoopMonitorFut = eventLoopMonitorLoop(
+ proc(lagTooHigh: bool) {.gcsafe, raises: [].} =
+ hm.eventLoopLagExceeded = lagTooHigh
+ hm.healthUpdateEvent.fire()
+ )
hm.startKeepalive().isOkOr:
return err("startHealthMonitor: failed starting keep alive: " & error)
+
return ok()
proc stopHealthMonitor*(hm: NodeHealthMonitor) {.async.} =
@@ -709,6 +722,9 @@ proc stopHealthMonitor*(hm: NodeHealthMonitor) {.async.} =
if not isNil(hm.healthLoopFut):
await hm.healthLoopFut.cancelAndWait()
+ if not isNil(hm.eventLoopMonitorFut):
+ await hm.eventLoopMonitorFut.cancelAndWait()
+
WakuPeerEvent.dropListener(hm.node.brokerCtx, hm.peerEventListener)
EventShardTopicHealthChange.dropListener(hm.node.brokerCtx, hm.shardHealthListener)
diff --git a/waku/node/kernel_api/relay.nim b/waku/node/kernel_api/relay.nim
index c5a11ff02..fe46f5bd2 100644
--- a/waku/node/kernel_api/relay.nim
+++ b/waku/node/kernel_api/relay.nim
@@ -263,7 +263,8 @@ proc mountRelay*(
node.wakuRelay.routingRecordsHandler.add(peerExchangeHandler.get())
if node.started:
- await node.startRelay()
+ await node.wakuRelay.start()
+ await node.reconnectRelayPeers()
node.switch.mount(node.wakuRelay, protocolMatcher(WakuRelayCodec))
diff --git a/waku/node/waku_metrics.nim b/waku/node/waku_metrics.nim
index 8d38624c1..af74b1532 100644
--- a/waku/node/waku_metrics.nim
+++ b/waku/node/waku_metrics.nim
@@ -2,8 +2,7 @@
import chronicles, chronos, metrics, metrics/chronos_httpserver
import
- ../waku_rln_relay/protocol_metrics as rln_metrics,
- ../utils/collector,
+ waku/[net/auto_port, waku_rln_relay/protocol_metrics as rln_metrics, utils/collector],
./peer_manager,
./waku_node
@@ -57,27 +56,36 @@ proc startMetricsLog*() =
discard setTimer(Moment.fromNow(LogInterval), logMetrics)
+type StartedMetricsServer* = tuple[server: MetricsHttpServerRef, port: Port]
+
proc startMetricsServer(
serverIp: IpAddress, serverPort: Port
-): Future[Result[MetricsHttpServerRef, string]] {.async.} =
- info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $serverPort
+): Future[Result[StartedMetricsServer, string]] {.async.} =
+ proc attempt(
+ port: Port
+ ): Future[Result[StartedMetricsServer, string]] {.async: (raises: []).} =
+ info "Starting metrics HTTP server", serverIp = $serverIp, serverPort = $port
- let server = MetricsHttpServerRef.new($serverIp, serverPort).valueOr:
- return err("metrics HTTP server start failed: " & $error)
+ let server = MetricsHttpServerRef.new($serverIp, port).valueOr:
+ return err("fail to start service metrics server, attempt:" & $error)
- try:
- await server.start()
- except CatchableError:
- return err("metrics HTTP server start failed: " & getCurrentExceptionMsg())
+ try:
+ await server.start()
+ except CatchableError:
+ return
+ err("exception while startMetricsServer, attempt: " & getCurrentExceptionMsg())
- info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $serverPort
- return ok(server)
+ info "Metrics HTTP server started", serverIp = $serverIp, serverPort = $port
+ return ok((server: server, port: port))
+
+ let started = (await tryWithAutoPort[StartedMetricsServer](serverPort, attempt)).valueOr:
+ return err("metrics HTTP server start failed: " & error)
+ return ok(started)
proc startMetricsServerAndLogging*(
conf: MetricsServerConf, portsShift: uint16
-): Future[Result[MetricsHttpServerRef, string]] {.async.} =
- var metricsServer: MetricsHttpServerRef
- metricsServer = (
+): Future[Result[StartedMetricsServer, string]] {.async.} =
+ let started = (
await (
startMetricsServer(conf.httpAddress, Port(conf.httpPort.uint16 + portsShift))
)
@@ -87,4 +95,4 @@ proc startMetricsServerAndLogging*(
if conf.logging:
startMetricsLog()
- return ok(metricsServer)
+ return ok(started)
diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim
index 506a3e592..7cd334b53 100644
--- a/waku/node/waku_node.nim
+++ b/waku/node/waku_node.nim
@@ -62,7 +62,7 @@ import
events/message_events,
],
waku/discovery/waku_kademlia,
- ./net_config,
+ waku/net/[bound_ports, net_config],
./peer_manager,
./health_monitor/health_status,
./health_monitor/topic_health
@@ -140,6 +140,7 @@ type
wakuMix*: WakuMix
kademliaDiscoveryLoop*: Future[void]
wakuKademlia*: WakuKademlia
+ ports*: BoundPorts
proc deduceRelayShard(
node: WakuNode,
@@ -224,6 +225,7 @@ proc new*(
announcedAddresses: netConfig.announcedAddresses,
topicSubscriptionQueue: queue,
rateLimitSettings: rateLimitSettings,
+ ports: BoundPorts.init(),
)
peerManager.setShardGetter(node.getShardsGetter(@[]))
@@ -369,30 +371,16 @@ proc mountStoreSync*(
return ok()
-proc startRelay*(node: WakuNode) {.async.} =
- ## Setup and start relay protocol
- info "starting relay protocol"
-
+proc reconnectRelayPeers*(node: WakuNode) {.async.} =
+ ## Reconnect to previously-seen WakuRelay peers.
if node.wakuRelay.isNil():
- error "Failed to start relay. Not mounted."
return
-
- ## Setup relay protocol
-
- # Resume previous relay connections
- if node.peerManager.switch.peerStore.hasPeers(protocolMatcher(WakuRelayCodec)):
- info "Found previous WakuRelay peers. Reconnecting."
-
- # Reconnect to previous relay peers. This will respect a backoff period, if necessary
- let backoffPeriod =
- node.wakuRelay.parameters.pruneBackoff + chronos.seconds(BackoffSlackTime)
-
- await node.peerManager.reconnectPeers(WakuRelayCodec, backoffPeriod)
-
- # Start the WakuRelay protocol
- await node.wakuRelay.start()
-
- info "relay started successfully"
+ if not node.peerManager.switch.peerStore.hasPeers(protocolMatcher(WakuRelayCodec)):
+ return
+ info "Found previous WakuRelay peers. Reconnecting."
+ let backoffPeriod =
+ node.wakuRelay.parameters.pruneBackoff + chronos.seconds(BackoffSlackTime)
+ await node.peerManager.reconnectPeers(WakuRelayCodec, backoffPeriod)
proc selectRandomPeers*(peers: seq[PeerId], numRandomPeers: int): seq[PeerId] =
var randomPeers = peers
@@ -430,7 +418,10 @@ proc mountRendezvous*(
return
if node.started:
- await node.wakuRendezvous.start()
+ try:
+ await node.wakuRendezvous.start()
+ except CancelledError as exc:
+ error "failed to start wakuRendezvous", error = exc.msg
try:
node.switch.mount(node.wakuRendezvous, protocolMatcher(WakuRendezVousCodec))
@@ -578,31 +569,12 @@ proc start*(node: WakuNode) {.async.} =
if isBindIpWithZeroPort(address):
zeroPortPresent = true
- # Perform relay-specific startup tasks TODO: this should be rethought
- if not node.wakuRelay.isNil():
- await node.startRelay()
-
- if not node.wakuMix.isNil():
- node.wakuMix.start()
-
- if not node.wakuMetadata.isNil():
- node.wakuMetadata.start()
-
if not node.wakuStoreResume.isNil():
await node.wakuStoreResume.start()
- if not node.wakuRendezvous.isNil():
- await node.wakuRendezvous.start()
-
if not node.wakuRendezvousClient.isNil():
await node.wakuRendezvousClient.start()
- if not node.wakuStoreReconciliation.isNil():
- node.wakuStoreReconciliation.start()
-
- if not node.wakuStoreTransfer.isNil():
- node.wakuStoreTransfer.start()
-
## The switch uses this mapper to update peer info addrs
## with announced addrs after start
let addressMapper = proc(
@@ -612,8 +584,12 @@ proc start*(node: WakuNode) {.async.} =
node.switch.peerInfo.addressMappers.add(addressMapper)
## The switch will update addresses after start using the addressMapper
+ ## NOTE: This will dispatch gossipsub start to the WakuRelay.start method override
await node.switch.start()
+ # After switch.start, run custom Logos Delivery relay start logic
+ await node.reconnectRelayPeers()
+
node.started = true
if not node.wakuFilterClient.isNil():
@@ -637,6 +613,7 @@ proc stop*(node: WakuNode) {.async.} =
node.stopProvidersAndListeners()
+ ## NOTE: This will dispatch gossipsub stop to the WakuRelay.stop method override
await node.switch.stop()
node.peerManager.stop()
@@ -653,12 +630,6 @@ proc stop*(node: WakuNode) {.async.} =
if not node.wakuStoreResume.isNil():
await node.wakuStoreResume.stopWait()
- if not node.wakuStoreReconciliation.isNil():
- node.wakuStoreReconciliation.stop()
-
- if not node.wakuStoreTransfer.isNil():
- node.wakuStoreTransfer.stop()
-
if not node.wakuPeerExchangeClient.isNil() and
not node.wakuPeerExchangeClient.pxLoopHandle.isNil():
await node.wakuPeerExchangeClient.pxLoopHandle.cancelAndWait()
@@ -666,9 +637,6 @@ proc stop*(node: WakuNode) {.async.} =
if not node.wakuKademlia.isNil():
await node.wakuKademlia.stop()
- if not node.wakuRendezvous.isNil():
- await node.wakuRendezvous.stopWait()
-
if not node.wakuRendezvousClient.isNil():
await node.wakuRendezvousClient.stopWait()
diff --git a/waku/rest_api/endpoint/admin/handlers.nim b/waku/rest_api/endpoint/admin/handlers.nim
index 1904d43f9..304fdabf8 100644
--- a/waku/rest_api/endpoint/admin/handlers.nim
+++ b/waku/rest_api/endpoint/admin/handlers.nim
@@ -344,7 +344,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) =
for ps in relayPeers:
totalRelayPeers += ps.peers.len
stat[$ps.shard] = ps.peers.len
- stat["Total relay peers"] = relayPeers.len
+ stat["Total relay peers"] = totalRelayPeers
stat
# stats of mesh peers
@@ -355,7 +355,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) =
for ps in meshPeers:
totalMeshPeers += ps.peers.len
stat[$ps.shard] = ps.peers.len
- stat["Total mesh peers"] = meshPeers.len
+ stat["Total mesh peers"] = totalMeshPeers
stat
var protoStats = initOrderedTable[string, int]()
diff --git a/waku/waku_enr/multiaddr.nim b/waku/waku_enr/multiaddr.nim
index c343fff51..4d6e9baa7 100644
--- a/waku/waku_enr/multiaddr.nim
+++ b/waku/waku_enr/multiaddr.nim
@@ -74,7 +74,7 @@ func stripPeerId(multiaddr: MultiAddress): MultiAddress =
return cleanAddr
func withMultiaddrs*(builder: var EnrBuilder, multiaddrs: seq[MultiAddress]) =
- let multiaddrs = multiaddrs.map(stripPeerId)
+ let multiaddrs = deduplicate(multiaddrs.map(stripPeerId))
let value = encodeMultiaddrs(multiaddrs)
builder.addFieldPair(MultiaddrEnrField, value)
diff --git a/waku/waku_filter_v2/protocol.nim b/waku/waku_filter_v2/protocol.nim
index 451bf5cb2..35620b6cd 100644
--- a/waku/waku_filter_v2/protocol.nim
+++ b/waku/waku_filter_v2/protocol.nim
@@ -244,7 +244,8 @@ proc handleMessage*(
) {.async.} =
let msgHash = computeMessageHash(pubsubTopic, message).to0xHex()
- info "handling message", pubsubTopic = pubsubTopic, msg_hash = msgHash
+ info "handling message",
+ pubsubTopic = pubsubTopic, contentTopic = message.contentTopic, msg_hash = msgHash
let handleMessageStartTime = Moment.now()
diff --git a/waku/waku_lightpush/protocol.nim b/waku/waku_lightpush/protocol.nim
index ecbff8461..8336f4dfc 100644
--- a/waku/waku_lightpush/protocol.nim
+++ b/waku/waku_lightpush/protocol.nim
@@ -68,6 +68,7 @@ proc handleRequest(
peer_id = peerId,
requestId = pushRequest.requestId,
pubsubTopic = pushRequest.pubsubTopic,
+ contentTopic = pushRequest.message.contentTopic,
msg_hash = msg_hash,
receivedTime = getNowInNanosecondTime()
diff --git a/waku/waku_lightpush_legacy/protocol.nim b/waku/waku_lightpush_legacy/protocol.nim
index 72fc963ee..f5ed60134 100644
--- a/waku/waku_lightpush_legacy/protocol.nim
+++ b/waku/waku_lightpush_legacy/protocol.nim
@@ -50,6 +50,7 @@ proc handleRequest*(
peer_id = peerId,
requestId = requestId,
pubsubTopic = pubsubTopic,
+ contentTopic = message.contentTopic,
msg_hash = msg_hash,
receivedTime = getNowInNanosecondTime()
diff --git a/waku/waku_metadata/protocol.nim b/waku/waku_metadata/protocol.nim
index 623cbb6c3..7c72a6934 100644
--- a/waku/waku_metadata/protocol.nim
+++ b/waku/waku_metadata/protocol.nim
@@ -108,9 +108,3 @@ proc new*(T: type WakuMetadata, clusterId: uint32, getShards: GetShards): T =
clusterId = wm.clusterId, shards = wm.getShards()
return wm
-
-proc start*(wm: WakuMetadata) =
- wm.started = true
-
-proc stop*(wm: WakuMetadata) =
- wm.started = false
diff --git a/waku/waku_mix/protocol.nim b/waku/waku_mix/protocol.nim
index e31929b71..ac8b69eaf 100644
--- a/waku/waku_mix/protocol.nim
+++ b/waku/waku_mix/protocol.nim
@@ -104,10 +104,4 @@ proc new*(
proc poolSize*(mix: WakuMix): int =
mix.nodePool.len
-method start*(mix: WakuMix) =
- info "starting waku mix protocol"
-
-method stop*(mix: WakuMix) {.async.} =
- discard
-
# Mix Protocol
diff --git a/waku/waku_node.nim b/waku/waku_node.nim
index e782e616b..c8b13d4ea 100644
--- a/waku/waku_node.nim
+++ b/waku/waku_node.nim
@@ -1,5 +1,5 @@
import
- ./node/net_config,
+ ./net/net_config,
./node/waku_switch as switch,
./node/waku_node as node,
./node/health_monitor as health_monitor,
diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim
index 490feae87..e7b2c99cb 100644
--- a/waku/waku_relay/protocol.nim
+++ b/waku/waku_relay/protocol.nim
@@ -223,6 +223,7 @@ proc logMessageInfo*(
msg_id = msg_id_short,
from_peer_id = remotePeerId,
topic = topic,
+ contentTopic = msg.contentTopic,
receivedTime = getNowInNanosecondTime(),
payloadSizeBytes = payloadSize
else:
@@ -232,6 +233,7 @@ proc logMessageInfo*(
msg_id = msg_id_short,
to_peer_id = remotePeerId,
topic = topic,
+ contentTopic = msg.contentTopic,
sentTime = getNowInNanosecondTime(),
payloadSizeBytes = payloadSize
@@ -515,12 +517,12 @@ proc topicsHealthLoop(w: WakuRelay) {.async.} =
# safety cooldown to protect from edge cases
await sleepAsync(100.milliseconds)
-method start*(w: WakuRelay) {.async, base.} =
+method start*(w: WakuRelay) {.async: (raises: [CancelledError]).} =
info "start"
await procCall GossipSub(w).start()
w.topicHealthLoopHandle = w.topicsHealthLoop()
-method stop*(w: WakuRelay) {.async, base.} =
+method stop*(w: WakuRelay) {.async: (raises: []).} =
info "stop"
await procCall GossipSub(w).stop()
@@ -616,7 +618,7 @@ proc subscribe*(w: WakuRelay, pubsubTopic: PubsubTopic, handler: WakuRelayHandle
# Otherwise this might lead to unintended behaviour.
if not w.topicValidator.hasKey(pubSubTopic):
let newValidator = w.generateOrderedValidator()
- procCall GossipSub(w).addValidator(pubSubTopic, w.generateOrderedValidator())
+ procCall GossipSub(w).addValidator(pubSubTopic, newValidator)
w.topicValidator[pubSubTopic] = newValidator
# set this topic parameters for scoring
@@ -680,7 +682,8 @@ proc publish*(
let data = message.encode().buffer
let msgHash = computeMessageHash(pubsubTopic, message).to0xHex()
- notice "start publish Waku message", msg_hash = msgHash, pubsubTopic = pubsubTopic
+ notice "start publish Waku message",
+ msg_hash = msgHash, pubsubTopic = pubsubTopic, contentTopic = message.contentTopic
let relayedPeerCount = await procCall GossipSub(w).publish(pubsubTopic, data)
diff --git a/waku/waku_rendezvous/protocol.nim b/waku/waku_rendezvous/protocol.nim
index 00b5f1a5c..89433f533 100644
--- a/waku/waku_rendezvous/protocol.nim
+++ b/waku/waku_rendezvous/protocol.nim
@@ -211,29 +211,22 @@ proc new*(
return ok(wrv)
-proc start*(self: WakuRendezVous) {.async: (raises: []).} =
+method start*(self: WakuRendezVous) {.async: (raises: [CancelledError]).} =
# Start the parent GenericRendezVous (starts the register deletion loop)
if self.started:
warn "waku rendezvous already started"
return
- try:
- await procCall GenericRendezVous[WakuPeerRecord](self).start()
- except CancelledError as exc:
- error "failed to start GenericRendezVous", cause = exc.msg
- return
+ await procCall GenericRendezVous[WakuPeerRecord](self).start()
# start registering forever
self.periodicRegistrationFut = self.periodicRegistration()
info "waku rendezvous discovery started"
-proc stopWait*(self: WakuRendezVous) {.async: (raises: []).} =
+method stop*(self: WakuRendezVous) {.async: (raises: []).} =
if not self.periodicRegistrationFut.isNil():
await self.periodicRegistrationFut.cancelAndWait()
# Stop the parent GenericRendezVous (stops the register deletion loop)
- await GenericRendezVous[WakuPeerRecord](self).stop()
-
- # Stop the parent GenericRendezVous (stops the register deletion loop)
- await GenericRendezVous[WakuPeerRecord](self).stop()
+ await procCall GenericRendezVous[WakuPeerRecord](self).stop()
info "waku rendezvous discovery stopped"
diff --git a/waku/waku_rln_relay/rln_relay.nim b/waku/waku_rln_relay/rln_relay.nim
index 8559dcd66..ac128b5bc 100644
--- a/waku/waku_rln_relay/rln_relay.nim
+++ b/waku/waku_rln_relay/rln_relay.nim
@@ -201,14 +201,18 @@ proc validateMessage*(
if timeDiff > rlnPeer.rlnMaxTimestampGap:
warn "invalid message: timestamp difference exceeds threshold",
- timeDiff = timeDiff, maxTimestampGap = rlnPeer.rlnMaxTimestampGap
+ timeDiff = timeDiff,
+ maxTimestampGap = rlnPeer.rlnMaxTimestampGap,
+ contentTopic = msg.contentTopic
waku_rln_invalid_messages_total.inc(labelValues = ["invalid_timestamp"])
return MessageValidationResult.Invalid
let computedEpoch = rlnPeer.calcEpoch(messageTime)
if proof.epoch != computedEpoch:
warn "invalid message: timestamp mismatches epoch",
- proofEpoch = fromEpoch(proof.epoch), computedEpoch = fromEpoch(computedEpoch)
+ proofEpoch = fromEpoch(proof.epoch),
+ computedEpoch = fromEpoch(computedEpoch),
+ contentTopic = msg.contentTopic
waku_rln_invalid_messages_total.inc(labelValues = ["timestamp_mismatch"])
return MessageValidationResult.Invalid
@@ -216,7 +220,8 @@ proc validateMessage*(
if not rootValidationRes:
warn "invalid message: provided root does not belong to acceptable window of roots",
provided = proof.merkleRoot.inHex(),
- validRoots = rlnPeer.groupManager.validRoots.mapIt(it.inHex())
+ validRoots = rlnPeer.groupManager.validRoots.mapIt(it.inHex()),
+ contentTopic = msg.contentTopic
waku_rln_invalid_messages_total.inc(labelValues = ["invalid_root"])
return MessageValidationResult.Invalid
@@ -233,12 +238,14 @@ proc validateMessage*(
proofVerificationRes.isOkOr:
waku_rln_errors_total.inc(labelValues = ["proof_verification"])
- warn "invalid message: proof verification failed", payloadLen = msg.payload.len
+ warn "invalid message: proof verification failed",
+ payloadLen = msg.payload.len, contentTopic = msg.contentTopic
return MessageValidationResult.Invalid
if not proofVerificationRes.value():
# invalid proof
- warn "invalid message: invalid proof", payloadLen = msg.payload.len
+ warn "invalid message: invalid proof",
+ payloadLen = msg.payload.len, contentTopic = msg.contentTopic
waku_rln_invalid_messages_total.inc(labelValues = ["invalid_proof"])
return MessageValidationResult.Invalid
@@ -252,11 +259,13 @@ proc validateMessage*(
if hasDup.isErr():
waku_rln_errors_total.inc(labelValues = ["duplicate_check"])
elif hasDup.value == true:
- trace "invalid message: message is spam", payloadLen = msg.payload.len
+ trace "invalid message: message is spam",
+ payloadLen = msg.payload.len, contentTopic = msg.contentTopic
waku_rln_spam_messages_total.inc()
return MessageValidationResult.Spam
- trace "message is valid", payloadLen = msg.payload.len
+ trace "message is valid",
+ payloadLen = msg.payload.len, contentTopic = msg.contentTopic
# Metric increment moved to validator to include shard label
return MessageValidationResult.Valid
diff --git a/waku/waku_store_sync/reconciliation.nim b/waku/waku_store_sync/reconciliation.nim
index 23f513322..b18251fff 100644
--- a/waku/waku_store_sync/reconciliation.nim
+++ b/waku/waku_store_sync/reconciliation.nim
@@ -145,7 +145,7 @@ proc preProcessPayload(
# convert to skip range before processing
for i in 0 ..< payload.ranges.len:
let rangeType = payload.ranges[i][1]
- if rangeType != RangeType.Skip:
+ if rangeType == RangeType.Skip:
continue
let upperBound = payload.ranges[i][0].b.time
@@ -468,7 +468,7 @@ proc idsReceiverLoop(self: SyncReconciliation) {.async.} =
self.messageIngress(id, pubsub, content)
-proc start*(self: SyncReconciliation) =
+method start*(self: SyncReconciliation) {.async: (raises: [CancelledError]).} =
if self.started:
return
@@ -484,13 +484,16 @@ proc start*(self: SyncReconciliation) =
info "Store Sync Reconciliation protocol started"
-proc stop*(self: SyncReconciliation) =
- if self.syncInterval > ZeroDuration:
- self.periodicSyncFut.cancelSoon()
+method stop*(self: SyncReconciliation) {.async: (raises: []).} =
+ defer:
+ self.started = false
if self.syncInterval > ZeroDuration:
- self.periodicPruneFut.cancelSoon()
+ await self.periodicSyncFut.cancelAndWait()
- self.idsReceiverFut.cancelSoon()
+ if self.syncInterval > ZeroDuration:
+ await self.periodicPruneFut.cancelAndWait()
+
+ await self.idsReceiverFut.cancelAndWait()
info "Store Sync Reconciliation protocol stopped"
diff --git a/waku/waku_store_sync/transfer.nim b/waku/waku_store_sync/transfer.nim
index 6a600b4e3..5d20afb18 100644
--- a/waku/waku_store_sync/transfer.nim
+++ b/waku/waku_store_sync/transfer.nim
@@ -217,7 +217,7 @@ proc new*(
return transfer
-proc start*(self: SyncTransfer) =
+method start*(self: SyncTransfer) {.async: (raises: [CancelledError]).} =
if self.started:
return
@@ -228,10 +228,11 @@ proc start*(self: SyncTransfer) =
info "Store Sync Transfer protocol started"
-proc stop*(self: SyncTransfer) =
- self.started = false
+method stop*(self: SyncTransfer) {.async: (raises: []).} =
+ defer:
+ self.started = false
- self.localWantsRxFut.cancelSoon()
- self.remoteNeedsRxFut.cancelSoon()
+ await self.localWantsRxFut.cancelAndWait()
+ await self.remoteNeedsRxFut.cancelAndWait()
info "Store Sync Transfer protocol stopped"