logos-messaging/logos-delivery
1
0
Fork 0
mirror of https://github.com/logos-messaging/logos-delivery.git synced 2026-06-04 05:00:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/api-consistency

Browse Source
This commit is contained in:
Fabiana Cecin 2026-06-01 20:32:00 -03:00
commit 03d2b5f68d
No known key found for this signature in database
GPG Key ID: BCAB8A55CB51B6C7
138 changed files with 6619 additions and 4470 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/ISSUE_TEMPLATE/prepare_release.md vendored
View File

@ -18,7 +18,7 @@ For detailed info on the release process refer to https://github.com/logos-messa
All items below are to be completed by the owner of the given release.
- [ ] Create release branch with major and minor only ( e.g. release/v0.X ) if it doesn't exist.
- [ ] Update the `version` field in `waku.nimble` to match the release version (e.g. `version = "0.X.0"`).
- [ ] Update the `version` field in `waku.nimble` to match the release version (e.g. `version = "0.X.0"`) **and merge it before assigning any tag** - the `release-assets` workflow gates artifact build/upload.
- [ ] Assign release candidate tag to the release branch HEAD (e.g. `v0.X.0-rc.0`, `v0.X.0-rc.1`, ... `v0.X.0-rc.N`).
- [ ] Generate and edit release notes in CHANGELOG.md.

9
.github/workflows/ci-daily.yml vendored
View File

@ -3,11 +3,13 @@ name: Daily logos-delivery CI
on:
schedule:
- cron: '30 6 * * *'
workflow_dispatch:
env:
NPROC: 2
MAKEFLAGS: "-j${NPROC}"
NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none"
NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none -d:disableMarchNative"
NIM_PARAMS: "-d:disableMarchNative"
jobs:
build:
@ -77,3 +79,8 @@ jobs:
}" \
"$DISCORD_WEBHOOK_URL"
# RLN end-to-end against the simulator. Defaults from tests/simulator/rln-sim.env.
rln-simulator:
uses: ./.github/workflows/ci-rln-simulator.yml
secrets: inherit

271
.github/workflows/ci-rln-simulator.yml vendored Normal file
View File

@ -0,0 +1,271 @@
name: RLN E2E — Simulator
# Validates the full RLN flow end-to-end against logos-delivery-simulator:
# keystore generation, on-chain registration, gossipsub propagation,
# per-epoch rate-limit enforcement, and epoch-boundary recovery.
#
# Why this exists: logos-dev runs with RLN disabled, so there is no
# production traffic exercising RLN. Until RLN is enabled there, this is
# the only end-to-end coverage of the RLN + zerokit path.
#
# The image is built ON the runner and tested ON the same runner, so the
# AVX-512 portability issue in container-image.yml does not apply here.
#
# No own schedule: ci-daily.yml is the single daily entry point and calls
# this via workflow_call. workflow_dispatch allows manual runs.
# Run defaults live in tests/simulator/rln-sim.env; inputs override per-run.
on:
workflow_call:
inputs:
branch:
type: string
default: ''
num_nodes:
type: string
default: ''
msg_limit:
type: string
default: ''
epoch_sec:
type: string
default: ''
workflow_dispatch:
inputs:
branch:
description: 'logos-delivery branch to build & test (blank = use rln-sim.env)'
type: string
default: ''
num_nodes:
description: 'Number of nwaku nodes (blank = use rln-sim.env)'
type: string
default: ''
msg_limit:
description: 'RLN_RELAY_MSG_LIMIT, must be >= contract min ~20 (blank = use rln-sim.env)'
type: string
default: ''
epoch_sec:
description: 'RLN_RELAY_EPOCH_SEC, large enough a burst cannot straddle an epoch (blank = use rln-sim.env)'
type: string
default: ''
env:
NPROC: 2
MAKEFLAGS: "-j2"
NIM_VERSION: '2.2.4'
NIMBLE_VERSION: '0.22.3'
jobs:
rln-e2e:
runs-on: ubuntu-22.04
timeout-minutes: 120
name: rln-e2e
steps:
# First checkout: the ref that triggered this workflow (CI branch /
# master). This is where the e2e test script and rln-sim.env live —
# the build branch may not contain them.
- name: Checkout CI ref (for the test script)
uses: actions/checkout@v4
with:
submodules: false
# Defaults come from tests/simulator/rln-sim.env (single source of truth);
# a non-blank input (dispatch or workflow_call) overrides the matching value.
- name: Resolve parameters
id: cfg
env:
IN_BRANCH: ${{ inputs.branch }}
IN_NUM_NODES: ${{ inputs.num_nodes }}
IN_MSG_LIMIT: ${{ inputs.msg_limit }}
IN_EPOCH_SEC: ${{ inputs.epoch_sec }}
run: |
set -euo pipefail
set -a; . tests/simulator/rln-sim.env; set +a
{
echo "branch=${IN_BRANCH:-$BRANCH}"
echo "num_nodes=${IN_NUM_NODES:-$NUM_NODES}"
echo "msg_limit=${IN_MSG_LIMIT:-$MSG_LIMIT}"
echo "epoch_sec=${IN_EPOCH_SEC:-$EPOCH_SEC}"
} >> "$GITHUB_OUTPUT"
- name: Stash e2e test script outside the workspace
run: |
test -f tests/simulator/rln-e2e-test.py \
|| { echo "tests/simulator/rln-e2e-test.py missing on CI ref"; exit 1; }
cp tests/simulator/rln-e2e-test.py "$RUNNER_TEMP/rln-e2e-test.py"
# Second checkout: the branch to build & test. Overwrites the workspace;
# the stashed test script in RUNNER_TEMP survives.
- name: Checkout logos-delivery (${{ steps.cfg.outputs.branch }})
uses: actions/checkout@v4
with:
ref: ${{ steps.cfg.outputs.branch }}
submodules: false
clean: true
- name: Get submodules hash
id: submodules
run: echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
- name: Cache submodules
uses: actions/cache@v3
with:
path: |
vendor/
.git/modules
key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
- name: Install Nim ${{ env.NIM_VERSION }}
uses: jiro4989/setup-nim-action@v2
with:
nim-version: ${{ env.NIM_VERSION }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Nimble ${{ env.NIMBLE_VERSION }}
run: |
cd /tmp && nimble install "nimble@${{ env.NIMBLE_VERSION }}" -y
echo "$HOME/.nimble/bin" >> $GITHUB_PATH
- name: Cache nimble deps
id: cache-nimbledeps
uses: actions/cache@v3
with:
path: |
nimbledeps/
nimble.paths
key: ${{ runner.os }}-nimbledeps-nimble${{ env.NIMBLE_VERSION }}-${{ hashFiles('nimble.lock', 'BearSSL.mk', 'Nat.mk') }}
- name: Install nimble deps
if: steps.cache-nimbledeps.outputs.cache-hit != 'true'
run: |
nimble setup --localdeps -y
make rebuild-nat-libs-nimbledeps
make rebuild-bearssl-nimbledeps
touch nimbledeps/.nimble-setup
- name: Build wakunode2
run: |
make -j${NPROC} V=1 POSTGRES=1 \
NIMFLAGS="-d:disableMarchNative -d:chronicles_colors:none" \
wakunode2
- name: Build local Docker image
run: |
docker build -t nwaku-rln-ci:test -f docker/binaries/Dockerfile.bn.amd64 .
- name: Clone logos-delivery-simulator
run: |
git clone --depth 1 https://github.com/logos-messaging/logos-delivery-simulator.git "$RUNNER_TEMP/logos-delivery-simulator"
- name: Write simulator .env
working-directory: ${{ runner.temp }}/logos-delivery-simulator
run: |
cat > .env <<EOF
LD_IMAGE=nwaku-rln-ci:test
NUM_LD_NODES=${{ steps.cfg.outputs.num_nodes }}
MSG_SIZE_KBYTES=1
TRAFFIC_DELAY_SECONDS=5
RLN_RELAY_EPOCH_SEC=${{ steps.cfg.outputs.epoch_sec }}
RLN_RELAY_MSG_LIMIT=${{ steps.cfg.outputs.msg_limit }}
MAX_MESSAGE_LIMIT=100
RPC_URL=http://foundry:8545
PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
ETH_FROM=0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266
RLN_CONTRACT_REPO_COMMIT=e75ac913e579ad872f54b2225eec35d1de3d98b0
WATCHTOWER_ENABLED=false
EOF
- name: Bring up simulator (RLN subset)
working-directory: ${{ runner.temp }}/logos-delivery-simulator
run: |
docker compose up -d foundry contract-repo-deployer nwaku-token-init bootstrap nwaku
- name: Wait for contract deployer
working-directory: ${{ runner.temp }}/logos-delivery-simulator
run: |
for _ in $(seq 1 60); do
st=$(docker inspect logos-delivery-simulator-contract-repo-deployer-1 --format='{{.State.Status}}' 2>/dev/null || echo missing)
[ "$st" = "exited" ] && break
echo "deployer status: $st"; sleep 15
done
ec=$(docker inspect logos-delivery-simulator-contract-repo-deployer-1 --format='{{.State.ExitCode}}')
echo "deployer exit code: $ec"
if [ "$ec" != "0" ]; then
docker logs logos-delivery-simulator-contract-repo-deployer-1 2>&1 | tail -50
exit 1
fi
- name: Wait for nwaku fleet to register
working-directory: ${{ runner.temp }}/logos-delivery-simulator
run: |
N=${{ steps.cfg.outputs.num_nodes }}
for _ in $(seq 1 60); do
up=$(docker ps --filter 'name=logos-delivery-simulator-nwaku-' --filter 'status=running' --format '{{.Names}}' | wc -l)
echo "nwaku running: $up/$N"
[ "$up" -ge "$N" ] && break
sleep 15
done
# nwaku-1 must reach the "registered + started" marker
timeout 300 docker logs -f logos-delivery-simulator-nwaku-1 2>&1 \
| grep -m1 -E "Segmentation fault|Illegal instruction|Failed to register on-chain|I am a nwaku node" \
| tee /tmp/nwaku1.verdict
grep -q "I am a nwaku node" /tmp/nwaku1.verdict
- name: Run RLN e2e scenarios
run: |
TEST_SCRIPT="$RUNNER_TEMP/rln-e2e-test.py"
test -f "$TEST_SCRIPT" \
|| { echo "stashed test script missing at $TEST_SCRIPT"; exit 1; }
docker run --rm \
--network logos-delivery-simulator_simulation \
-v "$TEST_SCRIPT:/test.py:ro" \
python:3.11-slim \
sh -c "pip install --quiet --disable-pip-version-check requests && \
python /test.py \
--hostname-prefix logos-delivery-simulator-nwaku- \
--num-nodes ${{ steps.cfg.outputs.num_nodes }} \
--msg-limit ${{ steps.cfg.outputs.msg_limit }} \
--epoch-sec ${{ steps.cfg.outputs.epoch_sec }} \
--health-deadline-sec 600"
- name: Collect logs on failure
if: failure()
working-directory: ${{ runner.temp }}/logos-delivery-simulator
run: |
mkdir -p "$RUNNER_TEMP/logs"
for c in $(docker ps -a --filter 'name=logos-delivery-simulator-' --format '{{.Names}}'); do
docker logs "$c" > "$RUNNER_TEMP/logs/$c.log" 2>&1 || true
done
- name: Upload logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: simulator-logs
path: ${{ runner.temp }}/logs
retention-days: 7
- name: Tear down
if: always()
working-directory: ${{ runner.temp }}/logos-delivery-simulator
run: docker compose down -v || true
- name: Notify Discord
if: always()
env:
DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
run: |
[ -z "$DISCORD_WEBHOOK_URL" ] && exit 0
STATUS="${{ job.status }}"
BRANCH="${{ steps.cfg.outputs.branch }}"
RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
if [ "$STATUS" = "success" ]; then COLOR=3066993; TITLE="✅ RLN E2E passed"; else COLOR=15158332; TITLE="❌ RLN E2E failed"; fi
curl -H "Content-Type: application/json" -X POST -d "{
\"embeds\":[{\"title\":\"$TITLE\",\"color\":$COLOR,
\"fields\":[
{\"name\":\"Branch\",\"value\":\"$BRANCH\",\"inline\":true},
{\"name\":\"Status\",\"value\":\"$STATUS\",\"inline\":true}],
\"url\":\"$RUN_URL\",
\"footer\":{\"text\":\"Daily RLN simulator E2E\"}}]}" \
"$DISCORD_WEBHOOK_URL"

23
.github/workflows/ci.yml vendored
View File

@ -13,7 +13,8 @@ concurrency:
env:
NPROC: 2
MAKEFLAGS: "-j${NPROC}"
NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none"
NIMFLAGS: "--parallelBuild:${NPROC} --colors:off -d:chronicles_colors:none -d:disableMarchNative"
NIM_PARAMS: "-d:disableMarchNative"
NIM_VERSION: '2.2.4'
NIMBLE_VERSION: '0.22.3'
@ -35,6 +36,9 @@ jobs:
- 'nimble.lock'
- 'waku.nimble'
- 'Makefile'
- 'scripts/**'
- 'flake.nix'
- 'flake.lock'
- 'library/**'
- 'liblogosdelivery/**'
v2:
@ -43,6 +47,7 @@ jobs:
- 'tools/**'
- 'tests/all_tests_v2.nim'
- 'tests/**'
- 'channels/**'
docker:
- 'docker/**'
@ -156,7 +161,7 @@ jobs:
fi
export MAKEFLAGS="-j1"
export NIMFLAGS="--colors:off -d:chronicles_colors:none"
export NIMFLAGS="--colors:off -d:chronicles_colors:none -d:disableMarchNative"
export USE_LIBBACKTRACE=0
make V=1 POSTGRES=$postgres_enabled test
@ -176,20 +181,6 @@ jobs:
secrets: inherit
js-waku-node:
needs: build-docker-image
uses: logos-messaging/logos-delivery-js/.github/workflows/test-node.yml@master
with:
nim_wakunode_image: ${{ needs.build-docker-image.outputs.image }}
test_type: node
js-waku-node-optional:
needs: build-docker-image
uses: logos-messaging/logos-delivery-js/.github/workflows/test-node.yml@master
with:
nim_wakunode_image: ${{ needs.build-docker-image.outputs.image }}
test_type: node-optional
lint:
name: "Lint"
runs-on: ubuntu-22.04

28
.github/workflows/release-assets.yml vendored
View File

@ -11,7 +11,35 @@ env:
NPROC: 2
jobs:
# Release gate: the pushed tag MUST exactly match waku.nimble's version,
# so every published artifact reports the correct getNodeInfo Version.
# CI cannot reject/remove a tag, so we gate artifact build & upload on
# this instead: a mismatched tag yields no released artifacts.
verify-version:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Assert pushed tag equals waku.nimble version
if: startsWith(github.ref, 'refs/tags/')
run: |
set -euo pipefail
NIMBLE_VERSION=$(grep -m1 '^version = ' waku.nimble | sed -E 's/version = "([^"]+)"/\1/')
# Strip leading v and any prerelease suffix (e.g. v0.38.0-rc.1 ->
# 0.38.0) so release-candidate tags build against the same
# waku.nimble version as the final tag.
TAG_VERSION="${GITHUB_REF_NAME#v}"
BASE_VERSION="${TAG_VERSION%%-*}"
echo "tag: ${GITHUB_REF_NAME} (base ${BASE_VERSION})"
echo "waku.nimble version: ${NIMBLE_VERSION}"
if [ "${BASE_VERSION}" != "${NIMBLE_VERSION}" ]; then
echo "::error::Tag ${GITHUB_REF_NAME} (base ${BASE_VERSION}) does not match"
echo "::error::waku.nimble version (${NIMBLE_VERSION}). Bump waku.nimble before tagging."
exit 1
fi
echo "OK: tag base matches waku.nimble."
build-and-upload:
needs: verify-version
strategy:
matrix:
os: [ubuntu-22.04, macos-15]

52
.github/workflows/version-check.yml vendored Normal file
View File

@ -0,0 +1,52 @@
name: version check
permissions:
contents: read
on:
pull_request:
branches: [master]
jobs:
# PR check: waku.nimble version must be >= the nearest tag reachable from
# this branch (`git describe --tags --abbrev=0`, i.e. ancestor-aware).
# Because we check out the PR HEAD (not the simulated merge ref), a branch
# that predates a release tag does not see that tag in its history, so a
# newly pushed tag does NOT break in-flight PRs. Once the branch merges/
# rebases past the tag, the bump is then enforced. This keeps waku.nimble
# fixed as early as possible, independent of whether a release is cut.
# The exact tag==nimble guarantee at release time lives in
# release-assets.yml, which gates artifact publishing on it.
nimble-not-behind-tag:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Compare waku.nimble version with nearest ancestor tag
run: |
set -euo pipefail
NIMBLE_VERSION=$(grep -m1 '^version = ' waku.nimble | sed -E 's/version = "([^"]+)"/\1/')
# Nearest tag reachable from HEAD; --abbrev=0 drops the -<n>-g<sha>
# suffix so we get the bare tag (e.g. v0.38.0). `--match 'v*'` skips
# the moving `nightly` tag (auto-updated by the daily CI to point at
# master HEAD), which would otherwise be picked as the nearest tag
# and break the version-sort comparison below.
BASE_TAG=$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || echo "")
BASE_TAG=${BASE_TAG#v}
# Compare on the base version, ignoring any -rc.N prerelease suffix.
BASE_TAG=${BASE_TAG%%-*}
echo "waku.nimble version: ${NIMBLE_VERSION}"
echo "ancestor git tag: ${BASE_TAG:-<none>}"
if [ -z "${BASE_TAG}" ]; then
echo "No ancestor release tag; skipping."
exit 0
fi
# lowest of the two by version sort must be the tag => nimble >= tag
LOWEST=$(printf '%s\n%s\n' "${NIMBLE_VERSION}" "${BASE_TAG}" | sort -V | head -1)
if [ "${LOWEST}" != "${BASE_TAG}" ] && [ "${NIMBLE_VERSION}" != "${BASE_TAG}" ]; then
echo "::error::waku.nimble version (${NIMBLE_VERSION}) is behind its"
echo "::error::ancestor git tag (v${BASE_TAG}). Bump 'version' in waku.nimble."
exit 1
fi
echo "OK: waku.nimble is not behind its ancestor tag."

5
.gitignore vendored
View File

@ -85,3 +85,8 @@ nimble.paths
nimbledeps
**/anvil_state/state-deployed-contracts-mint-and-approved.json
.gitnexus
# Python bytecode from tests/simulator
__pycache__/
*.pyc

95
AGENTS.md
View File

@ -16,7 +16,7 @@ Key architectural decisions:
Resource-restricted first: Protocols differentiate between full nodes (relay) and light clients (filter, lightpush, store). Light clients can participate without maintaining full message history or relay capabilities. This explains the client/server split in protocol implementations.
Privacy through unlinkability: RLN (Rate Limiting Nullifier) provides DoS protection while preserving sender anonymity. Messages are routed through pubsub topics with automatic sharding across 8 shards. Code prioritizes metadata privacy alongside content encryption.
Privacy through unlinkability: RLN (Rate Limiting Nullifier) provides DoS protection while preserving sender anonymity. Messages are routed through pubsub topics with automatic content-topic-based sharding (shard count is configurable; generation-zero defaults to 8 shards on cluster 0). Code prioritizes metadata privacy alongside content encryption.
Scalability via sharding: The network uses automatic content-topic-based sharding to distribute traffic. This is why you'll see sharding logic throughout the codebase and why pubsub topic selection is protocol-level, not application-level.
@ -36,7 +36,10 @@ See [documentation](https://docs.waku.org/learn/) for architectural details.
### Key Terminology
- ENR (Ethereum Node Record): Node identity and capability advertisement
- Multiaddr: libp2p addressing format (e.g., `/ip4/127.0.0.1/tcp/60000/p2p/16Uiu2...`)
- PubsubTopic: Gossipsub topic for message routing (e.g., `/waku/2/default-waku/proto`)
- PubsubTopic: Gossipsub topic for message routing (shard-based, e.g., `/waku/2/rs/<cluster-id>/<shard-id>`; the default is `/waku/2/rs/0/0`)
- cluster-id: network id
- shard-id: shard differentiator inside the network - drivers mesh forming.
- autosharding: network supports n (configured) shards [0..n-1], shard derived from ContentTopic
- ContentTopic: Application-level message categorization (e.g., `/my-app/1/chat/proto`)
- Sharding: Partitioning network traffic across topics (static or auto-sharding)
- RLN (Rate Limiting Nullifier): Zero-knowledge proof system for spam prevention
@ -77,29 +80,29 @@ type WakuFilter* = ref object of LPProtocol
### Build Requirements
- Nim 2.x (check `waku.nimble` for minimum version)
- Rust toolchain (required for RLN dependencies)
- Build system: Make with nimbus-build-system
- Build system: Make driven by Nimble (dependencies pinned in `nimble.lock`)
### Build System
The project uses Makefile with nimbus-build-system (Status's Nim build framework):
The project uses a Makefile that drives Nimble. Dependencies are resolved from
`nimble.lock` into a local `nimbledeps/` directory (tracked by the
`NIMBLEDEPS_STAMP` target).
```bash
# Initial build (updates submodules)
# Initial build (resolves Nimble deps automatically)
make wakunode2
# After git pull, update submodules
make update
# Build with custom flags
make wakunode2 NIMFLAGS="-d:chronicles_log_level=DEBUG"
```
Note: The build system uses `--mm:refc` memory management (automatically enforced). Only relevant if compiling outside the standard build system.
Note: The build uses `--mm:refc` memory management (passed automatically by the Nimble tasks in `waku.nimble`). Only relevant if compiling outside the standard build system.
### Common Make Targets
```bash
make wakunode2 # Build main node binary
make test # Run all tests
make testcommon # Run common tests only
make libwakuStatic # Build static C library
make libwaku # Build the legacy C library (libwaku)
make liblogosdelivery. # Build actual C FFI library
make chat2 # Build chat example
make install-nph # Install git hook for auto-formatting
```
@ -127,7 +130,7 @@ suite "Waku ENR - Capabilities":
test "check capabilities support":
## Given
let bitfield: CapabilitiesBitfield = 0b0000_1101u8
## Then
check:
bitfield.supportsCapability(Capabilities.Relay)
@ -135,7 +138,7 @@ suite "Waku ENR - Capabilities":
```
### Code Formatting
Mandatory: All code must be formatted with `nph` (vendored in `vendor/nph`)
Mandatory: All code must be formatted with `nph` (installed via `make build-nph`, which fetches a pinned `nph` version with Nimble)
```bash
# Format specific file
make nph/waku/waku_core.nim
@ -162,7 +165,6 @@ Compile with log level:
nim c -d:chronicles_log_level=TRACE myfile.nim
```
## Code Conventions
Common pitfalls:
@ -181,8 +183,13 @@ Common pitfalls:
- Exceptions: `XxxError` for CatchableError, `XxxDefect` for Defect
- ref object types: `XxxRef` suffix
### Calls and Member Access
- Prefer dot call syntax for predicates: `x.isNil()` instead of `isNil(x)`
- Use parentheses for "verbs" (operations/actions): `isSome()`, `handleRequest()`
- Omit parentheses for "nouns" (properties/values): `.len`, `.high`
### Imports Organization
Group imports: stdlib, external libs, internal modules:
Stdlib + external in one `import` block, internal modules in a separate block:
```nim
import
std/[options, sequtils], # stdlib
@ -214,11 +221,11 @@ proc subscribe(
): Future[FilterSubscribeResult] {.async.} =
if contentTopics.len > MaxContentTopicsPerRequest:
return err(FilterSubscribeError.badRequest("exceeds maximum"))
# Handle Result with isOkOr
(await wf.subscriptions.addSubscription(peerId, criteria)).isOkOr:
return err(FilterSubscribeError.serviceUnavailable(error))
ok()
```
@ -460,8 +467,7 @@ nim c -r \
### Vendor Directory
- Never edit files directly in vendor - it is auto-generated from git submodules
- Always run `make update` after pulling changes
- Managed by `nimbus-build-system`
- Nimble dependencies are resolved from `nimble.lock` into `nimbledeps/`
### Chronicles Performance
- Log levels are configured at compile time for performance
@ -475,7 +481,7 @@ nim c -r \
### RLN Dependencies
- RLN code requires a Rust toolchain, which explains Rust imports in some modules
- Pre-built `librln` libraries are checked into the repository
- `librln` is built from the vendored `zerokit` submodule via the `librln`/`rln-deps` Make targets
## Quick Reference
@ -483,18 +489,19 @@ Language: Nim 2.x | License: MIT or Apache 2.0
### Important Files
- `Makefile` - Primary build interface
- `waku.nimble` - Package definition and build tasks (called via nimbus-build-system)
- `vendor/nimbus-build-system/` - Status's build framework
- `waku.nimble` - Package definition and build tasks (invoked by the Makefile via Nimble)
- `nimble.lock` - Pinned dependency versions resolved into `nimbledeps/`
- `waku/node/waku_node.nim` - Core node implementation
- `apps/wakunode2/wakunode2.nim` - Main CLI application
- `waku/factory/waku_conf.nim` - Configuration types
- `library/libwaku.nim` - C bindings entry point
- `liblogosdelivery/liblogosdelivery.nim` - C bindings entry point
### Testing Entry Points
- `tests/all_tests_waku.nim` - All Waku protocol tests
- `tests/all_tests_wakunode2.nim` - Node application tests
- `tests/all_tests_common.nim` - Common utilities tests
#### in-flight testing
- any test can be run separately by issuing `make test tests/<relativepath>/<unit-test-source>.nim`
### Key Dependencies
- `chronos` - Async framework
- `nim-results` - Result type for error handling
@ -506,4 +513,46 @@ Language: Nim 2.x | License: MIT or Apache 2.0
Note: For specific version requirements, check `waku.nimble`.
<!-- gitnexus:start -->
# GitNexus — Code Intelligence
This project is indexed by GitNexus as **logos-delivery** (2076 symbols, 2564 relationships, 12 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first.
## Always Do
- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user.
- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows.
- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits.
- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance.
- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`.
## Never Do
- NEVER edit a function, class, or method without first running `gitnexus_impact` on it.
- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis.
- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph.
- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope.
## Resources
| Resource | Use for |
|----------|---------|
| `gitnexus://repo/logos-delivery/context` | Codebase overview, check index freshness |
| `gitnexus://repo/logos-delivery/clusters` | All functional areas |
| `gitnexus://repo/logos-delivery/processes` | All execution flows |
| `gitnexus://repo/logos-delivery/process/{name}` | Step-by-step execution trace |
## CLI
| Task | Read this skill file |
|------|---------------------|
| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
<!-- gitnexus:end -->

15
BearSSL.mk
View File

@ -9,7 +9,7 @@
## bearssl (nimbledeps) ##
###########################
# Rebuilds libbearssl.a from the package installed by nimble under
# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP).
# nimbledeps/pkgs2/. Invoked via $(NIMBLEDEPS_STAMP) / build-deps.
#
# BEARSSL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that
# depend on it must be invoked via a recursive $(MAKE) call so the sub-make
@ -29,18 +29,11 @@ else
PORTABLE_BEARSSL_CFLAGS := -W -Wall -Os -fPIC
endif
.PHONY: clean-bearssl-nimbledeps rebuild-bearssl-nimbledeps
.PHONY: rebuild-bearssl-nimbledeps
clean-bearssl-nimbledeps:
rebuild-bearssl-nimbledeps:
ifeq ($(BEARSSL_NIMBLEDEPS_DIR),)
$(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first)
endif
+ [ -e "$(BEARSSL_CSOURCES_DIR)/build" ] && \
"$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" clean || true
rebuild-bearssl-nimbledeps: | clean-bearssl-nimbledeps
ifeq ($(BEARSSL_NIMBLEDEPS_DIR),)
$(error No bearssl package found under nimbledeps/pkgs2/ — run 'make update' first)
$(error No bearssl package found under nimbledeps/pkgs2/ — run 'make build-deps' first)
endif
@echo "Rebuilding bearssl from $(BEARSSL_CSOURCES_DIR)"
+ "$(MAKE)" -C "$(BEARSSL_CSOURCES_DIR)" CFLAGS="$(PORTABLE_BEARSSL_CFLAGS)" lib

8
CHANGELOG.md
View File

@ -1,3 +1,11 @@
## v0.38.1 (2026-05-07)
### Changes
- Evict peer instead of abrupt disconnect and avoid sending unnecessary store requests ([#3857](https://github.com/logos-messaging/logos-delivery/pull/3857)) ([75dbeb1b](https://github.com/logos-messaging/logos-delivery/commit/75dbeb1be785df5e61c9ab0bcf8393349b9d0f5e) and [7e59b2c2](https://github.com/logos-messaging/logos-delivery/commit/7e59b2c2))
- RecvService now delivers store-recovered messages via MessageReceivedEvent and includes check for missed hashes before processing ([#3805](https://github.com/logos-messaging/logos-delivery/pull/3805)) ([494ea946](https://github.com/logos-messaging/logos-delivery/commit/494ea946))
## v0.38.0 (2026-03-16)
### Notes

1
CLAUDE.md Normal file
View File

@ -0,0 +1 @@
@AGENTS.md

73
Makefile
View File

@ -24,6 +24,7 @@ export PATH := $(HOME)/.nimble/bin:$(PATH)
# NIM binary location
NIM_BINARY := $(shell which nim 2>/dev/null)
NPH := $(HOME)/.nimble/bin/nph
NIMBLE := $(HOME)/.nimble/bin/nimble
NIMBLEDEPS_STAMP := nimbledeps/.nimble-setup
# Compilation parameters
@ -42,7 +43,8 @@ endif
##########
## Main ##
##########
.PHONY: all test update clean examples deps nimble install-nim install-nimble
# The Makefile automatically bootstraps dependency setup when needed for build and test targets.
.PHONY: all test clean examples deps nimble install-nim install-nimble
# default target
all: | wakunode2 libwaku liblogosdelivery
@ -69,18 +71,16 @@ endif
waku.nims:
ln -s waku.nimble $@
$(NIMBLEDEPS_STAMP): nimble.lock | waku.nims
$(MAKE) install-nimble
nimble setup --localdeps
$(MAKE) build-nph
$(MAKE) rebuild-bearssl-nimbledeps
$(MAKE) rebuild-nat-libs-nimbledeps
$(NIMBLEDEPS_STAMP): nimble.lock | install-nimble build-nph waku.nims
$(NIMBLE) setup --localdeps
touch $@
update:
rm -f $(NIMBLEDEPS_STAMP)
$(MAKE) $(NIMBLEDEPS_STAMP)
nimble lock
# Must be phony so the recipe always runs and the sub-make re-evaluates
# BEARSSL_NIMBLEDEPS_DIR / NAT_TRAVERSAL_NIMBLEDEPS_DIR (parse-time variables)
# after nimble setup has populated nimbledeps/.
.PHONY: build-deps
build-deps: | $(NIMBLEDEPS_STAMP)
$(MAKE) rebuild-bearssl-nimbledeps rebuild-nat-libs-nimbledeps
clean:
rm -rf build 2> /dev/null || true
@ -93,15 +93,14 @@ REQUIRED_NIM_VERSION := $(shell grep -E '^const RequiredNimVersion\s*=' waku.
REQUIRED_NIMBLE_VERSION := $(shell grep -E '^const RequiredNimbleVersion\s*=' waku.nimble | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"')
install-nim:
ifneq ($(detected_OS),Windows)
scripts/install_nim.sh $(REQUIRED_NIM_VERSION)
endif
install-nimble: install-nim
@nimble_ver=$$(nimble --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1); \
if [ "$$nimble_ver" = "$(REQUIRED_NIMBLE_VERSION)" ]; then \
echo "nimble $(REQUIRED_NIMBLE_VERSION) already installed, skipping."; \
else \
cd $$(mktemp -d) && nimble install "nimble@$(REQUIRED_NIMBLE_VERSION)" -y; \
fi
ifneq ($(detected_OS),Windows)
scripts/install_nimble.sh $(REQUIRED_NIMBLE_VERSION)
endif
build:
mkdir -p build
@ -176,7 +175,7 @@ deps: | nimble
.PHONY: librln
LIBRLN_BUILDDIR := $(CURDIR)/vendor/zerokit
LIBRLN_VERSION := v0.9.0
LIBRLN_VERSION := v2.0.2
ifeq ($(detected_OS),Windows)
LIBRLN_FILE ?= rln.lib
@ -203,7 +202,7 @@ clean: | clean-librln
#################
.PHONY: testcommon
testcommon: | $(NIMBLEDEPS_STAMP) build
testcommon: | build-deps build
echo -e $(BUILD_MSG) "build/$@" && \
nimble testcommon
@ -212,59 +211,59 @@ testcommon: | $(NIMBLEDEPS_STAMP) build
##########
.PHONY: testwaku wakunode2 testwakunode2 example2 chat2 chat2bridge liteprotocoltester
testwaku: | $(NIMBLEDEPS_STAMP) build rln-deps librln
testwaku: | build-deps build rln-deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble test
wakunode2: | $(NIMBLEDEPS_STAMP) build deps librln
wakunode2: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble wakunode2
benchmarks: | $(NIMBLEDEPS_STAMP) build deps librln
benchmarks: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble benchmarks
testwakunode2: | $(NIMBLEDEPS_STAMP) build deps librln
testwakunode2: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble testwakunode2
example2: | $(NIMBLEDEPS_STAMP) build deps librln
example2: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble example2
chat2: | $(NIMBLEDEPS_STAMP) build deps librln
chat2: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble chat2
chat2mix: | $(NIMBLEDEPS_STAMP) build deps librln
chat2mix: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble chat2mix
rln-db-inspector: | $(NIMBLEDEPS_STAMP) build deps librln
rln-db-inspector: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble rln_db_inspector
chat2bridge: | $(NIMBLEDEPS_STAMP) build deps librln
chat2bridge: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble chat2bridge
liteprotocoltester: | $(NIMBLEDEPS_STAMP) build deps librln
liteprotocoltester: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble liteprotocoltester
lightpushwithmix: | $(NIMBLEDEPS_STAMP) build deps librln
lightpushwithmix: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble lightpushwithmix
api_example: | $(NIMBLEDEPS_STAMP) build deps librln
api_example: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) nim api_example $(NIM_PARAMS) waku.nims
build/%: | $(NIMBLEDEPS_STAMP) build deps librln
build/%: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$*" && \
nimble buildone $*
compile-test: | $(NIMBLEDEPS_STAMP) build deps librln
compile-test: | build-deps build deps librln
echo -e $(BUILD_MSG) "$(TEST_FILE)" "\"$(TEST_NAME)\"" && \
nimble buildTest $(TEST_FILE) && \
nimble execTest $(TEST_FILE) "\"$(TEST_NAME)\""
@ -276,11 +275,11 @@ compile-test: | $(NIMBLEDEPS_STAMP) build deps librln
tools: networkmonitor wakucanary
wakucanary: | $(NIMBLEDEPS_STAMP) build deps librln
wakucanary: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble wakucanary
networkmonitor: | $(NIMBLEDEPS_STAMP) build deps librln
networkmonitor: | build-deps build deps librln
echo -e $(BUILD_MSG) "build/$@" && \
nimble networkmonitor
@ -424,10 +423,10 @@ else ifeq ($(detected_OS),Linux)
BUILD_COMMAND := $(BUILD_COMMAND)Linux
endif
libwaku: | $(NIMBLEDEPS_STAMP) librln
libwaku: | build-deps librln
nimble --verbose libwaku$(BUILD_COMMAND) waku.nimble
liblogosdelivery: | $(NIMBLEDEPS_STAMP) librln
liblogosdelivery: | build-deps librln
nimble --verbose liblogosdelivery$(BUILD_COMMAND) waku.nimble
logosdelivery_example: | build liblogosdelivery

19
Nat.mk
View File

@ -9,7 +9,7 @@
## nat-libs (nimbledeps) ##
###########################
# Builds miniupnpc and libnatpmp from the package installed by nimble under
# nimbledeps/pkgs2/. Used by `make update` / $(NIMBLEDEPS_STAMP).
# nimbledeps/pkgs2/. Invoked via $(NIMBLEDEPS_STAMP) / build-deps.
#
# NAT_TRAVERSAL_NIMBLEDEPS_DIR is evaluated at parse time, so targets that
# depend on it must be invoked via a recursive $(MAKE) call so the sub-make
@ -28,20 +28,11 @@ else
PORTABLE_NAT_MARCH :=
endif
.PHONY: clean-cross-nimbledeps rebuild-nat-libs-nimbledeps
.PHONY: rebuild-nat-libs-nimbledeps
clean-cross-nimbledeps:
rebuild-nat-libs-nimbledeps:
ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),)
$(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first)
endif
+ [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" ] && \
"$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/miniupnp/miniupnpc" CC=$(CC) clean $(HANDLE_OUTPUT) || true
+ [ -e "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" ] && \
"$(MAKE)" -C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" CC=$(CC) clean $(HANDLE_OUTPUT) || true
rebuild-nat-libs-nimbledeps: | clean-cross-nimbledeps
ifeq ($(NAT_TRAVERSAL_NIMBLEDEPS_DIR),)
$(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make update' first)
$(error No nat_traversal package found under nimbledeps/pkgs2/ — run 'make build-deps' first)
endif
@echo "Rebuilding nat-libs from $(NAT_TRAVERSAL_NIMBLEDEPS_DIR)"
ifeq ($(OS), Windows_NT)
@ -58,4 +49,4 @@ else
+ "$(MAKE)" CFLAGS="-Wall -Wno-cpp -Os -fPIC $(PORTABLE_NAT_MARCH) -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 $(CFLAGS)" \
-C "$(NAT_TRAVERSAL_NIMBLEDEPS_DIR)/vendor/libnatpmp-upstream" \
CC=$(CC) libnatpmp.a $(HANDLE_OUTPUT)
endif
endif

42
README.md
View File

@ -5,9 +5,9 @@
This repository implements a set of libp2p protocols aimed to bring
private communications.
- Nim implementation of [these specs](https://github.com/vacp2p/rfc-index/tree/main/waku).
- Nim implementation of [these specs](https://github.com/logos-co/logos-lips/tree/master/docs/messaging).
- C library that exposes the implemented protocols.
- CLI application that allows you to run an lmn node.
- CLI application that allows you to run a logos-delivery node.
- Examples.
- Various tests of above.
@ -17,15 +17,20 @@ For more details see the [source code](waku/README.md)
These instructions are generic. For more detailed instructions, see the source code above.
Recommended and tested toolchain versions (these are installed when you follow the build instructions below):
- Nim 2.2.4
- Nimble 0.22.3
### Prerequisites
The standard developer tools, including a C compiler, GNU Make, Bash, and Git. More information on these installations can be found [here](https://docs.waku.org/guides/nwaku/build-source#install-dependencies).
The standard developer tools, including a C compiler, GNU Make, Bash, and Git.
> In some distributions (Fedora linux for example), you may need to install `which` utility separately. Nimbus build system is relying on it.
You'll also need an installation of Rust and its toolchain (specifically `rustc` and `cargo`).
The easiest way to install these, is using `rustup`:
Rust:
```bash
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
```
@ -33,8 +38,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
### Wakunode
```bash
# The first `make` invocation will update all Git submodules.
# You'll run `make update` after each `git pull` in the future to keep those submodules updated.
# The first `make` invocation will initialize the local dependency state.
make wakunode2
# Build with custom compilation flags. Do not use NIM_PARAMS unless you know what you are doing.
@ -48,12 +52,12 @@ make wakunode2 NIMFLAGS="-d:chronicles_colors:none -d:disableMarchNative"
./build/wakunode2 --help
```
To join the network, you need to know the address of at least one bootstrap node.
Please refer to the [Waku README](https://github.com/waku-org/nwaku/blob/master/waku/README.md) for more information.
Please refer to the [Waku README](https://github.com/logos-messaging/logos-delivery/blob/master/waku/README.md) for more information.
For more on how to run `wakunode2`, refer to:
- [Run using binaries](https://docs.waku.org/guides/nwaku/build-source)
- [Run using docker](https://docs.waku.org/guides/nwaku/run-docker)
- [Run using docker-compose](https://docs.waku.org/guides/nwaku/run-docker-compose)
- [Run using binaries](https://docs.waku.org/run-node/build-source)
- [Run using docker](https://docs.waku.org/run-node/run-docker)
- [Run using docker-compose](https://docs.waku.org/run-node/run-docker-compose)
#### Issues
##### WSL
@ -104,13 +108,9 @@ If `wakunode2.exe` isn't generated:
This repository is bundled with a Nim runtime that includes the necessary dependencies for the project.
Before you can utilize the runtime you'll need to build the project, as detailed in a previous section.
This will generate a `vendor` directory containing various dependencies, including the `nimbus-build-system` which has the bundled nim runtime.
This will generate a `nimbledeps/pkgs2` directory containing various dependencies.
After successfully building the project, you may bring the bundled runtime into scope by running:
```bash
source env.sh
```
If everything went well, you should see your prompt suffixed with `[Nimbus env]$`. Now you can run `nim` commands as usual.
If everything went well, you should see your prompt suffixed with `[SuccessX]`. Now you can run `nim` commands as usual.
### Test Suite
@ -144,7 +144,7 @@ make test/tests/common/test_enr_builder.nim
```
### Testing against `js-waku`
Refer to [js-waku repo](https://github.com/waku-org/js-waku/tree/master/packages/tests) for instructions.
Refer to [logos-delivery-js repo](https://github.com/logos-messaging/logos-delivery-js/tree/master/packages/tests) for instructions.
## Formatting
@ -175,14 +175,14 @@ Different tools and their corresponding how-to guides can be found in the `tools
### Bugs, Questions & Features
For an inquiry, or if you would like to propose new features, feel free to [open a general issue](https://github.com/waku-org/nwaku/issues/new).
For an inquiry, or if you would like to propose new features, feel free to [open a general issue](https://github.com/logos-messaging/logos-delivery/issues/new).
For bug reports, please [tag your issue with the `bug` label](https://github.com/waku-org/nwaku/issues/new).
For bug reports, please [tag your issue with the `bug` label](https://github.com/logos-messaging/logos-delivery/issues/new).
If you believe the reported issue requires critical attention, please [use the `critical` label](https://github.com/waku-org/nwaku/issues/new?labels=critical,bug) to assist with triaging.
If you believe the reported issue requires critical attention, please [use the `critical` label](https://github.com/logos-messaging/logos-delivery/issues/new?labels=critical,bug) to assist with triaging.
To get help, or participate in the conversation, join the [Waku Discord](https://discord.waku.org/) server.
To get help, or participate in the conversation, join the [Logos Discord](https://discord.gg/logosnetwork) server.
### Docs
* [REST API Documentation](https://waku-org.github.io/waku-rest-api/)
* [REST API Documentation](https://logos-messaging.github.io/logos-delivery-rest-api/)

3
apps/chat2/chat2.nim
View File

@ -13,7 +13,8 @@ import
chronos,
eth/keys,
bearssl,
stew/[byteutils, results],
stew/[byteutils],
results,
metrics,
metrics/chronos_httpserver
import

10
apps/chat2/config_chat2.nim
View File

@ -140,7 +140,8 @@ type
metricsServerAddress* {.
desc: "Listening address of the metrics server.",
defaultValue: parseIpAddress("127.0.0.1"),
defaultValue:
IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "metrics-server-address"
.}: IpAddress
@ -173,7 +174,10 @@ type
dnsAddrsNameServers* {.
desc:
"DNS name server IPs to query for DNS multiaddrs resolution. Argument may be repeated.",
defaultValue: @[parseIpAddress("1.1.1.1"), parseIpAddress("1.0.0.1")],
defaultValue: @[
IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1]),
IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 0, 0, 1]),
],
name: "dns-addrs-name-server"
.}: seq[IpAddress]
@ -348,4 +352,4 @@ proc parseCmdArg*(T: type EthRpcUrl, s: string): T =
func defaultListenAddress*(conf: Chat2Conf): IpAddress =
# TODO: How should we select between IPv4 and IPv6
# Maybe there should be a config option for this.
(static parseIpAddress("0.0.0.0"))
(static IpAddress(family: IpAddressFamily.IPv4, address_v4: [0'u8, 0, 0, 0]))

10
apps/chat2bridge/chat2bridge.nim
View File

@ -1,7 +1,7 @@
{.push raises: [].}
import
std/[tables, times, strutils, hashes, sequtils, json],
std/[tables, times, strutils, hashes, sequtils, json, options],
chronos,
confutils,
chronicles,
@ -267,10 +267,16 @@ when isMainModule:
else:
nodev2ExtPort
let nodev2Key =
if conf.nodekey.isSome():
conf.nodekey.get()
else:
crypto.PrivateKey.random(Secp256k1, rng[]).tryGet()
let bridge = Chat2Matterbridge.new(
mbHostUri = "http://" & $initTAddress(conf.mbHostAddress, Port(conf.mbHostPort)),
mbGateway = conf.mbGateway,
nodev2Key = conf.nodekey,
nodev2Key = nodev2Key,
nodev2BindIp = conf.listenAddress,
nodev2BindPort = Port(uint16(conf.libp2pTcpPort) + conf.portsShift),
nodev2ExtIp = nodev2ExtIp,

11
apps/chat2bridge/config_chat2bridge.nim
View File

@ -1,4 +1,5 @@
import
std/options,
confutils,
confutils/defs,
confutils/std/net,
@ -45,7 +46,7 @@ type Chat2MatterbridgeConf* = object
metricsServerAddress* {.
desc: "Listening address of the metrics server",
defaultValue: parseIpAddress("127.0.0.1"),
defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "metrics-server-address"
.}: IpAddress
@ -62,10 +63,8 @@ type Chat2MatterbridgeConf* = object
.}: seq[string]
nodekey* {.
desc: "P2P node private key as hex",
defaultValue: crypto.PrivateKey.random(Secp256k1, newRng()[]).tryGet(),
name: "nodekey"
.}: crypto.PrivateKey
desc: "P2P node private key as hex", defaultValueDesc: "random", name: "nodekey"
.}: Option[crypto.PrivateKey]
store* {.
desc: "Flag whether to start store protocol", defaultValue: true, name: "store"
@ -94,7 +93,7 @@ type Chat2MatterbridgeConf* = object
# Matterbridge options
mbHostAddress* {.
desc: "Listening address of the Matterbridge host",
defaultValue: parseIpAddress("127.0.0.1"),
defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "mb-host-address"
.}: IpAddress

8
apps/chat2mix/config_chat2mix.nim
View File

@ -162,7 +162,8 @@ type
metricsServerAddress* {.
desc: "Listening address of the metrics server.",
defaultValue: parseIpAddress("127.0.0.1"),
defaultValue:
IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "metrics-server-address"
.}: IpAddress
@ -194,7 +195,10 @@ type
dnsDiscoveryNameServers* {.
desc: "DNS name server IPs to query. Argument may be repeated.",
defaultValue: @[parseIpAddress("1.1.1.1"), parseIpAddress("1.0.0.1")],
defaultValue: @[
IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1]),
IpAddress(family: IpAddressFamily.IPv4, address_v4: [1'u8, 0, 0, 1]),
],
name: "dns-discovery-name-server"
.}: seq[IpAddress]

2
apps/liteprotocoltester/tester_config.nim
View File

@ -133,7 +133,7 @@ type LiteProtocolTesterConf* = object
## Tester REST service configuration
restAddress* {.
desc: "Listening address of the REST HTTP server.",
defaultValue: parseIpAddress("127.0.0.1"),
defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "rest-address"
.}: IpAddress

2
apps/networkmonitor/networkmonitor_config.nim
View File

@ -116,7 +116,7 @@ type NetworkMonitorConf* = object
metricsServerAddress* {.
desc: "Listening address of the metrics server.",
defaultValue: parseIpAddress("127.0.0.1"),
defaultValue: IpAddress(family: IpAddressFamily.IPv4, address_v4: [127'u8, 0, 0, 1]),
name: "metrics-server-address"
.}: IpAddress

25
channels/encryption/encryption.nim Normal file
View File

@ -0,0 +1,25 @@
## Optional encryption hooks for the Reliable Channel API.
##
## Modelled as `RequestBroker`s: the broker pattern lets the channel
## delegate work to a provider that may live in any module without
## introducing a direct dependency. If no provider is registered the
## broker returns an error, so installing the noop providers from
## `noop_encryption` is required when the application does not want
## actual encryption.
##
## Applied per-segment after SDS processing on outgoing, and before
## SDS processing on incoming. No specific scheme is mandated.
##
## See: https://lip.logos.co/messaging/raw/reliable-channel-api.html
import brokers/request_broker
export request_broker
RequestBroker:
type Encrypt* = seq[byte]
proc signature*(payload: seq[byte]): Future[Result[Encrypt, string]] {.async.}
RequestBroker:
type Decrypt* = seq[byte]
proc signature*(payload: seq[byte]): Future[Result[Decrypt, string]] {.async.}

18
channels/encryption/noop_encryption.nim Normal file
View File

@ -0,0 +1,18 @@
## No-op encryption providers. Install these when the application does
## not want actual encryption so the `Encrypt` / `Decrypt` brokers have
## something to dispatch to.
import results
import chronos
import ./encryption
proc setNoopEncryption*() =
discard Encrypt.setProvider(
proc(payload: seq[byte]): Future[Result[Encrypt, string]] {.async.} =
return ok(Encrypt(payload))
)
discard Decrypt.setProvider(
proc(payload: seq[byte]): Future[Result[Decrypt, string]] {.async.} =
return ok(Decrypt(payload))
)

39
channels/events.nim Normal file
View File

@ -0,0 +1,39 @@
## Reliable Channel event types emitted to API consumers.
##
## Lifecycle events for individual segments (sent / propagated / errored)
## are the same as the network-level ones the DeliveryService already
## emits — `requestId` is shared across layers — so we just re-export
## `waku/events/message_events` and avoid declaring duplicates.
##
## Only the channel-level `MessageReceivedEvent` carries data that has
## no analogue in the lower layer (reassembled application payload,
## senderId, channelId), so it lives here.
import waku/events/message_events as waku_message_events
import brokers/event_broker
import ./types as channel_types
export waku_message_events, channel_types, event_broker
EventBroker:
type ChannelMessageReceivedEvent* = object
channelId*: ChannelId
senderId*: SdsParticipantID
payload*: seq[byte]
EventBroker:
## Emitted when every segment of a channel-level `send()` reached
## `Confirmed`. Channel-level analogue of `MessageSentEvent`; the
## `requestId` is the channel-layer parent returned by `send()`.
type ChannelMessageSentEvent* = object
channelId*: ChannelId
requestId*: RequestId
EventBroker:
## Emitted when a channel-level `send()` finalises with at least one
## segment in `Failed`. Channel-level analogue of `MessageErrorEvent`.
type ChannelMessageErrorEvent* = object
channelId*: ChannelId
requestId*: RequestId
error*: string

80
channels/rate_limit_manager/rate_limit_manager.nim Normal file
View File

@ -0,0 +1,80 @@
## Rate Limit Manager for the Reliable Channel API.
##
## Tracks messages sent per RLN epoch and delays dispatch when the
## limit is approached, ensuring RLN compliance on enforcing relays.
##
## For the skeleton this is a pass-through: messages are immediately
## released as ready-to-send. Real epoch budgeting will be added later.
##
## See: https://lip.logos.co/messaging/raw/reliable-channel-api.html
import std/times
import message
import brokers/event_broker
import brokers/broker_context
export event_broker, broker_context
export message.SdsChannelID
const
DefaultEpochPeriodSec* = 600
DefaultMessagesPerEpoch* = 1
EventBroker:
## Emitted by `enqueueToSend` carrying the batch of opaque message
## blobs that may now leave the rate limiter and continue down the
## outgoing pipeline (encryption -> dispatch). Bytes only: the rate
## limiter is intentionally agnostic of SDS, so anything serialisable
## can flow through it.
##
## `channelId` lets listeners filter to their own channel, since all
## reliable channels share the underlying Waku node's broker context.
type ReadyToSendEvent* = ref object
channelId*: SdsChannelID
msgs*: seq[seq[byte]]
type
RateLimitConfig* = object
enabled*: bool ## spec: rate limiting opt-in; SHOULD be true when RLN active
epochPeriodSec*: int
messagesPerEpoch*: int
RateLimitManager* = ref object
config*: RateLimitConfig
queue*: seq[seq[byte]]
currentEpochStart*: Time
sentInCurrentEpoch*: int
channelId*: SdsChannelID ## tag for the emitted `ReadyToSendEvent`
brokerCtx: BrokerContext
proc new*(
T: type RateLimitManager,
config: RateLimitConfig,
channelId: SdsChannelID,
brokerCtx: BrokerContext = globalBrokerContext(),
): T =
return T(
config: config,
queue: @[],
currentEpochStart: getTime(),
sentInCurrentEpoch: 0,
channelId: channelId,
brokerCtx: brokerCtx,
)
proc enqueueToSend*(self: RateLimitManager, msg: seq[byte]) =
## Skeleton behaviour: enqueue and immediately release as a single
## ready batch. Real per-epoch budgeting will park messages on
## `self.queue` and emit only when the budget allows.
ReadyToSendEvent.emit(
self.brokerCtx, ReadyToSendEvent(channelId: self.channelId, msgs: @[msg])
)
proc dequeueReady*(self: RateLimitManager): seq[seq[byte]] =
## Returns the set of queued messages that may be dispatched now
## without exceeding the configured rate limit.
discard
proc resetEpoch*(self: RateLimitManager) =
self.currentEpochStart = getTime()
self.sentInCurrentEpoch = 0

453
channels/reliable_channel.nim Normal file
View File

@ -0,0 +1,453 @@
## Reliable Channel type.
##
## A `ReliableChannel` orchestrates segmentation, SDS (end-to-end
## reliability), optional encryption, and rate-limited dispatch on top
## of the Messaging API for a single channel.
##
## Outgoing pipeline: Segment -> SDS -> Rate Limit -> Encrypt -> Dispatch
## Incoming pipeline: Decrypt -> SDS -> Reassemble -> Emit event
##
## Channels are owned by a `ReliableChannelManager`. Lifecycle and send
## operations are addressed by `ChannelId`, so callers only need to keep
## an opaque handle around.
##
## See: https://lip.logos.co/messaging/raw/reliable-channel-api.html
import std/[options, sets, tables]
import results
import chronos
import bearssl/rand
import stew/byteutils
import libp2p/crypto/crypto as libp2p_crypto
import waku/api/api
import waku/factory/waku as waku_factory
import waku/node/delivery_service/send_service
import waku/waku_core/topics
import ./events
import ./segmentation/segmentation
import ./scalable_data_sync/scalable_data_sync
import ./rate_limit_manager/rate_limit_manager
import ./encryption/encryption
export
api, waku_factory, events, segmentation, scalable_data_sync, rate_limit_manager,
encryption
const LipWireReliableChannelVersion* = "RELIABLE-CHANNEL-API/1"
## Wire-format spec marker for the Reliable Channel layer, as defined
## in the reliable-channel-api LIP (`Wire Format / Spec Marker`).
## A `WakuMessage` whose `meta` field does not equal these bytes is
## not addressed to this layer and is silently dropped on ingress.
## The trailing `/N` is the wire-format version and is bumped only
## on breaking on-the-wire changes; implementations pin one version.
type
SendHandler* = proc(envelope: MessageEnvelope): Future[Result[RequestId, string]] {.
async: (raises: [CatchableError]), gcsafe
.}
## Egress dispatch boundary. Defaults to `waku.send`; tests inject a
## fake that records calls and returns canned `RequestId`s so the
## send state machine can be exercised end-to-end without a network.
MessagePersistence {.pure.} = enum
Persistent
Ephemeral
SegmentSendState {.pure.} = enum
## Lifecycle of a single segment as tracked by the channel. The
## messaging layer has its own richer `DeliveryState` (retries,
## propagated-vs-validated); here we only model what's needed to
## decide when a `channelReqId` is fully accounted for.
AwaitingRateLimit ## Pushed by `send`; not yet released by rate_limit_manager.
InFlight
## Released by rate_limit_manager and handed to delivery_service;
## `messagingReqId` is now set.
Confirmed ## `MessageSentEvent` arrived for `messagingReqId`.
Failed
## `MessageErrorEvent` arrived for `messagingReqId`, or the local
## delivery-task construction failed before any id was reachable.
PendingMessagingRequest = object
## One entry per segment (i.e. per messaging-layer request). The
## relative order of `AwaitingRateLimit` entries must match the
## order in which `rate_limit_manager` re-emits messages, which is
## FIFO with `send()`.
channelReqId*: RequestId
## The channel-layer parent id returned to the caller of `send()` in channel layer.
## One channel request maps to N pending messaging requests.
messagingReqId*: Option[RequestId]
## Per-segment messaging layer id. `none` until `onReadyToSend` assigns it.
persistenceReqType: MessagePersistence
segmentSendState*: SegmentSendState
ReliableChannel* = ref object
## Spec-defined public type. Fields are private so callers cannot
## mutate internals and break invariants. Getters are added below
## for the few values consumers may need.
sendHandler: SendHandler
channelId: ChannelId
contentTopic: ContentTopic
senderId: SdsParticipantID
rng: ref HmacDrbgContext
segmentation: SegmentationHandler
sdsHandler: SdsHandler
rateLimit: RateLimitManager
requestIds: Table[RequestId, seq[RequestId]]
pendingMessagingRequests: seq[PendingMessagingRequest]
## Entries are kept until the matching segment reaches a final
## state (`Confirmed` or `Failed`); a whole channel request is
## then pruned in one pass once all its segments are final.
brokerCtx: BrokerContext
func getChannelId*(self: ReliableChannel): ChannelId {.inline.} =
self.channelId
func getContentTopic*(self: ReliableChannel): ContentTopic {.inline.} =
self.contentTopic
func getSenderId*(self: ReliableChannel): SdsParticipantID {.inline.} =
self.senderId
func isFinal(state: SegmentSendState): bool {.inline.} =
return state in {SegmentSendState.Confirmed, SegmentSendState.Failed}
proc pruneCompletedChannelReqs(self: ReliableChannel) =
## Drop every `pendingMessagingRequests` entry whose `channelReqId`
## has all of its segments in a final state. A single failing
## segment doesn't trigger a drop on its own — we wait until siblings
## are also accounted for, so the channel-level outcome is decided
## from a complete picture. For each fully-final `channelReqId`, emit
## the channel-level final event before the entries are dropped:
## `ChannelMessageSentEvent` if every sibling Confirmed,
## `ChannelMessageErrorEvent` if any sibling Failed.
var hasPending = initHashSet[RequestId]()
var anyFailed = initHashSet[RequestId]()
for entry in self.pendingMessagingRequests:
if not entry.segmentSendState.isFinal():
hasPending.incl(entry.channelReqId)
elif entry.segmentSendState == SegmentSendState.Failed:
anyFailed.incl(entry.channelReqId)
var emitted = initHashSet[RequestId]()
for entry in self.pendingMessagingRequests:
if entry.channelReqId in hasPending or entry.channelReqId in emitted:
continue
emitted.incl(entry.channelReqId)
if entry.channelReqId in anyFailed:
ChannelMessageErrorEvent.emit(
self.brokerCtx,
ChannelMessageErrorEvent(
channelId: self.channelId,
requestId: entry.channelReqId,
error: "one or more segments failed",
),
)
else:
ChannelMessageSentEvent.emit(
self.brokerCtx,
ChannelMessageSentEvent(
channelId: self.channelId, requestId: entry.channelReqId
),
)
self.pendingMessagingRequests.keepItIf(it.channelReqId in hasPending)
proc onMessageSent(self: ReliableChannel, messagingReqId: RequestId) =
## Invoked from this channel's `MessageSentEvent` listener. Flips
## the matching `InFlight` segment to `Confirmed` and prunes. The
## listener routes every event through here; entries that don't
## belong to this channel simply don't match and are no-ops.
self.pendingMessagingRequests.applyItIf(
it.segmentSendState == SegmentSendState.InFlight and
it.messagingReqId == some(messagingReqId)
):
it.segmentSendState = SegmentSendState.Confirmed
self.pruneCompletedChannelReqs()
proc onMessageError(self: ReliableChannel, messagingReqId: RequestId) =
## Symmetric to `onMessageSent` but for `MessageErrorEvent`.
self.pendingMessagingRequests.applyItIf(
it.segmentSendState == SegmentSendState.InFlight and
it.messagingReqId == some(messagingReqId)
):
it.segmentSendState = SegmentSendState.Failed
self.pruneCompletedChannelReqs()
proc onReadyToSend(
self: ReliableChannel, readyToSendEvent: ReadyToSendEvent
) {.async: (raises: []).} =
## Tail of the outgoing pipeline. Invoked from the `ReadyToSendEvent`
## listener once `rate_limit_manager` releases a batch of opaque
## blobs (already-encoded SDS messages):
##
## ... -> rate_limit_manager -> [encryption] -> dispatch
var idx = 0
for m in readyToSendEvent.msgs:
## The first `AwaitingRateLimit` entry in push order is the one
## this `m` belongs to: `send()` adds one entry per segment, and
## `rate_limit_manager` re-emits them in the same FIFO order, so
## the two sequences advance in lockstep. Earlier entries may
## already be `InFlight` / `Confirmed` / `Failed` because they
## live on until every sibling of their `channelReqId` is final,
## so we walk past those to find the next one that was awaiting for this batch.
while idx < self.pendingMessagingRequests.len and
self.pendingMessagingRequests[idx].segmentSendState !=
SegmentSendState.AwaitingRateLimit
:
idx.inc()
if idx >= self.pendingMessagingRequests.len:
## rate_limit_manager emitted more messages than we have pending —
## should not happen given `send` pushes one entry per enqueued
## SDS payload. Drop silently rather than corrupt state.
break
let channelReqId = self.pendingMessagingRequests[idx].channelReqId
let isEphemeral =
self.pendingMessagingRequests[idx].persistenceReqType ==
MessagePersistence.Ephemeral
## TODO: revisit which fields of the SDS message must be encrypted.
## Encrypting the whole encoded blob forces every receiver to attempt
## decryption before it can route, which breaks selective dispatch.
## Leave routing metadata (channelId, causal-history references) in
## clear and encrypt only the application payload.
let encRes = await Encrypt.request(m)
let encrypted = encRes.valueOr:
MessageErrorEvent.emit(
self.brokerCtx,
MessageErrorEvent(
requestId: channelReqId, messageHash: "", error: "encryption failed: " & error
),
)
## Encryption failed *before* we could hand the segment to the
## delivery layer — no `messagingReqId` was minted and no
## `DeliveryTask` was queued on `sendService`. The delivery
## layer will therefore never emit a `MessageSentEvent` /
## `MessageErrorEvent` for this segment, so `onMessageError`
## won't fire either. Advance the state machine inline so the
## parent `channelReqId` can still be pruned once its siblings
## are also final.
self.pendingMessagingRequests[idx].segmentSendState = SegmentSendState.Failed
idx.inc()
continue
let wireBytes = seq[byte](encrypted)
## The `meta` field carries the Reliable Channel wire-format spec
## marker so the ingress side of any peer can route this WakuMessage
## to its Reliable Channel layer.
let envelope = MessageEnvelope(
contentTopic: self.contentTopic,
payload: wireBytes,
ephemeral: isEphemeral,
meta: LipWireReliableChannelVersion.toBytes(),
)
## `waku.send` is not annotated `(raises: [])`, but this listener is.
## Convert any raise to a Result error so the state machine handles
## both failure modes (Result.err and exception) through one path.
let sendRes =
try:
await self.sendHandler(envelope)
except CatchableError as e:
Result[RequestId, string].err("waku send raised: " & e.msg)
let messagingReqId = sendRes.valueOr:
MessageErrorEvent.emit(
self.brokerCtx,
MessageErrorEvent(
requestId: channelReqId, messageHash: "", error: "waku send failed: " & error
),
)
self.pendingMessagingRequests[idx].segmentSendState = SegmentSendState.Failed
idx.inc()
continue
self.pendingMessagingRequests[idx].messagingReqId = some(messagingReqId)
self.pendingMessagingRequests[idx].segmentSendState = SegmentSendState.InFlight
self.requestIds.mgetOrPut(channelReqId, @[]).add(messagingReqId)
idx.inc()
self.pruneCompletedChannelReqs()
proc send*(
self: ReliableChannel, payload: seq[byte], ephemeral: bool = false
): Result[RequestId, string] =
## Single application-level send. The first three stages of the
## outgoing pipeline are chained explicitly so the flow is visible
## at a glance:
##
## segmentation -> sds -> rate_limit_manager
##
## `rate_limit_manager.enqueueToSend` emits a `ReadyToSendEvent` with
## the SDS messages cleared for transmission; the channel's listener
## then runs the final stage (encryption -> dispatch). The
## `persistenceReqType` is carried alongside each segment in
## `pendingMessagingRequests` and stamped onto the eventual
## `MessageEnvelope`.
##
## The returned `RequestId` is the channel-level parent of one-or-more
## messaging-layer `RequestId`s; the mapping is recorded in
## `self.requestIds`.
if payload.len == 0:
return err("empty payload")
let channelReqId = RequestId.new(self.rng)
self.requestIds[channelReqId] = @[]
let persistenceReqType =
if ephemeral: MessagePersistence.Ephemeral else: MessagePersistence.Persistent
for segmentBytes in self.segmentation.performSegmentation(payload):
## Segments arrive already encoded; the segmentation module owns
## the wire format so SDS only ever sees opaque bytes.
let sdsBytes = self.sdsHandler.wrapOutgoing(
self.channelId, self.senderId, segmentBytes
).valueOr:
return err("SDS wrap failed: " & error)
self.pendingMessagingRequests.add(
PendingMessagingRequest(
channelReqId: channelReqId,
messagingReqId: none(RequestId),
persistenceReqType: persistenceReqType,
segmentSendState: SegmentSendState.AwaitingRateLimit,
)
)
self.rateLimit.enqueueToSend(sdsBytes)
return ok(channelReqId)
proc onMessageReceived(
self: ReliableChannel, messageHash: string, payload: seq[byte]
) {.async: (raises: []).} =
## Ingress pipeline made visible:
##
## payload -> decrypt -> sds -> reassemble -> emit
##
## Invoked from this channel's `MessageReceivedEvent` listener, which
## already filtered on the spec marker and on `contentTopic`. The
## channel only sees the raw payload bytes for itself.
## Notice that the following "request" is implemented implicitly as a broker call to
## the `Decrypt` request broker.
let decRes = await Decrypt.request(payload)
let plaintext = decRes.valueOr:
MessageErrorEvent.emit(
self.brokerCtx,
MessageErrorEvent(
requestId: RequestId(""),
messageHash: messageHash,
error: "decryption failed: " & error,
),
)
return
let plaintextBytes = seq[byte](plaintext)
let unwrapped = self.sdsHandler.handleIncoming(plaintextBytes)
if unwrapped.isErr():
return
let reassembled = self.segmentation.handleIncomingSegment(unwrapped.get().content)
if reassembled.isSome():
## Emit on the captured `brokerCtx` (the manager's), so the
## application listener that the manager has set up on that same
## context picks the event up.
ChannelMessageReceivedEvent.emit(
self.brokerCtx,
ChannelMessageReceivedEvent(
channelId: self.channelId,
senderId: self.senderId,
payload: reassembled.get().payload,
),
)
proc new*(
T: type ReliableChannel,
waku: Waku,
channelId: ChannelId,
contentTopic: ContentTopic,
senderId: SdsParticipantID,
segConfig: SegmentationConfig,
sdsConfig: SdsConfig,
rateConfig: RateLimitConfig,
brokerCtx: BrokerContext = globalBrokerContext(),
sendHandler: SendHandler = nil,
): T =
## Pipeline handlers (segmentation/SDS/rate-limit) are constructed
## inside the channel rather than handed in by the caller — they are
## implementation details of the channel, not knobs the API consumer
## should be wiring up. Encryption is delegated to the `Encrypt`/
## `Decrypt` request brokers, so the channel keeps no per-instance
## encryption state either.
##
## `sendHandler` defaults to `waku.send`; tests pass a fake to drive
## the send state machine without touching the network.
let resolvedSendHandler =
if sendHandler.isNil():
proc(
envelope: MessageEnvelope
): Future[Result[RequestId, string]] {.async: (raises: [CatchableError]), gcsafe.} =
return await waku.send(envelope)
else:
sendHandler
let chn = T(
sendHandler: resolvedSendHandler,
channelId: channelId,
contentTopic: contentTopic,
senderId: senderId,
rng: libp2p_crypto.newRng(),
segmentation: SegmentationHandler.new(segConfig),
sdsHandler: SdsHandler.new(sdsConfig, senderId),
rateLimit: RateLimitManager.new(rateConfig, channelId, brokerCtx),
requestIds: initTable[RequestId, seq[RequestId]](),
pendingMessagingRequests: @[],
brokerCtx: brokerCtx,
)
## Each channel owns its own egress + ingress + send-completion
## listeners on `chn.brokerCtx`, filtered to traffic addressed to
## this channel. Keeping the listeners (and the handler procs they
## call) inside the channel lets `onReadyToSend` /
## `onMessageReceived` / `onMessageSent` / `onMessageError` stay
## private — the manager doesn't need to know about them.
discard ReadyToSendEvent.listen(
chn.brokerCtx,
proc(evt: ReadyToSendEvent): Future[void] {.async: (raises: []).} =
if evt.channelId == chn.channelId:
await chn.onReadyToSend(evt)
,
)
discard MessageReceivedEvent.listen(
chn.brokerCtx,
proc(evt: MessageReceivedEvent): Future[void] {.async: (raises: []).} =
## Drop foreign traffic (non-Reliable-Channel `meta`) and traffic
## for other channels before doing any decode work.
if string.fromBytes(evt.message.meta) != LipWireReliableChannelVersion:
return
if evt.message.contentTopic != chn.contentTopic:
return
await chn.onMessageReceived(evt.messageHash, evt.message.payload)
,
)
## Send-completion events are tagged with the per-segment messaging
## `requestId` — globally unique, so we don't need any channel filter
## up front. The handler scans this channel's pending entries for a
## match and is a no-op when the id belongs to a different channel.
discard MessageSentEvent.listen(
chn.brokerCtx,
proc(evt: MessageSentEvent): Future[void] {.async: (raises: []).} =
chn.onMessageSent(evt.requestId),
)
discard MessageErrorEvent.listen(
chn.brokerCtx,
proc(evt: MessageErrorEvent): Future[void] {.async: (raises: []).} =
chn.onMessageError(evt.requestId),
)
return chn

141
channels/reliable_channel_manager.nim Normal file
View File

@ -0,0 +1,141 @@
## Reliable Channel API entry point.
##
## Owns the set of `ReliableChannel` instances and exposes lifecycle and
## send/receive operations addressed by `ChannelId`.
##
## See: https://lip.logos.co/messaging/raw/reliable-channel-api.html
import std/tables
import results
import chronos
import stew/byteutils
import waku/api/api
import waku/api/api_conf
import waku/events/message_events as waku_message_events
import waku/factory/waku as waku_factory
import waku/node/delivery_service/delivery_service
import waku/waku_core/topics
import ./reliable_channel
import ./encryption/noop_encryption
export reliable_channel
type ReliableChannelManager* = ref object
channels: Table[ChannelId, ReliableChannel]
waku: Waku
## Owned by the manager. The channel layer reaches the messaging
## API through `waku.send(envelope)`; constructing DeliveryTasks
## directly would breach the layer boundary.
brokerCtx: BrokerContext
proc new*(
T: type ReliableChannelManager,
conf: WakuNodeConf,
brokerCtx: BrokerContext = globalBrokerContext(),
): Future[Result[T, string]] {.async.} =
## TODO !! The proper ownership chain is:
## ReliableChannelManager -> DeliveryService (MessagingClient) -> Waku (Kernel/Protocols) -> WakuNode,
## and this will be implemented in the future. For now, `createNode`
## is called here to get a Waku instance, and the WakuNode is immediately discarded.
## This is a temporary workaround to get the API
let waku = ?(await createNode(conf))
let manager = T(
channels: initTable[ChannelId, ReliableChannel](), waku: waku, brokerCtx: brokerCtx
)
return ok(manager)
proc start*(self: ReliableChannelManager): Result[void, string] =
## Bring the owned DeliveryService up. Separated from `new` so callers
## can register encryption providers / create channels before traffic
## starts flowing.
self.waku.deliveryService.startDeliveryService()
proc stop*(self: ReliableChannelManager) {.async.} =
if not self.waku.isNil():
await self.waku.deliveryService.stopDeliveryService()
proc createReliableChannel*(
self: ReliableChannelManager,
channelId: ChannelId,
contentTopic: ContentTopic,
senderId: SdsParticipantID,
sendHandler: SendHandler = nil,
): Result[ChannelId, string] =
## Spec entry point. The `DeliveryService` and `rng` the channel needs
## are sourced from the owning `ReliableChannelManager` rather than
## passed per call. Encryption is wired up through the `Encrypt`/
## `Decrypt` request brokers — the application installs its own
## providers (or `setNoopEncryption()`) before traffic flows.
##
## Segmentation, SDS and rate-limit configs will eventually be read
## from the node's `NodeConfig`. Defaults for now.
##
## `sendHandler` is left `nil` in production so the channel uses the
## owned `waku.send`; tests pass a fake to bypass the network.
if self.channels.hasKey(channelId):
return err("channel already exists: " & channelId)
let segConfig = SegmentationConfig(
segmentSizeBytes: DefaultSegmentSizeBytes,
enableReedSolomon: false,
persistence: nil,
)
let sdsConfig = SdsConfig(
acknowledgementTimeoutMs: DefaultAcknowledgementTimeoutMs,
maxRetransmissions: DefaultMaxRetransmissions,
causalHistorySize: DefaultCausalHistorySize,
persistence: nil,
)
let rateConfig = RateLimitConfig(
epochPeriodSec: DefaultEpochPeriodSec, messagesPerEpoch: DefaultMessagesPerEpoch
)
let chn = ReliableChannel.new(
waku = self.waku,
channelId = channelId,
contentTopic = contentTopic,
senderId = senderId,
segConfig = segConfig,
sdsConfig = sdsConfig,
rateConfig = rateConfig,
brokerCtx = self.brokerCtx,
sendHandler = sendHandler,
)
self.channels[channelId] = chn
return ok(channelId)
proc closeChannel*(
self: ReliableChannelManager, channelId: ChannelId
): Result[void, string] =
## Flush state, persist outstanding SDS buffers, release resources.
if not self.channels.hasKey(channelId):
return err("unknown channel: " & channelId)
self.channels.del(channelId)
return ok()
proc send*(
self: ReliableChannelManager,
channelId: ChannelId,
appPayload: seq[byte],
ephemeral: bool = false,
): Result[RequestId, string] =
## Spec-level entry point. Looks the channel up by id and delegates
## to `ReliableChannel.send`, which exposes the visible pipeline
## segmentation -> sds -> rate_limit_manager -> encryption.
let chn = self.channels.getOrDefault(channelId)
if chn.isNil():
return err("unknown channel: " & channelId)
return chn.send(appPayload, ephemeral)
## Inbound messages are not handed to the manager by direct call. Each
## `ReliableChannel` installs its own `MessageReceivedEvent` listener
## in `ReliableChannel.new`, filters by spec marker and `contentTopic`,
## and routes to its private `onMessageReceived`. This keeps the lower
## layer (MessagingAPI/Waku) unaware of the existence of ReliableChannel
## and keeps the manager out of per-channel event dispatch.

62
channels/scalable_data_sync/scalable_data_sync.nim Normal file
View File

@ -0,0 +1,62 @@
## Scalable Data Sync (SDS) component for the Reliable Channel API.
##
## Provides end-to-end delivery guarantees via causal history tracking,
## acknowledgements, and retransmission of unacknowledged segments.
##
## Skeleton: `wrapOutgoing` and `handleIncoming` are pass-throughs so
## the send/receive circuit can exercise the surrounding pipeline.
## Real SDS wrapping will plug in via `nim-sds` later.
##
## See: https://lip.logos.co/messaging/raw/reliable-channel-api.html
import results
import message as sds_message
import ./sds_persistence
export sds_message, sds_persistence
const
DefaultAcknowledgementTimeoutMs* = 5_000
DefaultMaxRetransmissions* = 5
DefaultCausalHistorySize* = 2
type
SdsConfig* = object
acknowledgementTimeoutMs*: int
maxRetransmissions*: int
causalHistorySize*: int
persistence*: SdsPersistence
SdsHandler* = ref object
config*: SdsConfig
participantId*: SdsParticipantID
proc new*(
T: type SdsHandler,
config: SdsConfig,
participantId: SdsParticipantID = SdsParticipantID(""),
): T =
return T(config: config, participantId: participantId)
proc wrapOutgoing*(
self: SdsHandler,
channelId: SdsChannelID,
senderId: SdsParticipantID,
payload: seq[byte],
): Result[seq[byte], string] =
## Stage 2 of the outgoing pipeline (segmentation -> sds -> rate_limit_manager -> encryption).
## Skeleton: pass the encoded segment through unchanged. Real causal
## history / lamport / bloom-filter population will replace this.
return ok(payload)
proc handleIncoming*(
self: SdsHandler, msg: seq[byte]
): Result[tuple[content: seq[byte], channelId: SdsChannelID], string] =
## Skeleton: pass the bytes through; channel id is left empty until
## the real wire format provides it.
return ok((content: msg, channelId: SdsChannelID("")))
proc tickRetransmissions*(self: SdsHandler) =
## Drives retransmissions of unacknowledged messages.
discard

25
channels/scalable_data_sync/sds_persistence.nim Normal file
View File

@ -0,0 +1,25 @@
## Persistence backend for SDS outgoing buffer and causal history.
##
## TODO (raised in PR review): this surface is duplicating concerns that
## should come from the SDS module itself. Once the SDS module exposes a
## complete persistence contract, drop this file and import that surface
## instead of re-declaring it here.
import message
type
SdsPersistenceKind* {.pure.} = enum
InMemory
Sqlite
SdsPersistence* = ref object of RootObj
kind*: SdsPersistenceKind
method storeOutgoing*(self: SdsPersistence, msg: SdsMessage) {.base.} =
discard
method markAcknowledged*(self: SdsPersistence, messageId: SdsMessageID) {.base.} =
discard
method unackedOlderThan*(self: SdsPersistence, ageMs: int): seq[SdsMessage] {.base.} =
discard

34
channels/segmentation/segment_message_proto.nim Normal file
View File

@ -0,0 +1,34 @@
## Wire format for a single segment, per the Reliable Channel API spec.
##
## Skeleton: encode/decode treat the segment as just its payload bytes,
## since for now we only ever produce a single segment per send.
type SegmentMessageProto* = object
entireMessageHash*: seq[byte] ## Keccak256(original payload), 32 bytes
dataSegmentIndex*: uint32 ## zero-indexed sequence number for data segments
dataSegmentCount*: uint32 ## number of data segments (>= 1)
payload*: seq[byte] ## segment payload (data or parity shard)
paritySegmentIndex*: uint32 ## zero-based sequence number for parity segments
paritySegmentCount*: uint32 ## number of parity segments
isParity*: bool ## true for parity segments, false (default) for data segments
proc isParityMessage*(self: SegmentMessageProto): bool =
self.isParity
proc isValid*(self: SegmentMessageProto): bool =
## Validates hash length (32 bytes), segment indices and counts.
discard
proc encode*(self: SegmentMessageProto): seq[byte] =
self.payload
proc decode*(T: type SegmentMessageProto, buf: seq[byte]): T =
T(
entireMessageHash: @[],
dataSegmentIndex: 0,
dataSegmentCount: 1,
payload: buf,
paritySegmentIndex: 0,
paritySegmentCount: 0,
isParity: false,
)

70
channels/segmentation/segmentation.nim Normal file
View File

@ -0,0 +1,70 @@
## Segmentation component for the Reliable Channel API.
##
## Splits large application payloads into transmittable segments and
## reassembles them on reception. Supports optional Reed-Solomon parity
## segments for loss recovery, as per the Reliable Channel API spec.
##
## For the skeleton everything fits in a single segment: real chunking
## and Reed-Solomon parity will be plugged in later.
##
## See: https://lip.logos.co/messaging/raw/reliable-channel-api.html
import std/options
import ./segment_message_proto
import ./segmentation_persistence
export segment_message_proto, segmentation_persistence
const
DefaultSegmentSizeBytes* = 102_400
SegmentsParityRate* = 0.125
SegmentsReedSolomonMaxCount* = 256
type
SegmentationConfig* = object
segmentSizeBytes*: int
enableReedSolomon*: bool
persistence*: SegmentationPersistence
SegmentationHandler* = ref object
config*: SegmentationConfig
ReassemblyResult* = object
payload*: seq[byte]
entireMessageHash*: seq[byte]
proc new*(T: type SegmentationHandler, config: SegmentationConfig): T =
return T(config: config)
proc performSegmentation*(
self: SegmentationHandler, payload: seq[byte]
): seq[seq[byte]] =
## Skeleton behaviour: emit exactly one segment carrying the whole
## payload. Real chunking and Reed-Solomon parity will replace this.
let segment = SegmentMessageProto(
entireMessageHash: @[],
dataSegmentIndex: 0,
dataSegmentCount: 1,
payload: payload,
paritySegmentIndex: 0,
paritySegmentCount: 0,
isParity: false,
)
return @[segment.encode()]
proc handleIncomingSegment*(
self: SegmentationHandler, segmentBytes: seq[byte]
): Option[ReassemblyResult] =
## Skeleton behaviour: every segment is already a complete message
## (since `performSegmentation` always emits one), so just hand the
## payload straight back.
let segment = SegmentMessageProto.decode(segmentBytes)
return some(
ReassemblyResult(
payload: segment.payload, entireMessageHash: segment.entireMessageHash
)
)
proc cleanupSegments*(self: SegmentationHandler) =
## Drop expired partial-reassembly state.
discard

20
channels/segmentation/segmentation_persistence.nim Normal file
View File

@ -0,0 +1,20 @@
## Persistence backend interface for segmentation reassembly state.
##
## Allows partial reassembly state to survive process restarts.
type
SegmentationPersistenceKind* {.pure.} = enum
InMemory
Sqlite
SegmentationPersistence* = ref object of RootObj
kind*: SegmentationPersistenceKind
method put*(self: SegmentationPersistence, key: seq[byte], value: seq[byte]) {.base.} =
discard
method get*(self: SegmentationPersistence, key: seq[byte]): seq[byte] {.base.} =
discard
method delete*(self: SegmentationPersistence, key: seq[byte]) {.base.} =
discard

15
channels/types.nim Normal file
View File

@ -0,0 +1,15 @@
## Core identifier types for the Reliable Channel API.
import std/hashes
import waku/api/types as api_types
import ./scalable_data_sync/scalable_data_sync
export scalable_data_sync
export api_types
type ChannelId* = SdsChannelID
proc hash*(r: RequestId): Hash =
## Allows `RequestId` to be used as a `Table` key.
hash(string(r))

2
config.nims
View File

@ -117,7 +117,7 @@ if defined(android):
switch("passL", "--sysroot=" & sysRoot)
switch("cincludes", sysRoot & "/usr/include/")
# begin Nimble config (version 2)
--noNimblePath
when withDir(thisDir(), system.fileExists("nimble.paths")):
--noNimblePath
include "nimble.paths"
# end Nimble config

6
examples/api_example/api_example.nim
View File

@ -33,9 +33,9 @@ proc periodicSender(w: Waku): Future[void] {.async.} =
return
defer:
MessageSentEvent.dropListener(sentListener)
MessageErrorEvent.dropListener(errorListener)
MessagePropagatedEvent.dropListener(propagatedListener)
await MessageSentEvent.dropListener(sentListener)
await MessageErrorEvent.dropListener(errorListener)
await MessagePropagatedEvent.dropListener(propagatedListener)
## Periodically sends a Waku message every 30 seconds
var counter = 0

46
flake.lock generated
View File

@ -19,8 +19,7 @@
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"rust-overlay": "rust-overlay",
"zerokit": "zerokit"
"rust-overlay": "rust-overlay"
}
},
"rust-overlay": {
@ -42,49 +41,6 @@
"repo": "rust-overlay",
"type": "github"
}
},
"rust-overlay_2": {
"inputs": {
"nixpkgs": [
"zerokit",
"nixpkgs"
]
},
"locked": {
"lastModified": 1771211437,
"narHash": "sha256-lcNK438i4DGtyA+bPXXyVLHVmJjYpVKmpux9WASa3ro=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "c62195b3d6e1bb11e0c2fb2a494117d3b55d410f",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
},
"zerokit": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"rust-overlay": "rust-overlay_2"
},
"locked": {
"lastModified": 1771279884,
"narHash": "sha256-tzkQPwSl4vPTUo1ixHh6NCENjsBDroMKTjifg2q8QX8=",
"owner": "vacp2p",
"repo": "zerokit",
"rev": "53b18098e6d5d046e3eb1ac338a8f4f651432477",
"type": "github"
},
"original": {
"owner": "vacp2p",
"repo": "zerokit",
"rev": "53b18098e6d5d046e3eb1ac338a8f4f651432477",
"type": "github"
}
}
},
"root": "root",

108
flake.nix
View File

@ -17,16 +17,9 @@
url = "github:oxalica/rust-overlay";
inputs.nixpkgs.follows = "nixpkgs";
};
# External flake input: Zerokit pinned to a specific commit.
# Update the rev here when a new zerokit version is needed.
zerokit = {
url = "github:vacp2p/zerokit/53b18098e6d5d046e3eb1ac338a8f4f651432477";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { self, nixpkgs, rust-overlay, zerokit }:
outputs = { self, nixpkgs, rust-overlay }:
let
systems = [
"x86_64-linux" "aarch64-linux"
@ -36,6 +29,20 @@
forAllSystems = nixpkgs.lib.genAttrs systems;
lib = nixpkgs.lib;
# Single source of truth for the semver: the `version` field of
# waku.nimble. Kept in sync with git tags by the version-check CI.
nimbleVersion =
let line = lib.findFirst (l: lib.hasPrefix "version = " l)
"version = \"unknown\""
(lib.splitString "\n" (builtins.readFile ./waku.nimble));
in lib.removeSuffix "\"" (lib.removePrefix "version = \"" line);
# A flake sandbox has no .git, so `git describe` is impossible; the
# commit comes from the flake metadata instead.
shortRev = self.shortRev or self.dirtyShortRev or "dirty";
nimbleOverlay = final: prev: {
nimble = prev.nimble.overrideAttrs (_: {
version = "0.22.3";
@ -52,17 +59,98 @@
inherit system;
overlays = [ (import rust-overlay) nimbleOverlay ];
};
# Prebuilt zerokit librln, fetched from the upstream GitHub release
# rather than compiled from source. Compiling zerokit makes Nix download
# its many crate dependencies from crates.io in one parallel burst, which
# crates.io intermittently rejects with HTTP 403 (rate limiting from the
# self-hosted runners' shared IP), breaking the nix build. The release
# ships the exact `stateless` library this project links (see
# scripts/build_rln.sh), so we use it directly — no Rust toolchain and
# no crates.io access needed.
#
# Keep `rlnVersion` aligned with `LIBRLN_VERSION` in the Makefile and the
# vendor/zerokit submodule. Each hash is the sha256 of the release tarball
# for that platform; refresh all four when bumping the version.
rlnVersion = "v2.0.2";
rlnAssets = {
"x86_64-linux" = { triple = "x86_64-unknown-linux-gnu"; hash = "sha256-qbrUdaetYKFhjzxUP/QcwD3JHWJ8qk/tCMK3yXceIAk="; };
"aarch64-linux" = { triple = "aarch64-unknown-linux-gnu"; hash = "sha256-s4bWrmCcNTWHNyJwV73ilWNp58ZdAVG+TAgtWN1cTQs="; };
"x86_64-darwin" = { triple = "x86_64-apple-darwin"; hash = "sha256-ZaHP5CApN66FYY7jxwOmGcF9kJR78Fng3k1qE2W08Mk="; };
"aarch64-darwin" = { triple = "aarch64-apple-darwin"; hash = "sha256-f2YppkPsKFdN00j+IY8fpvsebWTIb9lW/V1/vOTiVKU="; };
};
mkZerokitRln = system: pkgs:
let
asset = rlnAssets.${system} or
(throw "zerokit ${rlnVersion} has no prebuilt rln asset for system '${system}'");
in pkgs.stdenv.mkDerivation {
pname = "librln";
version = lib.removePrefix "v" rlnVersion;
src = pkgs.fetchurl {
url = "https://github.com/vacp2p/zerokit/releases/download/"
+ "${rlnVersion}/${asset.triple}-stateless-rln.tar.gz";
hash = asset.hash;
};
# The tarball lays its files out under release/.
sourceRoot = "release";
dontConfigure = true;
dontBuild = true;
# The release .so was linked outside Nix, so it references system
# libraries (libgcc_s, libstdc++, glibc) by bare name. autoPatchelfHook
# points those at the Nix versions so the library loads correctly when
# used by the Nix build. It does nothing for the static .a, and the
# step is skipped on macOS (dylib paths are fixed in nix/default.nix).
nativeBuildInputs =
pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.autoPatchelfHook ];
buildInputs =
pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.stdenv.cc.cc.lib ];
installPhase = ''
runHook preInstall
mkdir -p $out/lib
cp librln.a $out/lib/ 2>/dev/null || true
cp librln.so $out/lib/ 2>/dev/null || true
cp librln.dylib $out/lib/ 2>/dev/null || true
runHook postInstall
'';
meta = with pkgs.lib; {
description = "Prebuilt zerokit RLN library (stateless flavor)";
homepage = "https://github.com/vacp2p/zerokit";
license = with licenses; [ mit asl20 ];
platforms = builtins.attrNames rlnAssets;
};
};
in {
packages = forAllSystems (system:
let
pkgs = pkgsFor system;
zerokitRln = mkZerokitRln system pkgs;
liblogosdelivery = pkgs.callPackage ./nix/default.nix {
inherit pkgs;
src = ./.;
zerokitRln = zerokit.packages.${system}.rln;
inherit zerokitRln;
gitVersion = "v${nimbleVersion}-g${builtins.substring 0 6 shortRev}";
};
wakucanary = pkgs.callPackage ./nix/default.nix {
inherit pkgs;
src = ./.;
targets = ["wakucanary"];
inherit zerokitRln;
};
in {
inherit liblogosdelivery;
inherit liblogosdelivery wakucanary;
# Expose the prebuilt librln so downstream consumers
# (e.g. logos-delivery-module) bundle the exact same librln this
# build links against.
rln = zerokitRln;
default = liblogosdelivery;
}
);

10
liblogosdelivery/logos_delivery_api/node_api.nim
View File

@ -136,11 +136,11 @@ proc logosdelivery_stop_node(
requireInitializedNode(ctx, "STOP_NODE"):
return err(errMsg)
MessageErrorEvent.dropAllListeners(ctx.myLib[].brokerCtx)
MessageSentEvent.dropAllListeners(ctx.myLib[].brokerCtx)
MessagePropagatedEvent.dropAllListeners(ctx.myLib[].brokerCtx)
MessageReceivedEvent.dropAllListeners(ctx.myLib[].brokerCtx)
EventConnectionStatusChange.dropAllListeners(ctx.myLib[].brokerCtx)
await MessageErrorEvent.dropAllListeners(ctx.myLib[].brokerCtx)
await MessageSentEvent.dropAllListeners(ctx.myLib[].brokerCtx)
await MessagePropagatedEvent.dropAllListeners(ctx.myLib[].brokerCtx)
await MessageReceivedEvent.dropAllListeners(ctx.myLib[].brokerCtx)
await EventConnectionStatusChange.dropAllListeners(ctx.myLib[].brokerCtx)
(await ctx.myLib[].stop()).isOkOr:
let errMsg = $error

36
nimble.lock
View File

@ -250,7 +250,7 @@
},
"confutils": {
"version": "0.1.0",
"vcsRevision": "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a",
"vcsRevision": "36f3115ca350f40841ac0eecc7dfa5fe7790c864",
"url": "https://github.com/status-im/nim-confutils",
"downloadMethod": "git",
"dependencies": [
@ -260,7 +260,22 @@
"results"
],
"checksums": {
"sha1": "8bc8c30b107fdba73b677e5f257c6c42ae1cdc8e"
"sha1": "2fbe6418ddd9f79fb11a0addd7666a3e787adbe0"
}
},
"cbor_serialization": {
"version": "0.3.0",
"vcsRevision": "1664160e04d153573373afddc552b9cbf6fbe4dc",
"url": "https://github.com/vacp2p/nim-cbor-serialization",
"downloadMethod": "git",
"dependencies": [
"nim",
"serialization",
"stew",
"results"
],
"checksums": {
"sha1": "ab126eae09a6e39c72972a6a0b83cb06a2ffe8f0"
}
},
"json_serialization": {
@ -312,6 +327,23 @@
"sha1": "8df97c45683abe2337bdff43b844c4fbcc124ca2"
}
},
"brokers": {
"version": "#v2.0.1",
"vcsRevision": "2093ca4d50e581adda73fee7fd16231f990f4cbe",
"url": "https://github.com/NagyZoltanPeter/nim-brokers.git",
"downloadMethod": "git",
"dependencies": [
"nim",
"chronos",
"results",
"chronicles",
"testutils",
"cbor_serialization"
],
"checksums": {
"sha1": "cc74c987af94537e9d44d1b0143aa417299040c5"
}
},
"stint": {
"version": "0.8.2",
"vcsRevision": "470b7892561b5179ab20bd389a69217d6213fe58",

153
nix/default.nix
View File

@ -1,6 +1,8 @@
{ pkgs
, src
, zerokitRln
, targets ? []
, gitVersion ? "n/a"
, enablePostgres ? true
, enableNimDebugDlOpen ? true
, chroniclesLogLevel ? null
@ -9,8 +11,11 @@
let
deps = import ./deps.nix { inherit pkgs; };
buildWakucanary = builtins.elem "wakucanary" targets;
nimDefineArgs = pkgs.lib.concatStringsSep " \\\n " (
[ "--define:disable_libbacktrace" ]
[ "--define:disable_libbacktrace"
"--define:git_version=${gitVersion}" ]
++ pkgs.lib.optional enablePostgres "--define:postgres"
++ pkgs.lib.optional enableNimDebugDlOpen "--define:nimDebugDlOpen"
++ pkgs.lib.optional (chroniclesLogLevel != null)
@ -32,9 +37,29 @@ let
if pkgs.stdenv.hostPlatform.isWindows then "dll"
else if pkgs.stdenv.hostPlatform.isDarwin then "dylib"
else "so";
# Shared `nim c` invocation. Callers vary the output, the source file and a
# few mode-specific flags (e.g. --app:lib, --noMain, --header); everything
# else (paths, defines, threading, gc, nimcache, rln linkage) is constant.
# $NAT_TRAV and $NIMCACHE are shell variables defined in buildPhase.
nimCompile = { outFile, sourceFile, extraArgs ? [] }: ''
nim c \
--noNimblePath \
${pathArgs} \
--path:$NAT_TRAV \
--path:$NAT_TRAV/src \
--passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \
${nimDefineArgs} \
--threads:on \
--mm:refc \
--nimcache:$NIMCACHE \
--out:${outFile} \
${pkgs.lib.concatStringsSep " \\\n " extraArgs} \
${sourceFile}
'';
in
pkgs.stdenv.mkDerivation {
pname = "liblogosdelivery";
pname = if buildWakucanary then "wakucanary" else "liblogosdelivery";
version = "dev";
inherit src;
@ -69,45 +94,47 @@ pkgs.stdenv.mkDerivation {
make -C $NAT_TRAV/vendor/libnatpmp-upstream \
CFLAGS="-Wall -Os -fPIC -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4" libnatpmp.a
${if buildWakucanary then ''
echo "== Building wakucanary =="
${nimCompile {
outFile = "build/wakucanary";
sourceFile = "apps/wakucanary/wakucanary.nim";
extraArgs = [ "--path:." ];
}}
'' else ''
echo "== Building liblogosdelivery (dynamic) =="
nim c \
--noNimblePath \
${pathArgs} \
--path:$NAT_TRAV \
--path:$NAT_TRAV/src \
--passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \
${nimDefineArgs} \
--out:build/liblogosdelivery.${libExt} \
--app:lib \
--threads:on \
--opt:size \
--noMain \
--mm:refc \
--header \
--nimMainPrefix:liblogosdelivery \
--nimcache:$NIMCACHE \
liblogosdelivery/liblogosdelivery.nim
${nimCompile {
outFile = "build/liblogosdelivery.${libExt}";
sourceFile = "liblogosdelivery/liblogosdelivery.nim";
extraArgs = [
"--app:lib"
"--opt:size"
"--noMain"
"--header"
"--nimMainPrefix:liblogosdelivery"
];
}}
echo "== Building liblogosdelivery (static) =="
nim c \
--noNimblePath \
${pathArgs} \
--path:$NAT_TRAV \
--path:$NAT_TRAV/src \
--passL:"-L${zerokitRln}/lib -lrln${pkgs.lib.optionalString pkgs.stdenv.isLinux " -lstdc++"}" \
${nimDefineArgs} \
--out:build/liblogosdelivery.a \
--app:staticlib \
--threads:on \
--opt:size \
--noMain \
--mm:refc \
--nimMainPrefix:liblogosdelivery \
--nimcache:$NIMCACHE \
liblogosdelivery/liblogosdelivery.nim
${nimCompile {
outFile = "build/liblogosdelivery.a";
sourceFile = "liblogosdelivery/liblogosdelivery.nim";
extraArgs = [
"--app:staticlib"
"--opt:size"
"--noMain"
"--nimMainPrefix:liblogosdelivery"
];
}}
''}
'';
installPhase = ''
installPhase = if buildWakucanary then ''
runHook preInstall
mkdir -p $out/bin $out/lib
cp build/wakucanary $out/bin/
runHook postInstall
'' else ''
runHook preInstall
mkdir -p $out/lib $out/include
cp build/liblogosdelivery.${libExt} $out/lib/ 2>/dev/null || true
@ -116,21 +143,47 @@ pkgs.stdenv.mkDerivation {
runHook postInstall
'';
# Bundle librln alongside liblogosdelivery so the output is self-contained.
# Bundle librln alongside the produced artifact so the output is self-contained.
# Use --add-rpath (not --set-rpath) so fixupPhase's stdenv RUNPATH injection
# for libstdc++ is preserved.
postInstall =
pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
cp ${zerokitRln}/lib/librln.dylib $out/lib/
chmod +w $out/lib/librln.dylib $out/lib/liblogosdelivery.dylib
install_name_tool -id @rpath/liblogosdelivery.dylib $out/lib/liblogosdelivery.dylib
install_name_tool -id @rpath/librln.dylib $out/lib/librln.dylib
old=$(otool -L $out/lib/liblogosdelivery.dylib | awk 'NR>1{print $1}' | grep librln)
install_name_tool -change "$old" @rpath/librln.dylib $out/lib/liblogosdelivery.dylib
install_name_tool -add_rpath @loader_path $out/lib/liblogosdelivery.dylib
''
+ pkgs.lib.optionalString pkgs.stdenv.isLinux ''
cp ${zerokitRln}/lib/librln.so $out/lib/
patchelf --add-rpath '$ORIGIN' $out/lib/liblogosdelivery.so
'';
if buildWakucanary then
pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
cp ${zerokitRln}/lib/librln.dylib $out/lib/
chmod +w $out/lib/librln.dylib $out/bin/wakucanary
install_name_tool -id @rpath/librln.dylib $out/lib/librln.dylib
old=$(otool -L $out/bin/wakucanary | awk 'NR>1{print $1}' | grep librln || true)
if [ -n "$old" ]; then
install_name_tool -change "$old" @rpath/librln.dylib $out/bin/wakucanary
fi
install_name_tool -add_rpath @loader_path/../lib $out/bin/wakucanary
''
+ pkgs.lib.optionalString pkgs.stdenv.isLinux ''
cp ${zerokitRln}/lib/librln.so $out/lib/
patchelf --add-rpath '$ORIGIN/../lib' $out/bin/wakucanary
''
else
pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
cp ${zerokitRln}/lib/librln.dylib $out/lib/
chmod +w $out/lib/librln.dylib $out/lib/liblogosdelivery.dylib
install_name_tool -id @rpath/liblogosdelivery.dylib $out/lib/liblogosdelivery.dylib
install_name_tool -id @rpath/librln.dylib $out/lib/librln.dylib
old=$(otool -L $out/lib/liblogosdelivery.dylib | awk 'NR>1{print $1}' | grep librln)
install_name_tool -change "$old" @rpath/librln.dylib $out/lib/liblogosdelivery.dylib
install_name_tool -add_rpath @loader_path $out/lib/liblogosdelivery.dylib
''
+ pkgs.lib.optionalString pkgs.stdenv.isLinux ''
cp ${zerokitRln}/lib/librln.so $out/lib/
patchelf --add-rpath '$ORIGIN' $out/lib/liblogosdelivery.so
'';
meta = with pkgs.lib; {
description =
if buildWakucanary
then "Waku network canary tool"
else "logos-delivery shared/static library";
homepage = "https://github.com/logos-messaging/logos-delivery";
license = licenses.mit;
platforms = platforms.unix;
};
}

25
nix/deps.nix
View File

@ -124,8 +124,15 @@
confutils = pkgs.fetchgit {
url = "https://github.com/status-im/nim-confutils";
rev = "7728f6bd81a1eedcfe277d02ea85fdb805bcc05a";
sha256 = "18bj1ilx10jm2vmqx2wy2xl9rzy7alymi2m4n9jgpa4sbxnfh0x3";
rev = "36f3115ca350f40841ac0eecc7dfa5fe7790c864";
sha256 = "1vppqplwlpl7a61r8iki5hlzvhd8lnq41ixpqslv35dnm482c55j";
fetchSubmodules = true;
};
cbor_serialization = pkgs.fetchgit {
url = "https://github.com/vacp2p/nim-cbor-serialization";
rev = "1664160e04d153573373afddc552b9cbf6fbe4dc";
sha256 = "0c1rj4fk0fcqvsf0yqhxvm8h10aww75gi4yfsjhlczh88ypywii2";
fetchSubmodules = true;
};
@ -150,6 +157,13 @@
fetchSubmodules = true;
};
brokers = pkgs.fetchgit {
url = "https://github.com/NagyZoltanPeter/nim-brokers.git";
rev = "2093ca4d50e581adda73fee7fd16231f990f4cbe";
sha256 = "0a4ix2q6riqfrd0hfnajisy159qdmk5imwzymppj23rwc8n7d2dx";
fetchSubmodules = true;
};
stint = pkgs.fetchgit {
url = "https://github.com/status-im/nim-stint";
rev = "470b7892561b5179ab20bd389a69217d6213fe58";
@ -262,6 +276,13 @@
fetchSubmodules = true;
};
sds = pkgs.fetchgit {
url = "https://github.com/logos-messaging/nim-sds.git";
rev = "2e9a7683f0e180bf112135fae3a3803eed8490d4";
sha256 = "1dbpvp3zhvdlfxdyggz5waga1vg3b6ndd3acfzhnx8k1wdr01c6f";
fetchSubmodules = true;
};
ffi = pkgs.fetchgit {
url = "https://github.com/logos-messaging/nim-ffi";
rev = "06111de155253b34e47ed2aaed1d61d08d62cc1b";

44
scripts/build_rln.sh
View File

@ -1,8 +1,15 @@
#!/usr/bin/env bash
# This script is used to build the rln library for the current platform.
# Previously downloaded prebuilt binaries, but due to compatibility issues
# we now always build from source.
# Provides the rln static library for the current platform.
#
# If zerokit publishes a prebuilt `stateless` release asset for this platform,
# download and use it: that is faster than compiling and avoids fetching
# zerokit's many crate dependencies from crates.io. The asset is selected by
# the Rust host target triple (the platform identifier reported by rustc,
# e.g. x86_64-unknown-linux-gnu or aarch64-apple-darwin).
#
# When no matching asset exists (e.g. Windows), build from the vendored
# zerokit submodule instead.
set -e
@ -15,8 +22,26 @@ output_filename=$3
[[ -z "${rln_version}" ]] && { echo "No rln version specified"; exit 1; }
[[ -z "${output_filename}" ]] && { echo "No output filename specified"; exit 1; }
echo "Building RLN library from source (version ${rln_version})..."
# --- Prefer the prebuilt release asset --------------------------------------
# Host target triple, e.g. x86_64-unknown-linux-gnu / aarch64-apple-darwin.
host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}')
tarball="${host_triplet}-stateless-rln.tar.gz"
url="https://github.com/vacp2p/zerokit/releases/download/${rln_version}/${tarball}"
echo "Looking for prebuilt RLN: ${url}"
if curl --silent --fail-with-body -L "${url}" -o "${tarball}"; then
echo "Downloaded prebuilt ${tarball}"
tar -xzf "${tarball}"
mv "release/librln.a" "${output_filename}"
rm -rf "${tarball}" release
echo "Using prebuilt ${output_filename}"
exit 0
fi
# curl --fail-with-body writes the error body to the file on HTTP failure.
rm -f "${tarball}"
echo "No prebuilt asset for ${host_triplet} at ${rln_version}; building from source."
# --- Fall back to building from the vendored submodule ----------------------
# Check if submodule version = version in Makefile
cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml"
@ -33,8 +58,15 @@ if [[ "v${submodule_version}" != "${rln_version}" ]]; then
exit 1
fi
# Build rln from source
cargo build --release -p rln --manifest-path "${build_dir}/rln/Cargo.toml"
# `stateless` feature: logos-delivery does not maintain a local Merkle tree
# (post-PR #3312); the contract is the source of truth and the path is fetched
# via getMerkleProof(index). The stateless build compiles out tree code.
#
# --no-default-features is required because zerokit's default features include
# `pmtree-ft` (a Merkle tree backend); `stateless` and any Merkle-tree feature
# are mutually exclusive (rln/src/lib.rs:32 compile_error).
cargo build --release -p rln --manifest-path "${build_dir}/rln/Cargo.toml" \
--no-default-features --features stateless
cp "${build_dir}/target/release/librln.a" "${output_filename}"
echo "Successfully built ${output_filename}"

44
scripts/install_nim.sh
View File

@ -17,26 +17,36 @@ if [ -z "${NIM_VERSION}" ]; then
exit 1
fi
# Check if the right version is already installed
NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}"
# 1. A matching Nim is already on PATH (e.g. provided by CI's setup-nim-action,
# choosenim, or a previous run of this script). Use it as-is: installing over it
# would symlink a freshly downloaded Nim into ~/.nimble/bin (first on PATH) and
# shadow a known-good toolchain, which has caused C-backend build failures.
nim_ver=$(nim --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
if [ "${nim_ver}" = "${NIM_VERSION}" ]; then
echo "Nim ${NIM_VERSION} already installed, skipping."
echo "Nim ${NIM_VERSION} already on PATH ($(command -v nim)), skipping install."
exit 0
fi
# 2. Already installed at our expected location from a previous run, but not on PATH.
# Re-link binaries into ~/.nimble/bin.
if [ -f "${NIM_DEST}/lib/system.nim" ]; then
echo "Nim ${NIM_VERSION} already installed at ${NIM_DEST}, re-linking binaries."
mkdir -p "${HOME}/.nimble/bin"
for bin_path in "${NIM_DEST}/bin/"*; do
ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")"
done
exit 0
fi
if [ -n "${nim_ver}" ]; then
newer=$(printf '%s\n%s\n' "${NIM_VERSION}" "${nim_ver}" | sort -V | tail -1)
if [ "${newer}" = "${nim_ver}" ]; then
echo "WARNING: Nim ${nim_ver} is installed; this repo is validated against ${NIM_VERSION}." >&2
echo "WARNING: The build will proceed but may behave differently." >&2
exit 0
fi
echo "INFO: Nim ${nim_ver} found in PATH; installing Nim ${NIM_VERSION} to ${NIM_DEST}." >&2
fi
OS=$(uname -s | tr 'A-Z' 'a-z' | sed 's/darwin/macosx/')
ARCH=$(uname -m | sed 's/x86_64/x64/;s/aarch64/arm64/')
NIM_DEST="${HOME}/.nim/nim-${NIM_VERSION}"
BINARY_URL="https://nim-lang.org/download/nim-${NIM_VERSION}-${OS}_${ARCH}.tar.xz"
WORK_DIR=$(mktemp -d)
trap 'rm -rf "${WORK_DIR}"' EXIT
@ -48,9 +58,7 @@ if [ "${HTTP_STATUS}" = "200" ]; then
echo "Downloading pre-built binary from ${BINARY_URL}..."
curl -fL "${BINARY_URL}" -o "${WORK_DIR}/nim.tar.xz"
tar -xJf "${WORK_DIR}/nim.tar.xz" -C "${WORK_DIR}"
rm -rf "${NIM_DEST}"
mkdir -p "${HOME}/.nim"
cp -r "${WORK_DIR}/nim-${NIM_VERSION}" "${NIM_DEST}"
SRC_DIR="${WORK_DIR}/nim-${NIM_VERSION}"
else
echo "No pre-built binary found for ${OS}_${ARCH}. Building from source..."
SRC_URL="https://github.com/nim-lang/Nim/archive/refs/tags/v${NIM_VERSION}.tar.gz"
@ -58,15 +66,19 @@ else
tar -xzf "${WORK_DIR}/nim-src.tar.gz" -C "${WORK_DIR}"
cd "${WORK_DIR}/Nim-${NIM_VERSION}"
sh build_all.sh
rm -rf "${NIM_DEST}"
mkdir -p "${HOME}/.nim"
cp -r "${WORK_DIR}/Nim-${NIM_VERSION}" "${NIM_DEST}"
SRC_DIR="${WORK_DIR}/Nim-${NIM_VERSION}"
fi
# rm -rf can fail with "Directory not empty" on overlay filesystems (e.g. Docker).
# Using cp -r src/. dst/ handles both cases: dst absent (clean) or partially present.
rm -rf "${NIM_DEST}" 2>/dev/null || true
mkdir -p "${NIM_DEST}"
cp -r "${SRC_DIR}/." "${NIM_DEST}/"
mkdir -p "${HOME}/.nimble/bin"
for bin_path in "${NIM_DEST}/bin/"*; do
ln -sf "${bin_path}" "${HOME}/.nimble/bin/$(basename "${bin_path}")"
done
echo "Nim ${NIM_VERSION} installed to ${NIM_DEST}"
echo "Binaries symlinked in ~/.nimble/bin — ensure it is in your PATH."
echo "Binaries symlinked in ~/.nimble/bin — ensure it is in your PATH."

70
scripts/install_nimble.sh Executable file
View File

@ -0,0 +1,70 @@
#!/usr/bin/env bash
# Installs a specific nimble version without using `nimble install nimble`.
#
# `nimble install nimble` is inherently fragile:
# - ETXTBSY: overwriting the running nimble binary in pkgs2/
# - JSON parse failures with older nimble versions reading packages_official.json
#
# Strategy:
# 1. If the right version is already at ~/.nimble/bin/nimble → done.
# 2. If a previously-compiled binary exists in pkgs2/ → re-link it.
# 3. Otherwise: clone the nimble git repo, init submodules, build with nim,
# and atomically replace the target (mv avoids ETXTBSY on the old binary).
set -e
NIMBLE_VERSION="${1:-}"
if [ -z "${NIMBLE_VERSION}" ]; then
echo "Usage: $0 <nimble-version>" >&2
exit 1
fi
NIMBLE_BIN="${HOME}/.nimble/bin/nimble"
# 1. Already installed at the right version?
if [ -x "${NIMBLE_BIN}" ]; then
nimble_ver=$("${NIMBLE_BIN}" --version 2>/dev/null \
| head -1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)
if [ "${nimble_ver}" = "${NIMBLE_VERSION}" ]; then
echo "Nimble ${NIMBLE_VERSION} already installed, skipping."
exit 0
fi
fi
# 2. Already compiled into pkgs2/ from a previous (possibly partial) run?
PKGS2_NIMBLE=$(ls -dt "${HOME}/.nimble/pkgs2/nimble-${NIMBLE_VERSION}-"*/nimble \
2>/dev/null | head -1 || true)
if [ -n "${PKGS2_NIMBLE}" ] && [ -x "${PKGS2_NIMBLE}" ]; then
echo "Nimble ${NIMBLE_VERSION} found in pkgs2, re-linking to ${NIMBLE_BIN}."
mkdir -p "${HOME}/.nimble/bin"
ln -sf "${PKGS2_NIMBLE}" "${NIMBLE_BIN}"
exit 0
fi
# 3. Build from source.
NIM_BIN="${HOME}/.nimble/bin/nim"
if [ ! -x "${NIM_BIN}" ]; then
NIM_BIN="$(command -v nim)"
fi
WORK_DIR="$(mktemp -d)"
trap 'rm -rf "${WORK_DIR}"' EXIT
echo "Cloning nimble v${NIMBLE_VERSION} with submodules..."
git clone --depth=1 --branch "v${NIMBLE_VERSION}" \
--recurse-submodules --shallow-submodules \
https://github.com/nim-lang/nimble.git \
"${WORK_DIR}/nimble"
echo "Building nimble ${NIMBLE_VERSION} with $("${NIM_BIN}" --version | head -1)..."
cd "${WORK_DIR}/nimble"
# nim reads nim.cfg / config.nims in the current dir, which sets vendor paths.
"${NIM_BIN}" c -d:release --path:src \
-o:"${WORK_DIR}/nimble_new" src/nimble.nim
mkdir -p "${HOME}/.nimble/bin"
# Atomic rename: avoids ETXTBSY when the old binary at NIMBLE_BIN is still running.
cp "${WORK_DIR}/nimble_new" "${NIMBLE_BIN}.new.$$"
mv -f "${NIMBLE_BIN}.new.$$" "${NIMBLE_BIN}"
echo "Nimble ${NIMBLE_VERSION} installed to ${NIMBLE_BIN}"

6
tests/all_tests_waku.nim
View File

@ -85,3 +85,9 @@ import ./api/test_all
# Waku tools tests
import ./tools/test_all
# Persistency library tests
import ./persistency/test_all
# Reliable Channel API tests
import ./channels/test_all

7
tests/api/test_api_health.nim
View File

@ -2,11 +2,12 @@
import std/[options, sequtils, times]
import chronos, testutils/unittests, stew/byteutils, libp2p/[switch, peerinfo]
import brokers/broker_context
import ../testlib/[common, wakucore, wakunode, testasync]
import
waku,
waku/[waku_node, waku_core, waku_relay/protocol, common/broker/broker_context],
waku/[waku_node, waku_core, waku_relay/protocol],
waku/node/health_monitor/[topic_health, health_status, protocol_health, health_report],
waku/requests/health_requests,
waku/requests/node_requests,
@ -43,7 +44,7 @@ proc waitForConnectionStatus(
if not await future.withTimeout(TestTimeout):
raiseAssert "Timeout waiting for status: " & $expected
finally:
EventConnectionStatusChange.dropListener(brokerCtx, handle)
await EventConnectionStatusChange.dropListener(brokerCtx, handle)
proc waitForShardHealthy(
brokerCtx: BrokerContext
@ -67,7 +68,7 @@ proc waitForShardHealthy(
else:
raiseAssert "Timeout waiting for shard health event"
finally:
EventShardTopicHealthChange.dropListener(brokerCtx, handle)
await EventShardTopicHealthChange.dropListener(brokerCtx, handle)
suite "LM API health checking":
var

34
tests/api/test_api_receive.nim
View File

@ -3,6 +3,7 @@
import std/[options, sequtils, net, sets]
import chronos, testutils/unittests, stew/byteutils
import libp2p/[peerid, peerinfo, crypto/crypto]
import brokers/broker_context
import ../testlib/[common, wakucore, wakunode, testasync]
import ../waku_archive/archive_utils
@ -11,7 +12,6 @@ import
waku/[
waku_node,
waku_core,
common/broker/broker_context,
events/message_events,
waku_relay/protocol,
waku_archive,
@ -52,8 +52,8 @@ proc newReceiveEventListenerManager(
return manager
proc teardown(manager: ReceiveEventListenerManager) =
MessageReceivedEvent.dropListener(manager.brokerCtx, manager.receivedListener)
proc teardown(manager: ReceiveEventListenerManager) {.async.} =
await MessageReceivedEvent.dropListener(manager.brokerCtx, manager.receivedListener)
proc waitForEvents(
manager: ReceiveEventListenerManager, timeout: Duration
@ -138,7 +138,20 @@ suite "Messaging API, Receive Service (store recovery)":
break
await sleepAsync(100.milliseconds)
# publish before subscriber exists, gets archived
# create the subscriber before publishing.
# RecvService captures startTimeToCheck at construction time; the
# message's timestamp must land after that point to fall inside
# checkStore's time window.
var subscriber: Waku
lockNewGlobalBrokerContext:
subscriber = (await createNode(createApiNodeConf(numShards))).expect(
"Failed to create subscriber"
)
(await startWaku(addr subscriber)).expect("Failed to start subscriber")
# publish after the subscriber exists but before it connects to the
# store; the message reaches the archive but the subscriber doesn't
# see it via live relay.
let missedPayload = "This message was missed".toBytes()
let missedMsg = WakuMessage(
payload: missedPayload, contentTopic: testTopic, version: 0, timestamp: now()
@ -159,15 +172,8 @@ suite "Messaging API, Receive Service (store recovery)":
await sleepAsync(100.milliseconds)
raiseAssert "Message was not archived in time"
# create subscriber
var subscriber: Waku
lockNewGlobalBrokerContext:
subscriber = (await createNode(createApiNodeConf(numShards))).expect(
"Failed to create subscriber"
)
(await startWaku(addr subscriber)).expect("Failed to start subscriber")
# connect subscriber to store (not publisher, so msg won't come via relay to it)
# connect subscriber to store after the message is already archived so
# gossipsub doesn't replay it via the live path
await subscriber.node.connectToNodes(@[storeNodePeerInfo])
# subscribe to content topic
@ -176,7 +182,7 @@ suite "Messaging API, Receive Service (store recovery)":
# listen before triggering store check
let eventManager = newReceiveEventListenerManager(subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
# trigger store check, should recover and deliver via MessageReceivedEvent
await subscriber.deliveryService.recvService.checkStore()

24
tests/api/test_api_send.nim
View File

@ -2,10 +2,10 @@
import std/strutils
import chronos, testutils/unittests, stew/byteutils, libp2p/[switch, peerinfo]
import brokers/broker_context
import ../testlib/[common, wakucore, wakunode, testasync]
import ../waku_archive/archive_utils
import
waku, waku/[waku_node, waku_core, waku_relay/protocol, common/broker/broker_context]
import waku, waku/[waku_node, waku_core, waku_relay/protocol]
import waku/factory/waku_conf
import tools/confutils/cli_args
@ -77,10 +77,12 @@ proc newSendEventListenerManager(brokerCtx: BrokerContext): SendEventListenerMan
return manager
proc teardown(manager: SendEventListenerManager) =
MessageSentEvent.dropListener(manager.brokerCtx, manager.sentListener)
MessageErrorEvent.dropListener(manager.brokerCtx, manager.errorListener)
MessagePropagatedEvent.dropListener(manager.brokerCtx, manager.propagatedListener)
proc teardown(manager: SendEventListenerManager) {.async.} =
await MessageSentEvent.dropListener(manager.brokerCtx, manager.sentListener)
await MessageErrorEvent.dropListener(manager.brokerCtx, manager.errorListener)
await MessagePropagatedEvent.dropListener(
manager.brokerCtx, manager.propagatedListener
)
proc waitForEvents(
manager: SendEventListenerManager, timeout: Duration
@ -270,7 +272,7 @@ suite "Waku API - Send":
let eventManager = newSendEventListenerManager(node.brokerCtx)
defer:
eventManager.teardown()
await eventManager.teardown()
let envelope = MessageEnvelope.init(
ContentTopic("/waku/2/default-content/proto"), "test payload"
@ -302,7 +304,7 @@ suite "Waku API - Send":
let eventManager = newSendEventListenerManager(node.brokerCtx)
defer:
eventManager.teardown()
await eventManager.teardown()
let envelope = MessageEnvelope.init(
ContentTopic("/waku/2/default-content/proto"), "test payload"
@ -332,7 +334,7 @@ suite "Waku API - Send":
let eventManager = newSendEventListenerManager(node.brokerCtx)
defer:
eventManager.teardown()
await eventManager.teardown()
let envelope = MessageEnvelope.init(
ContentTopic("/waku/2/default-content/proto"), "test payload"
@ -362,7 +364,7 @@ suite "Waku API - Send":
let eventManager = newSendEventListenerManager(node.brokerCtx)
defer:
eventManager.teardown()
await eventManager.teardown()
let envelope = MessageEnvelope.init(
ContentTopic("/waku/2/default-content/proto"), "test payload"
@ -416,7 +418,7 @@ suite "Waku API - Send":
let eventManager = newSendEventListenerManager(node.brokerCtx)
defer:
eventManager.teardown()
await eventManager.teardown()
let envelope = MessageEnvelope.init(
ContentTopic("/waku/2/default-content/proto"), "test payload"

42
tests/api/test_api_subscription.nim
View File

@ -3,6 +3,7 @@
import std/[strutils, sequtils, net, options, sets, tables]
import chronos, testutils/unittests, stew/byteutils
import libp2p/[peerid, peerinfo, multiaddress, crypto/crypto]
import brokers/broker_context
import ../testlib/[common, wakucore, wakunode, testasync]
import
@ -10,7 +11,6 @@ import
waku/[
waku_node,
waku_core,
common/broker/broker_context,
events/message_events,
waku_relay/protocol,
node/kernel_api/filter,
@ -51,8 +51,8 @@ proc newReceiveEventListenerManager(
return manager
proc teardown(manager: ReceiveEventListenerManager) =
MessageReceivedEvent.dropListener(manager.brokerCtx, manager.receivedListener)
proc teardown(manager: ReceiveEventListenerManager) {.async.} =
await MessageReceivedEvent.dropListener(manager.brokerCtx, manager.receivedListener)
proc waitForEvents(
manager: ReceiveEventListenerManager, timeout: Duration
@ -208,7 +208,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(testTopic, "Hello, world!".toBytes())).expect(
"Publish failed"
@ -229,7 +229,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(ignoredTopic, "Ghost Msg".toBytes())).expect(
"Publish failed"
@ -250,7 +250,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(testTopic, "Should be dropped".toBytes())).expect(
"Publish failed"
@ -271,7 +271,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
net.subscriber.unsubscribe(topicA).expect("failed to unsub A")
@ -298,7 +298,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(glitchTopic, "Ghost Msg".toBytes())).expect(
"Publish failed"
@ -322,7 +322,7 @@ suite "Messaging API, SubscriptionManager":
(await net.publishToMesh(testTopic, "Msg 1".toBytes())).expect("Pub 1 failed")
require await eventManager.waitForEvents(TestTimeout)
eventManager.teardown()
await eventManager.teardown()
# Unsubscribe and verify teardown
net.subscriber.unsubscribe(testTopic).expect("Unsub failed")
@ -332,7 +332,7 @@ suite "Messaging API, SubscriptionManager":
(await net.publishToMesh(testTopic, "Ghost".toBytes())).expect("Ghost pub failed")
check not await eventManager.waitForEvents(NegativeTestTimeout)
eventManager.teardown()
await eventManager.teardown()
# Resubscribe
(await net.subscriber.subscribe(testTopic)).expect("Resub failed")
@ -364,7 +364,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 2)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(topicA, "Msg on Shard A".toBytes())).expect(
"Publish A failed"
@ -400,7 +400,7 @@ suite "Messaging API, SubscriptionManager":
# here we just give a chance for any messages that we don't expect to arrive
await sleepAsync(1.seconds)
eventManager.teardown()
await eventManager.teardown()
# weak check (but catches most bugs)
require eventManager.receivedMessages.len == expected.len
@ -451,7 +451,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMeshAfterEdgeReady(testTopic, "Hello, edge!".toBytes())).expect(
"Publish failed"
@ -472,7 +472,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(ignoredTopic, "Ghost Msg".toBytes())).expect(
"Publish failed"
@ -493,7 +493,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
discard (await net.publishToMesh(testTopic, "Should be dropped".toBytes())).expect(
"Publish failed"
@ -517,7 +517,7 @@ suite "Messaging API, SubscriptionManager":
let eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
defer:
eventManager.teardown()
await eventManager.teardown()
net.subscriber.unsubscribe(topicA).expect("failed to unsub A")
@ -546,7 +546,7 @@ suite "Messaging API, SubscriptionManager":
)
require await eventManager.waitForEvents(TestTimeout)
eventManager.teardown()
await eventManager.teardown()
net.subscriber.unsubscribe(testTopic).expect("Unsub failed")
eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
@ -555,7 +555,7 @@ suite "Messaging API, SubscriptionManager":
(await net.publishToMesh(testTopic, "Ghost".toBytes())).expect("Ghost pub failed")
check not await eventManager.waitForEvents(NegativeTestTimeout)
eventManager.teardown()
await eventManager.teardown()
(await net.subscriber.subscribe(testTopic)).expect("Resub failed")
eventManager = newReceiveEventListenerManager(net.subscriber.brokerCtx, 1)
@ -653,7 +653,7 @@ suite "Messaging API, SubscriptionManager":
require await eventManager.waitForEvents(TestTimeout)
check eventManager.receivedMessages[0].payload == "Before failover".toBytes()
eventManager.teardown()
await eventManager.teardown()
# Disconnect meshBuddy from edge (keeps relay mesh alive for publishing)
await subscriber.node.disconnectNode(meshBuddyPeerInfo)
@ -678,7 +678,7 @@ suite "Messaging API, SubscriptionManager":
require await eventManager.waitForEvents(TestTimeout)
check eventManager.receivedMessages[0].payload == "After failover".toBytes()
eventManager.teardown()
await eventManager.teardown()
(await subscriber.stop()).expect("Failed to stop subscriber")
await meshBuddy.stop()
@ -801,7 +801,7 @@ suite "Messaging API, SubscriptionManager":
require await eventManager.waitForEvents(TestTimeout)
check eventManager.receivedMessages[0].payload == "After replacement".toBytes()
eventManager.teardown()
await eventManager.teardown()
(await subscriber.stop()).expect("Failed to stop subscriber")
await sparePeer.stop()

16
tests/api/test_node_conf.nim
View File

@ -219,6 +219,22 @@ suite "WakuNodeConf - preset integration":
check:
wakuConf.clusterId == 2
test "LogosTest preset applies LogosTestConf":
## Given
var conf = defaultWakuNodeConf().valueOr:
raiseAssert error
conf.preset = "logostest"
## When
let wakuConfRes = conf.toWakuConf()
## Then
require wakuConfRes.isOk()
let wakuConf = wakuConfRes.get()
require wakuConf.validate().isOk()
check:
wakuConf.clusterId == 2
test "Invalid preset returns error":
## Given
var conf = defaultWakuNodeConf().valueOr:

3
tests/channels/test_all.nim Normal file
View File

@ -0,0 +1,3 @@
{.used.}
import ./test_reliable_channel_send_receive

317
tests/channels/test_reliable_channel_send_receive.nim Normal file
View File

@ -0,0 +1,317 @@
{.used.}
import std/[net]
import chronos, testutils/unittests, stew/byteutils
import brokers/broker_context
import ../testlib/[common, wakucore, wakunode, testasync]
import waku
import waku/[waku_node, waku_core]
import waku/factory/waku_conf
import waku/events/message_events as waku_message_events
import tools/confutils/cli_args
import channels/reliable_channel_manager
import channels/encryption/noop_encryption
const TestTimeout = chronos.seconds(15)
proc createApiNodeConf(): WakuNodeConf =
var conf = defaultWakuNodeConf().valueOr:
raiseAssert error
conf.mode = cli_args.WakuMode.Core
conf.listenAddress = parseIpAddress("0.0.0.0")
conf.tcpPort = Port(0)
conf.discv5UdpPort = Port(0)
conf.clusterId = 3'u16
conf.numShardsInNetwork = 1
conf.reliabilityEnabled = true
conf.rest = false
return conf
suite "Reliable Channel - ingress":
asyncTest "manager dispatches marked WakuMessage to the right channel":
## Unit test for the receive side of the API: instead of standing
## up two libp2p nodes and a relay mesh, we drive the manager
## directly by emitting a `MessageReceivedEvent` (the exact event
## the DeliveryService emits when a `WakuMessage` arrives off the
## wire). The manager must:
## - drop traffic missing the Reliable Channel spec marker
## - dispatch the matching channel's `onMessageReceived`
## - emit `ChannelMessageReceivedEvent` with the payload
const
channelId = ChannelId("test-channel")
contentTopic = ContentTopic("/reliable-channel/test/proto")
let appPayload = "hello reliable channel".toBytes()
var manager: ReliableChannelManager
var brokerCtx: BrokerContext
lockNewGlobalBrokerContext:
brokerCtx = globalBrokerContext()
manager = (await ReliableChannelManager.new(createApiNodeConf())).expect(
"Failed to create manager"
)
## Noop encryption providers so the Encrypt/Decrypt brokers have
## something to dispatch to; without this the channel falls back to
## plaintext anyway, but installing them is the documented setup.
setNoopEncryption()
discard manager
.createReliableChannel(channelId, contentTopic, SdsParticipantID("local"))
.expect("createReliableChannel")
let received = newFuture[seq[byte]]("channel-message-received")
discard ChannelMessageReceivedEvent
.listen(
brokerCtx,
proc(evt: ChannelMessageReceivedEvent) {.async: (raises: []).} =
if not received.finished() and evt.channelId == channelId:
received.complete(evt.payload)
,
)
.expect("listen ChannelMessageReceivedEvent")
## Build a `WakuMessage` that looks like one that came in off the
## wire from a peer: the spec marker on `meta` plus the right content
## topic. The manager's ingress listener should pick it up,
## decrypt (noop), unwrap SDS (pass-through), reassemble (one
## segment), and finally emit `ChannelMessageReceivedEvent`.
let inboundMsg = WakuMessage(
payload: appPayload,
contentTopic: contentTopic,
version: 0,
meta: LipWireReliableChannelVersion.toBytes(),
)
waku_message_events.MessageReceivedEvent.emit(
brokerCtx,
waku_message_events.MessageReceivedEvent(messageHash: "", message: inboundMsg),
)
let arrived = await received.withTimeout(TestTimeout)
check arrived
if arrived:
check received.read() == appPayload
await manager.stop()
asyncTest "manager drops unmarked WakuMessage":
## Mirror of the above: same content topic, but `meta` is empty
## (i.e. foreign traffic). The channel-level event must NOT fire.
const
channelId = ChannelId("test-channel-2")
contentTopic = ContentTopic("/reliable-channel/test/proto")
let appPayload = "foreign payload".toBytes()
var manager: ReliableChannelManager
var brokerCtx: BrokerContext
lockNewGlobalBrokerContext:
brokerCtx = globalBrokerContext()
manager = (await ReliableChannelManager.new(createApiNodeConf())).expect(
"Failed to create manager"
)
setNoopEncryption()
discard manager
.createReliableChannel(channelId, contentTopic, SdsParticipantID("local"))
.expect("createReliableChannel")
var fired = false
discard ChannelMessageReceivedEvent
.listen(
brokerCtx,
proc(evt: ChannelMessageReceivedEvent) {.async: (raises: []).} =
if evt.channelId == channelId:
fired = true
,
)
.expect("listen ChannelMessageReceivedEvent")
let inboundMsg = WakuMessage(
payload: appPayload,
contentTopic: contentTopic,
version: 0,
meta: @[], ## no Reliable Channel spec marker
)
waku_message_events.MessageReceivedEvent.emit(
brokerCtx,
waku_message_events.MessageReceivedEvent(messageHash: "", message: inboundMsg),
)
## Give the event broker a chance to fan out.
await sleepAsync(100.milliseconds)
check not fired
await manager.stop()
suite "Reliable Channel - send state machine":
asyncTest "MessageSentEvent finalises the channelReqId as Sent":
## Drives the real send pipeline (`send` -> segmentation -> SDS ->
## rate_limit -> encrypt -> dispatch) via a fake `SendHandler` that
## returns a canned `RequestId` instead of hitting the network.
## Emitting the delivery-layer `MessageSentEvent` must drive the
## channel-level state machine through `Confirmed` and produce a
## `ChannelMessageSentEvent` (channel-level terminal event) for the
## `channelReqId` returned by `send()`.
const
channelId = ChannelId("sm-success-channel")
contentTopic = ContentTopic("/reliable-channel/test/sm-success")
fakeMsgReqId = RequestId("fake-msg-req-1")
var manager: ReliableChannelManager
var brokerCtx: BrokerContext
lockNewGlobalBrokerContext:
brokerCtx = globalBrokerContext()
manager = (await ReliableChannelManager.new(createApiNodeConf())).expect(
"Failed to create manager"
)
setNoopEncryption()
var sendCalls = 0
let fakeSend: SendHandler = proc(
env: MessageEnvelope
): Future[Result[RequestId, string]] {.async: (raises: [CatchableError]), gcsafe.} =
sendCalls.inc
return ok(fakeMsgReqId)
discard manager
.createReliableChannel(
channelId, contentTopic, SdsParticipantID("local"), sendHandler = fakeSend
)
.expect("createReliableChannel")
let sentFut = newFuture[RequestId]("channel-sent")
discard ChannelMessageSentEvent
.listen(
brokerCtx,
proc(evt: ChannelMessageSentEvent) {.async: (raises: []).} =
if not sentFut.finished() and evt.channelId == channelId:
sentFut.complete(evt.requestId)
,
)
.expect("listen ChannelMessageSentEvent")
let channelReqId = manager.send(channelId, "hello".toBytes()).expect("send")
let dispatchDeadline = Moment.now() + 1.seconds
while Moment.now() < dispatchDeadline and sendCalls == 0:
await sleepAsync(5.milliseconds)
check sendCalls == 1
waku_message_events.MessageSentEvent.emit(
brokerCtx,
waku_message_events.MessageSentEvent(requestId: fakeMsgReqId, messageHash: ""),
)
let finalised = await sentFut.withTimeout(1.seconds)
check finalised
if finalised:
check sentFut.read() == channelReqId
await manager.stop()
asyncTest "two independent channelReqIds are finalised independently":
## Two `send()` calls -> two independent `channelReqId`s, each with
## one segment under the current segmentation skeleton
## (`performSegmentation` always emits exactly one segment). The
## fake `SendHandler` returns distinct `messagingReqId`s; finalising
## the first emits `ChannelMessageSentEvent` for its `channelReqId`,
## finalising the second as a failure emits `ChannelMessageErrorEvent`
## for the other.
const
channelId = ChannelId("sm-multi-channel")
contentTopic = ContentTopic("/reliable-channel/test/sm-multi")
var manager: ReliableChannelManager
var brokerCtx: BrokerContext
lockNewGlobalBrokerContext:
brokerCtx = globalBrokerContext()
manager = (await ReliableChannelManager.new(createApiNodeConf())).expect(
"Failed to create manager"
)
setNoopEncryption()
var msgReqIds: seq[RequestId]
let fakeSend: SendHandler = proc(
env: MessageEnvelope
): Future[Result[RequestId, string]] {.async: (raises: [CatchableError]), gcsafe.} =
let id = RequestId("fake-msg-req-" & $(msgReqIds.len + 1))
msgReqIds.add(id)
return ok(id)
discard manager
.createReliableChannel(
channelId, contentTopic, SdsParticipantID("local"), sendHandler = fakeSend
)
.expect("createReliableChannel")
let sentFut = newFuture[RequestId]("channel-sent")
let erroredFut = newFuture[RequestId]("channel-errored")
discard ChannelMessageSentEvent
.listen(
brokerCtx,
proc(evt: ChannelMessageSentEvent) {.async: (raises: []).} =
if not sentFut.finished() and evt.channelId == channelId:
sentFut.complete(evt.requestId)
,
)
.expect("listen ChannelMessageSentEvent")
discard ChannelMessageErrorEvent
.listen(
brokerCtx,
proc(evt: ChannelMessageErrorEvent) {.async: (raises: []).} =
if not erroredFut.finished() and evt.channelId == channelId:
erroredFut.complete(evt.requestId)
,
)
.expect("listen ChannelMessageErrorEvent")
let channelReqId1 = manager.send(channelId, "first".toBytes()).expect("send 1")
let channelReqId2 = manager.send(channelId, "second".toBytes()).expect("send 2")
let dispatchDeadline = Moment.now() + 1.seconds
while Moment.now() < dispatchDeadline and msgReqIds.len < 2:
await sleepAsync(5.milliseconds)
check msgReqIds.len == 2
waku_message_events.MessageSentEvent.emit(
brokerCtx,
waku_message_events.MessageSentEvent(requestId: msgReqIds[0], messageHash: ""),
)
let sentArrived = await sentFut.withTimeout(1.seconds)
check sentArrived
if sentArrived:
check sentFut.read() == channelReqId1
## The second `channelReqId` must NOT have finalised yet — its
## segment is still `InFlight`.
check not erroredFut.finished()
waku_message_events.MessageErrorEvent.emit(
brokerCtx,
waku_message_events.MessageErrorEvent(
requestId: msgReqIds[1], messageHash: "", error: "synthetic"
),
)
let erroredArrived = await erroredFut.withTimeout(1.seconds)
check erroredArrived
if erroredArrived:
check erroredFut.read() == channelReqId2
await manager.stop()
asyncTest "TODO: channelReqId not pruned until ALL its segments are final":
## Placeholder for the multi-sibling prune rule. Today's
## `performSegmentation` (segmentation skeleton) always emits
## exactly one segment per `send()`, so multiple siblings under one
## `channelReqId` cannot be produced through the real pipeline.
## Implement once segmentation does real chunking: send a payload
## larger than `DefaultSegmentSizeBytes`, capture the N
## `messagingReqId`s from a fake `SendHandler`, finalise some, and
## assert prune only fires once every sibling is final.
skip()

5
tests/common/test_all.nim
View File

@ -8,7 +8,4 @@ import
./test_parse_size,
./test_requestratelimiter,
./test_ratelimit_setting,
./test_timed_map,
./test_event_broker,
./test_request_broker,
./test_multi_request_broker
./test_timed_map

201
tests/common/test_event_broker.nim
View File

@ -1,201 +0,0 @@
import chronos
import std/sequtils
import testutils/unittests
import waku/common/broker/event_broker
type ExternalDefinedEventType = object
label*: string
EventBroker:
type IntEvent = int
EventBroker:
type ExternalAliasEvent = distinct ExternalDefinedEventType
EventBroker:
type SampleEvent = object
value*: int
label*: string
EventBroker:
type BinaryEvent = object
flag*: bool
EventBroker:
type RefEvent = ref object
payload*: seq[int]
template waitForListeners() =
waitFor sleepAsync(1.milliseconds)
suite "EventBroker":
test "delivers events to all listeners":
var seen: seq[(int, string)] = @[]
discard SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
seen.add((evt.value, evt.label))
)
discard SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
seen.add((evt.value * 2, evt.label & "!"))
)
let evt = SampleEvent(value: 5, label: "hi")
SampleEvent.emit(evt)
waitForListeners()
check seen.len == 2
check seen.anyIt(it == (5, "hi"))
check seen.anyIt(it == (10, "hi!"))
SampleEvent.dropAllListeners()
test "forget removes a single listener":
var counter = 0
let handleA = SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
inc counter
)
let handleB = SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
inc(counter, 2)
)
SampleEvent.dropListener(handleA.get())
let eventVal = SampleEvent(value: 1, label: "one")
SampleEvent.emit(eventVal)
waitForListeners()
check counter == 2
SampleEvent.dropAllListeners()
test "forgetAll clears every listener":
var triggered = false
let handle1 = SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
triggered = true
)
let handle2 = SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
discard
)
SampleEvent.dropAllListeners()
SampleEvent.emit(42, "noop")
SampleEvent.emit(label = "noop", value = 42)
waitForListeners()
check not triggered
let freshHandle = SampleEvent.listen(
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
discard
)
check freshHandle.get().id > 0'u64
SampleEvent.dropListener(freshHandle.get())
test "broker helpers operate via typedesc":
var toggles: seq[bool] = @[]
let handle = BinaryEvent.listen(
proc(evt: BinaryEvent): Future[void] {.async: (raises: []).} =
toggles.add(evt.flag)
)
BinaryEvent(flag: true).emit()
waitForListeners()
let binaryEvent = BinaryEvent(flag: false)
BinaryEvent.emit(binaryEvent)
waitForListeners()
check toggles == @[true, false]
BinaryEvent.dropAllListeners()
test "ref typed event":
var counter: int = 0
let handle = RefEvent.listen(
proc(evt: RefEvent): Future[void] {.async: (raises: []).} =
for n in evt.payload:
counter += n
)
RefEvent(payload: @[1, 2, 3]).emit()
waitForListeners()
RefEvent.emit(payload = @[4, 5, 6])
waitForListeners()
check counter == 21 # 1+2+3 + 4+5+6
RefEvent.dropAllListeners()
test "supports BrokerContext-scoped listeners":
SampleEvent.dropAllListeners()
let ctxA = NewBrokerContext()
let ctxB = NewBrokerContext()
var seenA: seq[int] = @[]
var seenB: seq[int] = @[]
discard SampleEvent.listen(
ctxA,
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
seenA.add(evt.value),
)
discard SampleEvent.listen(
ctxB,
proc(evt: SampleEvent): Future[void] {.async: (raises: []).} =
seenB.add(evt.value),
)
SampleEvent.emit(ctxA, SampleEvent(value: 1, label: "a"))
SampleEvent.emit(ctxB, SampleEvent(value: 2, label: "b"))
waitForListeners()
check seenA == @[1]
check seenB == @[2]
SampleEvent.dropAllListeners(ctxA)
SampleEvent.emit(ctxA, SampleEvent(value: 3, label: "a2"))
SampleEvent.emit(ctxB, SampleEvent(value: 4, label: "b2"))
waitForListeners()
check seenA == @[1]
check seenB == @[2, 4]
SampleEvent.dropAllListeners(ctxB)
test "supports non-object event types (auto-distinct)":
var seen: seq[int] = @[]
discard IntEvent.listen(
proc(evt: IntEvent): Future[void] {.async: (raises: []).} =
seen.add(int(evt))
)
IntEvent.emit(IntEvent(42))
waitForListeners()
check seen == @[42]
IntEvent.dropAllListeners()
test "supports externally-defined type aliases (auto-distinct)":
var seen: seq[string] = @[]
discard ExternalAliasEvent.listen(
proc(evt: ExternalAliasEvent): Future[void] {.async: (raises: []).} =
let base = ExternalDefinedEventType(evt)
seen.add(base.label)
)
ExternalAliasEvent.emit(ExternalAliasEvent(ExternalDefinedEventType(label: "x")))
waitForListeners()
check seen == @["x"]
ExternalAliasEvent.dropAllListeners()

343
tests/common/test_multi_request_broker.nim
View File

@ -1,343 +0,0 @@
{.used.}
import testutils/unittests
import chronos
import std/sequtils
import std/strutils
import waku/common/broker/multi_request_broker
MultiRequestBroker:
type NoArgResponse = object
label*: string
proc signatureFetch*(): Future[Result[NoArgResponse, string]] {.async.}
MultiRequestBroker:
type ArgResponse = object
id*: string
proc signatureFetch*(
suffix: string, numsuffix: int
): Future[Result[ArgResponse, string]] {.async.}
MultiRequestBroker:
type DualResponse = ref object
note*: string
suffix*: string
proc signatureBase*(): Future[Result[DualResponse, string]] {.async.}
proc signatureWithInput*(
suffix: string
): Future[Result[DualResponse, string]] {.async.}
type ExternalBaseType = string
MultiRequestBroker:
type NativeIntResponse = int
proc signatureFetch*(): Future[Result[NativeIntResponse, string]] {.async.}
MultiRequestBroker:
type ExternalAliasResponse = ExternalBaseType
proc signatureFetch*(): Future[Result[ExternalAliasResponse, string]] {.async.}
MultiRequestBroker:
type AlreadyDistinctResponse = distinct int
proc signatureFetch*(): Future[Result[AlreadyDistinctResponse, string]] {.async.}
suite "MultiRequestBroker":
test "aggregates zero-argument providers":
discard NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
ok(NoArgResponse(label: "one"))
)
discard NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
discard catch:
await sleepAsync(1.milliseconds)
ok(NoArgResponse(label: "two"))
)
let responses = waitFor NoArgResponse.request()
check responses.get().len == 2
check responses.get().anyIt(it.label == "one")
check responses.get().anyIt(it.label == "two")
NoArgResponse.clearProviders()
test "aggregates argument providers":
discard ArgResponse.setProvider(
proc(suffix: string, num: int): Future[Result[ArgResponse, string]] {.async.} =
ok(ArgResponse(id: suffix & "-a-" & $num))
)
discard ArgResponse.setProvider(
proc(suffix: string, num: int): Future[Result[ArgResponse, string]] {.async.} =
ok(ArgResponse(id: suffix & "-b-" & $num))
)
let keyed = waitFor ArgResponse.request("topic", 1)
check keyed.get().len == 2
check keyed.get().anyIt(it.id == "topic-a-1")
check keyed.get().anyIt(it.id == "topic-b-1")
ArgResponse.clearProviders()
test "clearProviders resets both provider lists":
discard DualResponse.setProvider(
proc(): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "base", suffix: ""))
)
discard DualResponse.setProvider(
proc(suffix: string): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "base" & suffix, suffix: suffix))
)
let noArgs = waitFor DualResponse.request()
check noArgs.get().len == 1
let param = waitFor DualResponse.request("-extra")
check param.get().len == 1
check param.get()[0].suffix == "-extra"
DualResponse.clearProviders()
let emptyNoArgs = waitFor DualResponse.request()
check emptyNoArgs.get().len == 0
let emptyWithArgs = waitFor DualResponse.request("-extra")
check emptyWithArgs.get().len == 0
test "request returns empty seq when no providers registered":
let empty = waitFor NoArgResponse.request()
check empty.get().len == 0
test "failed providers will fail the request":
NoArgResponse.clearProviders()
discard NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
err("boom")
)
discard NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
ok(NoArgResponse(label: "survivor"))
)
let filtered = waitFor NoArgResponse.request()
check filtered.isErr()
NoArgResponse.clearProviders()
test "deduplicates identical zero-argument providers":
NoArgResponse.clearProviders()
var invocations = 0
let sharedHandler = proc(): Future[Result[NoArgResponse, string]] {.async.} =
inc invocations
ok(NoArgResponse(label: "dup"))
let first = NoArgResponse.setProvider(sharedHandler)
let second = NoArgResponse.setProvider(sharedHandler)
check first.get().id == second.get().id
check first.get().kind == second.get().kind
let dupResponses = waitFor NoArgResponse.request()
check dupResponses.get().len == 1
check invocations == 1
NoArgResponse.clearProviders()
test "removeProvider deletes registered handlers":
var removedCalled = false
var keptCalled = false
let removable = NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
removedCalled = true
ok(NoArgResponse(label: "removed"))
)
discard NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
keptCalled = true
ok(NoArgResponse(label: "kept"))
)
NoArgResponse.removeProvider(removable.get())
let afterRemoval = (waitFor NoArgResponse.request()).valueOr:
assert false, "request failed"
@[]
check afterRemoval.len == 1
check afterRemoval[0].label == "kept"
check not removedCalled
check keptCalled
NoArgResponse.clearProviders()
test "removeProvider works for argument signatures":
var invoked: seq[string] = @[]
discard ArgResponse.setProvider(
proc(suffix: string, num: int): Future[Result[ArgResponse, string]] {.async.} =
invoked.add("first" & suffix)
ok(ArgResponse(id: suffix & "-one-" & $num))
)
let handle = ArgResponse.setProvider(
proc(suffix: string, num: int): Future[Result[ArgResponse, string]] {.async.} =
invoked.add("second" & suffix)
ok(ArgResponse(id: suffix & "-two-" & $num))
)
ArgResponse.removeProvider(handle.get())
let single = (waitFor ArgResponse.request("topic", 1)).valueOr:
assert false, "request failed"
@[]
check single.len == 1
check single[0].id == "topic-one-1"
check invoked == @["firsttopic"]
ArgResponse.clearProviders()
test "catches exception from providers and report error":
let firstHandler = NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
raise newException(ValueError, "first handler raised")
)
discard NoArgResponse.setProvider(
proc(): Future[Result[NoArgResponse, string]] {.async.} =
ok(NoArgResponse(label: "just ok"))
)
let afterException = waitFor NoArgResponse.request()
check afterException.isErr()
check afterException.error().contains("first handler raised")
NoArgResponse.clearProviders()
test "ref providers returning nil fail request":
DualResponse.clearProviders()
test "supports native request types":
NativeIntResponse.clearProviders()
discard NativeIntResponse.setProvider(
proc(): Future[Result[NativeIntResponse, string]] {.async.} =
ok(NativeIntResponse(1))
)
discard NativeIntResponse.setProvider(
proc(): Future[Result[NativeIntResponse, string]] {.async.} =
ok(NativeIntResponse(2))
)
let res = waitFor NativeIntResponse.request()
check res.isOk()
check res.get().len == 2
check res.get().anyIt(int(it) == 1)
check res.get().anyIt(int(it) == 2)
NativeIntResponse.clearProviders()
test "supports external request types":
ExternalAliasResponse.clearProviders()
discard ExternalAliasResponse.setProvider(
proc(): Future[Result[ExternalAliasResponse, string]] {.async.} =
ok(ExternalAliasResponse("hello"))
)
let res = waitFor ExternalAliasResponse.request()
check res.isOk()
check res.get().len == 1
check ExternalBaseType(res.get()[0]) == "hello"
ExternalAliasResponse.clearProviders()
test "supports already-distinct request types":
AlreadyDistinctResponse.clearProviders()
discard AlreadyDistinctResponse.setProvider(
proc(): Future[Result[AlreadyDistinctResponse, string]] {.async.} =
ok(AlreadyDistinctResponse(7))
)
let res = waitFor AlreadyDistinctResponse.request()
check res.isOk()
check res.get().len == 1
check int(res.get()[0]) == 7
AlreadyDistinctResponse.clearProviders()
test "context-aware providers are isolated":
NoArgResponse.clearProviders()
let ctxA = NewBrokerContext()
let ctxB = NewBrokerContext()
discard NoArgResponse.setProvider(
ctxA,
proc(): Future[Result[NoArgResponse, string]] {.async.} =
ok(NoArgResponse(label: "a")),
)
discard NoArgResponse.setProvider(
ctxB,
proc(): Future[Result[NoArgResponse, string]] {.async.} =
ok(NoArgResponse(label: "b")),
)
let resA = waitFor NoArgResponse.request(ctxA)
check resA.isOk()
check resA.get().len == 1
check resA.get()[0].label == "a"
let resB = waitFor NoArgResponse.request(ctxB)
check resB.isOk()
check resB.get().len == 1
check resB.get()[0].label == "b"
let resDefault = waitFor NoArgResponse.request()
check resDefault.isOk()
check resDefault.get().len == 0
NoArgResponse.clearProviders(ctxA)
let clearedA = waitFor NoArgResponse.request(ctxA)
check clearedA.isOk()
check clearedA.get().len == 0
let stillB = waitFor NoArgResponse.request(ctxB)
check stillB.isOk()
check stillB.get().len == 1
check stillB.get()[0].label == "b"
NoArgResponse.clearProviders(ctxB)
discard DualResponse.setProvider(
proc(): Future[Result[DualResponse, string]] {.async.} =
let nilResponse: DualResponse = nil
ok(nilResponse)
)
let zeroArg = waitFor DualResponse.request()
check zeroArg.isErr()
DualResponse.clearProviders()
discard DualResponse.setProvider(
proc(suffix: string): Future[Result[DualResponse, string]] {.async.} =
let nilResponse: DualResponse = nil
ok(nilResponse)
)
let withInput = waitFor DualResponse.request("-extra")
check withInput.isErr()
DualResponse.clearProviders()

675
tests/common/test_request_broker.nim
View File

@ -1,675 +0,0 @@
{.used.}
import testutils/unittests
import chronos
import std/strutils
import waku/common/broker/request_broker
## ---------------------------------------------------------------------------
## Async-mode brokers + tests
## ---------------------------------------------------------------------------
RequestBroker:
type SimpleResponse = object
value*: string
proc signatureFetch*(): Future[Result[SimpleResponse, string]] {.async.}
RequestBroker:
type KeyedResponse = object
key*: string
payload*: string
proc signatureFetchWithKey*(
key: string, subKey: int
): Future[Result[KeyedResponse, string]] {.async.}
RequestBroker:
type DualResponse = object
note*: string
count*: int
proc signatureNoInput*(): Future[Result[DualResponse, string]] {.async.}
proc signatureWithInput*(
suffix: string
): Future[Result[DualResponse, string]] {.async.}
RequestBroker(async):
type ImplicitResponse = ref object
note*: string
static:
doAssert typeof(SimpleResponse.request()) is Future[Result[SimpleResponse, string]]
suite "RequestBroker macro (async mode)":
test "serves zero-argument providers":
check SimpleResponse
.setProvider(
proc(): Future[Result[SimpleResponse, string]] {.async.} =
ok(SimpleResponse(value: "hi"))
)
.isOk()
let res = waitFor SimpleResponse.request()
check res.isOk()
check res.value.value == "hi"
SimpleResponse.clearProvider()
test "zero-argument request errors when unset":
let res = waitFor SimpleResponse.request()
check res.isErr()
check res.error.contains("no zero-arg provider")
test "serves input-based providers":
var seen: seq[string] = @[]
check KeyedResponse
.setProvider(
proc(
key: string, subKey: int
): Future[Result[KeyedResponse, string]] {.async.} =
seen.add(key)
ok(KeyedResponse(key: key, payload: key & "-payload+" & $subKey))
)
.isOk()
let res = waitFor KeyedResponse.request("topic", 1)
check res.isOk()
check res.value.key == "topic"
check res.value.payload == "topic-payload+1"
check seen == @["topic"]
KeyedResponse.clearProvider()
test "catches provider exception":
check KeyedResponse
.setProvider(
proc(
key: string, subKey: int
): Future[Result[KeyedResponse, string]] {.async.} =
raise newException(ValueError, "simulated failure")
)
.isOk()
let res = waitFor KeyedResponse.request("neglected", 11)
check res.isErr()
check res.error.contains("simulated failure")
KeyedResponse.clearProvider()
test "input request errors when unset":
let res = waitFor KeyedResponse.request("foo", 2)
check res.isErr()
check res.error.contains("input signature")
test "supports both provider types simultaneously":
check DualResponse
.setProvider(
proc(): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "base", count: 1))
)
.isOk()
check DualResponse
.setProvider(
proc(suffix: string): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "base" & suffix, count: suffix.len))
)
.isOk()
let noInput = waitFor DualResponse.request()
check noInput.isOk()
check noInput.value.note == "base"
let withInput = waitFor DualResponse.request("-extra")
check withInput.isOk()
check withInput.value.note == "base-extra"
check withInput.value.count == 6
DualResponse.clearProvider()
test "clearProvider resets both entries":
check DualResponse
.setProvider(
proc(): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "temp", count: 0))
)
.isOk()
DualResponse.clearProvider()
let res = waitFor DualResponse.request()
check res.isErr()
test "implicit zero-argument provider works by default":
check ImplicitResponse
.setProvider(
proc(): Future[Result[ImplicitResponse, string]] {.async.} =
ok(ImplicitResponse(note: "auto"))
)
.isOk()
let res = waitFor ImplicitResponse.request()
check res.isOk()
ImplicitResponse.clearProvider()
check res.value.note == "auto"
test "implicit zero-argument request errors when unset":
let res = waitFor ImplicitResponse.request()
check res.isErr()
check res.error.contains("no zero-arg provider")
test "no provider override":
check DualResponse
.setProvider(
proc(): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "base", count: 1))
)
.isOk()
check DualResponse
.setProvider(
proc(suffix: string): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "base" & suffix, count: suffix.len))
)
.isOk()
let overrideProc = proc(): Future[Result[DualResponse, string]] {.async.} =
ok(DualResponse(note: "something else", count: 1))
check DualResponse.setProvider(overrideProc).isErr()
let noInput = waitFor DualResponse.request()
check noInput.isOk()
check noInput.value.note == "base"
let stillResponse = waitFor DualResponse.request(" still works")
check stillResponse.isOk()
check stillResponse.value.note.contains("base still works")
DualResponse.clearProvider()
let noResponse = waitFor DualResponse.request()
check noResponse.isErr()
check noResponse.error.contains("no zero-arg provider")
let noResponseArg = waitFor DualResponse.request("Should not work")
check noResponseArg.isErr()
check noResponseArg.error.contains("no provider")
check DualResponse.setProvider(overrideProc).isOk()
let nowSuccWithOverride = waitFor DualResponse.request()
check nowSuccWithOverride.isOk()
check nowSuccWithOverride.value.note == "something else"
check nowSuccWithOverride.value.count == 1
DualResponse.clearProvider()
test "supports keyed providers (async, zero-arg)":
SimpleResponse.clearProvider()
check SimpleResponse
.setProvider(
proc(): Future[Result[SimpleResponse, string]] {.async.} =
ok(SimpleResponse(value: "default"))
)
.isOk()
check SimpleResponse
.setProvider(
BrokerContext(0x11111111'u32),
proc(): Future[Result[SimpleResponse, string]] {.async.} =
ok(SimpleResponse(value: "one")),
)
.isOk()
check SimpleResponse
.setProvider(
BrokerContext(0x22222222'u32),
proc(): Future[Result[SimpleResponse, string]] {.async.} =
ok(SimpleResponse(value: "two")),
)
.isOk()
let defaultRes = waitFor SimpleResponse.request()
check defaultRes.isOk()
check defaultRes.value.value == "default"
let res1 = waitFor SimpleResponse.request(BrokerContext(0x11111111'u32))
check res1.isOk()
check res1.value.value == "one"
let res2 = waitFor SimpleResponse.request(BrokerContext(0x22222222'u32))
check res2.isOk()
check res2.value.value == "two"
let missing = waitFor SimpleResponse.request(BrokerContext(0x33333333'u32))
check missing.isErr()
check missing.error.contains("no provider registered for broker context")
check SimpleResponse
.setProvider(
BrokerContext(0x11111111'u32),
proc(): Future[Result[SimpleResponse, string]] {.async.} =
ok(SimpleResponse(value: "dup")),
)
.isErr()
SimpleResponse.clearProvider()
test "supports keyed providers (async, with args)":
KeyedResponse.clearProvider()
check KeyedResponse
.setProvider(
proc(
key: string, subKey: int
): Future[Result[KeyedResponse, string]] {.async.} =
ok(KeyedResponse(key: "default-" & key, payload: $subKey))
)
.isOk()
check KeyedResponse
.setProvider(
BrokerContext(0xABCDEF01'u32),
proc(
key: string, subKey: int
): Future[Result[KeyedResponse, string]] {.async.} =
ok(KeyedResponse(key: "k1-" & key, payload: "p" & $subKey)),
)
.isOk()
check KeyedResponse
.setProvider(
BrokerContext(0xABCDEF02'u32),
proc(
key: string, subKey: int
): Future[Result[KeyedResponse, string]] {.async.} =
ok(KeyedResponse(key: "k2-" & key, payload: "q" & $subKey)),
)
.isOk()
let d = waitFor KeyedResponse.request("topic", 7)
check d.isOk()
check d.value.key == "default-topic"
let k1 = waitFor KeyedResponse.request(BrokerContext(0xABCDEF01'u32), "topic", 7)
check k1.isOk()
check k1.value.key == "k1-topic"
check k1.value.payload == "p7"
let k2 = waitFor KeyedResponse.request(BrokerContext(0xABCDEF02'u32), "topic", 7)
check k2.isOk()
check k2.value.key == "k2-topic"
check k2.value.payload == "q7"
let miss = waitFor KeyedResponse.request(BrokerContext(0xDEADBEEF'u32), "topic", 7)
check miss.isErr()
check miss.error.contains("no provider registered for broker context")
KeyedResponse.clearProvider()
## ---------------------------------------------------------------------------
## Sync-mode brokers + tests
## ---------------------------------------------------------------------------
RequestBroker(sync):
type SimpleResponseSync = object
value*: string
proc signatureFetch*(): Result[SimpleResponseSync, string]
RequestBroker(sync):
type KeyedResponseSync = object
key*: string
payload*: string
proc signatureFetchWithKey*(
key: string, subKey: int
): Result[KeyedResponseSync, string]
RequestBroker(sync):
type DualResponseSync = object
note*: string
count*: int
proc signatureNoInput*(): Result[DualResponseSync, string]
proc signatureWithInput*(suffix: string): Result[DualResponseSync, string]
RequestBroker(sync):
type ImplicitResponseSync = ref object
note*: string
static:
doAssert typeof(SimpleResponseSync.request()) is Result[SimpleResponseSync, string]
doAssert not (
typeof(SimpleResponseSync.request()) is Future[Result[SimpleResponseSync, string]]
)
doAssert typeof(KeyedResponseSync.request("topic", 1)) is
Result[KeyedResponseSync, string]
suite "RequestBroker macro (sync mode)":
test "serves zero-argument providers (sync)":
check SimpleResponseSync
.setProvider(
proc(): Result[SimpleResponseSync, string] =
ok(SimpleResponseSync(value: "hi"))
)
.isOk()
let res = SimpleResponseSync.request()
check res.isOk()
check res.value.value == "hi"
SimpleResponseSync.clearProvider()
test "zero-argument request errors when unset (sync)":
let res = SimpleResponseSync.request()
check res.isErr()
check res.error.contains("no zero-arg provider")
test "serves input-based providers (sync)":
var seen: seq[string] = @[]
check KeyedResponseSync
.setProvider(
proc(key: string, subKey: int): Result[KeyedResponseSync, string] =
seen.add(key)
ok(KeyedResponseSync(key: key, payload: key & "-payload+" & $subKey))
)
.isOk()
let res = KeyedResponseSync.request("topic", 1)
check res.isOk()
check res.value.key == "topic"
check res.value.payload == "topic-payload+1"
check seen == @["topic"]
KeyedResponseSync.clearProvider()
test "catches provider exception (sync)":
check KeyedResponseSync
.setProvider(
proc(key: string, subKey: int): Result[KeyedResponseSync, string] =
raise newException(ValueError, "simulated failure")
)
.isOk()
let res = KeyedResponseSync.request("neglected", 11)
check res.isErr()
check res.error.contains("simulated failure")
KeyedResponseSync.clearProvider()
test "input request errors when unset (sync)":
let res = KeyedResponseSync.request("foo", 2)
check res.isErr()
check res.error.contains("input signature")
test "supports both provider types simultaneously (sync)":
check DualResponseSync
.setProvider(
proc(): Result[DualResponseSync, string] =
ok(DualResponseSync(note: "base", count: 1))
)
.isOk()
check DualResponseSync
.setProvider(
proc(suffix: string): Result[DualResponseSync, string] =
ok(DualResponseSync(note: "base" & suffix, count: suffix.len))
)
.isOk()
let noInput = DualResponseSync.request()
check noInput.isOk()
check noInput.value.note == "base"
let withInput = DualResponseSync.request("-extra")
check withInput.isOk()
check withInput.value.note == "base-extra"
check withInput.value.count == 6
DualResponseSync.clearProvider()
test "clearProvider resets both entries (sync)":
check DualResponseSync
.setProvider(
proc(): Result[DualResponseSync, string] =
ok(DualResponseSync(note: "temp", count: 0))
)
.isOk()
DualResponseSync.clearProvider()
let res = DualResponseSync.request()
check res.isErr()
test "implicit zero-argument provider works by default (sync)":
check ImplicitResponseSync
.setProvider(
proc(): Result[ImplicitResponseSync, string] =
ok(ImplicitResponseSync(note: "auto"))
)
.isOk()
let res = ImplicitResponseSync.request()
check res.isOk()
ImplicitResponseSync.clearProvider()
check res.value.note == "auto"
test "implicit zero-argument request errors when unset (sync)":
let res = ImplicitResponseSync.request()
check res.isErr()
check res.error.contains("no zero-arg provider")
test "implicit zero-argument provider raises error (sync)":
check ImplicitResponseSync
.setProvider(
proc(): Result[ImplicitResponseSync, string] =
raise newException(ValueError, "simulated failure")
)
.isOk()
let res = ImplicitResponseSync.request()
check res.isErr()
check res.error.contains("simulated failure")
ImplicitResponseSync.clearProvider()
test "supports keyed providers (sync, zero-arg)":
SimpleResponseSync.clearProvider()
check SimpleResponseSync
.setProvider(
proc(): Result[SimpleResponseSync, string] =
ok(SimpleResponseSync(value: "default"))
)
.isOk()
check SimpleResponseSync
.setProvider(
BrokerContext(0x10101010'u32),
proc(): Result[SimpleResponseSync, string] =
ok(SimpleResponseSync(value: "ten")),
)
.isOk()
let defaultRes = SimpleResponseSync.request()
check defaultRes.isOk()
check defaultRes.value.value == "default"
let keyedRes = SimpleResponseSync.request(BrokerContext(0x10101010'u32))
check keyedRes.isOk()
check keyedRes.value.value == "ten"
let miss = SimpleResponseSync.request(BrokerContext(0x20202020'u32))
check miss.isErr()
check miss.error.contains("no provider registered for broker context")
SimpleResponseSync.clearProvider()
test "supports keyed providers (sync, with args)":
KeyedResponseSync.clearProvider()
check KeyedResponseSync
.setProvider(
proc(key: string, subKey: int): Result[KeyedResponseSync, string] =
ok(KeyedResponseSync(key: "default-" & key, payload: $subKey))
)
.isOk()
check KeyedResponseSync
.setProvider(
BrokerContext(0xA0A0A0A0'u32),
proc(key: string, subKey: int): Result[KeyedResponseSync, string] =
ok(KeyedResponseSync(key: "k-" & key, payload: "p" & $subKey)),
)
.isOk()
let d = KeyedResponseSync.request("topic", 2)
check d.isOk()
check d.value.key == "default-topic"
let keyed = KeyedResponseSync.request(BrokerContext(0xA0A0A0A0'u32), "topic", 2)
check keyed.isOk()
check keyed.value.key == "k-topic"
check keyed.value.payload == "p2"
let miss = KeyedResponseSync.request(BrokerContext(0xB0B0B0B0'u32), "topic", 2)
check miss.isErr()
check miss.error.contains("no provider registered for broker context")
KeyedResponseSync.clearProvider()
## ---------------------------------------------------------------------------
## POD / external type brokers + tests (distinct/alias behavior)
## ---------------------------------------------------------------------------
type ExternalDefinedTypeAsync = object
label*: string
type ExternalDefinedTypeSync = object
label*: string
type ExternalDefinedTypeShared = object
label*: string
RequestBroker:
type PodResponse = int
proc signatureFetch*(): Future[Result[PodResponse, string]] {.async.}
RequestBroker:
type ExternalAliasedResponse = ExternalDefinedTypeAsync
proc signatureFetch*(): Future[Result[ExternalAliasedResponse, string]] {.async.}
RequestBroker(sync):
type ExternalAliasedResponseSync = ExternalDefinedTypeSync
proc signatureFetch*(): Result[ExternalAliasedResponseSync, string]
RequestBroker(sync):
type DistinctStringResponseA = distinct string
RequestBroker(sync):
type DistinctStringResponseB = distinct string
RequestBroker(sync):
type ExternalDistinctResponseA = distinct ExternalDefinedTypeShared
RequestBroker(sync):
type ExternalDistinctResponseB = distinct ExternalDefinedTypeShared
suite "RequestBroker macro (POD/external types)":
test "supports non-object response types (async)":
check PodResponse
.setProvider(
proc(): Future[Result[PodResponse, string]] {.async.} =
ok(PodResponse(123))
)
.isOk()
let res = waitFor PodResponse.request()
check res.isOk()
check int(res.value) == 123
PodResponse.clearProvider()
test "supports aliased external types (async)":
check ExternalAliasedResponse
.setProvider(
proc(): Future[Result[ExternalAliasedResponse, string]] {.async.} =
ok(ExternalAliasedResponse(ExternalDefinedTypeAsync(label: "ext")))
)
.isOk()
let res = waitFor ExternalAliasedResponse.request()
check res.isOk()
check ExternalDefinedTypeAsync(res.value).label == "ext"
ExternalAliasedResponse.clearProvider()
test "supports aliased external types (sync)":
check ExternalAliasedResponseSync
.setProvider(
proc(): Result[ExternalAliasedResponseSync, string] =
ok(ExternalAliasedResponseSync(ExternalDefinedTypeSync(label: "ext")))
)
.isOk()
let res = ExternalAliasedResponseSync.request()
check res.isOk()
check ExternalDefinedTypeSync(res.value).label == "ext"
ExternalAliasedResponseSync.clearProvider()
test "distinct response types avoid overload ambiguity (sync)":
check DistinctStringResponseA
.setProvider(
proc(): Result[DistinctStringResponseA, string] =
ok(DistinctStringResponseA("a"))
)
.isOk()
check DistinctStringResponseB
.setProvider(
proc(): Result[DistinctStringResponseB, string] =
ok(DistinctStringResponseB("b"))
)
.isOk()
check ExternalDistinctResponseA
.setProvider(
proc(): Result[ExternalDistinctResponseA, string] =
ok(ExternalDistinctResponseA(ExternalDefinedTypeShared(label: "ea")))
)
.isOk()
check ExternalDistinctResponseB
.setProvider(
proc(): Result[ExternalDistinctResponseB, string] =
ok(ExternalDistinctResponseB(ExternalDefinedTypeShared(label: "eb")))
)
.isOk()
let resA = DistinctStringResponseA.request()
let resB = DistinctStringResponseB.request()
check resA.isOk()
check resB.isOk()
check string(resA.value) == "a"
check string(resB.value) == "b"
let resEA = ExternalDistinctResponseA.request()
let resEB = ExternalDistinctResponseB.request()
check resEA.isOk()
check resEB.isOk()
check ExternalDefinedTypeShared(resEA.value).label == "ea"
check ExternalDefinedTypeShared(resEB.value).label == "eb"
DistinctStringResponseA.clearProvider()
DistinctStringResponseB.clearProvider()
ExternalDistinctResponseA.clearProvider()
ExternalDistinctResponseB.clearProvider()

4
tests/node/peer_manager/peer_store/test_migrations.nim
View File

@ -1,11 +1,11 @@
import std/[options], stew/results, testutils/unittests
import std/[options], results, testutils/unittests
import
waku/node/peer_manager/peer_store/migrations,
../../waku_archive/archive_utils,
../../testlib/[simple_mock]
import std/[tables, strutils, os], stew/results, chronicles
import std/[tables, strutils, os], results, chronicles
import waku/common/databases/db_sqlite, waku/common/databases/common

2
tests/node/peer_manager/peer_store/test_peer_storage.nim
View File

@ -1,4 +1,4 @@
import stew/results, testutils/unittests
import results, testutils/unittests
import waku/node/peer_manager/peer_store/peer_storage, waku/waku_core/peers

8
tests/node/test_wakunode_health_monitor.nim
View File

@ -2,6 +2,7 @@
import
std/[json, options, sequtils, strutils, tables], testutils/unittests, chronos, results
import brokers/broker_context
import
waku/[
@ -23,7 +24,6 @@ import
events/health_events,
events/peer_events,
waku_archive,
common/broker/broker_context,
]
import ../testlib/[wakunode, wakucore], ../waku_archive/archive_utils
@ -277,7 +277,7 @@ suite "Health Monitor - events":
await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()])
let metadataOk = await metadataFut.withTimeout(TestConnectivityTimeLimit)
WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis)
await WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis)
require metadataOk
let connectTimeLimit = Moment.now() + TestConnectivityTimeLimit
@ -380,7 +380,7 @@ suite "Health Monitor - events":
await nodeA.connectToNodes(@[nodeB.switch.peerInfo.toRemotePeerInfo()])
let metadataOk = await metadataFut.withTimeout(TestConnectivityTimeLimit)
WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis)
await WakuPeerEvent.dropListener(nodeA.brokerCtx, metadataLis)
require metadataOk
var deadline = Moment.now() + TestConnectivityTimeLimit
@ -413,7 +413,7 @@ suite "Health Monitor - events":
subMgr.subscribe(contentTopic).expect("Failed to subscribe")
let shardHealthOk = await shardHealthFut.withTimeout(TestConnectivityTimeLimit)
EventShardTopicHealthChange.dropListener(nodeA.brokerCtx, shardHealthLis)
await EventShardTopicHealthChange.dropListener(nodeA.brokerCtx, shardHealthLis)
check shardHealthOk == true
check subMgr.edgeFilterSubStates.len > 0

2
tests/node/test_wakunode_relay_rln.nim
View File

@ -2,7 +2,7 @@
import
std/[tempfiles, strutils, options],
stew/results,
results,
testutils/unittests,
chronos,
libp2p/switch,

9
tests/persistency/test_all.nim Normal file
View File

@ -0,0 +1,9 @@
{.used.}
import ./test_keys
import ./test_backend
import ./test_lifecycle
import ./test_facade
import ./test_encoding
import ./test_string_lookup
import ./test_singleton

195
tests/persistency/test_backend.nim Normal file
View File

@ -0,0 +1,195 @@
{.used.}
import std/options
import results
import testutils/unittests
import waku/persistency/[types, keys, backend_sqlite]
template str(b: seq[byte]): string =
var s = newString(b.len)
for i, x in b:
s[i] = char(x)
s
proc payload(s: string): seq[byte] =
result = newSeq[byte](s.len)
for i, c in s:
result[i] = byte(c)
suite "Persistency SQLite backend":
test "open in-memory backend and round-trip a single value":
let b = openBackendInMemory().get()
defer:
b.close()
b
.applyOps(
[
TxOp(
category: "msg",
key: key("c1", 1'i64),
kind: txPut,
payload: payload("hello"),
)
]
)
.get()
let got = b.getOne("msg", key("c1", 1'i64)).get()
check got.isSome
check str(got.get) == "hello"
check b.existsOne("msg", key("c1", 1'i64)).get()
check not b.existsOne("msg", key("c1", 2'i64)).get()
test "INSERT OR REPLACE overwrites payload for the same key":
let b = openBackendInMemory().get()
defer:
b.close()
let k = key("c1", 1'i64)
b.applyOps([TxOp(category: "msg", key: k, kind: txPut, payload: payload("v1"))]).get()
b.applyOps([TxOp(category: "msg", key: k, kind: txPut, payload: payload("v2"))]).get()
check str(b.getOne("msg", k).get().get) == "v2"
test "deleteOne reports whether the row existed":
let b = openBackendInMemory().get()
defer:
b.close()
let k = key("c1", 1'i64)
check not b.deleteOne("msg", k).get()
b.applyOps([TxOp(category: "msg", key: k, kind: txPut, payload: payload("x"))]).get()
check b.deleteOne("msg", k).get()
check not b.existsOne("msg", k).get()
test "applyOps batches multiple ops atomically":
let b = openBackendInMemory().get()
defer:
b.close()
b
.applyOps(
[
TxOp(
category: "msg", key: key("c1", 1'i64), kind: txPut, payload: payload("a")
),
TxOp(
category: "msg", key: key("c1", 2'i64), kind: txPut, payload: payload("b")
),
TxOp(
category: "msg", key: key("c1", 3'i64), kind: txPut, payload: payload("c")
),
]
)
.get()
check b.countRange("msg", prefixRange(key("c1"))).get() == 3
test "scanRange ascending yields rows in key order":
let b = openBackendInMemory().get()
defer:
b.close()
let inserts = @[5'i64, 1, 4, 2, 3]
var ops: seq[TxOp] = @[]
for i in inserts:
ops.add(
TxOp(category: "msg", key: key("c1", i), kind: txPut, payload: payload($i))
)
b.applyOps(ops).get()
let rows = b.scanRange("msg", prefixRange(key("c1"))).get()
check rows.len == 5
var seenOrder: seq[string]
for r in rows:
seenOrder.add(str(r.payload))
check seenOrder == @["1", "2", "3", "4", "5"]
test "scanRange descending yields rows in reverse key order":
let b = openBackendInMemory().get()
defer:
b.close()
for i in [1'i64, 2, 3]:
b
.applyOps(
[TxOp(category: "msg", key: key("c1", i), kind: txPut, payload: payload($i))]
)
.get()
let rows = b.scanRange("msg", prefixRange(key("c1")), reverse = true).get()
check rows.len == 3
check str(rows[0].payload) == "3"
check str(rows[2].payload) == "1"
test "scanRange respects half-open [start, stop) bounds":
let b = openBackendInMemory().get()
defer:
b.close()
for i in [1'i64, 2, 3, 4, 5]:
b
.applyOps(
[TxOp(category: "msg", key: key("c1", i), kind: txPut, payload: payload($i))]
)
.get()
let rng = KeyRange(start: key("c1", 2'i64), stop: key("c1", 4'i64))
let rows = b.scanRange("msg", rng).get()
check rows.len == 2 # 2 and 3, not 4
check str(rows[0].payload) == "2"
check str(rows[1].payload) == "3"
test "scanRange with empty stop is open-ended":
let b = openBackendInMemory().get()
defer:
b.close()
for i in [1'i64, 2, 3]:
b
.applyOps(
[TxOp(category: "msg", key: key("c1", i), kind: txPut, payload: payload($i))]
)
.get()
let rng = KeyRange(start: key("c1", 2'i64), stop: rawKey(@[]))
let rows = b.scanRange("msg", rng).get()
check rows.len == 2
check str(rows[1].payload) == "3"
test "categories isolate keyspaces":
let b = openBackendInMemory().get()
defer:
b.close()
let k = key("c1", 1'i64)
b
.applyOps(
[
TxOp(category: "log", key: k, kind: txPut, payload: payload("log-1")),
TxOp(
category: "outgoing", key: k, kind: txPut, payload: payload("outgoing-1")
),
]
)
.get()
check str(b.getOne("log", k).get().get) == "log-1"
check str(b.getOne("outgoing", k).get().get) == "outgoing-1"
check b.countRange("log", prefixRange(key("c1"))).get() == 1
check b.countRange("outgoing", prefixRange(key("c1"))).get() == 1
test "txDelete inside a batch removes the row":
let b = openBackendInMemory().get()
defer:
b.close()
let k = key("c1", 1'i64)
b
.applyOps(
[
TxOp(category: "msg", key: k, kind: txPut, payload: payload("v")),
TxOp(category: "msg", key: k, kind: txDelete),
]
)
.get()
check not b.existsOne("msg", k).get()
test "missing key returns none":
let b = openBackendInMemory().get()
defer:
b.close()
check b.getOne("msg", key("nope")).get().isNone
test "countRange of empty category is zero":
let b = openBackendInMemory().get()
defer:
b.close()
check b.countRange("msg", prefixRange(key("c1"))).get() == 0

154
tests/persistency/test_encoding.nim Normal file
View File

@ -0,0 +1,154 @@
{.used.}
import std/[algorithm, options, os, times]
import chronos, results
import testutils/unittests
import waku/persistency/persistency
# Reusable byte-wise comparator (Key has its own `<`, but we sometimes
# want to sort `seq[Key]` here without relying on it for double-checking).
proc cmpBytes(a, b: Key): int =
let ab = bytes(a)
let bb = bytes(b)
let n = min(ab.len, bb.len)
for i in 0 ..< n:
if ab[i] != bb[i]:
return cmp(ab[i], bb[i])
cmp(ab.len, bb.len)
template str(b: seq[byte]): string =
var s = newString(b.len)
for i, x in b:
s[i] = char(x)
s
# Shared payload types used by multiple tests.
type
Mood = enum
moodCalm
moodHappy
moodAngry
Header = object
sender: string
epoch: int64
Msg = object
header: Header
mood: Mood
body: seq[byte]
suite "Persistency generic encoding":
# ── Key macro: composite types ────────────────────────────────────────
test "key macro accepts plain tuples":
let k1 = key(("ch", 1'i64))
let k2 = key("ch", 1'i64)
# A plain tuple is encoded field-by-field, so the result is identical
# to passing the fields directly.
check k1 == k2
test "key macro accepts named tuples":
type Coord = tuple[lane: string, seqNum: int64]
let k = key((lane: "a", seqNum: 7'i64))
let kFlat = key("a", 7'i64)
check k == kFlat
test "key macro accepts a user object":
let k1 = key(Header(sender: "alice", epoch: 5'i64))
let k2 = key("alice", 5'i64)
check k1 == k2
test "key macro accepts nested object inside another arg":
let k1 = key("v1", Header(sender: "alice", epoch: 5'i64))
let k2 = key("v1", "alice", 5'i64)
check k1 == k2
test "key macro encodes enums":
let k1 = key(moodAngry)
let k2 = key(int64(ord(moodAngry)))
check k1 == k2
test "toKey is equivalent to single-arg key()":
check toKey("x") == key("x")
check toKey(42'i64) == key(42'i64)
check toKey(Header(sender: "a", epoch: 1)) == key("a", 1'i64)
test "tuple-encoded keys preserve field-major sort order":
let inputs = @[
key(("a", 0'i64)),
key(("a", 1'i64)),
key(("a", int64.high)),
key(("b", int64.low)),
key(("b", 0'i64)),
]
var shuffled = @[inputs[3], inputs[0], inputs[4], inputs[2], inputs[1]]
shuffled.sort(cmpBytes)
check shuffled == inputs
test "embedded Key encodes verbatim":
let inner = key("a", 7'i64)
let outer = key("prefix", inner)
# Expanded: bytes of "prefix" + raw bytes of inner.
let expanded = key("prefix", "a", 7'i64)
check outer == expanded
# ── Payload macro / toPayload ─────────────────────────────────────────
test "toPayload encodes primitives":
check str(toPayload("hi")).len == 4 # 2-byte len prefix + 2 chars
check toPayload(42'i64).len == 8
check toPayload(true) == @[1'u8]
check toPayload(false) == @[0'u8]
test "toPayload encodes objects field-by-field":
let m = Msg(
header: Header(sender: "alice", epoch: 9'i64),
mood: moodHappy,
body: @[0xAA'u8, 0xBB, 0xCC],
)
let p = toPayload(m)
let pManual = payload("alice", 9'i64, int64(ord(moodHappy)), @[0xAA'u8, 0xBB, 0xCC])
check p == pManual
test "payload macro concatenates parts":
let p = payload("v1", 1'i64, @[0xDE'u8, 0xAD])
# Same as building each piece separately.
var expected: seq[byte] = @[]
encodePart(expected, "v1")
encodePart(expected, 1'i64)
encodePart(expected, @[0xDE'u8, 0xAD])
check p == expected
# ── End-to-end through the facade ─────────────────────────────────────
asyncTest "persistEncoded round-trips a struct through SQLite":
let root = getTempDir() / ("persistency_enc_" & $epochTime().int)
removeDir(root)
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let job = p.openJob("t").get()
let m = Msg(
header: Header(sender: "alice", epoch: 1'i64),
mood: moodHappy,
body: @[1'u8, 2, 3],
)
let k = key("channel-42", m.header.epoch)
await job.persistEncoded("msg", k, m)
# Poll for the row, then read it back as raw bytes.
let deadline = epochTime() + 1.0
var got: Option[seq[byte]]
while epochTime() < deadline:
let r = await job.get("msg", k)
check r.isOk
got = r.get()
if got.isSome:
break
await sleepAsync(chronos.milliseconds(2))
check got.isSome
check got.get == toPayload(m)

196
tests/persistency/test_facade.nim Normal file
View File

@ -0,0 +1,196 @@
{.used.}
import std/[options, os, strutils, times]
import chronos, results
import testutils/unittests
import waku/persistency/persistency
proc payload(s: string): seq[byte] =
result = newSeq[byte](s.len)
for i, c in s:
result[i] = byte(c)
template str(b: seq[byte]): string =
var s = newString(b.len)
for i, x in b:
s[i] = char(x)
s
proc tmpRoot(label: string): string =
let p = getTempDir() / ("persistency_facade_" & label & "_" & $epochTime().int)
removeDir(p)
p
# Bounded poll on exists() to bridge the documented persist->read race.
proc waitUntilExists(
t: Job, category: string, k: Key, timeoutMs = 1000
): Future[bool] {.async.} =
let deadline = epochTime() + (timeoutMs.float / 1000.0)
while epochTime() < deadline:
let r = await t.exists(category, k)
if r.isOk and r.get():
return true
await sleepAsync(chronos.milliseconds(2))
return false
suite "Persistency facade":
asyncTest "persistPut then get round-trips":
let root = tmpRoot("put_get")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
let k = key("c", 1'i64)
await t.persistPut("msg", k, payload("hi"))
let ckOk1 = await t.waitUntilExists("msg", k)
check ckOk1
let aw1 = await t.get("msg", k)
let got = aw1.get()
check got.isSome
check str(got.get) == "hi"
asyncTest "persist (batch) is atomic and visible together":
let root = tmpRoot("batch")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
var ops: seq[TxOp]
for i in 1'i64 .. 4:
ops.add(
TxOp(category: "msg", key: key("c", i), kind: txPut, payload: payload($i))
)
await t.persist(ops)
let ckOk2 = await t.waitUntilExists("msg", key("c", 4'i64))
check ckOk2
let aw2 = await t.count("msg", prefixRange(key("c")))
let cnt = aw2.get()
check cnt == 4
asyncTest "scanPrefix returns rows in key order":
let root = tmpRoot("scan")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
for i in [3'i64, 1, 4, 1, 5, 9, 2]:
await t.persistPut("msg", key("c", i), payload($i))
let ckOk3 = await t.waitUntilExists("msg", key("c", 9'i64))
check ckOk3
let aw3 = await t.scanPrefix("msg", key("c"))
let rows = aw3.get()
# 7 ops with duplicate key i=1 -> 6 distinct rows
check rows.len == 6
var seenOrder: seq[int]
for r in rows:
seenOrder.add(parseInt(str(r.payload)))
check seenOrder == @[1, 2, 3, 4, 5, 9]
asyncTest "scanPrefix reverse=true returns rows in reverse order":
let root = tmpRoot("scan_rev")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
for i in 1'i64 .. 3:
await t.persistPut("msg", key("c", i), payload($i))
let ckOk4 = await t.waitUntilExists("msg", key("c", 3'i64))
check ckOk4
let aw4 = await t.scanPrefix("msg", key("c"), reverse = true)
let rows = aw4.get()
check rows.len == 3
check str(rows[0].payload) == "3"
check str(rows[2].payload) == "1"
asyncTest "deleteAcked round-trips and reports row presence":
let root = tmpRoot("delete")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
let k = key("c", 1'i64)
let aw5 = await t.deleteAcked("msg", k)
let miss = aw5.get()
check miss == false
await t.persistPut("msg", k, payload("v"))
let ckOk5 = await t.waitUntilExists("msg", k)
check ckOk5
let aw6 = await t.deleteAcked("msg", k)
let hit = aw6.get()
check hit == true
let aw7 = await t.exists("msg", k)
check aw7.get() == false
asyncTest "persistDelete fire-and-forget removes the row":
let root = tmpRoot("fadel")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
let k = key("c", 1'i64)
await t.persistPut("msg", k, payload("v"))
let ckOk6 = await t.waitUntilExists("msg", k)
check ckOk6
await t.persistDelete("msg", k)
# Poll for absence.
let deadline = epochTime() + 1.0
var gone = false
while epochTime() < deadline:
let aw8 = await t.exists("msg", k)
if not aw8.get():
gone = true
break
await sleepAsync(chronos.milliseconds(2))
check gone
asyncTest "two jobs do not see each other's data via the facade":
let root = tmpRoot("iso")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let a = p.openJob("a").get()
let b = p.openJob("b").get()
let k = key("c", 1'i64)
await a.persistPut("msg", k, payload("A"))
await b.persistPut("msg", k, payload("B"))
let ckOk7 = await a.waitUntilExists("msg", k)
check ckOk7
let ckOk8 = await b.waitUntilExists("msg", k)
check ckOk8
let aw9 = await a.get("msg", k)
check str(aw9.get().get) == "A"
let aw10 = await b.get("msg", k)
check str(aw10.get().get) == "B"
let aw11 = await a.count("msg", prefixRange(key("c")))
check aw11.get() == 1
let aw12 = await b.count("msg", prefixRange(key("c")))
check aw12.get() == 1

135
tests/persistency/test_keys.nim Normal file
View File

@ -0,0 +1,135 @@
{.used.}
import std/[algorithm, sequtils]
import testutils/unittests
import waku/persistency/[types, keys]
proc cmpBytes(a, b: Key): int =
let ab = bytes(a)
let bb = bytes(b)
let n = min(ab.len, bb.len)
for i in 0 ..< n:
if ab[i] != bb[i]:
return cmp(ab[i], bb[i])
cmp(ab.len, bb.len)
suite "Persistency keys":
test "string components sort by length, then byte order":
var ks = @[key("ab"), key(""), key("a"), key("aa"), key("b")]
ks.sort(cmpBytes)
# length-prefix encoding => shorter strings always sort before longer
# ones; same-length strings sort in byte order.
check ks == @[key(""), key("a"), key("b"), key("aa"), key("ab")]
test "same-length strings sort in byte order":
var ks = @[key("delta"), key("alpha"), key("gamma"), key("bravo")]
ks.sort(cmpBytes)
check ks == @[key("alpha"), key("bravo"), key("delta"), key("gamma")]
test "int64 sign-flip preserves order across negative/zero/positive":
let inputs = @[
key("c", int64.low),
key("c", -2'i64),
key("c", -1'i64),
key("c", 0'i64),
key("c", 1'i64),
key("c", 2'i64),
key("c", int64.high),
]
var shuffled = inputs
# rotate so the natural order is not the input order
shuffled = @[
shuffled[3],
shuffled[6],
shuffled[0],
shuffled[5],
shuffled[1],
shuffled[4],
shuffled[2],
]
shuffled.sort(cmpBytes)
check shuffled == inputs
test "uint64 big-endian preserves order":
let inputs = @[
key("u", 0'u64),
key("u", 1'u64),
key("u", 256'u64),
key("u", 1_000_000'u64),
key("u", uint64.high - 1),
key("u", uint64.high),
]
var shuffled = @[inputs[3], inputs[0], inputs[5], inputs[2], inputs[1], inputs[4]]
shuffled.sort(cmpBytes)
check shuffled == inputs
test "composite (string, string) tuple ordering":
# First component "a" / "b" — both length 1, so byte order applies.
# Second components grouped by first; within each group, again
# length-then-byte: "" (len 0) < "a","z" (len 1) < "ab" (len 2).
let inputs = @[
key("a", ""),
key("a", "a"),
key("a", "z"),
key("a", "ab"),
key("b", ""),
key("b", "a"),
]
var shuffled = inputs.reversed()
shuffled.sort(cmpBytes)
check shuffled == inputs
test "composite (string, int64) tuple ordering":
let inputs = @[
key("a", int64.low),
key("a", -1'i64),
key("a", 0'i64),
key("a", 1'i64),
key("b", int64.low),
key("b", 0'i64),
]
var shuffled = inputs.reversed()
shuffled.sort(cmpBytes)
check shuffled == inputs
test "shorter composite key precedes longer one sharing its prefix":
check key("a") < key("a", 0'i64)
check key("a") < key("a", "")
check key("a", "x") < key("a", "x", "y")
test "Key equality is byte-wise":
check key("a", 1'i64) == key("a", 1'i64)
check not (key("a", 1'i64) == key("a", 2'i64))
test "prefixRange.start equals prefix":
let r = prefixRange(key("a"))
check r.start == key("a")
test "prefixRange.stop excludes the prefix and admits all extensions":
let r = prefixRange(key("a"))
let extensions = @[
key("a"),
key("a", 0'i64),
key("a", int64.high),
key("a", "x"),
key("a", uint64.high),
]
for k in extensions:
check r.start <= k
check k < r.stop
test "prefixRange.stop excludes siblings outside the prefix":
let r = prefixRange(key("a"))
# "b" has the same encoded length as "a" but a higher last byte, so it
# should be at-or-above the exclusive stop.
check not (key("b") < r.stop)
# "ab" has more bytes — its 2-byte length prefix bumps it past stop.
check not (key("ab") < r.stop)
# The empty key sits before the start.
check key("") < r.start
test "prefixRange handles all-0xFF prefix as open-ended":
let prefix = rawKey(@[0xFF'u8, 0xFF, 0xFF])
let r = prefixRange(prefix)
check r.start == prefix
check bytes(r.stop).len == 0

302
tests/persistency/test_lifecycle.nim Normal file
View File

@ -0,0 +1,302 @@
{.used.}
import std/[options, os, times]
import chronos, results
import testutils/unittests
import brokers/[event_broker, request_broker]
import waku/persistency/persistency
import waku/persistency/backend_comm
proc payloadBytes(s: string): seq[byte] =
result = newSeq[byte](s.len)
for i, c in s:
result[i] = byte(c)
template str(b: seq[byte]): string =
var s = newString(b.len)
for i, x in b:
s[i] = char(x)
s
proc tmpRoot(label: string): string =
let p = getTempDir() / ("persistency_test_" & label & "_" & $epochTime().int)
removeDir(p)
p
# Cross-thread persist: emit a PersistEvent then poll until the row shows up
# via KvExists. The PersistEvent listener is fire-and-forget, so reads
# immediately after emit are racy by design (documented in v1).
proc pollExists(
t: Job, category: string, k: Key, timeoutMs = 1000
): Future[bool] {.async.} =
let deadline = epochTime() + (timeoutMs.float / 1000.0)
while epochTime() < deadline:
let r = await KvExists.request(t.context, category, k)
if r.isOk and r.get().value:
return true
await sleepAsync(chronos.milliseconds(2))
return false
suite "Persistency lifecycle":
test "Persistency.instance accepts a pre-existing rootDir":
let root = tmpRoot("preexisting")
defer:
removeDir(root)
createDir(root) # pretend a previous run left it
let marker = root / "do-not-touch.txt"
writeFile(marker, "hi")
defer:
removeFile(marker)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
# The pre-existing file is untouched.
check fileExists(marker)
check readFile(marker) == "hi"
test "Persistency.instance refuses a non-directory path":
let root = tmpRoot("collision")
defer:
removeFile(root)
writeFile(root, "im a file not a dir") # collide with rootDir name
let r = Persistency.instance(root)
check r.isErr
check r.error.kind == peInvalidArgument
test "Persistency.instance defers rootDir creation until first openJob":
let root = tmpRoot("lazy")
defer:
removeDir(root)
check not dirExists(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
# instance() must not have touched the filesystem
check not dirExists(root)
discard p.openJob("first").get()
# first openJob materialises the directory
check dirExists(root)
test "Persistency.instance refuses a path whose ancestor is not a directory":
let parent = tmpRoot("bad-parent")
defer:
removeFile(parent)
writeFile(parent, "not a directory")
let root = parent / "child"
let r = Persistency.instance(root)
check r.isErr
check r.error.kind == peInvalidArgument
asyncTest "openJob reuses an existing DB file across processes-of-one":
let root = tmpRoot("reopen")
defer:
removeDir(root)
# First "session": write something then close.
block firstSession:
let p = Persistency.instance(root).get()
let j = p.openJob("persist").get()
await j.persistPut("msg", key("c", 1'i64), payloadBytes("v1"))
let ckOk1 = await j.pollExists("msg", key("c", 1'i64))
check ckOk1
Persistency.reset()
check fileExists(root / "persist.db")
# Second "session": reopen and read the data back.
block secondSession:
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let j = p.openJob("persist").get()
let aw1 = await KvGet.request(j.context, "msg", key("c", 1'i64))
let got = aw1.get()
check got.value.isSome
check str(got.value.get) == "v1"
test "openJob is idempotent within a session":
let root = tmpRoot("idem")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let a = p.openJob("same").get()
let b = p.openJob("same").get()
check a.id == b.id
check a.context == b.context
test "openJob materialises rootDir and launches a worker":
let root = tmpRoot("basic")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("alpha").get()
check t.id == "alpha"
check t.running
check fileExists(root / "alpha.db")
asyncTest "persist then read round-trips via brokers":
let root = tmpRoot("rw")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t1").get()
let k = key("c", 1'i64)
let ev = PersistEvent(
ops: @[TxOp(category: "msg", key: k, kind: txPut, payload: payloadBytes("hello"))]
)
await PersistEvent.emit(t.context, ev)
let ckOk2 = await t.pollExists("msg", k)
check ckOk2
let aw2 = await KvGet.request(t.context, "msg", k)
let got = aw2.get()
check got.value.isSome
check str(got.value.get) == "hello"
asyncTest "two jobs run in parallel with isolated DBs":
let root = tmpRoot("isolation")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let a = p.openJob("alpha").get()
let b = p.openJob("beta").get()
check a.context != b.context
let k = key("shared", 1'i64)
await PersistEvent.emit(
a.context,
PersistEvent(
ops: @[
TxOp(
category: "msg", key: k, kind: txPut, payload: payloadBytes("from-alpha")
)
]
),
)
await PersistEvent.emit(
b.context,
PersistEvent(
ops: @[
TxOp(category: "msg", key: k, kind: txPut, payload: payloadBytes("from-beta"))
]
),
)
let ckOk3 = await a.pollExists("msg", k)
check ckOk3
let ckOk4 = await b.pollExists("msg", k)
check ckOk4
let aw3 = await KvGet.request(a.context, "msg", k)
let aGot = aw3.get()
let aw4 = await KvGet.request(b.context, "msg", k)
let bGot = aw4.get()
check str(aGot.value.get) == "from-alpha"
check str(bGot.value.get) == "from-beta"
# Each job has its own DB file.
check fileExists(root / "alpha.db")
check fileExists(root / "beta.db")
asyncTest "closeJob joins the worker and frees the slot":
let root = tmpRoot("close")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("x").get()
let ctx = t.context
p.closeJob("x")
check not t.running
# After close, requests on the old context have no provider.
let r = await KvExists.request(ctx, "msg", key("k"))
check r.isErr
test "dropJob removes the DB file":
let root = tmpRoot("drop")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
discard p.openJob("ephemeral").get()
check fileExists(root / "ephemeral.db")
p.dropJob("ephemeral")
check not fileExists(root / "ephemeral.db")
asyncTest "scan and count over a range":
let root = tmpRoot("scan")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
var ops: seq[TxOp]
for i in 1'i64 .. 5:
ops.add(
TxOp(category: "msg", key: key("c", i), kind: txPut, payload: payloadBytes($i))
)
await PersistEvent.emit(t.context, PersistEvent(ops: ops))
# Wait for the last insert to land.
let ckOk5 = await t.pollExists("msg", key("c", 5'i64))
check ckOk5
let rng = prefixRange(key("c"))
let aw5 = await KvCount.request(t.context, "msg", rng)
let cnt = aw5.get()
check cnt.n == 5
let aw6 = await KvScan.request(t.context, "msg", rng, false)
let scn = aw6.get()
check scn.rows.len == 5
check str(scn.rows[0].payload) == "1"
check str(scn.rows[4].payload) == "5"
asyncTest "acked delete reports whether the row existed":
let root = tmpRoot("delete")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let t = p.openJob("t").get()
let k = key("d", 1'i64)
let aw7 = await KvDelete.request(t.context, "msg", k)
let r1 = aw7.get()
check r1.existed == false
await PersistEvent.emit(
t.context,
PersistEvent(
ops: @[TxOp(category: "msg", key: k, kind: txPut, payload: payloadBytes("v"))]
),
)
let ckOk6 = await t.pollExists("msg", k)
check ckOk6
let aw8 = await KvDelete.request(t.context, "msg", k)
let r2 = aw8.get()
check r2.existed == true
let aw9 = await KvExists.request(t.context, "msg", k)
let r3 = aw9.get()
check r3.value == false

79
tests/persistency/test_singleton.nim Normal file
View File

@ -0,0 +1,79 @@
{.used.}
import std/[os, strutils, times]
import chronos, results
import testutils/unittests
import brokers/multi_request_broker
import waku/persistency/persistency
proc tmpRoot(label: string): string =
let p = getTempDir() / ("persistency_singleton_" & label & "_" & $epochTime().int)
removeDir(p)
p
suite "Persistency singleton":
test "instance(rootDir) is idempotent with the same rootDir":
let root = tmpRoot("idem")
defer:
removeDir(root)
defer:
Persistency.reset()
let p1 = Persistency.instance(root).get()
let p2 = Persistency.instance(root).get()
check p1 == p2
test "instance(rootDir) refuses re-init with a different rootDir":
let rootA = tmpRoot("a")
let rootB = tmpRoot("b")
defer:
removeDir(rootA)
defer:
removeDir(rootB)
defer:
Persistency.reset()
discard Persistency.instance(rootA).get()
let r = Persistency.instance(rootB)
check r.isErr
check r.error.kind == peInvalidArgument
test "no-arg instance() fails before init, succeeds after":
let root = tmpRoot("noarg")
defer:
removeDir(root)
defer:
Persistency.reset()
let before = Persistency.instance()
check before.isErr
check before.error.kind == peClosed
discard Persistency.instance(root).get()
let after = Persistency.instance()
check after.isOk
test "reset() makes the next instance() target a different rootDir":
let rootA = tmpRoot("rs-a")
let rootB = tmpRoot("rs-b")
defer:
removeDir(rootA)
defer:
removeDir(rootB)
defer:
Persistency.reset()
let pA = Persistency.instance(rootA).get()
check pA.rootDir == rootA
Persistency.reset()
let pB = Persistency.instance(rootB).get()
check pB.rootDir == rootB
check pA != pB
test "reset() is idempotent":
defer:
Persistency.reset()
Persistency.reset()
Persistency.reset()
check Persistency.instance().isErr

184
tests/persistency/test_string_lookup.nim Normal file
View File

@ -0,0 +1,184 @@
{.used.}
import std/[options, os, times]
import chronos, results
import testutils/unittests
import waku/persistency/persistency
proc payloadBytes(s: string): seq[byte] =
result = newSeq[byte](s.len)
for i, c in s:
result[i] = byte(c)
template str(b: seq[byte]): string =
var s = newString(b.len)
for i, x in b:
s[i] = char(x)
s
proc tmpRoot(label: string): string =
let p = getTempDir() / ("persistency_lookup_" & label & "_" & $epochTime().int)
removeDir(p)
p
# Bridge the persist->read race (writes are fire-and-forget in v1).
proc waitUntilExists(
p: Persistency, jobId, category: string, k: Key, timeoutMs = 1000
): Future[bool] {.async.} =
let deadline = epochTime() + (timeoutMs.float / 1000.0)
while epochTime() < deadline:
let r = await p.exists(jobId, category, k)
if r.isOk and r.get():
return true
await sleepAsync(chronos.milliseconds(2))
return false
suite "Persistency string-id lookup":
test "job(p, id) returns peJobNotFound when not open":
let root = tmpRoot("notfound")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let r = p.job("nope")
check r.isErr
check r.error.kind == peJobNotFound
test "job(p, id) returns the Job after openJob":
let root = tmpRoot("found")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let opened = p.openJob("alpha").get()
let looked = p.job("alpha").get()
check looked.id == "alpha"
check looked == opened # same ref, no need to peek at .context
test "hasJob mirrors p.job()":
let root = tmpRoot("has")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
check not p.hasJob("x")
discard p.openJob("x")
check p.hasJob("x")
p.closeJob("x")
check not p.hasJob("x")
test "subscript [] returns the open Job":
let root = tmpRoot("subscript")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
discard p.openJob("a").get()
let j = p["a"]
check j.id == "a"
asyncTest "string-lookup persistPut + get round-trips without a Job ref":
let root = tmpRoot("rw")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
discard p.openJob("svc").get()
let k = key("c", 1'i64)
await p.persistPut("svc", "msg", k, payloadBytes("hello"))
let ckOk1 = await p.waitUntilExists("svc", "msg", k)
check ckOk1
let aw1 = await p.get("svc", "msg", k)
let got = aw1.get()
check got.isSome
check str(got.get) == "hello"
asyncTest "string-lookup reads short-circuit with peJobNotFound":
let root = tmpRoot("missingread")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let g = await p.get("nope", "msg", key("k"))
check g.isErr
check g.error.kind == peJobNotFound
let c = await p.count("nope", "msg", prefixRange(key("k")))
check c.isErr
check c.error.kind == peJobNotFound
let d = await p.deleteAcked("nope", "msg", key("k"))
check d.isErr
check d.error.kind == peJobNotFound
asyncTest "string-lookup writes to an unknown job are dropped, not raised":
let root = tmpRoot("missingwrite")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
# Should not raise and should not leak any state.
await p.persistPut("ghost", "msg", key("k"), payloadBytes("v"))
await p.persistDelete("ghost", "msg", key("k"))
await p.persistEncoded("ghost", "msg", key("k"), 42'i64)
check not p.hasJob("ghost")
asyncTest "string-lookup persistEncoded round-trips a struct":
let root = tmpRoot("encoded")
defer:
removeDir(root)
type Item = object
tag: string
n: int64
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
discard p.openJob("e").get()
let k = key("items", 1'i64)
await p.persistEncoded("e", "msg", k, Item(tag: "alpha", n: 7))
let ckOk2 = await p.waitUntilExists("e", "msg", k)
check ckOk2
let aw2 = await p.get("e", "msg", k)
let got = aw2.get()
check got.isSome
check got.get == toPayload(Item(tag: "alpha", n: 7))
asyncTest "string-lookup scan returns the same rows as Job-form":
let root = tmpRoot("scan")
defer:
removeDir(root)
let p = Persistency.instance(root).get()
defer:
Persistency.reset()
let j = p.openJob("s").get()
for i in 1'i64 .. 3:
await p.persistPut("s", "msg", key("c", i), payloadBytes($i))
let ckOk3 = await p.waitUntilExists("s", "msg", key("c", 3'i64))
check ckOk3
let aw3 = await p.scanPrefix("s", "msg", key("c"))
let viaId = aw3.get()
let aw4 = await j.scanPrefix("msg", key("c"))
let viaRef = aw4.get()
check viaId.len == viaRef.len
for i in 0 ..< viaId.len:
check viaId[i].key == viaRef[i].key
check viaId[i].payload == viaRef[i].payload

388
tests/simulator/rln-e2e-test.py Executable file
View File

@ -0,0 +1,388 @@
#!/usr/bin/env python3
"""
RLN end-to-end test against a running logos-delivery-simulator stack.
Designed to run as a sidecar container on the simulator's Docker network so
hostnames like `logos-delivery-simulator-nwaku-1` resolve via Docker DNS.
Scenarios covered (in order):
1. HEALTH - every node responds to /debug/v1/info with an enrUri
2. SUBSCRIBE - every node REST-subscribes to the pubsub topic
3. WITHIN_LIMIT - every node concurrently sends msg_limit messages -> 200
4. PROPAGATION - one sender's message lands in all peers' inboxes
5. OVER_LIMIT - one extra message per node -> 500 (rate-limit hit)
6. EPOCH_RESET - after epoch_sec, every node can send 1 more -> 200
7. SAME_MESSAGE_ID - sending same message_id twice in same epoch is the
slashable signal (verified by checking node logs)
Exit code:
0 = all scenarios passed
N = number of scenarios that failed
Usage (typical):
docker run --rm \\
--network logos-delivery-simulator_simulation \\
-v /path/to/rln-e2e-test.py:/test.py \\
python:3.11-slim \\
sh -c 'pip install --quiet requests && python /test.py \\
--hostname-prefix logos-delivery-simulator-nwaku- \\
--num-nodes 30 --msg-limit 30 --epoch-sec 15'
"""
import argparse
import base64
import concurrent.futures as cf
import json
import os
import sys
import time
import urllib.parse
from dataclasses import dataclass
from typing import Optional
import requests
PUBSUB_TOPIC = "/waku/2/rs/66/0"
CONTENT_TOPIC = "/rln-test/1/probe/proto"
# ---------------------------------------------------------------------------
# helpers
# ---------------------------------------------------------------------------
def url_of(host: str, port: int = 8645) -> str:
return f"http://{host}:{port}"
def waku_publish(node_url: str, payload: bytes, timeout: float = 5.0) -> int:
body = {
"payload": base64.b64encode(payload).decode("ascii"),
"contentTopic": CONTENT_TOPIC,
"version": 1,
"timestamp": time.time_ns(),
}
enc = urllib.parse.quote(PUBSUB_TOPIC, safe="")
try:
r = requests.post(
f"{node_url}/relay/v1/messages/{enc}",
json=body,
timeout=timeout,
headers={"content-type": "application/json"},
)
return r.status_code
except requests.RequestException:
return -1
def waku_subscribe(node_url: str, timeout: float = 5.0) -> int:
try:
r = requests.post(
f"{node_url}/relay/v1/subscriptions",
json=[PUBSUB_TOPIC],
timeout=timeout,
headers={"content-type": "application/json"},
)
return r.status_code
except requests.RequestException:
return -1
def waku_get_messages(node_url: str, timeout: float = 5.0) -> Optional[list]:
enc = urllib.parse.quote(PUBSUB_TOPIC, safe="")
try:
r = requests.get(
f"{node_url}/relay/v1/messages/{enc}",
timeout=timeout,
)
if r.status_code != 200:
return None
return r.json()
except (requests.RequestException, json.JSONDecodeError):
return None
def node_healthy(node_url: str, timeout: float = 3.0) -> bool:
try:
r = requests.get(f"{node_url}/debug/v1/info", timeout=timeout)
return r.status_code == 200 and "enrUri" in r.json()
except (requests.RequestException, json.JSONDecodeError):
return False
# ---------------------------------------------------------------------------
# scenarios
# ---------------------------------------------------------------------------
@dataclass
class Result:
name: str
ok: bool
detail: str = ""
def __str__(self) -> str:
status = "PASS" if self.ok else "FAIL"
s = f"[{status}] {self.name}"
if self.detail:
s += f" — {self.detail}"
return s
def scenario_health(nodes: list[str], deadline_sec: int = 120) -> Result:
"""Every node must be reachable within deadline_sec."""
start = time.time()
unhealthy = list(nodes)
while time.time() - start < deadline_sec and unhealthy:
with cf.ThreadPoolExecutor(max_workers=min(32, len(unhealthy))) as ex:
results = list(ex.map(node_healthy, [url_of(n) for n in unhealthy]))
unhealthy = [n for n, ok in zip(unhealthy, results) if not ok]
if unhealthy:
time.sleep(3)
return Result(
"HEALTH",
not unhealthy,
f"{len(nodes) - len(unhealthy)}/{len(nodes)} healthy"
+ (f"; failing: {unhealthy[:5]}" if unhealthy else ""),
)
def scenario_subscribe(nodes: list[str]) -> Result:
"""REST-subscribe every node to the pubsub topic so GETs return cached msgs."""
with cf.ThreadPoolExecutor(max_workers=min(32, len(nodes))) as ex:
codes = list(ex.map(waku_subscribe, [url_of(n) for n in nodes]))
bad = [(n, c) for n, c in zip(nodes, codes) if c != 200]
return Result(
"SUBSCRIBE",
not bad,
f"{len(nodes) - len(bad)}/{len(nodes)} subscribed"
+ (f"; failing: {bad[:5]}" if bad else ""),
)
def _send_n(node_url: str, n: int) -> list[int]:
codes = []
for i in range(n):
codes.append(waku_publish(node_url, f"probe-{i}".encode()))
return codes
def _burst_until_blocked(node_url: str, msg_limit: int, overshoot: int = 3):
"""Send msg_limit+overshoot messages back-to-back, fast, recording codes.
Designed to complete inside a single epoch — keep epoch_sec large enough
that this burst can't straddle an epoch boundary.
Returns (n_200, n_500, n_transport_err, two_hundred_after_block) where
two_hundred_after_block flags a 200 appearing AFTER the first 500 (i.e.
quota reset mid-burst => epoch straddle)."""
codes = []
for i in range(msg_limit + overshoot):
codes.append(waku_publish(node_url, f"burst-{i}".encode(), timeout=10.0))
n_200 = sum(c == 200 for c in codes)
n_500 = sum(c == 500 for c in codes)
n_err = sum(c not in (200, 500) for c in codes) # -1, 4xx transient, etc.
first_block_idx = next((i for i, c in enumerate(codes) if c == 500), None)
two_hundred_after_block = (
first_block_idx is not None
and any(c == 200 for c in codes[first_block_idx + 1:])
)
return n_200, n_500, n_err, two_hundred_after_block
def _publish_until_ok(node_url: str, attempts: int = 20, spacing: float = 5.0) -> bool:
"""Retry a single publish until it returns 200 or attempts run out.
Tolerates the post-startup window where discv5/gossipsub mesh is still
forming and the RLN publish path transiently 500s."""
for _ in range(attempts):
if waku_publish(node_url, b"warmup", timeout=10.0) == 200:
return True
time.sleep(spacing)
return False
def scenario_warmup(nodes: list[str], attempts: int = 20) -> Result:
"""Readiness gate: every node must successfully publish at least once.
This absorbs mesh-formation churn so PROPAGATION/RATE_LIMIT aren't
judging a not-yet-connected fleet. Consumes 1 nonce/node — well within
msg_limit, and RATE_LIMIT's tolerance accounts for it."""
with cf.ThreadPoolExecutor(max_workers=min(8, len(nodes))) as ex:
ready = list(ex.map(lambda n: _publish_until_ok(url_of(n), attempts), nodes))
not_ready = [n for n, ok in zip(nodes, ready) if not ok]
return Result(
"WARMUP",
not not_ready,
f"{len(nodes) - len(not_ready)}/{len(nodes)} nodes publishing"
+ (f"; never ready: {not_ready[:5]}" if not_ready else ""),
)
def scenario_rate_limit(nodes: list[str], msg_limit: int, tolerance: int = 3) -> Result:
"""Per-node burst of msg_limit+3 messages within one epoch.
The RLN invariant being checked:
(a) a node must NEVER publish more than msg_limit in one epoch, and
(b) the node must enforce a 500 ceiling once the quota is exhausted.
Transient HTTP errors under concurrent load can lower the accepted count
below msg_limit — that does NOT violate the invariant, so we accept
successes in [msg_limit - tolerance, msg_limit]. successes > msg_limit OR
a 200 after the first 500 means the epoch rolled mid-burst (raise
RLN_RELAY_EPOCH_SEC) — reported as a timing skew, not an RLN failure."""
# Cap concurrency: firing len(nodes)*(msg_limit+3) publishes all at once
# saturates small CI runners (2 vCPU) and causes publish-path timeouts
# that masquerade as rate-limit failures.
with cf.ThreadPoolExecutor(max_workers=min(5, len(nodes))) as ex:
per_node = list(
ex.map(lambda n: _burst_until_blocked(url_of(n), msg_limit), nodes)
)
rate_failures = [] # genuine RLN misbehaviour
timing_skews = [] # epoch straddled mid-burst — inconclusive
for node, (n_200, n_500, n_err, after_block) in zip(nodes, per_node):
if n_200 > msg_limit or after_block:
timing_skews.append(
(node, f"{n_200} ok, epoch rolled mid-burst (raise epoch_sec)")
)
elif n_500 == 0:
rate_failures.append((node, f"no 500 ceiling ({n_200} ok, {n_err} err)"))
elif n_200 < msg_limit - tolerance:
rate_failures.append(
(node, f"only {n_200}/{msg_limit} ok ({n_err} transport err)")
)
if timing_skews and not rate_failures:
return Result(
"RATE_LIMIT",
False,
f"INCONCLUSIVE (timing) — raise RLN_RELAY_EPOCH_SEC; "
f"{len(timing_skews)} node(s) straddled an epoch: {timing_skews[:3]}",
)
ok = not rate_failures and not timing_skews
good = len(nodes) - len(rate_failures) - len(timing_skews)
return Result(
"RATE_LIMIT",
ok,
f"{good}/{len(nodes)} nodes enforced <= {msg_limit} then 500 "
f"(tolerance {tolerance} for transport noise)"
+ (f"; rate failures: {rate_failures[:3]}" if rate_failures else "")
+ (f"; timing skews: {timing_skews[:3]}" if timing_skews else ""),
)
def scenario_propagation(
sender: str, receivers: list[str], settle_sec: int = 5
) -> Result:
"""Send one message on `sender`, expect it visible in every receiver's
REST inbox within settle_sec."""
marker = f"propagation-marker-{time.time_ns()}".encode()
code = waku_publish(url_of(sender), marker)
if code != 200:
return Result("PROPAGATION", False, f"sender publish returned {code}")
time.sleep(settle_sec)
missing = []
with cf.ThreadPoolExecutor(max_workers=min(32, len(receivers))) as ex:
inboxes = list(ex.map(waku_get_messages, [url_of(r) for r in receivers]))
encoded_marker = base64.b64encode(marker).decode().rstrip("=")
for r, inbox in zip(receivers, inboxes):
if inbox is None:
missing.append((r, "GET failed"))
continue
# Look for our marker payload in any message
found = any(
(m.get("payload") or "").rstrip("=") == encoded_marker
for m in inbox
)
if not found:
missing.append((r, f"{len(inbox)} msgs, marker not present"))
return Result(
"PROPAGATION",
not missing,
f"{len(receivers) - len(missing)}/{len(receivers)} receivers got the message"
+ (f"; missing on {missing[:3]}" if missing else ""),
)
def scenario_epoch_reset(nodes: list[str], epoch_sec: int) -> Result:
"""After epoch_sec + slack, each node can send 1 more message — expect 200."""
sleep_s = epoch_sec + 3
print(f" sleeping {sleep_s}s for epoch reset...")
time.sleep(sleep_s)
with cf.ThreadPoolExecutor(max_workers=len(nodes)) as ex:
codes = list(
ex.map(
lambda n: waku_publish(url_of(n), b"post-epoch"),
nodes,
)
)
bad = [(n, c) for n, c in zip(nodes, codes) if c != 200]
return Result(
"EPOCH_RESET",
not bad,
f"{sum(c == 200 for c in codes)}/{len(nodes)} returned 200 after epoch reset"
+ (f"; failing: {bad[:3]}" if bad else ""),
)
# ---------------------------------------------------------------------------
# main
# ---------------------------------------------------------------------------
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--hostname-prefix", default="logos-delivery-simulator-nwaku-")
ap.add_argument("--num-nodes", type=int, default=30)
ap.add_argument("--msg-limit", type=int, default=30,
help="Must match RLN_RELAY_MSG_LIMIT in simulator .env")
ap.add_argument("--epoch-sec", type=int, default=15,
help="Must match RLN_RELAY_EPOCH_SEC in simulator .env")
ap.add_argument("--health-deadline-sec", type=int, default=180)
args = ap.parse_args()
nodes = [f"{args.hostname_prefix}{i}" for i in range(1, args.num_nodes + 1)]
print(f"Testing {len(nodes)} nodes: {nodes[0]} … {nodes[-1]}")
print(f"Config: msg_limit={args.msg_limit}, epoch_sec={args.epoch_sec}")
print()
results: list[Result] = []
def run(scenario_fn, *fn_args, **fn_kwargs) -> bool:
r = scenario_fn(*fn_args, **fn_kwargs)
results.append(r)
print(r)
return r.ok
if not run(scenario_health, nodes, deadline_sec=args.health_deadline_sec):
print("\nABORTING — nodes never reached healthy state.")
return _summarize(results)
if not run(scenario_subscribe, nodes):
print("\nABORTING — could not subscribe nodes to pubsub topic.")
return _summarize(results)
# Readiness gate: wait out mesh-formation churn before judging behaviour.
if not run(scenario_warmup, nodes):
print("\nABORTING — fleet never reached a publishable state.")
return _summarize(results)
run(scenario_propagation, nodes[0], nodes[1:])
# Rate limit: per-node burst, asserts exactly msg_limit then 500.
# Requires epoch_sec large enough that the burst can't straddle an epoch.
run(scenario_rate_limit, nodes, args.msg_limit)
run(scenario_epoch_reset, nodes, args.epoch_sec)
return _summarize(results)
def _summarize(results: list[Result]) -> int:
print()
print("=" * 64)
passed = sum(r.ok for r in results)
print(f" {passed}/{len(results)} scenarios passed")
for r in results:
print(f" {r}")
print("=" * 64)
return len(results) - passed
if __name__ == "__main__":
sys.exit(main())

6
tests/simulator/rln-sim.env Normal file
View File

@ -0,0 +1,6 @@
# Source of truth for the RLN simulator E2E run (ci-rln-simulator.yml).
# workflow_dispatch inputs override any value here per-run (blank input = use this file).
BRANCH=master
NUM_NODES=6
MSG_LIMIT=30
EPOCH_SEC=120

38
tests/test_peer_manager.nim
View File

@ -54,6 +54,44 @@ procSuite "Peer Manager":
nodes[0].peerManager.switch.peerStore.connectedness(nodes[1].peerInfo.peerId) ==
Connectedness.Connected
asyncTest "Peer manager tracks active store request state":
let nodes = toSeq(0 ..< 2).mapIt(
newTestWakuNode(generateSecp256k1Key(), parseIpAddress("0.0.0.0"), Port(0))
)
await allFutures(nodes.mapIt(it.start()))
await allFutures(nodes.mapIt(it.mountRelay()))
let peerId = nodes[1].peerInfo.peerId
require (
await nodes[0].peerManager.connectPeer(nodes[1].peerInfo.toRemotePeerInfo())
)
await sleepAsync(chronos.milliseconds(500))
nodes[0].peerManager.addActiveStoreRequest(peerId)
check:
nodes[0].peerManager.hasActiveStoreRequest(peerId)
await nodes[0].peerManager.evictPeer(peerId)
await sleepAsync(chronos.milliseconds(100))
check:
nodes[0].peerManager.switch.peerStore.connectedness(peerId) ==
Connectedness.Connected
nodes[0].peerManager.removeActiveStoreRequest(peerId)
check:
not nodes[0].peerManager.hasActiveStoreRequest(peerId)
await nodes[0].peerManager.evictPeer(peerId)
await sleepAsync(chronos.milliseconds(100))
check:
nodes[0].peerManager.switch.peerStore.connectedness(peerId) !=
Connectedness.Connected
await allFutures(nodes.mapIt(it.stop()))
asyncTest "dialPeer() works":
# Create 2 nodes
let nodes = toSeq(0 ..< 2).mapIt(

2
tests/test_utils_compat.nim
View File

@ -1,7 +1,7 @@
{.used.}
import testutils/unittests
import stew/results, waku/waku_core/message, waku/waku_core/time, ./testlib/common
import results, waku/waku_core/message, waku/waku_core/time, ./testlib/common
suite "Waku Payload":
test "Encode/Decode waku message with timestamp":

2
tests/waku_enr/test_sharding.nim
View File

@ -1,7 +1,7 @@
{.used.}
import
stew/results,
results,
chronos,
testutils/unittests,
libp2p/crypto/crypto as libp2p_keys,

5
tests/waku_lightpush_legacy/lightpush_utils.nim
View File

@ -1,6 +1,9 @@
{.used.}
import std/options, chronos, libp2p/crypto/crypto
import std/options, chronos, chronicles, libp2p/crypto/crypto
logScope:
topics = "test waku_lightpush_legacy"
import
waku/node/peer_manager,

10
tests/waku_relay/utils.nim
View File

@ -8,17 +8,13 @@ import
libp2p/switch,
libp2p/protocols/pubsub/pubsub
import brokers/broker_context
from std/times import epochTime
import
waku/[
waku_relay,
node/waku_node,
node/peer_manager,
waku_core,
waku_node,
waku_rln_relay,
common/broker/broker_context,
waku_relay, node/waku_node, node/peer_manager, waku_core, waku_node, waku_rln_relay
],
../waku_store/store_utils,
../waku_archive/archive_utils,

15
tests/waku_rln_relay/rln/buffer_utils.nim
View File

@ -1,11 +1,4 @@
import waku/waku_rln_relay/rln/rln_interface
proc `==`*(a: Buffer, b: seq[uint8]): bool =
if a.len != uint(b.len):
return false
let bufferArray = cast[ptr UncheckedArray[uint8]](a.ptr)
for i in 0 ..< b.len:
if bufferArray[i] != b[i]:
return false
return true
# buffer_utils.nim — intentionally empty.
# The v0.9 Buffer type and toBuffer helper were removed in the zerokit v2.0.1
# migration. This file is kept as a placeholder so that any future test imports
# do not break the build; the content that was here is no longer needed.

37
tests/waku_rln_relay/rln/test_rln_interface.nim
View File

@ -1,17 +1,36 @@
import testutils/unittests
import testutils/unittests, results
import waku/waku_rln_relay/rln/rln_interface, ./buffer_utils
import waku/waku_rln_relay/rln/rln_interface
import waku/waku_rln_relay/rln/wrappers
suite "Buffer":
suite "toBuffer":
suite "Vec_uint8":
suite "toVecUint8":
test "valid":
# Given
let bytes: seq[byte] = @[0x01, 0x02, 0x03]
# When
let buffer = bytes.toBuffer()
# When — wrap as a Vec_uint8 view then read the bytes back
var vec = toVecUint8(bytes)
let roundtrip = vecToSeq(vec)
# Then
let expectedBuffer: seq[uint8] = @[1, 2, 3]
# Then — byte values are preserved
check:
buffer == expectedBuffer
roundtrip == bytes
suite "RlnConfig":
suite "createRLNInstance":
test "ok":
# When we create the RLN instance (stateless build — no tree_depth arg)
let rlnRes = createRLNInstance()
# Then it succeeds
check:
rlnRes.isOk()
test "default":
# When we create the RLN instance
let rlnRes = createRLNInstance()
# Then it succeeds
check:
rlnRes.isOk()

117
tests/waku_rln_relay/rln/test_wrappers.nim
View File

@ -1,37 +1,6 @@
import
std/options,
testutils/unittests,
chronicles,
chronos,
eth/keys,
bearssl,
stew/[results],
metrics,
metrics/chronos_httpserver
import testutils/unittests, results
import
waku/waku_rln_relay,
waku/waku_rln_relay/rln,
waku/waku_rln_relay/rln/wrappers,
./waku_rln_relay_utils,
../../testlib/[simple_mock, assertions],
../../waku_keystore/utils,
../../testlib/testutils
from std/times import epochTime
const Empty32Array = default(array[32, byte])
proc valid(x: seq[byte]): bool =
if x.len != 32:
error "Length should be 32", length = x.len
return false
if x == Empty32Array:
error "Should not be empty array", array = x
return false
return true
import waku/waku_rln_relay/rln, waku/waku_rln_relay/rln/wrappers, ./waku_rln_relay_utils
suite "membershipKeyGen":
test "ok":
@ -41,60 +10,20 @@ suite "membershipKeyGen":
# Then it contains valid identity credentials
let identityCredentials = identityCredentialsRes.get()
proc nonEmpty(x: seq[byte]): bool =
x.len == 32 and x != newSeq[byte](32)
check:
identityCredentials.idTrapdoor.valid()
identityCredentials.idNullifier.valid()
identityCredentials.idSecretHash.valid()
identityCredentials.idCommitment.valid()
test "done is false":
# Given the key_gen function fails
let backup = key_gen
mock(key_gen):
proc keyGenMock(ctx: ptr RLN, output_buffer: ptr Buffer): bool =
return false
keyGenMock
# When we generate the membership keys
let identityCredentialsRes = membershipKeyGen()
# Then it fails
check:
identityCredentialsRes.error() == "error in key generation"
# Cleanup
mock(key_gen):
backup
test "generatedKeys length is not 128":
# Given the key_gen function succeeds with wrong values
let backup = key_gen
mock(key_gen):
proc keyGenMock(ctx: ptr RLN, output_buffer: ptr Buffer): bool =
echo "# RUNNING MOCK"
output_buffer.len = 0
output_buffer.ptr = cast[ptr uint8](newSeq[byte](0))
return true
keyGenMock
# When we generate the membership keys
let identityCredentialsRes = membershipKeyGen()
# Then it fails
check:
identityCredentialsRes.error() == "keysBuffer is of invalid length"
# Cleanup
mock(key_gen):
backup
identityCredentials.idTrapdoor.nonEmpty()
identityCredentials.idNullifier.nonEmpty()
identityCredentials.idSecretHash.nonEmpty()
identityCredentials.idCommitment.nonEmpty()
suite "RlnConfig":
suite "createRLNInstance":
test "ok":
# When we create the RLN instance
let rlnRes: RLNResult = createRLNInstance(15)
# When we create the RLN instance (stateless build — no tree_depth arg)
let rlnRes = createRLNInstance()
# Then it succeeds
check:
@ -102,30 +31,8 @@ suite "RlnConfig":
test "default":
# When we create the RLN instance
let rlnRes: RLNResult = createRLNInstance()
let rlnRes = createRLNInstance()
# Then it succeeds
check:
rlnRes.isOk()
test "new_circuit fails":
# Given the new_circuit function fails
let backup = new_circuit
mock(new_circuit):
proc newCircuitMock(
tree_height: uint, input_buffer: ptr Buffer, ctx: ptr (ptr RLN)
): bool =
return false
newCircuitMock
# When we create the RLN instance
let rlnRes: RLNResult = createRLNInstance(15)
# Then it fails
check:
rlnRes.error() == "error in parameters generation"
# Cleanup
mock(new_circuit):
backup

86
tests/waku_rln_relay/test_rln_group_manager_onchain.nim
View File

@ -3,7 +3,7 @@
{.push raises: [].}
import
std/[options, sequtils, deques, random, locks, osproc],
std/[options, sequtils, deques, random, locks, osproc, algorithm],
results,
stew/byteutils,
testutils/unittests,
@ -253,6 +253,9 @@ suite "Onchain group manager":
manager.merkleProofCache = newSeq[byte](640)
for i in 0 ..< 640:
manager.merkleProofCache[i] = byte(rand(255))
# chunk[0] becomes the MSB after reversal in group_manager; must be < 0x30
for i in 0 ..< 20:
manager.merkleProofCache[i * 32] = 0
let messageBytes = "Hello".toBytes()
@ -335,6 +338,9 @@ suite "Onchain group manager":
manager.merkleProofCache = newSeq[byte](640)
for i in 0 ..< 640:
manager.merkleProofCache[i] = byte(rand(255))
# chunk[0] becomes the MSB after reversal in group_manager; must be < 0x30
for i in 0 ..< 20:
manager.merkleProofCache[i * 32] = 0
let epoch = default(Epoch)
info "epoch in bytes", epochHex = epoch.inHex()
@ -419,3 +425,81 @@ suite "Onchain group manager":
check:
isReady == true
test "proof roundtrip: generateRlnProofWithWitness -> verifyRlnProof":
## Smoke test: proof gen -> wire serialize -> deserialize -> ffi_verify_with_roots.
let credentials = generateCredentials()
(waitFor manager.init()).isOkOr:
raiseAssert $error
(waitFor manager.register(credentials, UserMessageLimit(20))).isOkOr:
assert false, "register failed: " & error
discard waitFor manager.updateRoots()
let roots = manager.validRoots.items().toSeq()
require:
roots.len > 0
let proofElements = (waitFor manager.fetchMerkleProofElements()).valueOr:
raiseAssert "fetchMerkleProofElements failed: " & error
let signal = "Hello, RLN!".toBytes()
let epoch = default(Epoch)
# Build RLNWitnessInput the same way group_manager.generateProof does.
var pathElements = newSeq[byte]()
for i in 0 ..< proofElements.len div 32:
pathElements.add(proofElements[i * 32 .. (i + 1) * 32 - 1].reversed())
let xCfr = hashToFieldLe(signal).valueOr:
raiseAssert "hashToFieldLe failed: " & error
defer:
ffi_cfr_free(xCfr)
let x = cfrToBytesLe(xCfr).valueOr:
raiseAssert "cfrToBytesLe failed: " & error
let extNullifier = generateExternalNullifier(epoch, DefaultRlnIdentifier).valueOr:
raiseAssert "generateExternalNullifier failed: " & error
let witness = RLNWitnessInput(
identity_secret: seqToField(credentials.idSecretHash),
user_message_limit: uint64ToField(uint64(UserMessageLimit(20))),
message_id: uint64ToField(uint64(MessageId(1))),
path_elements: pathElements,
identity_path_index: uint64ToIndex(manager.membershipIndex.get(), 20),
x: x,
external_nullifier: extNullifier,
)
# Step 1: generate proof via the FFI wrapper
let proof = generateRlnProofWithWitness(
manager.rlnInstance, witness, epoch, DefaultRlnIdentifier
).valueOr:
raiseAssert "generateRlnProofWithWitness failed: " & error
let zeroField = default(array[32, byte])
check:
proof.merkleRoot != zeroField
proof.nullifier != zeroField
# Step 2: serialize -> deserialize -> verify (the actual roundtrip)
let verified = verifyRlnProof(manager.rlnInstance, proof, signal, roots).valueOr:
raiseAssert "verifyRlnProof failed: " & error
check verified == true
# Step 3: wrong signal -> x mismatch -> false
let wrongSignalVerified = verifyRlnProof(
manager.rlnInstance, proof, "wrong".toBytes(), roots
).valueOr:
raiseAssert "verifyRlnProof (wrong signal) failed: " & error
check wrongSignalVerified == false
# Step 4: bad root -> root not in set -> false
# byte[31] in LE is the MSB; 0x01 < 0x30 so this is a canonical field element.
var badRoot: MerkleNode
for i in 0 ..< 32:
badRoot[i] = 0x01
let badRootVerified = verifyRlnProof(manager.rlnInstance, proof, signal, @[badRoot]).valueOr:
raiseAssert "verifyRlnProof (bad root) failed: " & error
check badRootVerified == false

49
tests/waku_rln_relay/test_waku_rln_relay.nim
View File

@ -1,13 +1,16 @@
{.used.}
import
std/[options, os, sequtils, tempfiles, strutils, osproc],
std/[options, os, sequtils, tempfiles, strutils, osproc, algorithm],
stew/byteutils,
testutils/unittests,
chronos,
chronicles,
stint,
libp2p/crypto/crypto
import brokers/broker_context
import
waku/[
waku_core,
@ -15,7 +18,6 @@ import
waku_rln_relay/rln,
waku_rln_relay/protocol_metrics,
waku_keystore,
common/broker/broker_context,
],
./rln/waku_rln_relay_utils,
./utils_onchain,
@ -34,23 +36,16 @@ suite "Waku rln relay":
teardown:
stopAnvil(anvilProc)
test "key_gen Nim Wrappers":
let merkleDepth: csize_t = 20
test "ffi_extended_key_gen raw FFI":
# When we call the raw key-generation FFI
var vec = ffi_extended_key_gen()
# keysBufferPtr will hold the generated identity credential i.e., id trapdoor, nullifier, secret hash and commitment
var keysBuffer: Buffer
let
keysBufferPtr = addr(keysBuffer)
done = key_gen(keysBufferPtr, true)
require:
# check whether the keys are generated successfully
done
let generatedKeys = cast[ptr array[4 * 32, byte]](keysBufferPtr.`ptr`)[]
# Then it returns exactly 4 field elements
# (idTrapdoor, idNullifier, idSecretHash, idCommitment — each 32 bytes)
defer:
ffi_vec_cfr_free(vec)
check:
# the id trapdoor, nullifier, secert hash and commitment together are 4*32 bytes
generatedKeys.len == 4 * 32
info "generated keys: ", generatedKeys
int(ffi_vec_cfr_len(addr vec)) == 4
test "membership Key Generation":
let idCredentialsRes = membershipKeyGen()
@ -78,18 +73,22 @@ suite "Waku rln relay":
rlnInstance.isOk()
let rln = rlnInstance.get()
# prepare the input
let msg = @[
"126f4c026cd731979365f79bd345a46d673c5a3f6f588bdc718e6356d02b6fdc".toBytes(),
"1f0e5db2b69d599166ab16219a97b82b662085c93220382b39f9f911d3b943b1".toBytes(),
]
# prepare the input — hex-decoded then reversed to little-endian field elements
let
left = hexToSeqByte(
"126f4c026cd731979365f79bd345a46d673c5a3f6f588bdc718e6356d02b6fdc"
)
.reversed()
right = hexToSeqByte(
"1f0e5db2b69d599166ab16219a97b82b662085c93220382b39f9f911d3b943b1"
)
.reversed()
let hashRes = poseidon(msg)
let hashRes = poseidon(left, right)
# Value taken from zerokit
check:
hashRes.isOk()
"28a15a991fe3d2a014485c7fa905074bfb55c0909112f865ded2be0a26a932c3" ==
"180543bc9afb81d9c2282df9c9946f87b4596cf6d3fec2cc32b6637427685353" ==
hashRes.get().inHex()
test "RateLimitProof Protobuf encode/init test":

7
tests/waku_rln_relay/test_wakunode_rln_relay.nim
View File

@ -7,13 +7,14 @@ import
chronicles,
chronos,
libp2p/switch,
libp2p/protocols/pubsub/pubsub
libp2p/protocols/pubsub/pubsub,
brokers/broker_context
import
waku/[waku_core, waku_node, waku_rln_relay],
../testlib/[wakucore, futures, wakunode, testutils],
./utils_onchain,
./rln/waku_rln_relay_utils,
waku/common/broker/broker_context
./rln/waku_rln_relay_utils
from std/times import epochTime

23
tests/waku_store/test_client.nim
View File

@ -1,6 +1,6 @@
{.used.}
import std/options, testutils/unittests, chronos, libp2p/crypto/crypto
import std/[options, sets], testutils/unittests, chronos, libp2p/crypto/crypto
import
waku/[node/peer_manager, waku_core, waku_store, waku_store/client, common/paging],
@ -223,3 +223,24 @@ suite "Store Client":
not await handlerFuture.withTimeout(FUTURE_TIMEOUT)
queryResponse.isErr()
queryResponse.error.kind == ErrorCode.PEER_DIAL_FAILURE
asyncTest "queryToAny shuffles peers across calls":
# Register several fake store peers (no servers running) so every dial
# fails. PEER_DIAL_FAILURE carries the peerId of the last peer tried in
# the shuffled order, so observing different "last" peerIds across calls
# confirms shuffle is active inside queryToAny.
for _ in 0 ..< 3:
let fakeSwitch = newTestSwitch()
let peerInfo = fakeSwitch.peerInfo.toRemotePeerInfo()
peerInfo.protocols = @[WakuStoreCodec]
clientSwitch.peerStore.addPeer(peerInfo)
var observedLastPeers: HashSet[string]
for _ in 0 ..< 20:
let res = await client.queryToAny(storeQuery)
check:
res.isErr()
res.error.kind == ErrorCode.PEER_DIAL_FAILURE
observedLastPeers.incl(res.error.address)
check observedLastPeers.len >= 2

2
tests/wakunode_rest/test_rest_relay.nim
View File

@ -7,6 +7,7 @@ import
presto,
presto/client as presto_client,
libp2p/crypto/crypto
import brokers/broker_context
import
waku/[
common/base64,
@ -21,7 +22,6 @@ import
rest_api/endpoint/relay/client as relay_rest_client,
waku_relay,
waku_rln_relay,
common/broker/broker_context,
],
../testlib/wakucore,
../testlib/wakunode,

12
tools/confutils/cli_args.nim
View File

@ -168,7 +168,7 @@ type WakuNodeConf* = object
preset* {.
desc:
"Network preset to use. 'twn' is The RLN-protected Waku Network (cluster 1). 'logos.dev' is the Logos Dev Network (cluster 2). Overrides other values.",
"Network preset to use. 'twn' is The RLN-protected Waku Network (cluster 1). 'logos.dev' is the Logos Dev Network (cluster 2). 'logos.test' is the Logos Test Network (cluster 2). Overrides other values.",
defaultValue: "",
name: "preset"
.}: string
@ -717,6 +717,12 @@ hence would have reachability issues.""",
name: "rate-limit"
.}: seq[string]
localStoragePath* {.
desc: "Path to store local data.",
defaultValue: "./data",
name: "local-storage-path"
.}: string
## Parsing
# NOTE: Keys are different in nim-libp2p
@ -960,6 +966,8 @@ proc toNetworkConf(
ok(some(NetworkConf.TheWakuNetworkConf()))
of "logos.dev", "logosdev":
ok(some(NetworkConf.LogosDevConf()))
of "logos.test", "logostest":
ok(some(NetworkConf.LogosTestConf()))
else:
err("Invalid --preset value passed: " & lcPreset)
@ -1145,6 +1153,8 @@ proc toWakuConf*(n: WakuNodeConf): ConfResult[WakuConf] =
if n.rateLimits.len > 0:
b.rateLimitConf.withRateLimits(n.rateLimits)
b.withLocalStoragePath(n.localStoragePath)
if n.enableKadDiscovery.isSome():
b.kademliaDiscoveryConf.withEnabled(n.enableKadDiscovery.get())
b.kademliaDiscoveryConf.withBootstrapNodes(n.kadBootstrapNodes)

10
tools/gen-nix-deps.sh
View File

@ -36,7 +36,10 @@ fi
echo "[*] Generating $OUTFILE from $LOCKFILE"
mkdir -p "$(dirname "$OUTFILE")"
cat > "$OUTFILE" <<'EOF'
TMPFILE=$(mktemp)
trap 'rm -f "$TMPFILE"' EXIT
cat > "$TMPFILE" <<'EOF'
# AUTOGENERATED from nimble.lock — do not edit manually.
# Regenerate with: ./tools/gen-nix-deps.sh nimble.lock nix/deps.nix
{ pkgs }:
@ -62,7 +65,7 @@ jq -c '
--fetch-submodules \
| jq -r '.sha256')
cat >> "$OUTFILE" <<EOF
cat >> "$TMPFILE" <<EOF
${name} = pkgs.fetchgit {
url = "${url}";
rev = "${rev}";
@ -73,8 +76,9 @@ jq -c '
EOF
done
cat >> "$OUTFILE" <<'EOF'
cat >> "$TMPFILE" <<'EOF'
}
EOF
mv "$TMPFILE" "$OUTFILE"
echo "[✓] Wrote $OUTFILE"

2
vendor/zerokit vendored

@ -1 +1 @@
Subproject commit a4bb3feb5054e6fd24827adf204493e6e173437b
Subproject commit 5e64cb8822bee65eed6cf459f95ae72b80c6ba63

12
waku.nimble
View File

@ -4,7 +4,7 @@ import os
mode = ScriptMode.Verbose
### Package
version = "0.37.4"
version = "0.38.1"
author = "Status Research & Development GmbH"
description = "Waku, Private P2P Messaging for Resource-Restricted Devices"
license = "MIT or Apache License 2.0"
@ -63,6 +63,16 @@ requires "https://github.com/logos-messaging/nim-ffi"
requires "https://github.com/logos-messaging/nim-sds.git#2e9a7683f0e180bf112135fae3a3803eed8490d4"
# brokers: pinned by URL+commit rather than the bare `brokers >= 2.0.1`
# form because the nim-lang/packages registry entry for `brokers` only
# carries metadata for the original v0.1.0 publication. Until that
# registry entry is refreshed, the local SAT solver enumerates "0.1.0"
# as the only available version and cannot satisfy `>= 2.0.1`. The URL
# pin below bypasses the registry and locks the exact commit of the
# v2.0.1 tag. Revert to the bare form once nim-lang/packages is
# updated.
requires "https://github.com/NagyZoltanPeter/nim-brokers.git#v2.0.1"
requires "https://github.com/vacp2p/nim-lsquic"
requires "https://github.com/vacp2p/nim-jwt.git#057ec95eb5af0eea9c49bfe9025b3312c95dc5f2"

18
waku/README.md
View File

@ -45,15 +45,16 @@ Setting up a `wakunode2` on the smallest [digital ocean](https://docs.digitaloce
make test
```
To run a specific test.
To run a specific test file or test case:
```bash
# Get a shell with the right environment variables set
./env.sh bash
# Run a specific test
nim c -r ./tests/test_waku_filter_legacy.nim
# Run all tests in a specific file
make test tests/waku_filter_v2/test_waku_filter.nim
# Run a specific test case within a file
make test tests/waku_filter_v2/test_waku_filter.nim "specific test name"
```
You can also alter compile options. For example, if you want a less verbose output you can do the following. For more, refer to the [compiler flags](https://nim-lang.org/docs/nimc.html#compiler-usage) and [chronicles documentation](https://github.com/status-im/nim-chronicles#compile-time-configuration).
Alternatively, you can invoke the Nim compiler directly. For more on available flags, refer to the [compiler flags](https://nim-lang.org/docs/nimc.html#compiler-usage) and [chronicles documentation](https://github.com/status-im/nim-chronicles#compile-time-configuration).
```bash
nim c -r -d:chronicles_log_level=WARN --verbosity=0 --hints=off ./tests/waku_filter_v2/test_waku_filter.nim
@ -231,7 +232,4 @@ However, they can be used for local testing purposes:
mkdir -p ./ssl_dir/
openssl req -x509 -newkey rsa:4096 -keyout ./ssl_dir/key.pem -out ./ssl_dir/cert.pem -sha256 -nodes
wakunode2 --websocket-secure-support=true --websocket-secure-key-path="./ssl_dir/key.pem" --websocket-secure-cert-path="./ssl_dir/cert.pem"
```
```

15
waku/api/types.nim
View File

@ -11,6 +11,10 @@ type
contentTopic*: ContentTopic
payload*: seq[byte]
ephemeral*: bool
meta*: seq[byte]
## Opaque wire-format marker carried on the underlying WakuMessage.
## Higher layers (e.g. Reliable Channel) stamp this so peers can route
## ingress traffic to their corresponding layer. Empty by default.
RequestId* = distinct string
@ -34,12 +38,18 @@ proc init*(
contentTopic: ContentTopic,
payload: seq[byte] | string,
ephemeral: bool = false,
meta: seq[byte] = @[],
): MessageEnvelope =
when payload is seq[byte]:
MessageEnvelope(contentTopic: contentTopic, payload: payload, ephemeral: ephemeral)
MessageEnvelope(
contentTopic: contentTopic, payload: payload, ephemeral: ephemeral, meta: meta
)
else:
MessageEnvelope(
contentTopic: contentTopic, payload: payload.toBytes(), ephemeral: ephemeral
contentTopic: contentTopic,
payload: payload.toBytes(),
ephemeral: ephemeral,
meta: meta,
)
proc toWakuMessage*(envelope: MessageEnvelope): WakuMessage =
@ -48,6 +58,7 @@ proc toWakuMessage*(envelope: MessageEnvelope): WakuMessage =
contentTopic: envelope.contentTopic,
payload: envelope.payload,
ephemeral: envelope.ephemeral,
meta: envelope.meta,
timestamp: getNowInNanosecondTime(),
)

68
waku/common/broker/broker_context.nim
View File

@ -1,68 +0,0 @@
{.push raises: [].}
import std/[strutils, concurrency/atomics], chronos
type BrokerContext* = distinct uint32
func `==`*(a, b: BrokerContext): bool =
uint32(a) == uint32(b)
func `!=`*(a, b: BrokerContext): bool =
uint32(a) != uint32(b)
func `$`*(bc: BrokerContext): string =
toHex(uint32(bc), 8)
const DefaultBrokerContext* = BrokerContext(0xCAFFE14E'u32)
# Global broker context accessor.
#
# NOTE: This intentionally creates a *single* active BrokerContext per process
# (per event loop thread). Use only if you accept serialization of all broker
# context usage through the lock.
var globalBrokerContextLock {.threadvar.}: AsyncLock
globalBrokerContextLock = newAsyncLock()
var globalBrokerContextValue {.threadvar.}: BrokerContext
globalBrokerContextValue = DefaultBrokerContext
proc globalBrokerContext*(): BrokerContext =
## Returns the currently active global broker context.
##
## This is intentionally lock-free; callers should use it inside
## `withNewGlobalBrokerContext` / `withGlobalBrokerContext`.
globalBrokerContextValue
var gContextCounter: Atomic[uint32]
proc NewBrokerContext*(): BrokerContext =
var nextId = gContextCounter.fetchAdd(1, moRelaxed)
if nextId == uint32(DefaultBrokerContext):
nextId = gContextCounter.fetchAdd(1, moRelaxed)
return BrokerContext(nextId)
template lockGlobalBrokerContext*(brokerCtx: BrokerContext, body: untyped): untyped =
## Runs `body` while holding the global broker context lock with the provided
## `brokerCtx` installed as the globally accessible context.
##
## This template is intended for use from within `chronos` async procs.
block:
await noCancel(globalBrokerContextLock.acquire())
let previousBrokerCtx = globalBrokerContextValue
globalBrokerContextValue = brokerCtx
try:
body
finally:
globalBrokerContextValue = previousBrokerCtx
try:
globalBrokerContextLock.release()
except AsyncLockError:
doAssert false, "globalBrokerContextLock.release(): lock not held"
template lockNewGlobalBrokerContext*(body: untyped): untyped =
## Runs `body` while holding the global broker context lock with a freshly
## generated broker context installed as the global accessor.
##
## The previous global broker context (if any) is restored on exit.
lockGlobalBrokerContext(NewBrokerContext()):
body
{.pop.}

411
waku/common/broker/event_broker.nim
View File

@ -1,411 +0,0 @@
## EventBroker
## -------------------
## EventBroker represents a reactive decoupling pattern, that
## allows event-driven development without
## need for direct dependencies in between emitters and listeners.
## Worth considering using it in a single or many emitters to many listeners scenario.
##
## Generates a standalone, type-safe event broker for the declared type.
## The macro exports the value type itself plus a broker companion that manages
## listeners via thread-local storage.
##
## Type definitions:
## - Inline `object` / `ref object` definitions are supported.
## - Native types, aliases, and externally-defined types are also supported.
## In that case, EventBroker will automatically wrap the declared RHS type in
## `distinct` unless you already used `distinct`.
## This keeps event types unique even when multiple brokers share the same
## underlying base type.
##
## Default vs. context aware use:
## Every generated broker is a thread-local global instance. This means EventBroker
## enables decoupled event exchange threadwise.
##
## Sometimes we use brokers inside a context (e.g. within a component that has many
## modules or subsystems). If you instantiate multiple such components in a single
## thread, and each component must have its own listener set for the same EventBroker
## type, you can use context-aware EventBroker.
##
## Context awareness is supported through the `BrokerContext` argument for
## `listen`, `emit`, `dropListener`, and `dropAllListeners`.
## Listener stores are kept separate per broker context.
##
## Default broker context is defined as `DefaultBrokerContext`. If you don't need
## context awareness, you can keep using the interfaces without the context
## argument, which operate on `DefaultBrokerContext`.
##
## Usage:
## Declare your desired event type inside an `EventBroker` macro, add any number of fields.:
## ```nim
## EventBroker:
## type TypeName = object
## field1*: FieldType
## field2*: AnotherFieldType
## ```
##
## After this, you can register async listeners anywhere in your code with
## `TypeName.listen(...)`, which returns a handle to the registered listener.
## Listeners are async procs or lambdas that take a single argument of the event type.
## Any number of listeners can be registered in different modules.
##
## Events can be emitted from anywhere with no direct dependency on the listeners by
## calling `TypeName.emit(...)` with an instance of the event type.
## This will asynchronously notify all registered listeners with the emitted event.
##
## Whenever you no longer need a listener (or your object instance that listen to the event goes out of scope),
## you can remove it from the broker with the handle returned by `listen`.
## This is done by calling `TypeName.dropListener(handle)`.
## Alternatively, you can remove all registered listeners through `TypeName.dropAllListeners()`.
##
##
## Example:
## ```nim
## EventBroker:
## type GreetingEvent = object
## text*: string
##
## let handle = GreetingEvent.listen(
## proc(evt: GreetingEvent): Future[void] {.async.} =
## echo evt.text
## )
## GreetingEvent.emit(text= "hi")
## GreetingEvent.dropListener(handle)
## ```
## Example (non-object event type):
## ```nim
## EventBroker:
## type CounterEvent = int # exported as: `distinct int`
##
## discard CounterEvent.listen(
## proc(evt: CounterEvent): Future[void] {.async.} =
## echo int(evt)
## )
## CounterEvent.emit(CounterEvent(42))
## ```
import std/[macros, tables]
import chronos, chronicles, results
import ./helper/broker_utils, broker_context
export chronicles, results, chronos, broker_context
macro EventBroker*(body: untyped): untyped =
when defined(eventBrokerDebug):
echo body.treeRepr
let parsed = parseSingleTypeDef(body, "EventBroker", collectFieldInfo = true)
let typeIdent = parsed.typeIdent
let objectDef = parsed.objectDef
let fieldNames = parsed.fieldNames
let fieldTypes = parsed.fieldTypes
let hasInlineFields = parsed.hasInlineFields
let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*")
let sanitized = sanitizeIdentName(typeIdent)
let typeNameLit = newLit($typeIdent)
let handlerProcIdent = ident(sanitized & "ListenerProc")
let listenerHandleIdent = ident(sanitized & "Listener")
let brokerTypeIdent = ident(sanitized & "Broker")
let exportedHandlerProcIdent = postfix(copyNimTree(handlerProcIdent), "*")
let exportedListenerHandleIdent = postfix(copyNimTree(listenerHandleIdent), "*")
let exportedBrokerTypeIdent = postfix(copyNimTree(brokerTypeIdent), "*")
let bucketTypeIdent = ident(sanitized & "CtxBucket")
let findBucketIdxIdent = ident(sanitized & "FindBucketIdx")
let getOrCreateBucketIdxIdent = ident(sanitized & "GetOrCreateBucketIdx")
let accessProcIdent = ident("access" & sanitized & "Broker")
let globalVarIdent = ident("g" & sanitized & "Broker")
let listenImplIdent = ident("register" & sanitized & "Listener")
let dropListenerImplIdent = ident("drop" & sanitized & "Listener")
let dropAllListenersImplIdent = ident("dropAll" & sanitized & "Listeners")
let emitImplIdent = ident("emit" & sanitized & "Value")
let listenerTaskIdent = ident("notify" & sanitized & "Listener")
result = newStmtList()
result.add(
quote do:
type
`exportedTypeIdent` = `objectDef`
`exportedListenerHandleIdent` = object
id*: uint64
`exportedHandlerProcIdent` =
proc(event: `typeIdent`): Future[void] {.async: (raises: []), gcsafe.}
`bucketTypeIdent` = object
brokerCtx: BrokerContext
listeners: Table[uint64, `handlerProcIdent`]
nextId: uint64
`exportedBrokerTypeIdent` = ref object
buckets: seq[`bucketTypeIdent`]
)
result.add(
quote do:
var `globalVarIdent` {.threadvar.}: `brokerTypeIdent`
)
result.add(
quote do:
proc `accessProcIdent`(): `brokerTypeIdent` =
if `globalVarIdent`.isNil():
new(`globalVarIdent`)
`globalVarIdent`.buckets = @[
`bucketTypeIdent`(
brokerCtx: DefaultBrokerContext,
listeners: initTable[uint64, `handlerProcIdent`](),
nextId: 1'u64,
)
]
`globalVarIdent`
)
result.add(
quote do:
proc `findBucketIdxIdent`(
broker: `brokerTypeIdent`, brokerCtx: BrokerContext
): int =
if brokerCtx == DefaultBrokerContext:
return 0
for i in 1 ..< broker.buckets.len:
if broker.buckets[i].brokerCtx == brokerCtx:
return i
return -1
proc `getOrCreateBucketIdxIdent`(
broker: `brokerTypeIdent`, brokerCtx: BrokerContext
): int =
let idx = `findBucketIdxIdent`(broker, brokerCtx)
if idx >= 0:
return idx
broker.buckets.add(
`bucketTypeIdent`(
brokerCtx: brokerCtx,
listeners: initTable[uint64, `handlerProcIdent`](),
nextId: 1'u64,
)
)
return broker.buckets.high
proc `listenImplIdent`(
brokerCtx: BrokerContext, handler: `handlerProcIdent`
): Result[`listenerHandleIdent`, string] =
if handler.isNil():
return err("Must provide a non-nil event handler")
var broker = `accessProcIdent`()
let bucketIdx = `getOrCreateBucketIdxIdent`(broker, brokerCtx)
if broker.buckets[bucketIdx].nextId == 0'u64:
broker.buckets[bucketIdx].nextId = 1'u64
if broker.buckets[bucketIdx].nextId == high(uint64):
error "Cannot add more listeners: ID space exhausted",
nextId = $broker.buckets[bucketIdx].nextId
return err("Cannot add more listeners, listener ID space exhausted")
let newId = broker.buckets[bucketIdx].nextId
inc broker.buckets[bucketIdx].nextId
broker.buckets[bucketIdx].listeners[newId] = handler
return ok(`listenerHandleIdent`(id: newId))
)
result.add(
quote do:
proc `dropListenerImplIdent`(
brokerCtx: BrokerContext, handle: `listenerHandleIdent`
) =
if handle.id == 0'u64:
return
var broker = `accessProcIdent`()
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
if broker.buckets[bucketIdx].listeners.len == 0:
return
broker.buckets[bucketIdx].listeners.del(handle.id)
if brokerCtx != DefaultBrokerContext and
broker.buckets[bucketIdx].listeners.len == 0:
broker.buckets.delete(bucketIdx)
)
result.add(
quote do:
proc `dropAllListenersImplIdent`(brokerCtx: BrokerContext) =
var broker = `accessProcIdent`()
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
if broker.buckets[bucketIdx].listeners.len > 0:
broker.buckets[bucketIdx].listeners.clear()
if brokerCtx != DefaultBrokerContext:
broker.buckets.delete(bucketIdx)
)
result.add(
quote do:
proc listen*(
_: typedesc[`typeIdent`], handler: `handlerProcIdent`
): Result[`listenerHandleIdent`, string] =
return `listenImplIdent`(DefaultBrokerContext, handler)
proc listen*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
handler: `handlerProcIdent`,
): Result[`listenerHandleIdent`, string] =
return `listenImplIdent`(brokerCtx, handler)
)
result.add(
quote do:
proc dropListener*(_: typedesc[`typeIdent`], handle: `listenerHandleIdent`) =
`dropListenerImplIdent`(DefaultBrokerContext, handle)
proc dropListener*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
handle: `listenerHandleIdent`,
) =
`dropListenerImplIdent`(brokerCtx, handle)
proc dropAllListeners*(_: typedesc[`typeIdent`]) =
`dropAllListenersImplIdent`(DefaultBrokerContext)
proc dropAllListeners*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) =
`dropAllListenersImplIdent`(brokerCtx)
)
result.add(
quote do:
proc `listenerTaskIdent`(
callback: `handlerProcIdent`, event: `typeIdent`
) {.async: (raises: []), gcsafe.} =
if callback.isNil():
return
try:
await callback(event)
except Exception:
error "Failed to execute event listener", error = getCurrentExceptionMsg()
proc `emitImplIdent`(
brokerCtx: BrokerContext, event: `typeIdent`
): Future[void] {.async: (raises: []), gcsafe.} =
when compiles(event.isNil()):
if event.isNil():
error "Cannot emit uninitialized event object", eventType = `typeNameLit`
return
let broker = `accessProcIdent`()
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
# nothing to do as nobody is listening
return
if broker.buckets[bucketIdx].listeners.len == 0:
return
var callbacks: seq[`handlerProcIdent`] = @[]
for cb in broker.buckets[bucketIdx].listeners.values:
callbacks.add(cb)
for cb in callbacks:
asyncSpawn `listenerTaskIdent`(cb, event)
proc emit*(event: `typeIdent`) =
asyncSpawn `emitImplIdent`(DefaultBrokerContext, event)
proc emit*(_: typedesc[`typeIdent`], event: `typeIdent`) =
asyncSpawn `emitImplIdent`(DefaultBrokerContext, event)
proc emit*(
_: typedesc[`typeIdent`], brokerCtx: BrokerContext, event: `typeIdent`
) =
asyncSpawn `emitImplIdent`(brokerCtx, event)
)
if hasInlineFields:
# Typedesc emit constructor overloads for inline object/ref object types.
var emitCtorParams = newTree(nnkFormalParams, newEmptyNode())
let typedescParamType =
newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent))
emitCtorParams.add(
newTree(nnkIdentDefs, ident("_"), typedescParamType, newEmptyNode())
)
for i in 0 ..< fieldNames.len:
emitCtorParams.add(
newTree(
nnkIdentDefs,
copyNimTree(fieldNames[i]),
copyNimTree(fieldTypes[i]),
newEmptyNode(),
)
)
var emitCtorExpr = newTree(nnkObjConstr, copyNimTree(typeIdent))
for i in 0 ..< fieldNames.len:
emitCtorExpr.add(
newTree(
nnkExprColonExpr, copyNimTree(fieldNames[i]), copyNimTree(fieldNames[i])
)
)
let emitCtorCallDefault =
newCall(copyNimTree(emitImplIdent), ident("DefaultBrokerContext"), emitCtorExpr)
let emitCtorBodyDefault = quote:
asyncSpawn `emitCtorCallDefault`
let typedescEmitProcDefault = newTree(
nnkProcDef,
postfix(ident("emit"), "*"),
newEmptyNode(),
newEmptyNode(),
emitCtorParams,
newEmptyNode(),
newEmptyNode(),
emitCtorBodyDefault,
)
result.add(typedescEmitProcDefault)
var emitCtorParamsCtx = newTree(nnkFormalParams, newEmptyNode())
emitCtorParamsCtx.add(
newTree(nnkIdentDefs, ident("_"), typedescParamType, newEmptyNode())
)
emitCtorParamsCtx.add(
newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode())
)
for i in 0 ..< fieldNames.len:
emitCtorParamsCtx.add(
newTree(
nnkIdentDefs,
copyNimTree(fieldNames[i]),
copyNimTree(fieldTypes[i]),
newEmptyNode(),
)
)
let emitCtorCallCtx =
newCall(copyNimTree(emitImplIdent), ident("brokerCtx"), copyNimTree(emitCtorExpr))
let emitCtorBodyCtx = quote:
asyncSpawn `emitCtorCallCtx`
let typedescEmitProcCtx = newTree(
nnkProcDef,
postfix(ident("emit"), "*"),
newEmptyNode(),
newEmptyNode(),
emitCtorParamsCtx,
newEmptyNode(),
newEmptyNode(),
emitCtorBodyCtx,
)
result.add(typedescEmitProcCtx)
when defined(eventBrokerDebug):
echo result.repr

206
waku/common/broker/helper/broker_utils.nim
View File

@ -1,206 +0,0 @@
import std/macros
type ParsedBrokerType* = object
## Result of parsing the single `type` definition inside a broker macro body.
##
## - `typeIdent`: base identifier for the declared type name
## - `objectDef`: exported type definition RHS (inline object fields exported;
## non-object types wrapped in `distinct` unless already distinct)
## - `isRefObject`: true only for inline `ref object` definitions
## - `hasInlineFields`: true for inline `object` / `ref object`
## - `fieldNames`/`fieldTypes`: populated only when `collectFieldInfo = true`
typeIdent*: NimNode
objectDef*: NimNode
isRefObject*: bool
hasInlineFields*: bool
fieldNames*: seq[NimNode]
fieldTypes*: seq[NimNode]
proc sanitizeIdentName*(node: NimNode): string =
var raw = $node
var sanitizedName = newStringOfCap(raw.len)
for ch in raw:
case ch
of 'A' .. 'Z', 'a' .. 'z', '0' .. '9', '_':
sanitizedName.add(ch)
else:
sanitizedName.add('_')
sanitizedName
proc ensureFieldDef*(node: NimNode) =
if node.kind != nnkIdentDefs or node.len < 3:
error("Expected field definition of the form `name: Type`", node)
let typeSlot = node.len - 2
if node[typeSlot].kind == nnkEmpty:
error("Field `" & $node[0] & "` must declare a type", node)
proc exportIdentNode*(node: NimNode): NimNode =
case node.kind
of nnkIdent:
postfix(copyNimTree(node), "*")
of nnkPostfix:
node
else:
error("Unsupported identifier form in field definition", node)
proc baseTypeIdent*(defName: NimNode): NimNode =
case defName.kind
of nnkIdent:
defName
of nnkAccQuoted:
if defName.len != 1:
error("Unsupported quoted identifier", defName)
defName[0]
of nnkPostfix:
baseTypeIdent(defName[1])
of nnkPragmaExpr:
baseTypeIdent(defName[0])
else:
error("Unsupported type name in broker definition", defName)
proc ensureDistinctType*(rhs: NimNode): NimNode =
## For PODs / aliases / externally-defined types, wrap in `distinct` unless
## it's already distinct.
if rhs.kind == nnkDistinctTy:
return copyNimTree(rhs)
newTree(nnkDistinctTy, copyNimTree(rhs))
proc cloneParams*(params: seq[NimNode]): seq[NimNode] =
## Deep copy parameter definitions so they can be inserted in multiple places.
result = @[]
for param in params:
result.add(copyNimTree(param))
proc collectParamNames*(params: seq[NimNode]): seq[NimNode] =
## Extract all identifier symbols declared across IdentDefs nodes.
result = @[]
for param in params:
assert param.kind == nnkIdentDefs
for i in 0 ..< param.len - 2:
let nameNode = param[i]
if nameNode.kind == nnkEmpty:
continue
result.add(ident($nameNode))
proc parseSingleTypeDef*(
body: NimNode,
macroName: string,
allowRefToNonObject = false,
collectFieldInfo = false,
): ParsedBrokerType =
## Parses exactly one `type` definition from a broker macro body.
##
## Supported RHS:
## - inline `object` / `ref object` (fields are auto-exported)
## - non-object types / aliases / externally-defined types (wrapped in `distinct`)
## - optionally: `ref SomeType` when `allowRefToNonObject = true`
var typeIdent: NimNode = nil
var objectDef: NimNode = nil
var isRefObject = false
var hasInlineFields = false
var fieldNames: seq[NimNode] = @[]
var fieldTypes: seq[NimNode] = @[]
for stmt in body:
if stmt.kind != nnkTypeSection:
continue
for def in stmt:
if def.kind != nnkTypeDef:
continue
if not typeIdent.isNil():
error("Only one type may be declared inside " & macroName, def)
typeIdent = baseTypeIdent(def[0])
let rhs = def[2]
case rhs.kind
of nnkObjectTy:
let recList = rhs[2]
if recList.kind != nnkRecList:
error(macroName & " object must declare a standard field list", rhs)
var exportedRecList = newTree(nnkRecList)
for field in recList:
case field.kind
of nnkIdentDefs:
ensureFieldDef(field)
if collectFieldInfo:
let fieldTypeNode = field[field.len - 2]
for i in 0 ..< field.len - 2:
let baseFieldIdent = baseTypeIdent(field[i])
fieldNames.add(copyNimTree(baseFieldIdent))
fieldTypes.add(copyNimTree(fieldTypeNode))
var cloned = copyNimTree(field)
for i in 0 ..< cloned.len - 2:
cloned[i] = exportIdentNode(cloned[i])
exportedRecList.add(cloned)
of nnkEmpty:
discard
else:
error(
macroName & " object definition only supports simple field declarations",
field,
)
objectDef = newTree(
nnkObjectTy, copyNimTree(rhs[0]), copyNimTree(rhs[1]), exportedRecList
)
isRefObject = false
hasInlineFields = true
of nnkRefTy:
if rhs.len != 1:
error(macroName & " ref type must have a single base", rhs)
if rhs[0].kind == nnkObjectTy:
let obj = rhs[0]
let recList = obj[2]
if recList.kind != nnkRecList:
error(macroName & " object must declare a standard field list", obj)
var exportedRecList = newTree(nnkRecList)
for field in recList:
case field.kind
of nnkIdentDefs:
ensureFieldDef(field)
if collectFieldInfo:
let fieldTypeNode = field[field.len - 2]
for i in 0 ..< field.len - 2:
let baseFieldIdent = baseTypeIdent(field[i])
fieldNames.add(copyNimTree(baseFieldIdent))
fieldTypes.add(copyNimTree(fieldTypeNode))
var cloned = copyNimTree(field)
for i in 0 ..< cloned.len - 2:
cloned[i] = exportIdentNode(cloned[i])
exportedRecList.add(cloned)
of nnkEmpty:
discard
else:
error(
macroName & " object definition only supports simple field declarations",
field,
)
let exportedObjectType = newTree(
nnkObjectTy, copyNimTree(obj[0]), copyNimTree(obj[1]), exportedRecList
)
objectDef = newTree(nnkRefTy, exportedObjectType)
isRefObject = true
hasInlineFields = true
elif allowRefToNonObject:
## `ref SomeType` (SomeType can be defined elsewhere)
objectDef = ensureDistinctType(rhs)
isRefObject = false
hasInlineFields = false
else:
error(macroName & " ref object must wrap a concrete object definition", rhs)
else:
## Non-object type / alias.
objectDef = ensureDistinctType(rhs)
isRefObject = false
hasInlineFields = false
if typeIdent.isNil():
error(macroName & " body must declare exactly one type", body)
result = ParsedBrokerType(
typeIdent: typeIdent,
objectDef: objectDef,
isRefObject: isRefObject,
hasInlineFields: hasInlineFields,
fieldNames: fieldNames,
fieldTypes: fieldTypes,
)

743
waku/common/broker/multi_request_broker.nim
View File

@ -1,743 +0,0 @@
## MultiRequestBroker
## --------------------
## MultiRequestBroker represents a proactive decoupling pattern, that
## allows defining request-response style interactions between modules without
## need for direct dependencies in between.
## Worth considering using it for use cases where you need to collect data from multiple providers.
##
## Generates a standalone, type-safe request broker for the declared type.
## The macro exports the value type itself plus a broker companion that manages
## providers via thread-local storage.
##
## Unlike `RequestBroker`, every call to `request` fan-outs to every registered
## provider and returns all collected responses.
## The request succeeds only if all providers succeed, otherwise it fails.
##
## Type definitions:
## - Inline `object` / `ref object` definitions are supported.
## - Native types, aliases, and externally-defined types are also supported.
## In that case, MultiRequestBroker will automatically wrap the declared RHS
## type in `distinct` unless you already used `distinct`.
## This keeps request types unique even when multiple brokers share the same
## underlying base type.
##
## Default vs. context aware use:
## Every generated broker is a thread-local global instance.
## Sometimes you want multiple independent provider sets for the same request
## type within the same thread (e.g. multiple components). For that, you can use
## context-aware MultiRequestBroker.
##
## Context awareness is supported through the `BrokerContext` argument for
## `setProvider`, `request`, `removeProvider`, and `clearProviders`.
## Provider stores are kept separate per broker context.
##
## Default broker context is defined as `DefaultBrokerContext`. If you don't
## need context awareness, you can keep using the interfaces without the context
## argument, which operate on `DefaultBrokerContext`.
##
## Usage:
##
## Declare collectable request data type inside a `MultiRequestBroker` macro, add any number of fields:
## ```nim
## MultiRequestBroker:
## type TypeName = object
## field1*: Type1
## field2*: Type2
##
## ## Define the request and provider signature, that is enforced at compile time.
## proc signature*(): Future[Result[TypeName, string]] {.async: (raises: []).}
##
## ## Also possible to define signature with arbitrary input arguments.
## proc signature*(arg1: ArgType, arg2: AnotherArgType): Future[Result[TypeName, string]] {.async: (raises: []).}
##
## ```
##
## You can register a request processor (provider) anywhere without the need to
## know who will request.
## Register provider functions with `TypeName.setProvider(...)`.
## Providers are async procs or lambdas that return `Future[Result[TypeName, string]]`.
## `setProvider` returns a handle (or an error) that can later be used to remove
## the provider.
## Requests can be made from anywhere with no direct dependency on the provider(s)
## by calling `TypeName.request()` (with arguments respecting the declared signature).
## This will asynchronously call all registered providers and return the collected
## responses as `Future[Result[seq[TypeName], string]]`.
##
## Whenever you don't want to process requests anymore (or your object instance that provides the request goes out of scope),
## you can remove it from the broker with `TypeName.removeProvider(handle)`.
## Alternatively, you can remove all registered providers through `TypeName.clearProviders()`.
##
## Example:
## ```nim
## MultiRequestBroker:
## type Greeting = object
## text*: string
##
## ## Define the request and provider signature, that is enforced at compile time.
## proc signature*(): Future[Result[Greeting, string]] {.async: (raises: []).}
##
## ## Also possible to define signature with arbitrary input arguments.
## proc signature*(lang: string): Future[Result[Greeting, string]] {.async: (raises: []).}
##
## ...
## let handle = Greeting.setProvider(
## proc(): Future[Result[Greeting, string]] {.async: (raises: []).} =
## ok(Greeting(text: "hello"))
## )
##
## let anotherHandle = Greeting.setProvider(
## proc(): Future[Result[Greeting, string]] {.async: (raises: []).} =
## ok(Greeting(text: "szia"))
## )
##
## let responses = (await Greeting.request()).valueOr(@[Greeting(text: "default")])
##
## echo responses.len
## Greeting.clearProviders()
## ```
## If no `signature` proc is declared, a zero-argument form is generated
## automatically, so the caller only needs to provide the type definition.
import std/[macros, strutils, tables, sugar]
import chronos
import results
import ./helper/broker_utils
import ./broker_context
export results, chronos, broker_context
proc isReturnTypeValid(returnType, typeIdent: NimNode): bool =
## Accept Future[Result[TypeIdent, string]] as the contract.
if returnType.kind != nnkBracketExpr or returnType.len != 2:
return false
if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Future"):
return false
let inner = returnType[1]
if inner.kind != nnkBracketExpr or inner.len != 3:
return false
if inner[0].kind != nnkIdent or not inner[0].eqIdent("Result"):
return false
if inner[1].kind != nnkIdent or not inner[1].eqIdent($typeIdent):
return false
inner[2].kind == nnkIdent and inner[2].eqIdent("string")
proc makeProcType(returnType: NimNode, params: seq[NimNode]): NimNode =
var formal = newTree(nnkFormalParams)
formal.add(returnType)
for param in params:
formal.add(param)
let pragmas = quote:
{.async.}
newTree(nnkProcTy, formal, pragmas)
macro MultiRequestBroker*(body: untyped): untyped =
when defined(requestBrokerDebug):
echo body.treeRepr
let parsed = parseSingleTypeDef(body, "MultiRequestBroker")
let typeIdent = parsed.typeIdent
let objectDef = parsed.objectDef
let isRefObject = parsed.isRefObject
when defined(requestBrokerDebug):
echo "MultiRequestBroker generating type: ", $typeIdent
let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*")
let sanitized = sanitizeIdentName(typeIdent)
let typeNameLit = newLit($typeIdent)
let isRefObjectLit = newLit(isRefObject)
let uint64Ident = ident("uint64")
let providerKindIdent = ident(sanitized & "ProviderKind")
let providerHandleIdent = ident(sanitized & "ProviderHandle")
let exportedProviderHandleIdent = postfix(copyNimTree(providerHandleIdent), "*")
let bucketTypeIdent = ident(sanitized & "CtxBucket")
let findBucketIdxIdent = ident(sanitized & "FindBucketIdx")
let getOrCreateBucketIdxIdent = ident(sanitized & "GetOrCreateBucketIdx")
let zeroKindIdent = ident("pk" & sanitized & "NoArgs")
let argKindIdent = ident("pk" & sanitized & "WithArgs")
var zeroArgSig: NimNode = nil
var zeroArgProviderName: NimNode = nil
var zeroArgFieldName: NimNode = nil
var argSig: NimNode = nil
var argParams: seq[NimNode] = @[]
var argProviderName: NimNode = nil
var argFieldName: NimNode = nil
for stmt in body:
case stmt.kind
of nnkProcDef:
let procName = stmt[0]
let procNameIdent =
case procName.kind
of nnkIdent:
procName
of nnkPostfix:
procName[1]
else:
procName
let procNameStr = $procNameIdent
if not procNameStr.startsWith("signature"):
error("Signature proc names must start with `signature`", procName)
let params = stmt.params
if params.len == 0:
error("Signature must declare a return type", stmt)
let returnType = params[0]
if not isReturnTypeValid(returnType, typeIdent):
error(
"Signature must return Future[Result[`" & $typeIdent & "`, string]]", stmt
)
let paramCount = params.len - 1
if paramCount == 0:
if zeroArgSig != nil:
error("Only one zero-argument signature is allowed", stmt)
zeroArgSig = stmt
zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs")
zeroArgFieldName = ident("providerNoArgs")
elif paramCount >= 1:
if argSig != nil:
error("Only one argument-based signature is allowed", stmt)
argSig = stmt
argParams = @[]
for idx in 1 ..< params.len:
let paramDef = params[idx]
if paramDef.kind != nnkIdentDefs:
error(
"Signature parameter must be a standard identifier declaration", paramDef
)
let paramTypeNode = paramDef[paramDef.len - 2]
if paramTypeNode.kind == nnkEmpty:
error("Signature parameter must declare a type", paramDef)
var hasName = false
for i in 0 ..< paramDef.len - 2:
if paramDef[i].kind != nnkEmpty:
hasName = true
if not hasName:
error("Signature parameter must declare a name", paramDef)
argParams.add(copyNimTree(paramDef))
argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs")
argFieldName = ident("providerWithArgs")
of nnkTypeSection, nnkEmpty:
discard
else:
error("Unsupported statement inside MultiRequestBroker definition", stmt)
if zeroArgSig.isNil() and argSig.isNil():
zeroArgSig = newEmptyNode()
zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs")
zeroArgFieldName = ident("providerNoArgs")
var typeSection = newTree(nnkTypeSection)
typeSection.add(newTree(nnkTypeDef, exportedTypeIdent, newEmptyNode(), objectDef))
var kindEnum = newTree(nnkEnumTy, newEmptyNode())
if not zeroArgSig.isNil():
kindEnum.add(zeroKindIdent)
if not argSig.isNil():
kindEnum.add(argKindIdent)
typeSection.add(newTree(nnkTypeDef, providerKindIdent, newEmptyNode(), kindEnum))
var handleRecList = newTree(nnkRecList)
handleRecList.add(newTree(nnkIdentDefs, ident("id"), uint64Ident, newEmptyNode()))
handleRecList.add(
newTree(nnkIdentDefs, ident("kind"), providerKindIdent, newEmptyNode())
)
typeSection.add(
newTree(
nnkTypeDef,
exportedProviderHandleIdent,
newEmptyNode(),
newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), handleRecList),
)
)
let returnType = quote:
Future[Result[`typeIdent`, string]]
if not zeroArgSig.isNil():
let procType = makeProcType(returnType, @[])
typeSection.add(newTree(nnkTypeDef, zeroArgProviderName, newEmptyNode(), procType))
if not argSig.isNil():
let procType = makeProcType(returnType, cloneParams(argParams))
typeSection.add(newTree(nnkTypeDef, argProviderName, newEmptyNode(), procType))
var bucketRecList = newTree(nnkRecList)
bucketRecList.add(
newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode())
)
if not zeroArgSig.isNil():
bucketRecList.add(
newTree(
nnkIdentDefs,
zeroArgFieldName,
newTree(nnkBracketExpr, ident("seq"), zeroArgProviderName),
newEmptyNode(),
)
)
if not argSig.isNil():
bucketRecList.add(
newTree(
nnkIdentDefs,
argFieldName,
newTree(nnkBracketExpr, ident("seq"), argProviderName),
newEmptyNode(),
)
)
typeSection.add(
newTree(
nnkTypeDef,
bucketTypeIdent,
newEmptyNode(),
newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), bucketRecList),
)
)
var brokerRecList = newTree(nnkRecList)
brokerRecList.add(
newTree(
nnkIdentDefs,
ident("buckets"),
newTree(nnkBracketExpr, ident("seq"), bucketTypeIdent),
newEmptyNode(),
)
)
let brokerTypeIdent = ident(sanitizeIdentName(typeIdent) & "Broker")
typeSection.add(
newTree(
nnkTypeDef,
brokerTypeIdent,
newEmptyNode(),
newTree(
nnkRefTy, newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), brokerRecList)
),
)
)
result = newStmtList()
result.add(typeSection)
let globalVarIdent = ident("g" & sanitizeIdentName(typeIdent) & "Broker")
let accessProcIdent = ident("access" & sanitizeIdentName(typeIdent) & "Broker")
result.add(
quote do:
var `globalVarIdent` {.threadvar.}: `brokerTypeIdent`
proc `findBucketIdxIdent`(
broker: `brokerTypeIdent`, brokerCtx: BrokerContext
): int =
if brokerCtx == DefaultBrokerContext:
return 0
for i in 1 ..< broker.buckets.len:
if broker.buckets[i].brokerCtx == brokerCtx:
return i
return -1
proc `getOrCreateBucketIdxIdent`(
broker: `brokerTypeIdent`, brokerCtx: BrokerContext
): int =
let idx = `findBucketIdxIdent`(broker, brokerCtx)
if idx >= 0:
return idx
broker.buckets.add(`bucketTypeIdent`(brokerCtx: brokerCtx))
return broker.buckets.high
proc `accessProcIdent`(): `brokerTypeIdent` =
if `globalVarIdent`.isNil():
new(`globalVarIdent`)
`globalVarIdent`.buckets =
@[`bucketTypeIdent`(brokerCtx: DefaultBrokerContext)]
return `globalVarIdent`
)
var clearBody = newStmtList()
if not zeroArgSig.isNil():
result.add(
quote do:
proc setProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
handler: `zeroArgProviderName`,
): Result[`providerHandleIdent`, string] =
if handler.isNil():
return err("Provider handler must be provided")
let broker = `accessProcIdent`()
let bucketIdx = `getOrCreateBucketIdxIdent`(broker, brokerCtx)
for i, existing in broker.buckets[bucketIdx].`zeroArgFieldName`:
if not existing.isNil() and existing == handler:
return ok(`providerHandleIdent`(id: uint64(i + 1), kind: `zeroKindIdent`))
broker.buckets[bucketIdx].`zeroArgFieldName`.add(handler)
return ok(
`providerHandleIdent`(
id: uint64(broker.buckets[bucketIdx].`zeroArgFieldName`.len),
kind: `zeroKindIdent`,
)
)
proc setProvider*(
_: typedesc[`typeIdent`], handler: `zeroArgProviderName`
): Result[`providerHandleIdent`, string] =
return setProvider(`typeIdent`, DefaultBrokerContext, handler)
)
result.add(
quote do:
proc request*(
_: typedesc[`typeIdent`], brokerCtx: BrokerContext
): Future[Result[seq[`typeIdent`], string]] {.async: (raises: []), gcsafe.} =
var aggregated: seq[`typeIdent`] = @[]
let broker = `accessProcIdent`()
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return ok(aggregated)
let providers = broker.buckets[bucketIdx].`zeroArgFieldName`
if providers.len == 0:
return ok(aggregated)
# var providersFut: seq[Future[Result[`typeIdent`, string]]] = collect:
var providersFut = collect(newSeq):
for provider in providers:
if provider.isNil():
continue
provider()
let catchable = catch:
await allFinished(providersFut)
catchable.isOkOr:
return err("Some provider(s) failed:" & error.msg)
for fut in catchable.get():
if fut.failed():
return err("Some provider(s) failed:" & fut.error.msg)
elif fut.finished():
let providerResult = fut.value()
if providerResult.isOk:
let providerValue = providerResult.get()
when `isRefObjectLit`:
if providerValue.isNil():
return err(
"MultiRequestBroker(" & `typeNameLit` &
"): provider returned nil result"
)
aggregated.add(providerValue)
else:
return err("Some provider(s) failed:" & providerResult.error)
return ok(aggregated)
proc request*(
_: typedesc[`typeIdent`]
): Future[Result[seq[`typeIdent`], string]] =
return request(`typeIdent`, DefaultBrokerContext)
)
if not argSig.isNil():
result.add(
quote do:
proc setProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
handler: `argProviderName`,
): Result[`providerHandleIdent`, string] =
if handler.isNil():
return err("Provider handler must be provided")
let broker = `accessProcIdent`()
let bucketIdx = `getOrCreateBucketIdxIdent`(broker, brokerCtx)
for i, existing in broker.buckets[bucketIdx].`argFieldName`:
if not existing.isNil() and existing == handler:
return ok(`providerHandleIdent`(id: uint64(i + 1), kind: `argKindIdent`))
broker.buckets[bucketIdx].`argFieldName`.add(handler)
return ok(
`providerHandleIdent`(
id: uint64(broker.buckets[bucketIdx].`argFieldName`.len),
kind: `argKindIdent`,
)
)
proc setProvider*(
_: typedesc[`typeIdent`], handler: `argProviderName`
): Result[`providerHandleIdent`, string] =
return setProvider(`typeIdent`, DefaultBrokerContext, handler)
)
let requestParamDefs = cloneParams(argParams)
let argNameIdents = collectParamNames(requestParamDefs)
let providerSym = genSym(nskLet, "providerVal")
var providerCall = newCall(providerSym)
for argName in argNameIdents:
providerCall.add(argName)
var formalParams = newTree(nnkFormalParams)
formalParams.add(
quote do:
Future[Result[seq[`typeIdent`], string]]
)
formalParams.add(
newTree(
nnkIdentDefs,
ident("_"),
newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)),
newEmptyNode(),
)
)
formalParams.add(
newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode())
)
for paramDef in requestParamDefs:
formalParams.add(paramDef)
let requestPragmas = quote:
{.async: (raises: []), gcsafe.}
let requestBody = quote:
var aggregated: seq[`typeIdent`] = @[]
let broker = `accessProcIdent`()
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return ok(aggregated)
let providers = broker.buckets[bucketIdx].`argFieldName`
if providers.len == 0:
return ok(aggregated)
var providersFut = collect(newSeq):
for provider in providers:
if provider.isNil():
continue
let `providerSym` = provider
`providerCall`
let catchable = catch:
await allFinished(providersFut)
catchable.isOkOr:
return err("Some provider(s) failed:" & error.msg)
for fut in catchable.get():
if fut.failed():
return err("Some provider(s) failed:" & fut.error.msg)
elif fut.finished():
let providerResult = fut.value()
if providerResult.isOk:
let providerValue = providerResult.get()
when `isRefObjectLit`:
if providerValue.isNil():
return err(
"MultiRequestBroker(" & `typeNameLit` &
"): provider returned nil result"
)
aggregated.add(providerValue)
else:
return err("Some provider(s) failed:" & providerResult.error)
return ok(aggregated)
result.add(
newTree(
nnkProcDef,
postfix(ident("request"), "*"),
newEmptyNode(),
newEmptyNode(),
formalParams,
requestPragmas,
newEmptyNode(),
requestBody,
)
)
# Backward-compatible default-context overload (no brokerCtx parameter).
var formalParamsDefault = newTree(nnkFormalParams)
formalParamsDefault.add(
quote do:
Future[Result[seq[`typeIdent`], string]]
)
formalParamsDefault.add(
newTree(
nnkIdentDefs,
ident("_"),
newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)),
newEmptyNode(),
)
)
for paramDef in requestParamDefs:
formalParamsDefault.add(copyNimTree(paramDef))
var wrapperCall = newCall(ident("request"))
wrapperCall.add(copyNimTree(typeIdent))
wrapperCall.add(ident("DefaultBrokerContext"))
for argName in argNameIdents:
wrapperCall.add(copyNimTree(argName))
result.add(
newTree(
nnkProcDef,
postfix(ident("request"), "*"),
newEmptyNode(),
newEmptyNode(),
formalParamsDefault,
newEmptyNode(),
newEmptyNode(),
newStmtList(newTree(nnkReturnStmt, wrapperCall)),
)
)
let removeHandleCtxSym = genSym(nskParam, "handle")
let removeHandleDefaultSym = genSym(nskParam, "handle")
when true:
# Generate clearProviders / removeProvider with macro-time knowledge about which
# provider lists exist (zero-arg and/or arg providers).
if not zeroArgSig.isNil() and not argSig.isNil():
result.add(
quote do:
proc clearProviders*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) =
let broker = `accessProcIdent`()
if broker.isNil():
return
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
broker.buckets[bucketIdx].`zeroArgFieldName`.setLen(0)
broker.buckets[bucketIdx].`argFieldName`.setLen(0)
if brokerCtx != DefaultBrokerContext:
broker.buckets.delete(bucketIdx)
proc clearProviders*(_: typedesc[`typeIdent`]) =
clearProviders(`typeIdent`, DefaultBrokerContext)
proc removeProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
`removeHandleCtxSym`: `providerHandleIdent`,
) =
if `removeHandleCtxSym`.id == 0'u64:
return
let broker = `accessProcIdent`()
if broker.isNil():
return
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
if `removeHandleCtxSym`.kind == `zeroKindIdent`:
let idx = int(`removeHandleCtxSym`.id) - 1
if idx >= 0 and idx < broker.buckets[bucketIdx].`zeroArgFieldName`.len:
broker.buckets[bucketIdx].`zeroArgFieldName`[idx] = nil
elif `removeHandleCtxSym`.kind == `argKindIdent`:
let idx = int(`removeHandleCtxSym`.id) - 1
if idx >= 0 and idx < broker.buckets[bucketIdx].`argFieldName`.len:
broker.buckets[bucketIdx].`argFieldName`[idx] = nil
if brokerCtx != DefaultBrokerContext:
var hasAny = false
for p in broker.buckets[bucketIdx].`zeroArgFieldName`:
if not p.isNil():
hasAny = true
break
if not hasAny:
for p in broker.buckets[bucketIdx].`argFieldName`:
if not p.isNil():
hasAny = true
break
if not hasAny:
broker.buckets.delete(bucketIdx)
proc removeProvider*(
_: typedesc[`typeIdent`], `removeHandleDefaultSym`: `providerHandleIdent`
) =
removeProvider(`typeIdent`, DefaultBrokerContext, `removeHandleDefaultSym`)
)
elif not zeroArgSig.isNil():
result.add(
quote do:
proc clearProviders*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) =
let broker = `accessProcIdent`()
if broker.isNil():
return
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
broker.buckets[bucketIdx].`zeroArgFieldName`.setLen(0)
if brokerCtx != DefaultBrokerContext:
broker.buckets.delete(bucketIdx)
proc clearProviders*(_: typedesc[`typeIdent`]) =
clearProviders(`typeIdent`, DefaultBrokerContext)
proc removeProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
`removeHandleCtxSym`: `providerHandleIdent`,
) =
if `removeHandleCtxSym`.id == 0'u64:
return
let broker = `accessProcIdent`()
if broker.isNil():
return
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
if `removeHandleCtxSym`.kind != `zeroKindIdent`:
return
let idx = int(`removeHandleCtxSym`.id) - 1
if idx >= 0 and idx < broker.buckets[bucketIdx].`zeroArgFieldName`.len:
broker.buckets[bucketIdx].`zeroArgFieldName`[idx] = nil
if brokerCtx != DefaultBrokerContext:
var hasAny = false
for p in broker.buckets[bucketIdx].`zeroArgFieldName`:
if not p.isNil():
hasAny = true
break
if not hasAny:
broker.buckets.delete(bucketIdx)
proc removeProvider*(
_: typedesc[`typeIdent`], `removeHandleDefaultSym`: `providerHandleIdent`
) =
removeProvider(`typeIdent`, DefaultBrokerContext, `removeHandleDefaultSym`)
)
else:
result.add(
quote do:
proc clearProviders*(_: typedesc[`typeIdent`], brokerCtx: BrokerContext) =
let broker = `accessProcIdent`()
if broker.isNil():
return
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
broker.buckets[bucketIdx].`argFieldName`.setLen(0)
if brokerCtx != DefaultBrokerContext:
broker.buckets.delete(bucketIdx)
proc clearProviders*(_: typedesc[`typeIdent`]) =
clearProviders(`typeIdent`, DefaultBrokerContext)
proc removeProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
`removeHandleCtxSym`: `providerHandleIdent`,
) =
if `removeHandleCtxSym`.id == 0'u64:
return
let broker = `accessProcIdent`()
if broker.isNil():
return
let bucketIdx = `findBucketIdxIdent`(broker, brokerCtx)
if bucketIdx < 0:
return
if `removeHandleCtxSym`.kind != `argKindIdent`:
return
let idx = int(`removeHandleCtxSym`.id) - 1
if idx >= 0 and idx < broker.buckets[bucketIdx].`argFieldName`.len:
broker.buckets[bucketIdx].`argFieldName`[idx] = nil
if brokerCtx != DefaultBrokerContext:
var hasAny = false
for p in broker.buckets[bucketIdx].`argFieldName`:
if not p.isNil():
hasAny = true
break
if not hasAny:
broker.buckets.delete(bucketIdx)
proc removeProvider*(
_: typedesc[`typeIdent`], `removeHandleDefaultSym`: `providerHandleIdent`
) =
removeProvider(`typeIdent`, DefaultBrokerContext, `removeHandleDefaultSym`)
)
when defined(requestBrokerDebug):
echo result.repr

841
waku/common/broker/request_broker.nim
View File

@ -1,841 +0,0 @@
## RequestBroker
## --------------------
## RequestBroker represents a proactive decoupling pattern, that
## allows defining request-response style interactions between modules without
## need for direct dependencies in between.
## Worth considering using it in a single provider, many requester scenario.
##
## Provides a declarative way to define an immutable value type together with a
## thread-local broker that can register an asynchronous or synchronous provider,
## dispatch typed requests and clear provider.
##
## For consideration use `sync` mode RequestBroker when you need to provide simple value(s)
## where there is no long-running async operation involved.
## Typically it act as a accessor for the local state of generic setting.
##
## `async` mode is better to be used when you request date that may involve some long IO operation
## or action.
##
## Default vs. context aware use:
## Every generated broker is a thread-local global instance. This means each RequestBroker enables decoupled
## data exchange threadwise. Sometimes we use brokers inside a context - like inside a component that has many modules or subsystems.
## In case you would instantiate multiple such components in a single thread, and each component must has its own provider for the same RequestBroker type,
## in order to avoid provider collision, you can use context aware RequestBroker.
## Context awareness is supported through the `BrokerContext` argument for `setProvider`, `request`, `clearProvider` interfaces.
## Suce use requires generating a new unique `BrokerContext` value per component instance, and spread it to all modules using the brokers.
## Example, store the `BrokerContext` as a field inside the top level component instance, and spread around at initialization of the subcomponents..
##
## Default broker context is defined as `DefaultBrokerContext` constant. But if you don't need context awareness, you can use the
## interfaces without context argument.
##
## Usage:
## Declare your desired request type inside a `RequestBroker` macro, add any number of fields.
## Define the provider signature, that is enforced at compile time.
##
## ```nim
## RequestBroker:
## type TypeName = object
## field1*: FieldType
## field2*: AnotherFieldType
##
## proc signature*(): Future[Result[TypeName, string]]
## ## Also possible to define signature with arbitrary input arguments.
## proc signature*(arg1: ArgType, arg2: AnotherArgType): Future[Result[TypeName, string]]
##
## ```
##
## Sync mode (no `async` / `Future`) can be generated with:
##
## ```nim
## RequestBroker(sync):
## type TypeName = object
## field1*: FieldType
##
## proc signature*(): Result[TypeName, string]
## proc signature*(arg1: ArgType): Result[TypeName, string]
## ```
##
## Note: When the request type is declared as a native type / alias / externally-defined
## type (i.e. not an inline `object` / `ref object` definition), RequestBroker
## will wrap it in `distinct` automatically unless you already used `distinct`.
## This avoids overload ambiguity when multiple brokers share the same
## underlying base type (Nim overload resolution does not consider return type).
##
## This means that for non-object request types you typically:
## - construct values with an explicit cast/constructor, e.g. `MyType("x")`
## - unwrap with a cast when needed, e.g. `string(myVal)` or `BaseType(myVal)`
##
## Example (native response type):
## ```nim
## RequestBroker(sync):
## type MyCount = int # exported as: `distinct int`
##
## MyCount.setProvider(proc(): Result[MyCount, string] = ok(MyCount(42)))
## let res = MyCount.request()
## if res.isOk():
## let raw = int(res.get())
## ```
##
## Example (externally-defined type):
## ```nim
## type External = object
## label*: string
##
## RequestBroker:
## type MyExternal = External # exported as: `distinct External`
##
## MyExternal.setProvider(
## proc(): Future[Result[MyExternal, string]] {.async.} =
## ok(MyExternal(External(label: "hi")))
## )
## let res = await MyExternal.request()
## if res.isOk():
## let base = External(res.get())
## echo base.label
## ```
## The 'TypeName' object defines the requestable data (but also can be seen as request for action with return value).
## The 'signature' proc defines the provider(s) signature, that is enforced at compile time.
## One signature can be with no arguments, another with any number of arguments - where the input arguments are
## not related to the request type - but alternative inputs for the request to be processed.
##
## After this, you can register a provider anywhere in your code with
## `TypeName.setProvider(...)`, which returns error if already having a provider.
## Providers are async procs/lambdas in default mode and sync procs in sync mode.
##
## Providers are stored as a broker-context keyed list:
## - the default provider is always stored at index 0 (reserved broker context: 0)
## - additional providers can be registered under arbitrary non-zero broker contexts
##
## The original `setProvider(handler)` / `request(...)` APIs continue to operate
## on the default provider (broker context 0) for backward compatibility.
##
## Requests can be made from anywhere with no direct dependency on the provider by
## calling `TypeName.request()` - with arguments respecting the signature(s).
## In async mode, this returns a Future[Result[TypeName, string]]. In sync mode, it returns Result[TypeName, string].
##
## Whenever you no want to process requests (or your object instance that provides the request goes out of scope),
## you can remove it from the broker with `TypeName.clearProvider()`.
##
##
## Example:
## ```nim
## RequestBroker:
## type Greeting = object
## text*: string
##
## ## Define the request and provider signature, that is enforced at compile time.
## proc signature*(): Future[Result[Greeting, string]] {.async.}
##
## ## Also possible to define signature with arbitrary input arguments.
## proc signature*(lang: string): Future[Result[Greeting, string]] {.async.}
##
## ...
## Greeting.setProvider(
## proc(): Future[Result[Greeting, string]] {.async.} =
## ok(Greeting(text: "hello"))
## )
## let res = await Greeting.request()
##
##
## ...
## # using native type as response for a synchronous request.
## RequestBroker(sync):
## type NeedThatInfo = string
##
##...
## NeedThatInfo.setProvider(
## proc(): Result[NeedThatInfo, string] =
## ok("this is the info you wanted")
## )
## let res = NeedThatInfo.request().valueOr:
## echo "not ok due to: " & error
## NeedThatInfo(":-(")
##
## echo string(res)
## ```
## If no `signature` proc is declared, a zero-argument form is generated
## automatically, so the caller only needs to provide the type definition.
import std/[macros, strutils]
from std/sequtils import keepItIf
import chronos
import results
import ./helper/broker_utils, broker_context
export results, chronos, keepItIf, broker_context
proc errorFuture[T](message: string): Future[Result[T, string]] {.inline.} =
## Build a future that is already completed with an error result.
let fut = newFuture[Result[T, string]]("request_broker.errorFuture")
fut.complete(err(Result[T, string], message))
fut
type RequestBrokerMode = enum
rbAsync
rbSync
proc isAsyncReturnTypeValid(returnType, typeIdent: NimNode): bool =
## Accept Future[Result[TypeIdent, string]] as the contract.
if returnType.kind != nnkBracketExpr or returnType.len != 2:
return false
if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Future"):
return false
let inner = returnType[1]
if inner.kind != nnkBracketExpr or inner.len != 3:
return false
if inner[0].kind != nnkIdent or not inner[0].eqIdent("Result"):
return false
if inner[1].kind != nnkIdent or not inner[1].eqIdent($typeIdent):
return false
inner[2].kind == nnkIdent and inner[2].eqIdent("string")
proc isSyncReturnTypeValid(returnType, typeIdent: NimNode): bool =
## Accept Result[TypeIdent, string] as the contract.
if returnType.kind != nnkBracketExpr or returnType.len != 3:
return false
if returnType[0].kind != nnkIdent or not returnType[0].eqIdent("Result"):
return false
if returnType[1].kind != nnkIdent or not returnType[1].eqIdent($typeIdent):
return false
returnType[2].kind == nnkIdent and returnType[2].eqIdent("string")
proc isReturnTypeValid(returnType, typeIdent: NimNode, mode: RequestBrokerMode): bool =
case mode
of rbAsync:
isAsyncReturnTypeValid(returnType, typeIdent)
of rbSync:
isSyncReturnTypeValid(returnType, typeIdent)
proc makeProcType(
returnType: NimNode, params: seq[NimNode], mode: RequestBrokerMode
): NimNode =
var formal = newTree(nnkFormalParams)
formal.add(returnType)
for param in params:
formal.add(param)
case mode
of rbAsync:
let pragmas = newTree(nnkPragma, ident("async"))
newTree(nnkProcTy, formal, pragmas)
of rbSync:
let raisesPragma = newTree(
nnkExprColonExpr, ident("raises"), newTree(nnkBracket, ident("CatchableError"))
)
let pragmas = newTree(nnkPragma, raisesPragma, ident("gcsafe"))
newTree(nnkProcTy, formal, pragmas)
proc parseMode(modeNode: NimNode): RequestBrokerMode =
## Parses the mode selector for the 2-argument macro overload.
## Supported spellings: `sync` / `async` (case-insensitive).
let raw = ($modeNode).strip().toLowerAscii()
case raw
of "sync":
rbSync
of "async":
rbAsync
else:
error("RequestBroker mode must be `sync` or `async` (default is async)", modeNode)
proc generateRequestBroker(body: NimNode, mode: RequestBrokerMode): NimNode =
when defined(requestBrokerDebug):
echo body.treeRepr
echo "RequestBroker mode: ", $mode
let parsed = parseSingleTypeDef(body, "RequestBroker", allowRefToNonObject = true)
let typeIdent = parsed.typeIdent
let objectDef = parsed.objectDef
when defined(requestBrokerDebug):
echo "RequestBroker generating type: ", $typeIdent
let exportedTypeIdent = postfix(copyNimTree(typeIdent), "*")
let typeDisplayName = sanitizeIdentName(typeIdent)
let typeNameLit = newLit(typeDisplayName)
var zeroArgSig: NimNode = nil
var zeroArgProviderName: NimNode = nil
var argSig: NimNode = nil
var argParams: seq[NimNode] = @[]
var argProviderName: NimNode = nil
for stmt in body:
case stmt.kind
of nnkProcDef:
let procName = stmt[0]
let procNameIdent =
case procName.kind
of nnkIdent:
procName
of nnkPostfix:
procName[1]
else:
procName
let procNameStr = $procNameIdent
if not procNameStr.startsWith("signature"):
error("Signature proc names must start with `signature`", procName)
let params = stmt.params
if params.len == 0:
error("Signature must declare a return type", stmt)
let returnType = params[0]
if not isReturnTypeValid(returnType, typeIdent, mode):
case mode
of rbAsync:
error(
"Signature must return Future[Result[`" & $typeIdent & "`, string]]", stmt
)
of rbSync:
error("Signature must return Result[`" & $typeIdent & "`, string]", stmt)
let paramCount = params.len - 1
if paramCount == 0:
if zeroArgSig != nil:
error("Only one zero-argument signature is allowed", stmt)
zeroArgSig = stmt
zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs")
elif paramCount >= 1:
if argSig != nil:
error("Only one argument-based signature is allowed", stmt)
argSig = stmt
argParams = @[]
for idx in 1 ..< params.len:
let paramDef = params[idx]
if paramDef.kind != nnkIdentDefs:
error(
"Signature parameter must be a standard identifier declaration", paramDef
)
let paramTypeNode = paramDef[paramDef.len - 2]
if paramTypeNode.kind == nnkEmpty:
error("Signature parameter must declare a type", paramDef)
var hasName = false
for i in 0 ..< paramDef.len - 2:
if paramDef[i].kind != nnkEmpty:
hasName = true
if not hasName:
error("Signature parameter must declare a name", paramDef)
argParams.add(copyNimTree(paramDef))
argProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderWithArgs")
of nnkTypeSection, nnkEmpty:
discard
else:
error("Unsupported statement inside RequestBroker definition", stmt)
if zeroArgSig.isNil() and argSig.isNil():
zeroArgSig = newEmptyNode()
zeroArgProviderName = ident(sanitizeIdentName(typeIdent) & "ProviderNoArgs")
var typeSection = newTree(nnkTypeSection)
typeSection.add(newTree(nnkTypeDef, exportedTypeIdent, newEmptyNode(), objectDef))
let returnType =
case mode
of rbAsync:
quote:
Future[Result[`typeIdent`, string]]
of rbSync:
quote:
Result[`typeIdent`, string]
if not zeroArgSig.isNil():
let procType = makeProcType(returnType, @[], mode)
typeSection.add(newTree(nnkTypeDef, zeroArgProviderName, newEmptyNode(), procType))
if not argSig.isNil():
let procType = makeProcType(returnType, cloneParams(argParams), mode)
typeSection.add(newTree(nnkTypeDef, argProviderName, newEmptyNode(), procType))
var brokerRecList = newTree(nnkRecList)
if not zeroArgSig.isNil():
let zeroArgProvidersFieldName = ident("providersNoArgs")
let zeroArgProvidersTupleTy = newTree(
nnkTupleTy,
newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()),
newTree(nnkIdentDefs, ident("handler"), zeroArgProviderName, newEmptyNode()),
)
let zeroArgProvidersSeqTy =
newTree(nnkBracketExpr, ident("seq"), zeroArgProvidersTupleTy)
brokerRecList.add(
newTree(
nnkIdentDefs, zeroArgProvidersFieldName, zeroArgProvidersSeqTy, newEmptyNode()
)
)
if not argSig.isNil():
let argProvidersFieldName = ident("providersWithArgs")
let argProvidersTupleTy = newTree(
nnkTupleTy,
newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode()),
newTree(nnkIdentDefs, ident("handler"), argProviderName, newEmptyNode()),
)
let argProvidersSeqTy = newTree(nnkBracketExpr, ident("seq"), argProvidersTupleTy)
brokerRecList.add(
newTree(nnkIdentDefs, argProvidersFieldName, argProvidersSeqTy, newEmptyNode())
)
let brokerTypeIdent = ident(sanitizeIdentName(typeIdent) & "Broker")
let brokerTypeDef = newTree(
nnkTypeDef,
brokerTypeIdent,
newEmptyNode(),
newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), brokerRecList),
)
typeSection.add(brokerTypeDef)
result = newStmtList()
result.add(typeSection)
let globalVarIdent = ident("g" & sanitizeIdentName(typeIdent) & "Broker")
let accessProcIdent = ident("access" & sanitizeIdentName(typeIdent) & "Broker")
var brokerNewBody = newStmtList()
if not zeroArgSig.isNil():
brokerNewBody.add(
quote do:
result.providersNoArgs =
@[(brokerCtx: DefaultBrokerContext, handler: default(`zeroArgProviderName`))]
)
if not argSig.isNil():
brokerNewBody.add(
quote do:
result.providersWithArgs =
@[(brokerCtx: DefaultBrokerContext, handler: default(`argProviderName`))]
)
var brokerInitChecks = newStmtList()
if not zeroArgSig.isNil():
brokerInitChecks.add(
quote do:
if `globalVarIdent`.providersNoArgs.len == 0:
`globalVarIdent` = `brokerTypeIdent`.new()
)
if not argSig.isNil():
brokerInitChecks.add(
quote do:
if `globalVarIdent`.providersWithArgs.len == 0:
`globalVarIdent` = `brokerTypeIdent`.new()
)
result.add(
quote do:
var `globalVarIdent` {.threadvar.}: `brokerTypeIdent`
proc new(_: type `brokerTypeIdent`): `brokerTypeIdent` =
result = `brokerTypeIdent`()
`brokerNewBody`
proc `accessProcIdent`(): var `brokerTypeIdent` =
`brokerInitChecks`
`globalVarIdent`
)
var clearBodyKeyed = newStmtList()
let brokerCtxParamIdent = ident("brokerCtx")
if not zeroArgSig.isNil():
let zeroArgProvidersFieldName = ident("providersNoArgs")
result.add(
quote do:
proc setProvider*(
_: typedesc[`typeIdent`], handler: `zeroArgProviderName`
): Result[void, string] =
if not `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler.isNil():
return err("Zero-arg provider already set")
`accessProcIdent`().`zeroArgProvidersFieldName`[0].handler = handler
return ok()
)
result.add(
quote do:
proc setProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
handler: `zeroArgProviderName`,
): Result[void, string] =
if brokerCtx == DefaultBrokerContext:
return setProvider(`typeIdent`, handler)
for entry in `accessProcIdent`().`zeroArgProvidersFieldName`:
if entry.brokerCtx == brokerCtx:
return err(
"RequestBroker(" & `typeNameLit` &
"): provider already set for broker context " & $brokerCtx
)
`accessProcIdent`().`zeroArgProvidersFieldName`.add(
(brokerCtx: brokerCtx, handler: handler)
)
return ok()
)
clearBodyKeyed.add(
quote do:
if `brokerCtxParamIdent` == DefaultBrokerContext:
`accessProcIdent`().`zeroArgProvidersFieldName`[0].handler =
default(`zeroArgProviderName`)
else:
`accessProcIdent`().`zeroArgProvidersFieldName`.keepItIf(
it.brokerCtx != `brokerCtxParamIdent`
)
)
case mode
of rbAsync:
result.add(
quote do:
proc request*(
_: typedesc[`typeIdent`]
): Future[Result[`typeIdent`, string]] {.async: (raises: []).} =
return await request(`typeIdent`, DefaultBrokerContext)
)
result.add(
quote do:
proc request*(
_: typedesc[`typeIdent`], brokerCtx: BrokerContext
): Future[Result[`typeIdent`, string]] {.async: (raises: []).} =
var provider: `zeroArgProviderName`
if brokerCtx == DefaultBrokerContext:
provider = `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler
else:
for entry in `accessProcIdent`().`zeroArgProvidersFieldName`:
if entry.brokerCtx == brokerCtx:
provider = entry.handler
break
if provider.isNil():
if brokerCtx == DefaultBrokerContext:
return err(
"RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered"
)
return err(
"RequestBroker(" & `typeNameLit` &
"): no provider registered for broker context " & $brokerCtx
)
let catchedRes = catch:
await provider()
if catchedRes.isErr():
return err(
"RequestBroker(" & `typeNameLit` & "): provider threw exception: " &
catchedRes.error.msg
)
let providerRes = catchedRes.get()
if providerRes.isOk():
let resultValue = providerRes.get()
when compiles(resultValue.isNil()):
if resultValue.isNil():
return err(
"RequestBroker(" & `typeNameLit` & "): provider returned nil result"
)
return providerRes
)
of rbSync:
result.add(
quote do:
proc request*(
_: typedesc[`typeIdent`]
): Result[`typeIdent`, string] {.gcsafe, raises: [].} =
return request(`typeIdent`, DefaultBrokerContext)
)
result.add(
quote do:
proc request*(
_: typedesc[`typeIdent`], brokerCtx: BrokerContext
): Result[`typeIdent`, string] {.gcsafe, raises: [].} =
var provider: `zeroArgProviderName`
if brokerCtx == DefaultBrokerContext:
provider = `accessProcIdent`().`zeroArgProvidersFieldName`[0].handler
else:
for entry in `accessProcIdent`().`zeroArgProvidersFieldName`:
if entry.brokerCtx == brokerCtx:
provider = entry.handler
break
if provider.isNil():
if brokerCtx == DefaultBrokerContext:
return err(
"RequestBroker(" & `typeNameLit` & "): no zero-arg provider registered"
)
return err(
"RequestBroker(" & `typeNameLit` &
"): no provider registered for broker context " & $brokerCtx
)
var providerRes: Result[`typeIdent`, string]
try:
providerRes = provider()
except CatchableError as e:
return err(
"RequestBroker(" & `typeNameLit` & "): provider threw exception: " &
e.msg
)
if providerRes.isOk():
let resultValue = providerRes.get()
when compiles(resultValue.isNil()):
if resultValue.isNil():
return err(
"RequestBroker(" & `typeNameLit` & "): provider returned nil result"
)
return providerRes
)
if not argSig.isNil():
let argProvidersFieldName = ident("providersWithArgs")
result.add(
quote do:
proc setProvider*(
_: typedesc[`typeIdent`], handler: `argProviderName`
): Result[void, string] =
if not `accessProcIdent`().`argProvidersFieldName`[0].handler.isNil():
return err("Provider already set")
`accessProcIdent`().`argProvidersFieldName`[0].handler = handler
return ok()
)
result.add(
quote do:
proc setProvider*(
_: typedesc[`typeIdent`],
brokerCtx: BrokerContext,
handler: `argProviderName`,
): Result[void, string] =
if brokerCtx == DefaultBrokerContext:
return setProvider(`typeIdent`, handler)
for entry in `accessProcIdent`().`argProvidersFieldName`:
if entry.brokerCtx == brokerCtx:
return err(
"RequestBroker(" & `typeNameLit` &
"): provider already set for broker context " & $brokerCtx
)
`accessProcIdent`().`argProvidersFieldName`.add(
(brokerCtx: brokerCtx, handler: handler)
)
return ok()
)
clearBodyKeyed.add(
quote do:
if `brokerCtxParamIdent` == DefaultBrokerContext:
`accessProcIdent`().`argProvidersFieldName`[0].handler =
default(`argProviderName`)
else:
`accessProcIdent`().`argProvidersFieldName`.keepItIf(
it.brokerCtx != `brokerCtxParamIdent`
)
)
let requestParamDefs = cloneParams(argParams)
let argNameIdents = collectParamNames(requestParamDefs)
var formalParams = newTree(nnkFormalParams)
formalParams.add(copyNimTree(returnType))
formalParams.add(
newTree(
nnkIdentDefs,
ident("_"),
newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)),
newEmptyNode(),
)
)
for paramDef in requestParamDefs:
formalParams.add(paramDef)
let requestPragmas =
case mode
of rbAsync:
quote:
{.async: (raises: []).}
of rbSync:
quote:
{.gcsafe, raises: [].}
var forwardCall = newCall(ident("request"))
forwardCall.add(copyNimTree(typeIdent))
forwardCall.add(ident("DefaultBrokerContext"))
for argName in argNameIdents:
forwardCall.add(argName)
var requestBody = newStmtList()
case mode
of rbAsync:
requestBody.add(
quote do:
return await `forwardCall`
)
of rbSync:
requestBody.add(
quote do:
return `forwardCall`
)
result.add(
newTree(
nnkProcDef,
postfix(ident("request"), "*"),
newEmptyNode(),
newEmptyNode(),
formalParams,
requestPragmas,
newEmptyNode(),
requestBody,
)
)
# Keyed request variant for the argument-based signature.
let requestParamDefsKeyed = cloneParams(argParams)
let argNameIdentsKeyed = collectParamNames(requestParamDefsKeyed)
let providerSymKeyed = genSym(nskVar, "provider")
var formalParamsKeyed = newTree(nnkFormalParams)
formalParamsKeyed.add(copyNimTree(returnType))
formalParamsKeyed.add(
newTree(
nnkIdentDefs,
ident("_"),
newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)),
newEmptyNode(),
)
)
formalParamsKeyed.add(
newTree(nnkIdentDefs, ident("brokerCtx"), ident("BrokerContext"), newEmptyNode())
)
for paramDef in requestParamDefsKeyed:
formalParamsKeyed.add(paramDef)
let requestPragmasKeyed = requestPragmas
var providerCallKeyed = newCall(providerSymKeyed)
for argName in argNameIdentsKeyed:
providerCallKeyed.add(argName)
var requestBodyKeyed = newStmtList()
requestBodyKeyed.add(
quote do:
var `providerSymKeyed`: `argProviderName`
if brokerCtx == DefaultBrokerContext:
`providerSymKeyed` = `accessProcIdent`().`argProvidersFieldName`[0].handler
else:
for entry in `accessProcIdent`().`argProvidersFieldName`:
if entry.brokerCtx == brokerCtx:
`providerSymKeyed` = entry.handler
break
)
requestBodyKeyed.add(
quote do:
if `providerSymKeyed`.isNil():
if brokerCtx == DefaultBrokerContext:
return err(
"RequestBroker(" & `typeNameLit` &
"): no provider registered for input signature"
)
return err(
"RequestBroker(" & `typeNameLit` &
"): no provider registered for broker context " & $brokerCtx
)
)
case mode
of rbAsync:
requestBodyKeyed.add(
quote do:
let catchedRes = catch:
await `providerCallKeyed`
if catchedRes.isErr():
return err(
"RequestBroker(" & `typeNameLit` & "): provider threw exception: " &
catchedRes.error.msg
)
let providerRes = catchedRes.get()
if providerRes.isOk():
let resultValue = providerRes.get()
when compiles(resultValue.isNil()):
if resultValue.isNil():
return err(
"RequestBroker(" & `typeNameLit` & "): provider returned nil result"
)
return providerRes
)
of rbSync:
requestBodyKeyed.add(
quote do:
var providerRes: Result[`typeIdent`, string]
try:
providerRes = `providerCallKeyed`
except CatchableError as e:
return err(
"RequestBroker(" & `typeNameLit` & "): provider threw exception: " & e.msg
)
if providerRes.isOk():
let resultValue = providerRes.get()
when compiles(resultValue.isNil()):
if resultValue.isNil():
return err(
"RequestBroker(" & `typeNameLit` & "): provider returned nil result"
)
return providerRes
)
result.add(
newTree(
nnkProcDef,
postfix(ident("request"), "*"),
newEmptyNode(),
newEmptyNode(),
formalParamsKeyed,
requestPragmasKeyed,
newEmptyNode(),
requestBodyKeyed,
)
)
block:
var formalParamsClearKeyed = newTree(nnkFormalParams)
formalParamsClearKeyed.add(newEmptyNode())
formalParamsClearKeyed.add(
newTree(
nnkIdentDefs,
ident("_"),
newTree(nnkBracketExpr, ident("typedesc"), copyNimTree(typeIdent)),
newEmptyNode(),
)
)
formalParamsClearKeyed.add(
newTree(nnkIdentDefs, brokerCtxParamIdent, ident("BrokerContext"), newEmptyNode())
)
result.add(
newTree(
nnkProcDef,
postfix(ident("clearProvider"), "*"),
newEmptyNode(),
newEmptyNode(),
formalParamsClearKeyed,
newEmptyNode(),
newEmptyNode(),
clearBodyKeyed,
)
)
result.add(
quote do:
proc clearProvider*(_: typedesc[`typeIdent`]) =
clearProvider(`typeIdent`, DefaultBrokerContext)
)
when defined(requestBrokerDebug):
echo result.repr
return result
macro RequestBroker*(body: untyped): untyped =
## Default (async) mode.
generateRequestBroker(body, rbAsync)
macro RequestBroker*(mode: untyped, body: untyped): untyped =
## Explicit mode selector.
## Example:
## RequestBroker(sync):
## type Foo = object
## proc signature*(): Result[Foo, string]
generateRequestBroker(body, parseMode(mode))

3
waku/events/delivery_events.nim
View File

@ -1,4 +1,5 @@
import waku/waku_core/[message/message, message/digest], waku/common/broker/event_broker
import brokers/event_broker
import waku/waku_core/[message/message, message/digest]
EventBroker:
type OnFilterSubscribeEvent* = object

4
waku/events/events.nim
View File

@ -1,3 +1,3 @@
import ./[message_events, delivery_events, health_events, peer_events]
import ./[message_events, delivery_events, health_events, peer_events, lifecycle_events]
export message_events, delivery_events, health_events, peer_events
export message_events, delivery_events, health_events, peer_events, lifecycle_events

2
waku/events/health_events.nim
View File

@ -1,4 +1,4 @@
import waku/common/broker/event_broker
import brokers/event_broker
import waku/api/types
import waku/node/health_monitor/[protocol_health, topic_health]

4
waku/events/message_events.nim
View File

@ -1,5 +1,5 @@
import waku/[api/types, waku_core/message, waku_core/topics, common/broker/event_broker]
import brokers/event_broker
import waku/[api/types, waku_core/message, waku_core/topics]
export types
EventBroker:

2
waku/events/peer_events.nim
View File

@ -1,4 +1,4 @@
import waku/common/broker/event_broker
import brokers/event_broker
import libp2p/switch
type WakuPeerEventKind* {.pure.} = enum

7
waku/factory/builder.nim
View File

@ -8,15 +8,16 @@ import
libp2p/builders,
libp2p/nameresolving/nameresolver,
libp2p/transports/wstransport,
libp2p/protocols/connectivity/relay/relay
libp2p/protocols/connectivity/relay/relay,
brokers/broker_context
import
../waku_enr,
../discovery/waku_discv5,
../waku_node,
../node/peer_manager,
../common/rate_limit/setting,
../common/utils/parse_size_units,
../common/broker/broker_context
../common/utils/parse_size_units
type
WakuNodeBuilder* = object # General

Some files were not shown because too many files have changed in this diff Show More