mirror of
https://github.com/logos-messaging/nim-chat-poc.git
synced 2026-07-04 23:49:46 +00:00
feat: add mix+LEZ chat simulation with Docker support
End-to-end simulation: 4 mix nodes with RLN spam protection, LEZ sequencer, gifter service, and two logos-chat-module clients exchanging double-ratchet-encrypted messages through 3-hop Sphinx onion routes. Simulation scripts: - run_simulation.sh: orchestrates sequencer, mix nodes, chat clients, with poll-based readiness checks and 15 verification checks - setup_and_run.sh: one-shot build + run for fresh clones - run_in_docker.sh: Docker-based Linux testing with pre-built modules - README.md: configuration, troubleshooting, architecture guide Configurable via SIM_* env vars (ports, timeouts, log level). Cross-platform: macOS native + Linux via Docker. Docker image (Dockerfile.sim) uses multi-stage build: - Stage 1: builds all LEZ modules, sequencer, liblogoschat, chat-module - Stage 2: copies only runtime nix closure (~1.9GB) + output artifacts - Pre-built modules are symlinked at runtime, skipping build_all.sh
This commit is contained in:
parent
4060bb67c1
commit
695ecb42bc
3
.gitignore
vendored
3
.gitignore
vendored
@ -37,6 +37,9 @@ nimble.paths
|
||||
/metrics/prometheus
|
||||
/metrics/waku-sim-all-nodes-grafana-dashboard.json
|
||||
|
||||
# Simulation runtime state (logs, keystores, configs generated per run)
|
||||
/simulations/**/.sim_state/
|
||||
|
||||
*.log
|
||||
/package-lock.json
|
||||
/package.json
|
||||
|
||||
121
scripts/run_in_docker.sh
Executable file
121
scripts/run_in_docker.sh
Executable file
@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the mix+LEZ chat simulation inside a Docker container (Linux aarch64).
|
||||
#
|
||||
# The Docker image pre-builds EVERYTHING: LEZ modules, sequencer, logoscore,
|
||||
# liblogoschat, chat_module_plugin. Each sim run only clones the repo (for
|
||||
# scripts + configs), symlinks pre-built artifacts, and runs the simulation.
|
||||
#
|
||||
# First run: ~60 min (one-time docker build)
|
||||
# Subsequent runs: ~5 min (clone + submodule init + sim)
|
||||
#
|
||||
# Prerequisites: Docker Desktop running.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/run_in_docker.sh
|
||||
# BRANCH=my-branch bash scripts/run_in_docker.sh # test a different branch
|
||||
#
|
||||
# Environment variables:
|
||||
# BRANCH — git branch to clone (default: feat/logos-delivery)
|
||||
# REPO_URL — git repo URL
|
||||
# GUEST_BINARIES_DIR — path to pre-built guest .bin files (auto-detected)
|
||||
# REBUILD_IMAGE — set to 1 to force image rebuild
|
||||
# SIM_* — simulation parameters, passed through to run_simulation.sh
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DOCKERFILE="$ROOT/.github/Dockerfile.sim"
|
||||
IMAGE_NAME="logos-chat-sim"
|
||||
CONTAINER_NAME="logos-chat-sim-run"
|
||||
BRANCH="${BRANCH:-feat/logos-delivery}"
|
||||
REPO_URL="${REPO_URL:-https://github.com/adklempner/logos-chat.git}"
|
||||
|
||||
# Build image if it doesn't exist or REBUILD_IMAGE=1
|
||||
if [ "${REBUILD_IMAGE:-0}" = "1" ] || ! docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then
|
||||
echo "=== Building Docker image (one-time — ~60 min) ==="
|
||||
docker build -t "$IMAGE_NAME" -f "$DOCKERFILE" "$ROOT"
|
||||
else
|
||||
echo "=== Docker image cached ==="
|
||||
fi
|
||||
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||
trap 'echo "=== Rescuing logs ==="; docker cp "$CONTAINER_NAME:/root/logos-chat/simulations/mix_lez_chat/.sim_state" ./docker-sim-logs 2>/dev/null || true; docker rm -f "$CONTAINER_NAME" 2>/dev/null || true' EXIT
|
||||
|
||||
echo "=== Starting container ==="
|
||||
docker run --rm -d --name "$CONTAINER_NAME" "$IMAGE_NAME" tail -f /dev/null
|
||||
|
||||
# Stage guest binaries
|
||||
GUEST_REL="vendor/logos-lez-rln/lez-rln/methods/guest/target/riscv32im-risc0-zkvm-elf/docker"
|
||||
GUEST_SRC=""
|
||||
for candidate in \
|
||||
"${GUEST_BINARIES_DIR:-}" \
|
||||
"$ROOT/$GUEST_REL" \
|
||||
"$HOME/Waku/Logos/logos-chat/$GUEST_REL" \
|
||||
"../logos-chat/$GUEST_REL"; do
|
||||
[ -f "$candidate/rln_registration.bin" ] 2>/dev/null && GUEST_SRC="$candidate" && break
|
||||
done
|
||||
if [ -n "$GUEST_SRC" ]; then
|
||||
echo "=== Staging guest binaries ==="
|
||||
docker exec "$CONTAINER_NAME" mkdir -p "/tmp/guest-bins"
|
||||
docker cp "$GUEST_SRC/rln_registration.bin" "$CONTAINER_NAME:/tmp/guest-bins/"
|
||||
docker cp "$GUEST_SRC/incremental_merkle_tree.bin" "$CONTAINER_NAME:/tmp/guest-bins/"
|
||||
fi
|
||||
|
||||
# Collect SIM_* env vars
|
||||
SIM_ENVS=""
|
||||
for var in $(env | grep '^SIM_' | cut -d= -f1); do
|
||||
SIM_ENVS="$SIM_ENVS export $var='${!var}';"
|
||||
done
|
||||
|
||||
echo "=== Running simulation ==="
|
||||
docker exec "$CONTAINER_NAME" bash -c "
|
||||
$SIM_ENVS
|
||||
export RISC0_DEV_MODE=1
|
||||
|
||||
# Clone repo (just for scripts + configs, not for building)
|
||||
cd /root
|
||||
rm -rf logos-chat
|
||||
git clone --depth 1 -b $BRANCH $REPO_URL
|
||||
cd logos-chat
|
||||
# Only init top-level submodules + logos-lez-rln selectively (for sim script paths)
|
||||
git submodule update --init --depth 1
|
||||
(cd vendor/logos-lez-rln && git submodule update --init --depth 1 lssa logos-delivery logos-delivery-module logos-execution-zone-module)
|
||||
(cd vendor/logos-lez-rln/logos-delivery-module && git submodule update --init --depth 1 vendor/logos-delivery)
|
||||
|
||||
# Symlink ALL pre-built artifacts from the Docker image
|
||||
LEZ_DIR=vendor/logos-lez-rln
|
||||
ln -sf /root/lez-modules/result-rln \$LEZ_DIR/logos-rln-module/result-rln
|
||||
ln -sf /root/lez-modules/result-wallet \$LEZ_DIR/logos-rln-module/result-wallet
|
||||
mkdir -p \$LEZ_DIR/logos-delivery-module/build_plugin
|
||||
cp -r /root/lez-modules/delivery-plugin \$LEZ_DIR/logos-delivery-module/build_plugin/modules
|
||||
mkdir -p \$LEZ_DIR/logos-delivery-module/vendor/logos-delivery/build
|
||||
cp /root/lez-modules/delivery-build/* \$LEZ_DIR/logos-delivery-module/vendor/logos-delivery/build/ 2>/dev/null || true
|
||||
# Sequencer is built from source at runtime (pre-built crashes in risc0 dev prover).
|
||||
# Reuse cargo target cache from image to speed up (~30s vs ~2min).
|
||||
ln -sf /root/lez-modules/lssa-target-cache \$LEZ_DIR/lssa/target
|
||||
mkdir -p \$LEZ_DIR/lez-rln/target/debug
|
||||
cp /root/lez-modules/run_setup \$LEZ_DIR/lez-rln/target/debug/
|
||||
mkdir -p build
|
||||
cp /root/lez-modules/liblogoschat.so build/
|
||||
|
||||
# Restore guest binaries
|
||||
GUEST_DIR=\"\$LEZ_DIR/lez-rln/methods/guest/target/riscv32im-risc0-zkvm-elf/docker\"
|
||||
if [ -f /tmp/guest-bins/rln_registration.bin ]; then
|
||||
mkdir -p \"\$GUEST_DIR\"
|
||||
cp /tmp/guest-bins/*.bin \"\$GUEST_DIR/\"
|
||||
fi
|
||||
|
||||
# Chat module — symlink from pre-built in image
|
||||
ln -sf /root/lez-modules/chat-module-result /root/logos-chat-module/result 2>/dev/null || \
|
||||
(mkdir -p /root/logos-chat-module && ln -sf /root/lez-modules/chat-module-result /root/logos-chat-module/result)
|
||||
|
||||
# Set env vars for sim
|
||||
export LOGOSCORE=\"/root/lez-modules/logoscore-result/bin/logoscore\"
|
||||
CLANG_SO=\$(find /nix/store -maxdepth 3 -name 'libclang.so' 2>/dev/null | head -1 || true)
|
||||
[ -n \"\$CLANG_SO\" ] && export LIBCLANG_PATH=\$(dirname \"\$CLANG_SO\")
|
||||
|
||||
bash simulations/mix_lez_chat/run_simulation.sh --fresh
|
||||
"
|
||||
|
||||
echo "=== Done ==="
|
||||
# Container cleanup handled by EXIT trap (which also rescues logs on failure)
|
||||
162
simulations/mix_lez_chat/README.md
Normal file
162
simulations/mix_lez_chat/README.md
Normal file
@ -0,0 +1,162 @@
|
||||
# Mix + LEZ RLN Chat Simulation
|
||||
|
||||
End-to-end private chat between two `logos-chat-module` clients over a 4-node mix network with LEZ-backed RLN spam protection against a local LEZ sequencer.
|
||||
|
||||
Two logoscore instances (sender + receiver) load `chat_module` and establish an X3DH key agreement via an out-of-band intro bundle, then exchange double-ratchet-encrypted messages routed through 3-hop Sphinx onion routes with per-hop RLN proof generation and verification. Node 0 mounts the `rln_gifter` service; nodes 1-3 and both chat clients register RLN memberships on-chain via the gifter protocol. The chat clients run `mix: true, relay: false, filter: true` — the sender publishes via `lightpushPublish(mixify=true)`, the mix exit node verifies the RLN proof before fanning out via gossipsub relay to the shard, and the receiver consumes the message via a Waku filter subscription on one of the mix nodes.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
logoscore (per mix node) logoscore (per chat client)
|
||||
├── wallet_module (LEZ wallet) ├── wallet_module (LEZ wallet)
|
||||
├── liblogos_rln_module (RLN proofs) ├── liblogos_rln_module (RLN proofs)
|
||||
└── delivery_module (Waku mix relay) └── chat_module (logos-chat-module)
|
||||
├── liblogosdelivery.so ├── chat_module_plugin.so
|
||||
└── mix + relay + filter + gifter └── liblogoschat.so
|
||||
└── mix client + filter + gifter client
|
||||
```
|
||||
|
||||
Node 0 runs the RLN gifter service. Nodes 1-3 register via gifter on startup. Chat clients also register via gifter when `startChat()` runs.
|
||||
|
||||
## Quick start
|
||||
|
||||
**Prereqs:** nix (with flakes), Docker (for guest zkVM binaries), cargo-risczero, SSH access to GitHub.
|
||||
|
||||
**Run from scratch:**
|
||||
|
||||
```bash
|
||||
git clone -b feat/logos-delivery git@github.com:adklempner/logos-chat.git
|
||||
cd logos-chat && bash simulations/mix_lez_chat/setup_and_run.sh
|
||||
```
|
||||
|
||||
**Re-run (after initial build):**
|
||||
|
||||
```bash
|
||||
bash simulations/mix_lez_chat/run_simulation.sh --fresh
|
||||
```
|
||||
|
||||
Pass = **ALL 15 CHECKS PASSED**.
|
||||
|
||||
## Configuration
|
||||
|
||||
Override via environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `SIM_NUM_NODES` | `4` | Number of mix relay nodes |
|
||||
| `SIM_BASE_TCP_PORT` | `60001` | First node's TCP port (increments per node) |
|
||||
| `SIM_BASE_DISC_PORT` | `9001` | First node's discv5 UDP port (increments per node) |
|
||||
| `SIM_CLUSTER_ID` | `99` | Waku cluster ID |
|
||||
| `SIM_LOG_LEVEL` | `INFO` | Node log level (TRACE, DEBUG, INFO, WARN, ERROR) |
|
||||
| `SIM_CHAT_RECV_PORT` | `60010` | Chat receiver TCP port |
|
||||
| `SIM_CHAT_SEND_PORT` | `60011` | Chat sender TCP port |
|
||||
| `SIM_KADEMLIA_MIN_WAIT` | `30` | Minimum seconds to wait for kademlia propagation |
|
||||
| `SIM_RECEIVER_MIN_WAIT` | `15` | Minimum seconds to wait for receiver to join mix |
|
||||
| `SIM_DELIVERY_TIMEOUT` | `120` | Max seconds to wait for message delivery |
|
||||
|
||||
Example — fast iteration with verbose logging:
|
||||
|
||||
```bash
|
||||
SIM_LOG_LEVEL=TRACE SIM_KADEMLIA_MIN_WAIT=10 SIM_RECEIVER_MIN_WAIT=5 \
|
||||
bash simulations/mix_lez_chat/run_simulation.sh --fresh
|
||||
```
|
||||
|
||||
## `--fresh` behavior
|
||||
|
||||
When `--fresh` is passed:
|
||||
- Kills all existing `logos_host` processes
|
||||
- Cleans `/tmp/logos_*` Qt RemoteObjects sockets
|
||||
- Removes `.sim_state/` directory
|
||||
- Removes sequencer state (`rocksdb/`, `bedrock_signing_key`)
|
||||
- Rebuilds and restarts the sequencer
|
||||
- Redeploys RLN programs via `run_setup`
|
||||
|
||||
Without `--fresh`, reuses existing sequencer if port 3040 is already bound.
|
||||
|
||||
## Checks (15 total)
|
||||
|
||||
| Category | Check | What it verifies |
|
||||
|---|---|---|
|
||||
| Mix nodes (4) | Node N mounted mix | Mix protocol handler registered |
|
||||
| RLN gifter | Node 0 gifter service mounted | `/logos/rln-gifter/1.0.0` protocol handler |
|
||||
| LEZ RLN | LEZ root polling active | Nodes polling valid Merkle roots from LEZ |
|
||||
| Chat module (4) | Receiver/Sender initialized | `chatInitResult` event fired |
|
||||
| | Receiver/Sender started | `Waku client started` + `chatStartResult` |
|
||||
| | Receiver/Sender mounted mix+LEZ | Mix protocol + LEZ callbacks wired |
|
||||
| | Receiver created intro bundle | X3DH pre-key bundle generated |
|
||||
| Message exchange (2) | Sender sent message | `chatNewPrivateConversationResult` |
|
||||
| | Receiver received message | `chatNewMessage` via filter subscription |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"Sequencer failed to start"** — port 3040 already in use:
|
||||
```bash
|
||||
kill $(lsof -ti tcp:3040) && bash simulations/mix_lez_chat/run_simulation.sh --fresh
|
||||
```
|
||||
|
||||
**"run_setup failed" / "Timeout waiting for account"** — stale guest binaries or wallet state:
|
||||
```bash
|
||||
rm -rf vendor/logos-lez-rln/lez-rln/methods/guest/target
|
||||
rm -f vendor/logos-lez-rln/dev/wallet_config.json vendor/logos-lez-rln/dev/storage.json
|
||||
bash simulations/mix_lez_chat/setup_and_run.sh
|
||||
```
|
||||
|
||||
**"Sender started FAIL"** — stale Qt RemoteObjects sockets:
|
||||
```bash
|
||||
rm -f /tmp/logos_*
|
||||
bash simulations/mix_lez_chat/run_simulation.sh --fresh
|
||||
```
|
||||
|
||||
**"FAIL: Receiver received message (0)"** — timing issue, try re-running:
|
||||
```bash
|
||||
bash simulations/mix_lez_chat/run_simulation.sh --fresh
|
||||
```
|
||||
|
||||
## Adapting for other LEZ programs
|
||||
|
||||
This simulation provides a complete mix network infrastructure that other logos modules can reuse for testing. To test your own module:
|
||||
|
||||
### What the sim provides
|
||||
- 4 logoscore mix nodes with `delivery_module` (Waku relay + mix + RLN)
|
||||
- LEZ sequencer with deployed RLN programs
|
||||
- RLN gifter service on node 0
|
||||
- Wallet modules for on-chain transactions
|
||||
|
||||
### What you replace
|
||||
The chat_module sender/receiver instances (phase 5 of run_simulation.sh). Your module needs:
|
||||
|
||||
1. **A C++ Qt plugin** implementing `PluginInterface` (see `chat_module_plugin.cpp`)
|
||||
- `initLogos(LogosAPI*)` — receive the LogosAPI instance
|
||||
- `eventResponse(QString, QVariantList)` signal — mandatory per logos-liblogos contract
|
||||
- Methods exposed via `LOGOS_METHOD` for logoscore `-c` invocation
|
||||
2. **A shared library** with your program logic (like `liblogoschat.so`)
|
||||
3. **RLN integration** — wire `setRlnConfig` to pass RLN credentials from the C++ plugin to your library
|
||||
4. **EVENT: stderr fallback** — on Linux, Qt signal forwarding from plugin to logoscore doesn't work across the FFI thread boundary. Write event data to stderr in `EVENT:name:data` format for cross-platform reliability.
|
||||
|
||||
### How to stage your module
|
||||
|
||||
```bash
|
||||
MDIR=$(mktemp -d)
|
||||
mkdir -p "$MDIR/your_module"
|
||||
cp your_module_plugin.so "$MDIR/your_module/"
|
||||
cp libyour_library.so "$MDIR/your_module/"
|
||||
echo '{"name":"your_module","version":"1.0.0","type":"core",...}' > "$MDIR/your_module/manifest.json"
|
||||
|
||||
logoscore -m "$MDIR" \
|
||||
-l "liblogos_execution_zone_wallet_module,liblogos_rln_module,your_module" \
|
||||
-c "liblogos_execution_zone_wallet_module.open($WALLET_CONFIG,$WALLET_STORAGE)" \
|
||||
-c "your_module.init(@config.json)" \
|
||||
-c "your_module.start()"
|
||||
```
|
||||
|
||||
### Reference
|
||||
- `chat_module_plugin.cpp` — complete working example with RLN, gifter, mix, and event emission
|
||||
- `delivery_module_plugin.cpp` — more complex example with full RLN fetcher integration
|
||||
- `run_simulation.sh` — orchestration, module staging, and verification patterns
|
||||
|
||||
## Logs
|
||||
|
||||
All logs in `simulations/mix_lez_chat/.sim_state/`:
|
||||
- `node0.log` – `node3.log` — mix relay nodes
|
||||
- `chat_receiver.log` — receiver chat module
|
||||
- `chat_sender.log` — sender chat module
|
||||
BIN
simulations/mix_lez_chat/fixtures/gifter_auth/.build/derive_addr
Executable file
BIN
simulations/mix_lez_chat/fixtures/gifter_auth/.build/derive_addr
Executable file
Binary file not shown.
585
simulations/mix_lez_chat/run_simulation.sh
Executable file
585
simulations/mix_lez_chat/run_simulation.sh
Executable file
@ -0,0 +1,585 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mix + LEZ RLN simulation using logos-chat-module as sender/receiver.
|
||||
# Reuses the logoscore mix node infrastructure from logos-lez-rln and replaces
|
||||
# chat2mix with logoscore instances running the chat_module plugin.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - logos-lez-rln repo as sibling or set LEZ_RLN_DIR
|
||||
# - logos-chat-module built (nix build in ../logos-chat-module)
|
||||
# - logos-chat built (make liblogoschat in this repo)
|
||||
#
|
||||
# Usage: ./run_simulation.sh [--fresh]
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
LOGOS_CHAT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
CHAT_MODULE_DIR="${CHAT_MODULE_DIR:-$(cd "$LOGOS_CHAT_DIR/../logos-chat-module" && pwd)}"
|
||||
|
||||
# Use vendored logos-lez-rln submodule, or auto-detect as sibling
|
||||
LEZ_RLN_DIR="${LEZ_RLN_DIR:-}"
|
||||
if [ -z "$LEZ_RLN_DIR" ] && [ -d "$LOGOS_CHAT_DIR/vendor/logos-lez-rln/lez-rln" ]; then
|
||||
LEZ_RLN_DIR="$LOGOS_CHAT_DIR/vendor/logos-lez-rln"
|
||||
fi
|
||||
for candidate in "$LOGOS_CHAT_DIR/.." "$LOGOS_CHAT_DIR/../logos-lez-rln"; do
|
||||
[ -n "$LEZ_RLN_DIR" ] && break
|
||||
[ -d "$candidate/lez-rln" ] && [ -d "$candidate/lssa" ] && LEZ_RLN_DIR="$(cd "$candidate" && pwd)" && break
|
||||
done
|
||||
[ -z "$LEZ_RLN_DIR" ] && { echo "FATAL: Cannot find logos-lez-rln repo. Set LEZ_RLN_DIR or run: git submodule update --init --recursive"; exit 1; }
|
||||
|
||||
DELIVERY_MODULE_DIR="${DELIVERY_MODULE_DIR:-$LEZ_RLN_DIR/logos-delivery-module}"
|
||||
DELIVERY_DIR="$DELIVERY_MODULE_DIR/vendor/logos-delivery"
|
||||
|
||||
export RISC0_DEV_MODE=1
|
||||
export TMPDIR=/tmp
|
||||
export LOGOS_EVENT_STDERR=1 # Enable EVENT: stderr output for sim script event observation
|
||||
|
||||
die() { echo " FATAL: $*" >&2; exit 1; }
|
||||
log() { echo "[$(date '+%H:%M:%S')] $*"; }
|
||||
|
||||
# --- Node identity constants (4 mix nodes) ---
|
||||
NODEKEYS=(
|
||||
"f98e3fba96c32e8d1967d460f1b79457380e1a895f7971cecc8528abe733781a"
|
||||
"09e9d134331953357bd38bbfce8edb377f4b6308b4f3bfbe85c610497053d684"
|
||||
"ed54db994682e857d77cd6fb81be697382dc43aa5cd78e16b0ec8098549f860e"
|
||||
"42f96f29f2d6670938b0864aced65a332dcf5774103b4c44ec4d0ea4ef3c47d6"
|
||||
)
|
||||
PEER_IDS=(
|
||||
"16Uiu2HAmPiEs2ozjjJF2iN2Pe2FYeMC9w4caRHKYdLdAfjgbWM6o"
|
||||
"16Uiu2HAmLtKaFaSWDohToWhWUZFLtqzYZGPFuXwKrojFVF6az5UF"
|
||||
"16Uiu2HAmTEDHwAziWUSz6ZE23h5vxG2o4Nn7GazhMor4bVuMXTrA"
|
||||
"16Uiu2HAmPwRKZajXtfb1Qsv45VVfRZgK3ENdfmnqzSrVm3BczF6f"
|
||||
)
|
||||
MIXKEYS=(
|
||||
"c86029e02c05a7e25182974b519d0d52fcbafeca6fe191fbb64857fb05be1a53"
|
||||
"b858ac16bbb551c4b2973313b1c8c8f7ea469fca03f1608d200bbf58d388ec7f"
|
||||
"d8bd379bb394b0f22dd236d63af9f1a9bc45266beffc3fbbe19e8b6575f2535b"
|
||||
"780fff09e51e98df574e266bf3266ec6a3a1ddfcf7da826a349a29c137009d49"
|
||||
)
|
||||
MIX_PUBKEYS=(
|
||||
"9231e86da6432502900a84f867004ce78632ab52cd8e30b1ec322cd795710c2a"
|
||||
"275cd6889e1f29ca48e5b9edb800d1a94f49f13d393a0ecf1a07af753506de6c"
|
||||
"e0ed594a8d506681be075e8e23723478388fb182477f7a469309a25e7076fc18"
|
||||
"8fd7a1a7c19b403d231452a9b1ea40eb1cc76f455d918ef8980e7685f9eeeb1f"
|
||||
)
|
||||
# --- Configurable parameters (override via environment) ---
|
||||
NUM_NODES=${SIM_NUM_NODES:-4}
|
||||
BASE_TCP_PORT=${SIM_BASE_TCP_PORT:-60001}
|
||||
BASE_DISC_PORT=${SIM_BASE_DISC_PORT:-9001}
|
||||
CLUSTER_ID=${SIM_CLUSTER_ID:-99}
|
||||
NUM_SHARDS=1
|
||||
CONTENT_TOPIC="/logos-chat/1/mix-test/proto"
|
||||
TEST_MESSAGE_PREFIX="chatmixtest"
|
||||
LOG_LEVEL=${SIM_LOG_LEVEL:-INFO}
|
||||
CHAT_RECV_PORT=${SIM_CHAT_RECV_PORT:-60010}
|
||||
CHAT_SEND_PORT=${SIM_CHAT_SEND_PORT:-60011}
|
||||
KADEMLIA_MIN_WAIT=${SIM_KADEMLIA_MIN_WAIT:-30}
|
||||
RECEIVER_MIN_WAIT=${SIM_RECEIVER_MIN_WAIT:-15}
|
||||
DELIVERY_TIMEOUT=${SIM_DELIVERY_TIMEOUT:-120}
|
||||
|
||||
case "$(uname -s)-$(uname -m)" in
|
||||
Darwin-arm64) PLATFORM="darwin-arm64-dev"; EXT="dylib";;
|
||||
Linux-x86_64) PLATFORM="linux-x86_64-dev"; EXT="so";;
|
||||
Linux-aarch64) PLATFORM="linux-aarch64-dev"; EXT="so";;
|
||||
*) die "Unsupported platform";;
|
||||
esac
|
||||
|
||||
STATE_DIR="$SCRIPT_DIR/.sim_state"
|
||||
FRESH=0
|
||||
for arg in "$@"; do [ "$arg" = "--fresh" ] && FRESH=1; done
|
||||
[ "$FRESH" -eq 1 ] && rm -rf "$STATE_DIR"
|
||||
mkdir -p "$STATE_DIR"
|
||||
|
||||
SEQUENCER_PID=""
|
||||
OWN_SEQUENCER=0
|
||||
INSTANCE_PIDS=()
|
||||
MODULES_DIRS=()
|
||||
SENDER_PID=""
|
||||
RECEIVER_PID=""
|
||||
EXIT_CODE=1
|
||||
|
||||
cleanup() {
|
||||
set +u
|
||||
echo ""; echo "=== Shutting down ==="
|
||||
[ -n "$SENDER_PID" ] && kill "$SENDER_PID" 2>/dev/null || true
|
||||
[ -n "$RECEIVER_PID" ] && kill "$RECEIVER_PID" 2>/dev/null || true
|
||||
for pid in "${INSTANCE_PIDS[@]+"${INSTANCE_PIDS[@]}"}"; do [ -n "$pid" ] && kill "$pid" 2>/dev/null || true; done
|
||||
pkill -f 'logos_host' 2>/dev/null || true
|
||||
[ "$OWN_SEQUENCER" -eq 1 ] && [ -n "$SEQUENCER_PID" ] && kill "$SEQUENCER_PID" 2>/dev/null || true
|
||||
for mdir in "${MODULES_DIRS[@]+"${MODULES_DIRS[@]}"}"; do [ -n "$mdir" ] && rm -rf "$mdir"; done
|
||||
echo " Logs: $STATE_DIR"; echo "Done."; exit "$EXIT_CODE"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "=== Mix + LEZ RLN Chat Simulation ($NUM_NODES nodes) ==="
|
||||
echo " LEZ repo: $LEZ_RLN_DIR"
|
||||
echo " Chat module: $CHAT_MODULE_DIR"
|
||||
echo " Logos-chat: $LOGOS_CHAT_DIR"
|
||||
echo ""
|
||||
|
||||
pkill -f 'logos_host' 2>/dev/null || true; sleep 1
|
||||
# Clean stale QtRO LocalServer sockets from prior runs (can confuse capability_module lookups)
|
||||
rm -f /tmp/logos_* 2>/dev/null || true
|
||||
|
||||
# ---------- Phase 1: Sequencer ----------
|
||||
echo "[1/6] Sequencer..."
|
||||
if nc -z 127.0.0.1 3040 2>/dev/null && [ "$FRESH" -eq 0 ]; then
|
||||
SEQUENCER_PID=$(lsof -ti tcp:3040 2>/dev/null || true)
|
||||
echo " Already running (PID $SEQUENCER_PID)"
|
||||
else
|
||||
[ "$(nc -z 127.0.0.1 3040 2>/dev/null; echo $?)" = "0" ] && kill "$(lsof -ti tcp:3040 2>/dev/null)" 2>/dev/null || true; sleep 1
|
||||
rm -rf "$LEZ_RLN_DIR/lssa/rocksdb" "$LEZ_RLN_DIR/lssa/sequencer/service/bedrock_signing_key"
|
||||
|
||||
# Skip sequencer build if binary already exists (pre-built in Docker image).
|
||||
# Also skip lssa auto-sync (requires full git history, fails on shallow clones).
|
||||
if [ -x "$LEZ_RLN_DIR/lssa/target/debug/sequencer_service" ]; then
|
||||
log " Using pre-built sequencer"
|
||||
SEQ_BIN="./target/debug/sequencer_service"; SEQ_CFG="sequencer/service/configs/debug/sequencer_config.json"
|
||||
else
|
||||
# Sequencer guest binaries must match the lssa rev that lez-rln's host-side
|
||||
# client pins. Mismatch → DeserializeUnexpectedEnd (wire format divergence).
|
||||
LSSA_REV=$(grep -oE 'rev\s*=\s*"[0-9a-f]+"' "$LEZ_RLN_DIR/lez-rln/Cargo.toml" | head -1 | sed 's/.*"\([0-9a-f]*\)"/\1/')
|
||||
[ -z "$LSSA_REV" ] && die "Could not extract lssa rev from lez-rln/Cargo.toml"
|
||||
if ! git -C "$LEZ_RLN_DIR/lssa" merge-base --is-ancestor "$LSSA_REV" HEAD 2>/dev/null; then
|
||||
log " Pinning lssa to $LSSA_REV..."
|
||||
(cd "$LEZ_RLN_DIR/lssa" && git fetch --quiet origin && git checkout --quiet "$LSSA_REV") \
|
||||
|| die "lssa checkout $LSSA_REV failed"
|
||||
fi
|
||||
log " Building sequencer..."
|
||||
if (cd "$LEZ_RLN_DIR/lssa" && cargo build --features standalone -p sequencer_service 2>&1 | tail -3); then
|
||||
SEQ_BIN="./target/debug/sequencer_service"; SEQ_CFG="sequencer/service/configs/debug/sequencer_config.json"
|
||||
elif (cd "$LEZ_RLN_DIR/lssa" && cargo build --features standalone -p sequencer_runner 2>&1 | tail -3); then
|
||||
SEQ_BIN="./target/debug/sequencer_runner"; SEQ_CFG="sequencer_runner/configs/debug"
|
||||
else die "sequencer build failed"; fi
|
||||
fi
|
||||
(cd "$LEZ_RLN_DIR/lssa" && env RUST_LOG=info "$SEQ_BIN" "$SEQ_CFG") >"$STATE_DIR/sequencer.log" 2>&1 &
|
||||
SEQUENCER_PID=$!; OWN_SEQUENCER=1; echo " PID: $SEQUENCER_PID"
|
||||
for _ in $(seq 1 60); do nc -z 127.0.0.1 3040 2>/dev/null && break; sleep 1; done
|
||||
nc -z 127.0.0.1 3040 2>/dev/null || die "Sequencer failed to start"
|
||||
log " Ready."
|
||||
fi
|
||||
|
||||
# ---------- Phase 2: Deploy programs ----------
|
||||
echo "[2/6] Deploying programs..."
|
||||
export NSSA_WALLET_HOME_DIR="$LEZ_RLN_DIR/dev"
|
||||
export WALLET_CONFIG="$NSSA_WALLET_HOME_DIR/wallet_config.json"
|
||||
export WALLET_STORAGE="$NSSA_WALLET_HOME_DIR/storage.json"
|
||||
TREE_ID_HEX="000102030405060708090a0b0c0d0e0f1011121314151617"
|
||||
GIFTER_ACCOUNT_FILE="$HOME/.logos-lez-rln/payment_account_${TREE_ID_HEX}.txt"
|
||||
|
||||
rm -f "$WALLET_CONFIG" "$WALLET_STORAGE"
|
||||
# Use pre-built binary if available, otherwise cargo run
|
||||
if [ -x "$LEZ_RLN_DIR/lez-rln/target/debug/run_setup" ]; then
|
||||
SETUP_OUTPUT=$(cd "$LEZ_RLN_DIR/lez-rln" && ./target/debug/run_setup 2>&1) || die "run_setup failed"
|
||||
else
|
||||
SETUP_OUTPUT=$(cd "$LEZ_RLN_DIR/lez-rln" && cargo run --bin run_setup 2>&1) || die "run_setup failed"
|
||||
fi
|
||||
echo "$SETUP_OUTPUT" | tail -4
|
||||
CONFIG_ACCOUNT=$(echo "$SETUP_OUTPUT" | grep -oE 'Config account:\s+\S+' | awk '{print $NF}' || true)
|
||||
[ -z "$CONFIG_ACCOUNT" ] && die "Failed to parse config account"
|
||||
GIFTER_ACCOUNT=$(cat "$GIFTER_ACCOUNT_FILE" 2>/dev/null || true)
|
||||
[ -z "$GIFTER_ACCOUNT" ] && die "Gifter account not found at $GIFTER_ACCOUNT_FILE"
|
||||
|
||||
# ---------- Phase 3: Prerequisites ----------
|
||||
echo "[3/6] Verifying prerequisites..."
|
||||
LOGOSCORE="${LOGOSCORE:-$(nix build github:logos-co/logos-liblogos/7df6195 --override-input logos-cpp-sdk github:logos-co/logos-cpp-sdk/a4bd66c --no-link --print-out-paths)/bin/logoscore}"
|
||||
RLN_MODULE="$LEZ_RLN_DIR/logos-rln-module/result-rln/lib"
|
||||
WALLET_MODULE="$LEZ_RLN_DIR/logos-rln-module/result-wallet/lib"
|
||||
|
||||
# Delivery module plugin (for mix relay nodes)
|
||||
if [ -f "$DELIVERY_MODULE_DIR/build_plugin/modules/delivery_module_plugin.$EXT" ]; then
|
||||
DELIVERY_PLUGIN="$DELIVERY_MODULE_DIR/build_plugin/modules/delivery_module_plugin.$EXT"
|
||||
else
|
||||
DELIVERY_PLUGIN="$DELIVERY_MODULE_DIR/result/lib/delivery_module_plugin.$EXT"
|
||||
fi
|
||||
|
||||
# Chat module plugin (for sender/receiver)
|
||||
# Prefer locally-built liblogoschat over nix result (uses vendored Nim toolchain)
|
||||
CHAT_MODULE_RESULT="$CHAT_MODULE_DIR/result"
|
||||
if [ -f "$LOGOS_CHAT_DIR/build/liblogoschat.$EXT" ]; then
|
||||
CHAT_LIB="$LOGOS_CHAT_DIR/build/liblogoschat.$EXT"
|
||||
log " Using locally-built liblogoschat"
|
||||
else
|
||||
CHAT_LIB="$CHAT_MODULE_RESULT/lib/liblogoschat.$EXT"
|
||||
fi
|
||||
CHAT_PLUGIN="$CHAT_MODULE_RESULT/lib/chat_module_plugin.$EXT"
|
||||
|
||||
for check in \
|
||||
"$RLN_MODULE/liblogos_rln_module.$EXT" \
|
||||
"$WALLET_MODULE/liblogos_execution_zone_wallet_module.$EXT" \
|
||||
"$DELIVERY_PLUGIN" \
|
||||
"$CHAT_PLUGIN" \
|
||||
"$CHAT_LIB"; do
|
||||
[ -f "$check" ] || die "Missing: $check"
|
||||
done
|
||||
log " All modules present."
|
||||
|
||||
# ---------- Phase 4: Start mix nodes ----------
|
||||
echo "[4/6] Starting $NUM_NODES mix+LEZ nodes..."
|
||||
LOAD_ORDER="liblogos_execution_zone_wallet_module,liblogos_rln_module,delivery_module"
|
||||
WALLET_CALL="liblogos_execution_zone_wallet_module.open($WALLET_CONFIG,$WALLET_STORAGE)"
|
||||
BOOTSTRAP_PEER="/ip4/127.0.0.1/tcp/$BASE_TCP_PORT/p2p/${PEER_IDS[0]}"
|
||||
|
||||
# NOTE: Off-chain credential generation (setup_credentials) and pre-registration
|
||||
# (register_commitments) are not used. All nodes and chat clients register their
|
||||
# RLN memberships at runtime via the gifter protocol on node 0.
|
||||
|
||||
for i in $(seq 0 $((NUM_NODES - 1))); do
|
||||
TCP_PORT=$((BASE_TCP_PORT + i)); DISC_PORT=$((BASE_DISC_PORT + i))
|
||||
NODE_CONFIG="$STATE_DIR/node${i}_config.json"
|
||||
LOG_FILE="$STATE_DIR/node${i}.log"
|
||||
KAD_BOOTSTRAP="[]"; [ "$i" -gt 0 ] && KAD_BOOTSTRAP="[\"$BOOTSTRAP_PEER\"]"
|
||||
PEER_LIST=""
|
||||
for j in $(seq 0 $((NUM_NODES - 1))); do
|
||||
[ "$j" -eq "$i" ] && continue
|
||||
[ -n "$PEER_LIST" ] && PEER_LIST="$PEER_LIST,"
|
||||
PEER_LIST="$PEER_LIST\"/ip4/127.0.0.1/tcp/$((BASE_TCP_PORT + j))/p2p/${PEER_IDS[$j]}\""
|
||||
done
|
||||
STATIC_PEERS="[$PEER_LIST]"
|
||||
|
||||
GIFTER_FIELDS=""
|
||||
if [ "$i" -eq 0 ]; then
|
||||
GIFTER_FIELDS="\"mixGifterService\": true, \"mixGifterWalletAccount\": \"$GIFTER_ACCOUNT\","
|
||||
else
|
||||
GIFTER_FIELDS="\"mixGifterNode\": \"$BOOTSTRAP_PEER\", \"mixGifterWalletAccount\": \"$GIFTER_ACCOUNT\","
|
||||
fi
|
||||
|
||||
cat > "$NODE_CONFIG" <<EOF
|
||||
{
|
||||
"clusterId": $CLUSTER_ID,
|
||||
"numShardsInNetwork": $NUM_SHARDS,
|
||||
"listenAddress": "127.0.0.1",
|
||||
"tcpPort": $TCP_PORT,
|
||||
"discv5UdpPort": $DISC_PORT,
|
||||
"nat": "extip:127.0.0.1",
|
||||
"extMultiAddrs": ["/ip4/127.0.0.1/tcp/$TCP_PORT"],
|
||||
"extMultiAddrsOnly": true,
|
||||
"nodekey": "${NODEKEYS[$i]}",
|
||||
"staticnodes": $STATIC_PEERS,
|
||||
"relay": true,
|
||||
"lightpush": true,
|
||||
"filter": true,
|
||||
"mix": true,
|
||||
"mixkey": "${MIXKEYS[$i]}",
|
||||
"mixOnchainLEZ": true,
|
||||
$GIFTER_FIELDS
|
||||
"enableKadDiscovery": true,
|
||||
"kadBootstrapNodes": $KAD_BOOTSTRAP,
|
||||
"peerExchange": false,
|
||||
"rendezvous": false,
|
||||
"colocationLimit": 0,
|
||||
"logLevel": "$LOG_LEVEL"
|
||||
}
|
||||
EOF
|
||||
|
||||
MDIR=$(mktemp -d); MODULES_DIRS+=("$MDIR")
|
||||
# Stage wallet module
|
||||
mkdir -p "$MDIR/liblogos_execution_zone_wallet_module"
|
||||
cp -L "$WALLET_MODULE/liblogos_execution_zone_wallet_module.$EXT" "$MDIR/liblogos_execution_zone_wallet_module/"
|
||||
[ -f "$WALLET_MODULE/libwallet_ffi.$EXT" ] && cp -L "$WALLET_MODULE/libwallet_ffi.$EXT" "$MDIR/liblogos_execution_zone_wallet_module/"
|
||||
echo "{\"name\":\"liblogos_execution_zone_wallet_module\",\"version\":\"1.0.0\",\"type\":\"core\",\"main\":{\"$PLATFORM\":\"liblogos_execution_zone_wallet_module.$EXT\"},\"dependencies\":[],\"capabilities\":[]}" > "$MDIR/liblogos_execution_zone_wallet_module/manifest.json"
|
||||
# Stage RLN module
|
||||
mkdir -p "$MDIR/liblogos_rln_module"
|
||||
cp -L "$RLN_MODULE/liblogos_rln_module.$EXT" "$MDIR/liblogos_rln_module/"
|
||||
cp -L "$RLN_MODULE/liblez_rln_ffi.$EXT" "$MDIR/liblogos_rln_module/" 2>/dev/null || true
|
||||
echo "{\"name\":\"liblogos_rln_module\",\"version\":\"1.0.0\",\"type\":\"core\",\"main\":{\"$PLATFORM\":\"liblogos_rln_module.$EXT\"},\"dependencies\":[\"liblogos_execution_zone_wallet_module\"],\"capabilities\":[]}" > "$MDIR/liblogos_rln_module/manifest.json"
|
||||
# Stage delivery module
|
||||
mkdir -p "$MDIR/delivery_module"
|
||||
cp -L "$DELIVERY_PLUGIN" "$MDIR/delivery_module/"
|
||||
if [ -f "$DELIVERY_DIR/build/liblogosdelivery.$EXT" ]; then
|
||||
cp -L "$DELIVERY_DIR/build/liblogosdelivery.$EXT" "$MDIR/delivery_module/"
|
||||
else
|
||||
cp -L "$DELIVERY_MODULE_DIR/result/lib/liblogosdelivery.$EXT" "$MDIR/delivery_module/" 2>/dev/null || true
|
||||
fi
|
||||
for pq in "$DELIVERY_MODULE_DIR"/result/lib/libpq*; do [ -f "$pq" ] && cp -L "$pq" "$MDIR/delivery_module/"; done
|
||||
echo "{\"name\":\"delivery_module\",\"version\":\"1.0.0\",\"type\":\"core\",\"main\":{\"$PLATFORM\":\"delivery_module_plugin.$EXT\"},\"dependencies\":[],\"capabilities\":[]}" > "$MDIR/delivery_module/manifest.json"
|
||||
|
||||
log " Starting node $i (port $TCP_PORT)..."
|
||||
(cd "$STATE_DIR" && TMPDIR=/tmp "$LOGOSCORE" -m "$MDIR" -l "$LOAD_ORDER" \
|
||||
-c "$WALLET_CALL" \
|
||||
-c "delivery_module.createNode(@$NODE_CONFIG)" \
|
||||
-c "delivery_module.start()" \
|
||||
-c "delivery_module.setRlnConfig($CONFIG_ACCOUNT,$i)" \
|
||||
-c "delivery_module.subscribe($CONTENT_TOPIC)" \
|
||||
</dev/null >"$LOG_FILE" 2>&1) &
|
||||
EXPECTED_CALLS=5
|
||||
NODE_PID=$!; INSTANCE_PIDS+=($NODE_PID)
|
||||
WAIT_TIMEOUT=90
|
||||
for t in $(seq 1 $WAIT_TIMEOUT); do
|
||||
N=$(grep -c '^Method call successful' "$LOG_FILE" 2>/dev/null || true); N=${N:-0}
|
||||
[ "$N" -ge "$EXPECTED_CALLS" ] && break; sleep 1
|
||||
done
|
||||
if [ "${N:-0}" -ge "$EXPECTED_CALLS" ]; then
|
||||
log " Node $i ready ($N/$EXPECTED_CALLS calls) PID: $NODE_PID"
|
||||
else
|
||||
echo " WARNING: Node $i: $N/$EXPECTED_CALLS calls"
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo ""
|
||||
|
||||
# ---------- Phase 5: Chat module sender/receiver ----------
|
||||
echo "[5/6] Starting chat module instances..."
|
||||
|
||||
# Wait for all nodes to be fully ready (gifter registrations finalize during startup)
|
||||
for i in $(seq 0 $((NUM_NODES - 1))); do
|
||||
LOG_FILE="$STATE_DIR/node${i}.log"
|
||||
EC=5
|
||||
for t in $(seq 1 120); do
|
||||
N=$(grep -c '^Method call successful' "$LOG_FILE" 2>/dev/null || true); N=${N:-0}
|
||||
[ "$N" -ge "$EC" ] && break; sleep 2
|
||||
done
|
||||
done
|
||||
|
||||
# Wait for mix mesh + RLN root convergence. Requires: 3 gifter registrations on
|
||||
# node0, LEZ root polling events across all nodes, AND min 30s floor for mix
|
||||
# protocol handshakes to complete (no single log line signals this cleanly).
|
||||
echo " Waiting for kademlia propagation + RLN convergence..."
|
||||
KADEMLIA_T0=$SECONDS
|
||||
while true; do
|
||||
ELAPSED=$((SECONDS - KADEMLIA_T0))
|
||||
GR=$(sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/node0.log" 2>/dev/null | grep -c "RLN gifter registration succeeded" || true); GR=${GR:-0}
|
||||
LR=0
|
||||
for i in $(seq 0 $((NUM_NODES - 1))); do
|
||||
L=$(sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/node${i}.log" 2>/dev/null | grep -c "Polled valid roots\|Fetched roots from\|valid_roots\|OnchainLEZGroupManager initialized\|Wired LEZ callbacks" || true)
|
||||
LR=$((LR + L))
|
||||
done
|
||||
if [ "$ELAPSED" -ge "$KADEMLIA_MIN_WAIT" ] && [ "$GR" -ge 3 ] && [ "$LR" -ge 40 ]; then break; fi
|
||||
[ "$ELAPSED" -ge 120 ] && break
|
||||
sleep 1
|
||||
done
|
||||
log " Kademlia ready after $((SECONDS - KADEMLIA_T0))s ($GR gifter regs, $LR LEZ root events)"
|
||||
|
||||
RECEIVER_LOG="$STATE_DIR/chat_receiver.log"
|
||||
SENDER_LOG="$STATE_DIR/chat_sender.log"
|
||||
|
||||
# Build mix node list for chat config (multiaddr:mixPubKey format)
|
||||
MIXNODE_LIST=""
|
||||
for j in $(seq 0 $((NUM_NODES - 1))); do
|
||||
[ -n "$MIXNODE_LIST" ] && MIXNODE_LIST="$MIXNODE_LIST,"
|
||||
MIXNODE_LIST="$MIXNODE_LIST\"/ip4/127.0.0.1/tcp/$((BASE_TCP_PORT + j))/p2p/${PEER_IDS[$j]}:${MIX_PUBKEYS[$j]}\""
|
||||
done
|
||||
|
||||
# Helper: stage chat_module for a logoscore instance
|
||||
stage_chat_module() {
|
||||
local MDIR=$1
|
||||
mkdir -p "$MDIR/liblogos_execution_zone_wallet_module"
|
||||
cp -L "$WALLET_MODULE/liblogos_execution_zone_wallet_module.$EXT" "$MDIR/liblogos_execution_zone_wallet_module/"
|
||||
[ -f "$WALLET_MODULE/libwallet_ffi.$EXT" ] && cp -L "$WALLET_MODULE/libwallet_ffi.$EXT" "$MDIR/liblogos_execution_zone_wallet_module/"
|
||||
echo "{\"name\":\"liblogos_execution_zone_wallet_module\",\"version\":\"1.0.0\",\"type\":\"core\",\"main\":{\"$PLATFORM\":\"liblogos_execution_zone_wallet_module.$EXT\"},\"dependencies\":[],\"capabilities\":[]}" > "$MDIR/liblogos_execution_zone_wallet_module/manifest.json"
|
||||
|
||||
mkdir -p "$MDIR/liblogos_rln_module"
|
||||
cp -L "$RLN_MODULE/liblogos_rln_module.$EXT" "$MDIR/liblogos_rln_module/"
|
||||
cp -L "$RLN_MODULE/liblez_rln_ffi.$EXT" "$MDIR/liblogos_rln_module/" 2>/dev/null || true
|
||||
echo "{\"name\":\"liblogos_rln_module\",\"version\":\"1.0.0\",\"type\":\"core\",\"main\":{\"$PLATFORM\":\"liblogos_rln_module.$EXT\"},\"dependencies\":[\"liblogos_execution_zone_wallet_module\"],\"capabilities\":[]}" > "$MDIR/liblogos_rln_module/manifest.json"
|
||||
|
||||
mkdir -p "$MDIR/chat_module"
|
||||
cp -L "$CHAT_PLUGIN" "$MDIR/chat_module/"
|
||||
cp -L "$CHAT_LIB" "$MDIR/chat_module/"
|
||||
echo "{\"name\":\"chat_module\",\"version\":\"1.0.0\",\"type\":\"core\",\"main\":{\"$PLATFORM\":\"chat_module_plugin.$EXT\"},\"dependencies\":[],\"capabilities\":[]}" > "$MDIR/chat_module/manifest.json"
|
||||
}
|
||||
|
||||
CHAT_LOAD_ORDER="liblogos_execution_zone_wallet_module,liblogos_rln_module,chat_module"
|
||||
|
||||
# --- Receiver ---
|
||||
RECV_MDIR=$(mktemp -d); MODULES_DIRS+=("$RECV_MDIR")
|
||||
stage_chat_module "$RECV_MDIR"
|
||||
|
||||
RECV_CONFIG="$STATE_DIR/chat_receiver_config.json"
|
||||
# Build static peer list (ENR or multiaddr) for chat nodes to join relay mesh
|
||||
CHAT_STATIC_PEERS=""
|
||||
for j in $(seq 0 $((NUM_NODES - 1))); do
|
||||
[ -n "$CHAT_STATIC_PEERS" ] && CHAT_STATIC_PEERS="$CHAT_STATIC_PEERS,"
|
||||
CHAT_STATIC_PEERS="$CHAT_STATIC_PEERS\"/ip4/127.0.0.1/tcp/$((BASE_TCP_PORT + j))/p2p/${PEER_IDS[$j]}\""
|
||||
done
|
||||
cat > "$RECV_CONFIG" <<EOF
|
||||
{
|
||||
"name": "receiver",
|
||||
"clusterId": $CLUSTER_ID,
|
||||
"shardId": 0,
|
||||
"port": $CHAT_RECV_PORT,
|
||||
"mixEnabled": true,
|
||||
"mixNodes": [$MIXNODE_LIST],
|
||||
"destPeerAddr": "$BOOTSTRAP_PEER",
|
||||
"minMixPoolSize": 4,
|
||||
"gifterNodeAddr": "$BOOTSTRAP_PEER",
|
||||
"staticPeers": [$CHAT_STATIC_PEERS]
|
||||
}
|
||||
EOF
|
||||
|
||||
TEST_MSG_HEX=$(printf '%s' "$TEST_MESSAGE_PREFIX" | xxd -p | tr -d '\n')
|
||||
|
||||
log " Starting receiver..."
|
||||
(cd "$STATE_DIR" && TMPDIR=/tmp "$LOGOSCORE" -m "$RECV_MDIR" -l "$CHAT_LOAD_ORDER" \
|
||||
-c "$WALLET_CALL" \
|
||||
-c "chat_module.initChat(@$RECV_CONFIG)" \
|
||||
-c "chat_module.setEventCallback()" \
|
||||
-c "chat_module.startChat()" \
|
||||
-c "chat_module.setRlnConfig($CONFIG_ACCOUNT,5)" \
|
||||
-c "chat_module.createIntroBundle()" \
|
||||
</dev/null >"$RECEIVER_LOG" 2>&1) &
|
||||
RECEIVER_PID=$!; INSTANCE_PIDS+=($RECEIVER_PID)
|
||||
log " Receiver PID: $RECEIVER_PID"
|
||||
|
||||
# Wait for receiver to complete all method calls (6 calls now including createIntroBundle)
|
||||
RECV_EXPECTED=6
|
||||
for t in $(seq 1 180); do
|
||||
N=$(grep -c '^Method call successful' "$RECEIVER_LOG" 2>/dev/null || true); N=${N:-0}
|
||||
[ "$N" -ge "$RECV_EXPECTED" ] && break; sleep 2
|
||||
done
|
||||
N=$(grep -c '^Method call successful' "$RECEIVER_LOG" 2>/dev/null || true); N=${N:-0}
|
||||
log " Receiver method calls: $N/$RECV_EXPECTED"
|
||||
|
||||
# Extract intro bundle from receiver log (event callback delivers it as chatCreateIntroBundleResult)
|
||||
INTRO_BUNDLE=""
|
||||
for t in $(seq 1 30); do
|
||||
INTRO_BUNDLE=$(grep -oE 'logos_chatintro_[A-Za-z0-9_-]+' "$RECEIVER_LOG" 2>/dev/null | head -1 || true)
|
||||
[ -n "$INTRO_BUNDLE" ] && break; sleep 2
|
||||
done
|
||||
if [ -n "$INTRO_BUNDLE" ]; then
|
||||
log " Receiver intro bundle: ${INTRO_BUNDLE:0:40}..."
|
||||
else
|
||||
log " WARNING: Could not extract intro bundle from receiver log"
|
||||
fi
|
||||
|
||||
# Wait for receiver's async startChat to finish (Waku client started) AND give
|
||||
# filter subscription a 15s floor to propagate through the relay mesh.
|
||||
echo " Waiting for receiver to join mix network..."
|
||||
JOIN_T0=$SECONDS
|
||||
while true; do
|
||||
ELAPSED=$((SECONDS - JOIN_T0))
|
||||
RS=$(grep -c "Waku client started" "$RECEIVER_LOG" 2>/dev/null || true); RS=${RS:-0}
|
||||
[ "$ELAPSED" -ge "$RECEIVER_MIN_WAIT" ] && [ "$RS" -ge 1 ] && break
|
||||
[ "$ELAPSED" -ge 60 ] && break
|
||||
sleep 1
|
||||
done
|
||||
log " Receiver joined after $((SECONDS - JOIN_T0))s"
|
||||
|
||||
# --- Sender ---
|
||||
SEND_MDIR=$(mktemp -d); MODULES_DIRS+=("$SEND_MDIR")
|
||||
stage_chat_module "$SEND_MDIR"
|
||||
|
||||
SEND_CONFIG="$STATE_DIR/chat_sender_config.json"
|
||||
cat > "$SEND_CONFIG" <<EOF
|
||||
{
|
||||
"name": "sender",
|
||||
"clusterId": $CLUSTER_ID,
|
||||
"shardId": 0,
|
||||
"port": $CHAT_SEND_PORT,
|
||||
"mixEnabled": true,
|
||||
"mixNodes": [$MIXNODE_LIST],
|
||||
"destPeerAddr": "$BOOTSTRAP_PEER",
|
||||
"minMixPoolSize": 4,
|
||||
"gifterNodeAddr": "$BOOTSTRAP_PEER",
|
||||
"staticPeers": [$CHAT_STATIC_PEERS]
|
||||
}
|
||||
EOF
|
||||
|
||||
# Build sender -c calls: init, start, setRlnConfig, then if we have the intro bundle,
|
||||
# create a private conversation with the test message
|
||||
SENDER_CALLS="-c \"$WALLET_CALL\""
|
||||
SENDER_CALLS="$SENDER_CALLS -c \"chat_module.initChat(@$SEND_CONFIG)\""
|
||||
SENDER_CALLS="$SENDER_CALLS -c \"chat_module.setEventCallback()\""
|
||||
SENDER_CALLS="$SENDER_CALLS -c \"chat_module.startChat()\""
|
||||
# RLN leaf indices: 0-3 = mix nodes, 4 = gifter-reserved, 5 = receiver, 6 = sender
|
||||
SENDER_CALLS="$SENDER_CALLS -c \"chat_module.setRlnConfig($CONFIG_ACCOUNT,6)\""
|
||||
if [ -n "$INTRO_BUNDLE" ]; then
|
||||
SENDER_CALLS="$SENDER_CALLS -c \"chat_module.newPrivateConversation($INTRO_BUNDLE,$TEST_MSG_HEX)\""
|
||||
fi
|
||||
|
||||
log " Starting sender..."
|
||||
eval "(cd \"$STATE_DIR\" && TMPDIR=/tmp \"$LOGOSCORE\" -m \"$SEND_MDIR\" -l \"$CHAT_LOAD_ORDER\" \
|
||||
$SENDER_CALLS \
|
||||
</dev/null >\"$SENDER_LOG\" 2>&1) &"
|
||||
SENDER_PID=$!; INSTANCE_PIDS+=($SENDER_PID)
|
||||
log " Sender PID: $SENDER_PID"
|
||||
|
||||
# Wait for sender to complete method calls
|
||||
SEND_EXPECTED=6
|
||||
[ -n "$INTRO_BUNDLE" ] && SEND_EXPECTED=7
|
||||
for t in $(seq 1 180); do
|
||||
N=$(grep -c '^Method call successful' "$SENDER_LOG" 2>/dev/null || true); N=${N:-0}
|
||||
[ "$N" -ge "$SEND_EXPECTED" ] && break; sleep 2
|
||||
done
|
||||
|
||||
# On slower systems (Docker/ARM), the sender's async gifter registration may not
|
||||
# complete before newPrivateConversation runs. Wait for gifter + RLN readiness
|
||||
# before checking for message delivery. If newPrivateConversation initially failed,
|
||||
# the Nim async code will retry once credentials are valid.
|
||||
echo " Waiting for sender RLN readiness..."
|
||||
SENDER_RLN_T0=$SECONDS
|
||||
for t in $(seq 1 60); do
|
||||
SG=$(sed 's/\x1b\[[0-9;]*m//g' "$SENDER_LOG" 2>/dev/null | grep -c "Registered via RLN gifter\|Waku client started" || true)
|
||||
[ "${SG:-0}" -ge 2 ] && break
|
||||
sleep 1
|
||||
done
|
||||
log " Sender RLN ready after $((SECONDS - SENDER_RLN_T0))s"
|
||||
N=$(grep -c '^Method call successful' "$SENDER_LOG" 2>/dev/null || true); N=${N:-0}
|
||||
log " Sender method calls: $N/$SEND_EXPECTED"
|
||||
|
||||
# Poll receiver log for incoming message(s) instead of waiting a fixed 120s.
|
||||
echo " Waiting for message delivery via mix..."
|
||||
DELIVERY_T0=$SECONDS
|
||||
for t in $(seq 1 $DELIVERY_TIMEOUT); do
|
||||
RM=$(grep -c "chatNewMessage\|chatNewConversation\|New Message\|new_message" "$RECEIVER_LOG" 2>/dev/null || true); RM=${RM:-0}
|
||||
[ "$RM" -ge 1 ] && break
|
||||
sleep 1
|
||||
done
|
||||
log " Delivery check after $((SECONDS - DELIVERY_T0))s (messages: $RM)"
|
||||
|
||||
echo ""
|
||||
|
||||
# ---------- Phase 6: Verify ----------
|
||||
echo "[6/6] Verification"; echo ""
|
||||
PASS=0; FAIL=0
|
||||
check() { local c=$1 d=$2; if eval "$c"; then echo " PASS: $d"; PASS=$((PASS+1)); else echo " FAIL: $d"; FAIL=$((FAIL+1)); fi; }
|
||||
|
||||
echo " --- logos-core mix nodes ---"
|
||||
for i in $(seq 0 $((NUM_NODES - 1))); do
|
||||
M=$(sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/node${i}.log" 2>/dev/null | grep -c "mounting mix protocol" || true)
|
||||
check "[ ${M:-0} -ge 1 ]" "Node $i mounted mix ($M)"
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo " --- RLN gifter ---"
|
||||
GIFTER_MOUNTED=$(sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/node0.log" 2>/dev/null | grep -c "RLN gifter service mounted" || true)
|
||||
check "[ ${GIFTER_MOUNTED:-0} -ge 1 ]" "Node 0 gifter service mounted ($GIFTER_MOUNTED)"
|
||||
echo ""
|
||||
|
||||
echo " --- LEZ RLN ---"
|
||||
LEZ_ROOTS=0
|
||||
for i in $(seq 0 $((NUM_NODES - 1))); do
|
||||
R=$(sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/node${i}.log" 2>/dev/null | grep -c "Polled valid roots\|Fetched roots from\|valid_roots\|OnchainLEZGroupManager initialized\|Wired LEZ callbacks" || true)
|
||||
LEZ_ROOTS=$((LEZ_ROOTS + R))
|
||||
done
|
||||
check "[ $LEZ_ROOTS -ge 1 ]" "LEZ RLN active ($LEZ_ROOTS events across nodes)"
|
||||
echo ""
|
||||
|
||||
echo " --- chat module ---"
|
||||
RECV_INIT=$(grep -c "chatInitResult\|Chat context created" "$RECEIVER_LOG" 2>/dev/null || true)
|
||||
check "[ ${RECV_INIT:-0} -ge 1 ]" "Receiver initialized ($RECV_INIT)"
|
||||
RECV_START=$(grep -c "chatStartResult\|Waku client started" "$RECEIVER_LOG" 2>/dev/null || true)
|
||||
check "[ ${RECV_START:-0} -ge 1 ]" "Receiver started ($RECV_START)"
|
||||
SEND_INIT=$(grep -c "chatInitResult\|Chat context created" "$SENDER_LOG" 2>/dev/null || true)
|
||||
check "[ ${SEND_INIT:-0} -ge 1 ]" "Sender initialized ($SEND_INIT)"
|
||||
SEND_START=$(grep -c "chatStartResult\|Waku client started" "$SENDER_LOG" 2>/dev/null || true)
|
||||
check "[ ${SEND_START:-0} -ge 1 ]" "Sender started ($SEND_START)"
|
||||
|
||||
RECV_MIX=$(grep -c "mounting mix protocol\|Wired LEZ callbacks" "$RECEIVER_LOG" 2>/dev/null || true)
|
||||
check "[ ${RECV_MIX:-0} -ge 1 ]" "Receiver mounted mix+LEZ ($RECV_MIX)"
|
||||
SEND_MIX=$(grep -c "mounting mix protocol\|Wired LEZ callbacks" "$SENDER_LOG" 2>/dev/null || true)
|
||||
check "[ ${SEND_MIX:-0} -ge 1 ]" "Sender mounted mix+LEZ ($SEND_MIX)"
|
||||
|
||||
RECV_BUNDLE=$(grep -c "logos_chatintro_" "$RECEIVER_LOG" 2>/dev/null || true)
|
||||
check "[ ${RECV_BUNDLE:-0} -ge 1 ]" "Receiver created intro bundle ($RECV_BUNDLE)"
|
||||
echo ""
|
||||
|
||||
echo " --- message exchange ---"
|
||||
SEND_MSG=$(grep -c "chatNewPrivateConversationResult\|chatSendMessageResult\|Message sent via mix" "$SENDER_LOG" 2>/dev/null || true)
|
||||
check "[ ${SEND_MSG:-0} -ge 1 ]" "Sender sent message ($SEND_MSG)"
|
||||
RECV_MSG=$(grep -c "chatNewMessage\|chatNewConversation\|New Message\|new_message" "$RECEIVER_LOG" 2>/dev/null || true)
|
||||
check "[ ${RECV_MSG:-0} -ge 1 ]" "Receiver received message ($RECV_MSG)"
|
||||
|
||||
echo ""; echo " =========================================="
|
||||
if [ "$FAIL" -eq 0 ]; then echo " ALL $PASS CHECKS PASSED"; EXIT_CODE=0
|
||||
else echo " $FAIL FAILED, $PASS passed"; EXIT_CODE=1; fi
|
||||
echo " =========================================="
|
||||
95
simulations/mix_lez_chat/setup_and_run.sh
Executable file
95
simulations/mix_lez_chat/setup_and_run.sh
Executable file
@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-shot: init nested submodules, build all modules, run mix+LEZ chat sim.
|
||||
#
|
||||
# Prerequisites: nix (with flakes), Docker, cargo-risczero.
|
||||
#
|
||||
# Environment variables:
|
||||
# CHAT_MODULE_DIR — path to logos-chat-module checkout (default: ../logos-chat-module)
|
||||
# CHAT_MODULE_REPO — git URL to clone if CHAT_MODULE_DIR doesn't exist
|
||||
# CHAT_MODULE_BRANCH — branch to clone (default: feat/logos-delivery)
|
||||
# SIM_* — simulation parameters, see run_simulation.sh / README.md
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
CHAT_MODULE_DIR="${CHAT_MODULE_DIR:-$ROOT/../logos-chat-module}"
|
||||
CHAT_MODULE_REPO="${CHAT_MODULE_REPO:-git@github.com:adklempner/logos-chat-module.git}"
|
||||
CHAT_MODULE_BRANCH="${CHAT_MODULE_BRANCH:-feat/logos-delivery}"
|
||||
|
||||
log() { echo "[$(date '+%H:%M:%S')] $*"; }
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
# Docker/CI: nix can't sandbox inside containers. Remove /homeless-shelter
|
||||
# (nix's sandbox HOME stub) and export NIX_CONFIG to disable sandboxing.
|
||||
if [ -f /.dockerenv ] || grep -q docker /proc/1/cgroup 2>/dev/null; then
|
||||
rmdir /homeless-shelter 2>/dev/null || true
|
||||
export NIX_CONFIG="sandbox = false
|
||||
${NIX_CONFIG:-}"
|
||||
fi
|
||||
|
||||
# On Linux with nix, bindgen (used by rocksdb/sequencer) needs LIBCLANG_PATH
|
||||
# and system headers. This covers both Docker and native nix-on-Linux.
|
||||
if [ "$(uname -s)" = "Linux" ] && [ -d /nix/store ] && [ -z "${LIBCLANG_PATH:-}" ]; then
|
||||
CLANG_SO=$(find /nix/store -maxdepth 3 -name 'libclang.so' 2>/dev/null | head -1 || true)
|
||||
if [ -n "$CLANG_SO" ]; then
|
||||
export LIBCLANG_PATH=$(dirname "$CLANG_SO")
|
||||
STDBOOL=$(find "$LIBCLANG_PATH" -maxdepth 5 -name 'stdbool.h' 2>/dev/null | head -1 || true)
|
||||
[ -n "$STDBOOL" ] && export BINDGEN_EXTRA_CLANG_ARGS="-I$(dirname "$STDBOOL")"
|
||||
fi
|
||||
log "Docker detected — disabled nix sandbox, LIBCLANG_PATH=$LIBCLANG_PATH"
|
||||
fi
|
||||
|
||||
# 1. Init submodules. Top-level first (non-recursive to avoid circular refs
|
||||
# in vendor/logos-lez-rln), then nwaku/nimbus-build-system recursively,
|
||||
# then logos-lez-rln's nested submodules selectively.
|
||||
log "Initializing top-level submodules..."
|
||||
git submodule update --init
|
||||
(cd vendor/nwaku && git submodule update --init --recursive)
|
||||
(cd vendor/nimbus-build-system && git submodule update --init --recursive)
|
||||
|
||||
# Init nested submodules inside vendor/logos-lez-rln non-recursively
|
||||
# (recursive init hits circular submodule references).
|
||||
# Preserve pre-built guest binaries across submodule reset (they're
|
||||
# architecture-independent RISC-V ELFs that take ~10min to rebuild).
|
||||
GUEST_DIR="vendor/logos-lez-rln/lez-rln/methods/guest/target/riscv32im-risc0-zkvm-elf/docker"
|
||||
# Check repo tree first, then /tmp/guest-bins/ (staged by run_in_docker.sh)
|
||||
GUEST_TMP=""
|
||||
if [ -f "$GUEST_DIR/rln_registration.bin" ]; then
|
||||
GUEST_TMP=$(mktemp -d)
|
||||
cp "$GUEST_DIR"/*.bin "$GUEST_TMP/"
|
||||
log "Preserved guest binaries from repo"
|
||||
elif [ -f "/tmp/guest-bins/rln_registration.bin" ]; then
|
||||
GUEST_TMP="/tmp/guest-bins"
|
||||
log "Using guest binaries from /tmp/guest-bins/"
|
||||
fi
|
||||
log "Initializing nested submodules in vendor/logos-lez-rln..."
|
||||
(cd vendor/logos-lez-rln && \
|
||||
git submodule update --init lssa logos-delivery logos-delivery-module logos-execution-zone-module && \
|
||||
cd logos-delivery-module && git submodule update --init vendor/logos-delivery)
|
||||
if [ -d "${GUEST_TMP:-}" ]; then
|
||||
mkdir -p "$GUEST_DIR"
|
||||
cp "$GUEST_TMP"/*.bin "$GUEST_DIR/"
|
||||
rm -rf "$GUEST_TMP"
|
||||
log "Restored guest binaries"
|
||||
fi
|
||||
|
||||
# 2. Build LEZ modules (RLN, wallet, delivery plugin, guest zkVM binaries).
|
||||
log "Building LEZ modules via vendor/logos-lez-rln/build_all.sh..."
|
||||
bash vendor/logos-lez-rln/build_all.sh
|
||||
|
||||
# 3. Build liblogoschat (Nim shared library).
|
||||
log "Building liblogoschat..."
|
||||
make update
|
||||
make liblogoschat
|
||||
|
||||
# 4. Clone and build logos-chat-module (C++ Qt plugin).
|
||||
if [ ! -d "$CHAT_MODULE_DIR" ]; then
|
||||
log "Cloning logos-chat-module to $CHAT_MODULE_DIR..."
|
||||
git clone -b "$CHAT_MODULE_BRANCH" "$CHAT_MODULE_REPO" "$CHAT_MODULE_DIR"
|
||||
fi
|
||||
log "Building logos-chat-module..."
|
||||
(cd "$CHAT_MODULE_DIR" && nix build)
|
||||
|
||||
# 5. Run the simulation.
|
||||
log "Starting mix+LEZ chat simulation..."
|
||||
exec bash "$ROOT/simulations/mix_lez_chat/run_simulation.sh" --fresh "$@"
|
||||
Loading…
x
Reference in New Issue
Block a user