diff --git a/simulations/mixnet/README.md b/simulations/mixnet/README.md index 99b0ba50b..7db1d0579 100644 --- a/simulations/mixnet/README.md +++ b/simulations/mixnet/README.md @@ -24,6 +24,7 @@ The simulation includes: | `run_chat_mix.sh` | Chat app instance 1 | | `run_chat_mix1.sh` | Chat app instance 2 | | `build_setup.sh` | Build and generate RLN credentials | +| `check_cover_traffic.sh` | Monitor cover traffic metrics from all nodes | ## Prerequisites @@ -130,3 +131,35 @@ To exit the chat apps, enter `/exit`: >> /exit quitting... ``` + +## Running Without DoS Protection + +To test cover traffic without RLN spam protection (avoids heavy proof generation compute), the config files include two flags: + +```toml +mix-user-message-limit=2 # slots per epoch (reduce for lighter testing) +mix-disable-spam-protection=true # skip RLN proof generation/verification +``` + +These are already set in `config.toml` through `config4.toml`. To re-enable RLN, set `mix-disable-spam-protection=false` (or remove the line) and ensure credentials are generated via `./build_setup.sh`. + +When running without DoS protection, cover traffic uses an internal epoch timer and does not require RLN credentials or `rln_tree.db`. + +### Monitoring Cover Traffic + +Use the metrics script to verify cover traffic is working: + +```bash +./check_cover_traffic.sh +``` + +Key metrics to look for: +- `mix_cover_emitted_total` — cover messages generated per node (should increase each epoch) +- `mix_cover_received_total` — cover messages received back at origin after 3-hop mix path +- `mix_slots_exhausted_total` — expected when slots per epoch are low + +### Note on Rate Limit and Expected Errors + +The default `mix-user-message-limit=2` (R=2) with path length L=3 yields a fractional cover target of `R/(1+L) = 0.5` packets per epoch. Because this is not an integer, epoch boundary jitter can cause two cover emissions in one epoch, exhausting all slots and leaving none for forwarding. This produces `SLOT_EXHAUSTED` and `SPAM_PROOF_GEN_FAILED` errors at intermediate hops — these are expected with the default config. + +For a clean setup, R should be a multiple of `(1+L) = 4`. Setting `mix-user-message-limit=4` gives exactly 1 cover packet per epoch with 3 slots remaining for forwarding, eliminating these errors. diff --git a/simulations/mixnet/check_cover_traffic.sh b/simulations/mixnet/check_cover_traffic.sh new file mode 100755 index 000000000..a5763fbfc --- /dev/null +++ b/simulations/mixnet/check_cover_traffic.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Check cover traffic metrics from all mix nodes. +# Ports: 8008 + ports-shift (1-5) = 8009-8013 + +echo "=== Cover Traffic Metrics ===" +echo "" + +for i in 1 2 3 4 5; do + port=$((8008 + i)) + echo "--- Node $i (port $port) ---" + metrics=$(curl -s "http://127.0.0.1:$port/metrics" 2>/dev/null) + if [ -z "$metrics" ]; then + echo " (unreachable)" + else + echo "$metrics" | grep -E "mix_cover_|mix_slot_" | grep -v "^#" || echo " (no cover metrics yet)" + fi + echo "" +done diff --git a/simulations/mixnet/config.toml b/simulations/mixnet/config.toml index 5cd1aa936..fcf3eda39 100644 --- a/simulations/mixnet/config.toml +++ b/simulations/mixnet/config.toml @@ -7,6 +7,7 @@ lightpush = true max-connections = 150 peer-exchange = false metrics-logging = false +metrics-server = true cluster-id = 2 discv5-discovery = false discv5-udp-port = 9000 @@ -26,3 +27,5 @@ nat = "extip:127.0.0.1" ext-multiaddr = ["/ip4/127.0.0.1/tcp/60001"] ext-multiaddr-only = true ip-colocation-limit=0 +mix-user-message-limit=4 +mix-disable-spam-protection=false diff --git a/simulations/mixnet/config1.toml b/simulations/mixnet/config1.toml index 73cccb8c6..a65cc2f5a 100644 --- a/simulations/mixnet/config1.toml +++ b/simulations/mixnet/config1.toml @@ -7,6 +7,7 @@ lightpush = true max-connections = 150 peer-exchange = false metrics-logging = false +metrics-server = true cluster-id = 2 discv5-discovery = false discv5-udp-port = 9001 @@ -27,4 +28,6 @@ nat = "extip:127.0.0.1" ext-multiaddr = ["/ip4/127.0.0.1/tcp/60002"] ext-multiaddr-only = true ip-colocation-limit=0 +mix-user-message-limit=4 +mix-disable-spam-protection=false #staticnode = ["/ip4/127.0.0.1/tcp/60001/p2p/16Uiu2HAmPiEs2ozjjJF2iN2Pe2FYeMC9w4caRHKYdLdAfjgbWM6o", "/ip4/127.0.0.1/tcp/60003/p2p/16Uiu2HAmTEDHwAziWUSz6ZE23h5vxG2o4Nn7GazhMor4bVuMXTrA","/ip4/127.0.0.1/tcp/60004/p2p/16Uiu2HAmPwRKZajXtfb1Qsv45VVfRZgK3ENdfmnqzSrVm3BczF6f","/ip4/127.0.0.1/tcp/60005/p2p/16Uiu2HAmRhxmCHBYdXt1RibXrjAUNJbduAhzaTHwFCZT4qWnqZAu"] diff --git a/simulations/mixnet/config2.toml b/simulations/mixnet/config2.toml index 3acd2bf8a..b37bb9759 100644 --- a/simulations/mixnet/config2.toml +++ b/simulations/mixnet/config2.toml @@ -7,6 +7,7 @@ lightpush = true max-connections = 150 peer-exchange = false metrics-logging = false +metrics-server = true cluster-id = 2 discv5-discovery = false discv5-udp-port = 9002 @@ -27,4 +28,6 @@ nat = "extip:127.0.0.1" ext-multiaddr = ["/ip4/127.0.0.1/tcp/60003"] ext-multiaddr-only = true ip-colocation-limit=0 +mix-user-message-limit=4 +mix-disable-spam-protection=false #staticnode = ["/ip4/127.0.0.1/tcp/60001/p2p/16Uiu2HAmPiEs2ozjjJF2iN2Pe2FYeMC9w4caRHKYdLdAfjgbWM6o", "/ip4/127.0.0.1/tcp/60002/p2p/16Uiu2HAmLtKaFaSWDohToWhWUZFLtqzYZGPFuXwKrojFVF6az5UF","/ip4/127.0.0.1/tcp/60004/p2p/16Uiu2HAmPwRKZajXtfb1Qsv45VVfRZgK3ENdfmnqzSrVm3BczF6f","/ip4/127.0.0.1/tcp/60005/p2p/16Uiu2HAmRhxmCHBYdXt1RibXrjAUNJbduAhzaTHwFCZT4qWnqZAu"] diff --git a/simulations/mixnet/config3.toml b/simulations/mixnet/config3.toml index bd8e7c4e9..9b256ea08 100644 --- a/simulations/mixnet/config3.toml +++ b/simulations/mixnet/config3.toml @@ -7,6 +7,7 @@ lightpush = true max-connections = 150 peer-exchange = false metrics-logging = false +metrics-server = true cluster-id = 2 discv5-discovery = false discv5-udp-port = 9003 @@ -27,4 +28,6 @@ nat = "extip:127.0.0.1" ext-multiaddr = ["/ip4/127.0.0.1/tcp/60004"] ext-multiaddr-only = true ip-colocation-limit=0 +mix-user-message-limit=4 +mix-disable-spam-protection=false #staticnode = ["/ip4/127.0.0.1/tcp/60002/p2p/16Uiu2HAmLtKaFaSWDohToWhWUZFLtqzYZGPFuXwKrojFVF6az5UF", "/ip4/127.0.0.1/tcp/60003/p2p/16Uiu2HAmTEDHwAziWUSz6ZE23h5vxG2o4Nn7GazhMor4bVuMXTrA","/ip4/127.0.0.1/tcp/60001/p2p/16Uiu2HAmPiEs2ozjjJF2iN2Pe2FYeMC9w4caRHKYdLdAfjgbWM6o","/ip4/127.0.0.1/tcp/60005/p2p/16Uiu2HAmRhxmCHBYdXt1RibXrjAUNJbduAhzaTHwFCZT4qWnqZAu"] diff --git a/simulations/mixnet/config4.toml b/simulations/mixnet/config4.toml index f174250d5..d616896e6 100644 --- a/simulations/mixnet/config4.toml +++ b/simulations/mixnet/config4.toml @@ -7,6 +7,7 @@ lightpush = true max-connections = 150 peer-exchange = false metrics-logging = false +metrics-server = true cluster-id = 2 discv5-discovery = false discv5-udp-port = 9004 @@ -27,4 +28,6 @@ nat = "extip:127.0.0.1" ext-multiaddr = ["/ip4/127.0.0.1/tcp/60005"] ext-multiaddr-only = true ip-colocation-limit=0 +mix-user-message-limit=4 +mix-disable-spam-protection=false #staticnode = ["/ip4/127.0.0.1/tcp/60001/p2p/16Uiu2HAmPiEs2ozjjJF2iN2Pe2FYeMC9w4caRHKYdLdAfjgbWM6o", "/ip4/127.0.0.1/tcp/60003/p2p/16Uiu2HAmTEDHwAziWUSz6ZE23h5vxG2o4Nn7GazhMor4bVuMXTrA","/ip4/127.0.0.1/tcp/60004/p2p/16Uiu2HAmPwRKZajXtfb1Qsv45VVfRZgK3ENdfmnqzSrVm3BczF6f","/ip4/127.0.0.1/tcp/60002/p2p/16Uiu2HAmLtKaFaSWDohToWhWUZFLtqzYZGPFuXwKrojFVF6az5UF"] diff --git a/simulations/mixnet/setup_credentials.nim b/simulations/mixnet/setup_credentials.nim index 77c796354..cffd60e9e 100644 --- a/simulations/mixnet/setup_credentials.nim +++ b/simulations/mixnet/setup_credentials.nim @@ -20,8 +20,8 @@ import const KeystorePassword = "mix-rln-password" # Must match protocol.nim - DefaultUserMessageLimit = 100'u64 # Network-wide default rate limit - SpammerUserMessageLimit = 3'u64 # Lower limit for spammer testing + DefaultUserMessageLimit = 4'u64 # R=4 slots per 10s epoch + SpammerUserMessageLimit = 3'u64 # Higher limit for spammer testing # Peer IDs derived from nodekeys in config files # config.toml: nodekey = "f98e3fba96c32e8d1967d460f1b79457380e1a895f7971cecc8528abe733781a" diff --git a/tools/confutils/cli_args.nim b/tools/confutils/cli_args.nim index fe8c1617d..8db217429 100644 --- a/tools/confutils/cli_args.nim +++ b/tools/confutils/cli_args.nim @@ -642,6 +642,18 @@ with the drawback of consuming some more bandwidth.""", name: "mixnode" .}: seq[MixNodePubInfo] + mixUserMessageLimit* {. + desc: + "Maximum messages per RLN epoch for mix cover traffic. If not set, uses plugin default.", + name: "mix-user-message-limit" + .}: Option[int] + + mixDisableSpamProtection* {. + desc: "Disable RLN spam protection for mix protocol (for testing only).", + defaultValue: false, + name: "mix-disable-spam-protection" + .}: bool + # Kademlia Discovery config enableKadDiscovery* {. desc: @@ -1079,6 +1091,9 @@ proc toWakuConf*(n: WakuNodeConf): ConfResult[WakuConf] = b.withMix(n.mix) if n.mixkey.isSome(): b.mixConf.withMixKey(n.mixkey.get()) + if n.mixUserMessageLimit.isSome(): + b.mixConf.withUserMessageLimit(n.mixUserMessageLimit.get()) + b.mixConf.withDisableSpamProtection(n.mixDisableSpamProtection) b.filterServiceConf.withEnabled(n.filter) b.filterServiceConf.withSubscriptionTimeout(n.filterSubscriptionTimeout) diff --git a/waku/factory/conf_builder/mix_conf_builder.nim b/waku/factory/conf_builder/mix_conf_builder.nim index c04378884..a1aa2ced2 100644 --- a/waku/factory/conf_builder/mix_conf_builder.nim +++ b/waku/factory/conf_builder/mix_conf_builder.nim @@ -12,6 +12,8 @@ type MixConfBuilder* = object enabled: Option[bool] mixKey: Option[string] mixNodes: seq[MixNodePubInfo] + userMessageLimit: Option[int] + disableSpamProtection: bool proc init*(T: type MixConfBuilder): MixConfBuilder = MixConfBuilder() @@ -25,6 +27,12 @@ proc withMixKey*(b: var MixConfBuilder, mixKey: string) = proc withMixNodes*(b: var MixConfBuilder, mixNodes: seq[MixNodePubInfo]) = b.mixNodes = mixNodes +proc withUserMessageLimit*(b: var MixConfBuilder, limit: int) = + b.userMessageLimit = some(limit) + +proc withDisableSpamProtection*(b: var MixConfBuilder, disable: bool) = + b.disableSpamProtection = disable + proc build*(b: MixConfBuilder): Result[Option[MixConf], string] = if not b.enabled.get(false): return ok(none[MixConf]()) @@ -33,11 +41,27 @@ proc build*(b: MixConfBuilder): Result[Option[MixConf], string] = let mixPrivKey = intoCurve25519Key(ncrutils.fromHex(b.mixKey.get())) let mixPubKey = public(mixPrivKey) return ok( - some(MixConf(mixKey: mixPrivKey, mixPubKey: mixPubKey, mixNodes: b.mixNodes)) + some( + MixConf( + mixKey: mixPrivKey, + mixPubKey: mixPubKey, + mixNodes: b.mixNodes, + userMessageLimit: b.userMessageLimit, + disableSpamProtection: b.disableSpamProtection, + ) + ) ) else: let (mixPrivKey, mixPubKey) = generateKeyPair().valueOr: return err("Generate key pair error: " & $error) return ok( - some(MixConf(mixKey: mixPrivKey, mixPubKey: mixPubKey, mixNodes: b.mixNodes)) + some( + MixConf( + mixKey: mixPrivKey, + mixPubKey: mixPubKey, + mixNodes: b.mixNodes, + userMessageLimit: b.userMessageLimit, + disableSpamProtection: b.disableSpamProtection, + ) + ) ) diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index 5f99f7e17..b77975632 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -164,7 +164,12 @@ proc setupProtocols( #mount mix if conf.mixConf.isSome(): let mixConf = conf.mixConf.get() - (await node.mountMix(conf.clusterId, mixConf.mixKey, mixConf.mixnodes)).isOkOr: + ( + await node.mountMix( + conf.clusterId, mixConf.mixKey, mixConf.mixnodes, mixConf.userMessageLimit, + mixConf.disableSpamProtection, + ) + ).isOkOr: return err("failed to mount waku mix protocol: " & $error) # Setup extended kademlia discovery diff --git a/waku/factory/waku_conf.nim b/waku/factory/waku_conf.nim index 9edc12a44..d20dc263c 100644 --- a/waku/factory/waku_conf.nim +++ b/waku/factory/waku_conf.nim @@ -51,6 +51,8 @@ type MixConf* = ref object mixKey*: Curve25519Key mixPubKey*: Curve25519Key mixnodes*: seq[MixNodePubInfo] + userMessageLimit*: Option[int] + disableSpamProtection*: bool type KademliaDiscoveryConf* = object bootstrapNodes*: seq[(PeerId, seq[MultiAddress])] diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 70581e3d9..356e77e3e 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -317,6 +317,7 @@ proc mountMix*( mixPrivKey: Curve25519Key, mixnodes: seq[MixNodePubInfo], userMessageLimit: Option[int] = none(int), + disableSpamProtection: bool = false, ): Future[Result[void, string]] {.async.} = info "mounting mix protocol", nodeId = node.info #TODO log the config used @@ -349,7 +350,7 @@ proc mountMix*( node.wakuMix = WakuMix.new( localaddrStr, node.peerManager, clusterId, mixPrivKey, mixnodes, publishMessage, - userMessageLimit, + userMessageLimit, disableSpamProtection, ).valueOr: error "Waku Mix protocol initialization failed", err = error return diff --git a/waku/waku_mix/protocol.nim b/waku/waku_mix/protocol.nim index a504f94f4..7ad065454 100644 --- a/waku/waku_mix/protocol.nim +++ b/waku/waku_mix/protocol.nim @@ -11,6 +11,7 @@ import libp2p_mix/mix_metrics, libp2p_mix/delay_strategy, libp2p_mix/spam_protection, + libp2p_mix/cover_traffic, libp2p/[multiaddress, multicodec, peerid, peerinfo], eth/common/keys @@ -91,6 +92,7 @@ proc new*( bootnodes: seq[MixNodePubInfo], publishMessage: PublishMessage, userMessageLimit: Option[int] = none(int), + disableSpamProtection: bool = false, ): WakuMixResult[T] = let mixPubKey = public(mixPrivKey) trace "mixPubKey", mixPubKey = mixPubKey @@ -101,36 +103,50 @@ proc new*( peermgr.switch.peerInfo.publicKey.skkey, peermgr.switch.peerInfo.privateKey.skkey, ) - # Initialize spam protection with persistent credentials - # Use peerID in keystore path so multiple peers can run from same directory - # Tree path is shared across all nodes to maintain the full membership set - let peerId = peermgr.switch.peerInfo.peerId - var spamProtectionConfig = defaultConfig() - spamProtectionConfig.keystorePath = "rln_keystore_" & $peerId & ".json" - spamProtectionConfig.keystorePassword = "mix-rln-password" - if userMessageLimit.isSome(): - spamProtectionConfig.userMessageLimit = userMessageLimit.get() - # rlnResourcesPath left empty to use bundled resources (via "tree_height_/" placeholder) + let totalSlots = userMessageLimit.get(2) + let ct = ConstantRateCoverTraffic.new( + totalSlots = totalSlots, + epochDuration = 10.seconds, + useInternalEpochTimer = disableSpamProtection, + ) - let spamProtection = newMixRlnSpamProtection(spamProtectionConfig).valueOr: - return err("failed to create spam protection: " & error) + var spamProtectionOpt = default(Opt[SpamProtection]) + if not disableSpamProtection: + # Initialize spam protection with persistent credentials + let peerId = peermgr.switch.peerInfo.peerId + var spamProtectionConfig = defaultConfig() + spamProtectionConfig.keystorePath = "rln_keystore_" & $peerId & ".json" + spamProtectionConfig.keystorePassword = "mix-rln-password" + if userMessageLimit.isSome(): + spamProtectionConfig.userMessageLimit = userMessageLimit.get() + + let spamProtection = newMixRlnSpamProtection(spamProtectionConfig).valueOr: + return err("failed to create spam protection: " & error) + spamProtectionOpt = Opt.some(SpamProtection(spamProtection)) + else: + info "mix spam protection disabled" + + var mixRlnSpam: MixRlnSpamProtection + if spamProtectionOpt.isSome(): + mixRlnSpam = MixRlnSpamProtection(spamProtectionOpt.get()) var m = WakuMix( peerManager: peermgr, clusterId: clusterId, pubKey: mixPubKey, - mixRlnSpamProtection: spamProtection, publishMessage: publishMessage, + mixRlnSpamProtection: mixRlnSpam, ) procCall MixProtocol(m).init( localMixNodeInfo, peermgr.switch, - spamProtection = Opt.some(SpamProtection(spamProtection)), + spamProtection = spamProtectionOpt, delayStrategy = Opt.some( DelayStrategy( ExponentialDelayStrategy.new(meanDelay = 100, rng = crypto.newRng()) ) ), + coverTraffic = Opt.some(CoverTraffic(ct)), ) processBootNodes(bootnodes, peermgr, m) @@ -147,6 +163,8 @@ proc setupSpamProtectionCallbacks(mix: WakuMix) = ## Set up the publish callback for spam protection coordination. ## This enables the plugin to broadcast membership updates and proof metadata ## via Waku relay. + if mix.mixRlnSpamProtection.isNil(): + return if mix.publishMessage.isNil(): warn "PublishMessage callback not available, spam protection coordination disabled" return