More changes

.env

@@ -1,6 +1,6 @@
 export GOWAKU_IMAGE=statusteam/go-waku:latest
 export NWAKU_IMAGE=057383cb7328f525585f4cecb6a36c0d3859da94be84add8dab6a56af5eba75d
-export NUM_NWAKU_NODES=20
+export NUM_NWAKU_NODES=5
 export NUM_GOWAKU_NODES=0
 export MSG_PER_SECOND=10
 export MSG_SIZE_KBYTES=10

docker-compose.yml

@@ -18,16 +18,338 @@ services:
     volumes:
       - ./run_bootstrap.sh:/opt/run_bootstrap.sh:Z
   
-  nwaku:
+  nwaku_1:
+    container_name: waku-simulator_nwaku_1
+    environment:
+      REPLICA: 1
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_2:
+    container_name: waku-simulator_nwaku_2
+    environment:
+      REPLICA: 2
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_3:
+    container_name: waku-simulator_nwaku_3
+    environment:
+      REPLICA: 3
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_4:
+    container_name: waku-simulator_nwaku_4
+    environment:
+      REPLICA: 4
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_5:
+    container_name: waku-simulator_nwaku_5
+    environment:
+      REPLICA: 5
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_6:
+    container_name: waku-simulator_nwaku_6
+    environment:
+      REPLICA: 6
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_7:
+    container_name: waku-simulator_nwaku_7
+    environment:
+      REPLICA: 7
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_8:
+    container_name: waku-simulator_nwaku_8
+    environment:
+      REPLICA: 8
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_9:
+    container_name: waku-simulator_nwaku_9
+    environment:
+      REPLICA: 9
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_10:
+    container_name: waku-simulator_nwaku_10
+    environment:
+      REPLICA: 10
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_11:
+    container_name: waku-simulator_nwaku_11
+    environment:
+      REPLICA: 11
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_12:
+    container_name: waku-simulator_nwaku_12
+    environment:
+      REPLICA: 12
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_13:
+    container_name: waku-simulator_nwaku_13
+    environment:
+      REPLICA: 13
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_14:
+    container_name: waku-simulator_nwaku_14
+    environment:
+      REPLICA: 14
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_15:
+    container_name: waku-simulator_nwaku_15
+    environment:
+      REPLICA: 15
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_16:
+    container_name: waku-simulator_nwaku_16
+    environment:
+      REPLICA: 16
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_17:
+    container_name: waku-simulator_nwaku_17
+    environment:
+      REPLICA: 17
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_18:
+    container_name: waku-simulator_nwaku_18
+    environment:
+      REPLICA: 18
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_19:
+    container_name: waku-simulator_nwaku_19
+    environment:
+      REPLICA: 19
+    image: ${NWAKU_IMAGE}
+    restart: on-failure
+    entrypoint: sh
+    command:
+      - '/opt/run_nwaku.sh'
+    volumes:
+      - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
+      - ./rlnKeystore.json:/rlnKeystore.json:Z
+#      - ./rln_tree.db:/rln_tree.db:Z # this should not be done as its being written by all the replicas
+    depends_on:
+      - bootstrap
+
+
+  nwaku_20:
+    container_name: waku-simulator_nwaku_20
+    environment:
+      REPLICA: 20
     image: ${NWAKU_IMAGE}
     restart: on-failure
-    deploy:
-      replicas: ${NUM_NWAKU_NODES}
     entrypoint: sh
     command:
       - '/opt/run_nwaku.sh'
-    #ports:
-    #  - 127.0.0.1:8545:8545
     volumes:
       - ./run_nwaku.sh:/opt/run_nwaku.sh:Z
       - ./rlnKeystore.json:/rlnKeystore.json:Z

generate_keystores.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Ensure you have wakunode2 built with RLN
+# make wakunode2 EXPERIMENTAL=true
+echo "Generating rln memberships into a single keystore"
+
+# clean old keystore
+rm rlnKeystore.json
+
+# seems we dont populate the chain id, so infura considers it as replayable and rejects it.
+#ws://linux-01.ih-eu-mda1.nimbus.sepolia.wg:9557
+
+for i in {0..20}
+do
+   echo "Generating membership at index $i"
+   ./build/wakunode2 \
+   --rln-relay=true \
+   --rln-relay-dynamic=true \
+   --rln-relay-eth-account-private-key=d9f61e035e233e7baabb7ca806f7e9800cfa68397df2d844bf197c3a728cdcef \
+   --rln-relay-membership-index=$i \
+   --rln-relay-eth-contract-address=0x39558059411112732d73997712b75a865a697330 \
+   --rln-relay-eth-client-address=wss://sepolia.infura.io/ws/v3/4576482c0f474483ac709755f2663b20 &
+   last_pid=$!
+   echo "pid of process is: "
+   echo $last_pid
+
+   # this has to be enough to sync the tree and create the keystore. not very efficient 
+   sleep 120
+   echo "killing $last_pid and generating next membership"
+   kill -KILL $last_pid
+done
+
+
+# TODO: add this. run at the end. ensures the size of the keystore matches
+# grep -o salt rlnKeystore.json | wc -l

run_nwaku.sh

@@ -3,6 +3,7 @@
 IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
 
 echo "I am a nwaku node"
+echo "I am replica $REPLICA"
 
 RETRIES=${RETRIES:=10}
 
@@ -36,5 +37,6 @@ exec /usr/bin/wakunode\
       --rln-relay-dynamic=true \
       --rln-relay-cred-password=password \
       --rln-relay-cred-path=/rlnKeystore.json \
+      --rln-relay-membership-index=$REPLICA \
       --rln-relay-eth-contract-address=0x39558059411112732d73997712b75a865a697330  \
       --rln-relay-eth-client-address=ws://linux-01.ih-eu-mda1.nimbus.sepolia.wg:9557

traffic.py

@@ -62,7 +62,11 @@ if args.multiple_nodes:
 
   print("Injecting traffic to multiple nodes RPC") 
   for i in range(start, end+1):
-    print(node_placeholder.replace("{placeholder}", str(i)))
+    nodes.append(node_placeholder.replace("{placeholder}", str(i)))
+
+print("Injecting traffic to multiple nodes RPC")
+for node in nodes:
+  print(node)
 
 while True:
     # calls are blocking
@@ -75,4 +79,5 @@ while True:
       for node in nodes:
         send_waku_msg(node, args.msg_size_kbytes, args.pubsub_topic, args.content_topic)
 
+    print("sleeping: ", args.delay_seconds, " seconds")
     time.sleep(args.delay_seconds)