Connecting nodes with bootstrap node and adding go-waku store clients

One "go-waku" relay node is responsible for generating
traffic, which in turns creates "insert" events. And another
go-waku node that acts as a store client to generate "store"
requests to both "sqlite" and "postgres" nwaku store nodes.

docker/docker-compose-manual-binaries.yml

@@ -3,6 +3,15 @@
 ## folder as this file.
 
 version: "3.7"
+
+networks:
+  simulation:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: "10.1.0.0/22"
+
 x-logging: &logging
   logging:
     driver: json-file
@@ -29,25 +38,28 @@ x-pg-exporter-env: &pg_exp_env
 
 services:
 
-  nwaku_client_postgres:
+  bootstrap:
-    ## Waku node that acts as a Store client to the `nwaku_postgres` and listens
-    ## to REST-Store requests
     image: statusteam/nim-waku:v0.20.0
     restart: on-failure
-    <<:
+    ports:
-      - *logging
+      - 127.0.0.1:60000:60000
-    volumes:
+      - 127.0.0.1:8008:8008
-      - ./run_nwaku_store_client_postgres.sh:/opt/run_nwaku_store_client_postgres.sh:Z
+      - 127.0.0.1:9000:9000
+      - 127.0.0.1:8544:8544
     entrypoint: sh
     command:
-      - /opt/run_nwaku_store_client_postgres.sh
+    - '/opt/run_bootstrap.sh'
-    depends_on:
+    volumes:
-      - nwaku_postgres
+      - ./run_bootstrap.sh:/opt/run_bootstrap.sh:Z
+    networks:
+      - simulation
 
-  nwaku_sqlite:
+  nwaku-sqlite:
     ## Waku node with Store mounted & SQLite
     image: ubuntu
     restart: on-failure
+    ports:
+      - 0.0.0.0:8546:8546
     <<:
       - *logging
     volumes:
@@ -57,53 +69,60 @@ services:
     entrypoint: sh
     command:
       - /opt/run_nwaku_store_sqlite.sh
+    networks:
+      - simulation
 
-  nwaku_client_sqlite:
+  nwaku-postgres:
-    ## Waku node that acts as a Store client to the `nwaku_sqlite` and listens
-    ## to REST-Store requests
-    image: statusteam/nim-waku:v0.20.0
-    restart: on-failure
-    <<:
-      - *logging
-    volumes:
-      - ./run_nwaku_store_client_sqlite.sh:/opt/run_nwaku_store_client_sqlite.sh:Z
-    entrypoint: sh
-    command:
-      - /opt/run_nwaku_store_client_sqlite.sh
-    depends_on:
-      - nwaku_sqlite
-
-  nwaku_postgres:
     ## Waku node with Store mounted & Postgres
     image: ubuntu
     restart: on-failure
+    ports:
+      - 0.0.0.0:8545:8545
     <<:
       - *logging
       - *pg_env
     volumes:
     # This is expected to be run from metal-01.he-eu-hel1.wakudev.misc.statusim.net
-      - /home/shared/nwaku/build/wakunode2:/usr/bin/wakunode:Z
+      - /home/ivansete/workspace/status/nwaku/build/wakunode2:/usr/bin/wakunode:Z
-      - ./run_nwaku_store_postgres_ubuntu.sh:/opt/run_nwaku_store_postgres.sh:Z
+      - ./run_nwaku_store_postgres_ubuntu.sh:/opt/run_nwaku_store_postgres_ubuntu.sh:Z
     entrypoint: bash
     command:
-      - /opt/run_nwaku_store_postgres.sh
+      - /opt/run_nwaku_store_postgres_ubuntu.sh
     depends_on:
       - postgres
+    networks:
+      - simulation
 
-  # msg_publisher:
+  waku-store-query-generator:
-  #   ## Sends json-rpc 'post_waku_v2_relay_v1_message' messages infinitely
+    image: ivansete/waku-store-request-maker:91bb3ffd
-  #   image: alpine:3.16
+    restart: on-failure
-  #   restart: on-failure
+    entrypoint: sh
-  #   logging:
+      - 'opt/run_waku_store_query_maker.sh'
-  #     driver: "none"
+    deploy:
-  #   volumes:
+      replicas: 1
-  #     - ./msg_publisher.sh:/opt/msg_publisher.sh:Z
+    volumes:
-  #   entrypoint: sh
+      - ./run_waku_store_query_maker.sh:/opt/run_waku_store_query_maker.sh:Z
-  #   command:
+    environment:
-  #     - /opt/msg_publisher.sh
+      STORE_QUERIES_PER_SECOND: 10
-  #   depends_on:
+    networks:
-  #     - nwaku_postgres
+      - simulation
-  #     - nwaku_sqlite
+
+  waku-publisher:
+    image: alrevuelta/waku-publisher:9fb206c
+    entrypoint: sh
+      - 'opt/run_wakupublisher.sh'
+    deploy:
+      replicas: 1
+    volumes:
+      - ./run_wakupublisher.sh:/opt/run_wakupublisher.sh:Z
+    environment:
+      MSG_PER_SECOND: 10
+      MSG_SIZE_KBYTES: 10
+    depends_on:
+      - nwaku-postgres
+      - nwaku-sqlite
+    networks:
+      - simulation
 
   prometheus:
     image: docker.io/prom/prometheus:latest
@@ -114,11 +133,15 @@ services:
     restart: on-failure
     depends_on:
       - postgres-exporter
-      - nwaku_postgres
+      - nwaku-postgres
-      - nwaku_sqlite
+      - nwaku-sqlite
+    networks:
+      - simulation
 
   grafana:
     image: docker.io/grafana/grafana:latest
+    ports:
+      - 0.0.0.0:3000:3000
     env_file:
       - ./monitoring/configuration/grafana-plugins.env
     volumes:
@@ -132,12 +155,16 @@ services:
     restart: on-failure
     depends_on:
       - prometheus
+    networks:
+      - simulation
 
   postgres:
     # This service is used when the Waku node has the 'store' protocol enabled
     # and the store-message-db-url is set to use Postgres
     image: postgres:15.4-alpine3.18
     restart: on-failure
+    ports:
+      - 0.0.0.0:5432:5432
     <<: *pg_env
     volumes:
       - ./data/postgres-data:/var/lib/postgresql/data
@@ -150,6 +177,8 @@ services:
       timeout: 60s
       retries: 5
       start_period: 80s
+    networks:
+      - simulation
 
   postgres-exporter:
     # Service aimed to scrape information from Postgres and post it to Prometeus
@@ -164,3 +193,5 @@ services:
       - --config.file=/etc/pgexporter/postgres-exporter.yml
     depends_on:
       - postgres
+    networks:
+      - simulation

docker/monitoring/prometheus-config.yml

@@ -7,11 +7,11 @@ global:
 scrape_configs:
   - job_name: "nwaku_postgres"
     static_configs:
-    - targets: ['nwaku_postgres:8003']
+    - targets: ['nwaku-postgres:8003']
 
   - job_name: "nwaku_sqlite"
     static_configs:
-    - targets: ['nwaku_sqlite:8004']
+    - targets: ['nwaku-sqlite:8004']
 
   - job_name: postgres-exporter
     static_configs:

docker/run_bootstrap.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
+
+echo "I am a bootstrap node. Listening to: ${IP}"
+
+exec /usr/bin/wakunode\
+      --relay=true\
+      --rpc-admin=true\
+      --keep-alive=true\
+      --max-connections=300\
+      --dns-discovery=true\
+      --discv5-discovery=true\
+      --discv5-enr-auto-update=True\
+      --log-level=INFO\
+      --rpc-port=8544\
+      --rpc-address=0.0.0.0\
+      --metrics-server=True\
+      --metrics-server-address=0.0.0.0\
+      --nodekey=30348dd51465150e04a5d9d932c72864c8967f806cce60b5d26afeca1e77eb68\
+      --nat=extip:${IP}

docker/run_nwaku_store_postgres_ubuntu.sh

@@ -3,16 +3,32 @@
 apt-get update
 ## Install the `dig` command
 apt-get install dnsutils -y
+apt-get install wget -y
 
-peer_IP=$(dig +short nwaku_sqlite)
+bootstrap_IP=$(dig +short bootstrap)
 
 apt-get install libpq5 -y
-
 chmod +x /usr/bin/wakunode
 
+RETRIES=${RETRIES:=10}
+
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://${bootstrap_IP}:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready in ${bootstrap_IP}, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+IP=$(hostname -I)
+
+echo "I am postgres ubuntu. Listening on: ${IP}"
+
 ./usr/bin/wakunode\
-  --nodekey=1d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\
-  --staticnode=/ip4/${peer_IP}/tcp/30304/p2p/16Uiu2HAkxj3WzLiqBximSaHc8wV9Co87GyRGRYLVGsHZrzi3TL5W\
   --relay=true\
   --topic=/waku/2/default-waku/proto\
   --topic=/waku/2/dev-waku/proto\
@@ -21,10 +37,15 @@ chmod +x /usr/bin/wakunode
   --log-level=DEBUG\
   --rpc-port=8545\
   --rpc-address=0.0.0.0\
-  --tcp-port=30303\
   --metrics-server=True\
   --metrics-server-port=8003\
   --metrics-server-address=0.0.0.0\
+  --max-connections=250\
+  --dns-discovery=true\
+  --discv5-discovery=true\
+  --discv5-enr-auto-update=True\
+  --discv5-bootstrap-node=${BOOTSTRAP_ENR}\
+  --nat=extip:${IP}\
   --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres"\
   --store=true\
   --store-message-retention-policy=capacity:4000000

docker/run_nwaku_store_sqlite_ubuntu.sh

@@ -3,16 +3,32 @@
 apt-get update
 ## Install the `dig` command
 apt-get install dnsutils -y
+apt-get install wget -y
 
-peer_IP=$(dig +short nwaku_postgres)
+bootstrap_IP=$(dig +short bootstrap)
 
 apt-get install libpq5 -y
-
 chmod +x /usr/bin/wakunode
 
+RETRIES=${RETRIES:=10}
+
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://${bootstrap_IP}:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready in ${bootstrap_IP}, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+IP=$(hostname -I)
+
+echo "I am sqlite ubuntu. Listening on: ${IP}"
+
 ./usr/bin/wakunode\
-  --nodekey=2d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\
-  --staticnode=/ip4/${peer_IP}/tcp/30303/p2p/16Uiu2HAmJyLCRhiErTRFcW5GKPrpoMjGbbWdFMx4GCUnnhmxeYhd\
   --relay=true\
   --topic=/waku/2/default-waku/proto\
   --topic=/waku/2/dev-waku/proto\
@@ -21,10 +37,15 @@ chmod +x /usr/bin/wakunode
   --log-level=DEBUG\
   --rpc-port=8546\
   --rpc-address=0.0.0.0\
-  --tcp-port=30304\
   --metrics-server=True\
   --metrics-server-port=8004\
   --metrics-server-address=0.0.0.0\
+  --max-connections=250\
+  --dns-discovery=true\
+  --discv5-discovery=true\
+  --discv5-enr-auto-update=True\
+  --discv5-bootstrap-node=${BOOTSTRAP_ENR}\
+  --nat=extip:${IP}\
   --store-message-db-url="sqlite:///data/store.sqlite3"\
   --store=true\
   --store-message-retention-policy=capacity:4000000

docker/run_waku_store_query_maker.sh

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
+
+echo "I am a waku store query generator"
+
+## Getting the address of the Postgres node
+RETRIES=10
+while [ -z "${POSTGRES_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do
+  POSTGRES_ADDR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-postgres:8545/ 2> /dev/null | sed 's/.*"listenAddresses":\["\(.*\)"].*/\1/');
+  wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-postgres:8545/
+  echo "Store Postgres node node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${POSTGRES_ADDR}" ]; then
+   echo "Could not get POSTGRES_ADDR and none provided. Failing"
+   exit 1
+fi
+
+## Getting the address of the SQLite node
+RETRIES=10
+while [ -z "${SQLITE_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do
+  SQLITE_ADDR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-sqlite:8546/ 2> /dev/null | sed 's/.*"listenAddresses":\["\(.*\)"].*/\1/');
+  echo "Store SQLite node node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${SQLITE_ADDR}" ]; then
+   echo "Could not get SQLITE_ADDR and none provided. Failing"
+   exit 1
+fi
+
+## Getting the bootstrap node ENR
+RETRIES=10
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://bootstrap:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
+exec /main\
+    --pubsub-topic="/waku/2/default-waku/proto"\
+    --content-topic="my-ctopic"\
+    --queries-per-second=${STORE_QUERIES_PER_SECOND}\
+    --bootstrap-node=${BOOTSTRAP_ENR}\
+    --peer-store-postgres-addr="${POSTGRES_ADDR}"\
+    --peer-store-sqlite-addr="${SQLITE_ADDR}"\
+    --num-minutes-query=60

docker/run_wakupublisher.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
+
+echo "I am a traffic generator"
+
+RETRIES=${RETRIES:=10}
+
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://bootstrap:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
+exec /main\
+    --pubsub-topic="/waku/2/default-waku/proto"\
+    --content-topic="my-ctopic"\
+    --msg-per-second=${MSG_PER_SECOND}\
+    --msg-size-kb=${MSG_SIZE_KBYTES}\
+    --bootstrap-node=${BOOTSTRAP_ENR}\
+    --max-peers=50