Connecting nodes with bootstrap node and adding go-waku store clients

One "go-waku" relay node is responsible for generating traffic, which in turns creates "insert" events. And another go-waku node that acts as a store client to generate "store" requests to both "sqlite" and "postgres" nwaku store nodes.
2026-01-02 14:03:10 +00:00 · 2023-10-15 20:26:40 +02:00 · 2023-10-15 20:26:40 +02:00 · 2859cae961
commit 2859cae961
parent 33d47b44af
7 changed files with 238 additions and 58 deletions
--- a/docker/docker-compose-manual-binaries.yml
+++ b/docker/docker-compose-manual-binaries.yml
@ -3,6 +3,15 @@
 ## folder as this file.

 version: "3.7"
+
+networks:
+  simulation:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: "10.1.0.0/22"
+
 x-logging: &logging
  logging:
    driver: json-file
@ -29,25 +38,28 @@ x-pg-exporter-env: &pg_exp_env

 services:

-  nwaku_client_postgres:
-    ## Waku node that acts as a Store client to the `nwaku_postgres` and listens
-    ## to REST-Store requests
+  bootstrap:
    image: statusteam/nim-waku:v0.20.0
    restart: on-failure
-    <<:
-      - *logging
-    volumes:
-      - ./run_nwaku_store_client_postgres.sh:/opt/run_nwaku_store_client_postgres.sh:Z
+    ports:
+      - 127.0.0.1:60000:60000
+      - 127.0.0.1:8008:8008
+      - 127.0.0.1:9000:9000
+      - 127.0.0.1:8544:8544
    entrypoint: sh
    command:
-      - /opt/run_nwaku_store_client_postgres.sh
-    depends_on:
-      - nwaku_postgres
+    - '/opt/run_bootstrap.sh'
+    volumes:
+      - ./run_bootstrap.sh:/opt/run_bootstrap.sh:Z
+    networks:
+      - simulation

-  nwaku_sqlite:
+  nwaku-sqlite:
    ## Waku node with Store mounted & SQLite
    image: ubuntu
    restart: on-failure
+    ports:
+      - 0.0.0.0:8546:8546
    <<:
      - *logging
    volumes:
@ -57,53 +69,60 @@ services:
    entrypoint: sh
    command:
      - /opt/run_nwaku_store_sqlite.sh
+    networks:
+      - simulation

-  nwaku_client_sqlite:
-    ## Waku node that acts as a Store client to the `nwaku_sqlite` and listens
-    ## to REST-Store requests
-    image: statusteam/nim-waku:v0.20.0
-    restart: on-failure
-    <<:
-      - *logging
-    volumes:
-      - ./run_nwaku_store_client_sqlite.sh:/opt/run_nwaku_store_client_sqlite.sh:Z
-    entrypoint: sh
-    command:
-      - /opt/run_nwaku_store_client_sqlite.sh
-    depends_on:
-      - nwaku_sqlite
-
-  nwaku_postgres:
+  nwaku-postgres:
    ## Waku node with Store mounted & Postgres
    image: ubuntu
    restart: on-failure
+    ports:
+      - 0.0.0.0:8545:8545
    <<:
      - *logging
      - *pg_env
    volumes:
    # This is expected to be run from metal-01.he-eu-hel1.wakudev.misc.statusim.net
-      - /home/shared/nwaku/build/wakunode2:/usr/bin/wakunode:Z
-      - ./run_nwaku_store_postgres_ubuntu.sh:/opt/run_nwaku_store_postgres.sh:Z
+      - /home/ivansete/workspace/status/nwaku/build/wakunode2:/usr/bin/wakunode:Z
+      - ./run_nwaku_store_postgres_ubuntu.sh:/opt/run_nwaku_store_postgres_ubuntu.sh:Z
    entrypoint: bash
    command:
-      - /opt/run_nwaku_store_postgres.sh
+      - /opt/run_nwaku_store_postgres_ubuntu.sh
    depends_on:
      - postgres
+    networks:
+      - simulation

-  # msg_publisher:
-  #   ## Sends json-rpc 'post_waku_v2_relay_v1_message' messages infinitely
-  #   image: alpine:3.16
-  #   restart: on-failure
-  #   logging:
-  #     driver: "none"
-  #   volumes:
-  #     - ./msg_publisher.sh:/opt/msg_publisher.sh:Z
-  #   entrypoint: sh
-  #   command:
-  #     - /opt/msg_publisher.sh
-  #   depends_on:
-  #     - nwaku_postgres
-  #     - nwaku_sqlite
+  waku-store-query-generator:
+    image: ivansete/waku-store-request-maker:91bb3ffd
+    restart: on-failure
+    entrypoint: sh
+      - 'opt/run_waku_store_query_maker.sh'
+    deploy:
+      replicas: 1
+    volumes:
+      - ./run_waku_store_query_maker.sh:/opt/run_waku_store_query_maker.sh:Z
+    environment:
+      STORE_QUERIES_PER_SECOND: 10
+    networks:
+      - simulation
+
+  waku-publisher:
+    image: alrevuelta/waku-publisher:9fb206c
+    entrypoint: sh
+      - 'opt/run_wakupublisher.sh'
+    deploy:
+      replicas: 1
+    volumes:
+      - ./run_wakupublisher.sh:/opt/run_wakupublisher.sh:Z
+    environment:
+      MSG_PER_SECOND: 10
+      MSG_SIZE_KBYTES: 10
+    depends_on:
+      - nwaku-postgres
+      - nwaku-sqlite
+    networks:
+      - simulation

  prometheus:
    image: docker.io/prom/prometheus:latest
@ -114,11 +133,15 @@ services:
    restart: on-failure
    depends_on:
      - postgres-exporter
-      - nwaku_postgres
-      - nwaku_sqlite
+      - nwaku-postgres
+      - nwaku-sqlite
+    networks:
+      - simulation

  grafana:
    image: docker.io/grafana/grafana:latest
+    ports:
+      - 0.0.0.0:3000:3000
    env_file:
      - ./monitoring/configuration/grafana-plugins.env
    volumes:
@ -132,12 +155,16 @@ services:
    restart: on-failure
    depends_on:
      - prometheus
+    networks:
+      - simulation

  postgres:
    # This service is used when the Waku node has the 'store' protocol enabled
    # and the store-message-db-url is set to use Postgres
    image: postgres:15.4-alpine3.18
    restart: on-failure
+    ports:
+      - 0.0.0.0:5432:5432
    <<: *pg_env
    volumes:
      - ./data/postgres-data:/var/lib/postgresql/data
@ -150,6 +177,8 @@ services:
      timeout: 60s
      retries: 5
      start_period: 80s
+    networks:
+      - simulation

  postgres-exporter:
    # Service aimed to scrape information from Postgres and post it to Prometeus
@ -164,3 +193,5 @@ services:
      - --config.file=/etc/pgexporter/postgres-exporter.yml
    depends_on:
      - postgres
+    networks:
+      - simulation
--- a/docker/monitoring/prometheus-config.yml
+++ b/docker/monitoring/prometheus-config.yml
@ -7,11 +7,11 @@ global:
 scrape_configs:
  - job_name: "nwaku_postgres"
    static_configs:
-    - targets: ['nwaku_postgres:8003']
+    - targets: ['nwaku-postgres:8003']

  - job_name: "nwaku_sqlite"
    static_configs:
-    - targets: ['nwaku_sqlite:8004']
+    - targets: ['nwaku-sqlite:8004']

  - job_name: postgres-exporter
    static_configs:
--- a/docker/run_bootstrap.sh
+++ b/docker/run_bootstrap.sh
@ -0,0 +1,21 @@
+#!/bin/sh
+
+IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
+
+echo "I am a bootstrap node. Listening to: ${IP}"
+
+exec /usr/bin/wakunode\
+      --relay=true\
+      --rpc-admin=true\
+      --keep-alive=true\
+      --max-connections=300\
+      --dns-discovery=true\
+      --discv5-discovery=true\
+      --discv5-enr-auto-update=True\
+      --log-level=INFO\
+      --rpc-port=8544\
+      --rpc-address=0.0.0.0\
+      --metrics-server=True\
+      --metrics-server-address=0.0.0.0\
+      --nodekey=30348dd51465150e04a5d9d932c72864c8967f806cce60b5d26afeca1e77eb68\
+      --nat=extip:${IP}
--- a/docker/run_nwaku_store_postgres_ubuntu.sh
+++ b/docker/run_nwaku_store_postgres_ubuntu.sh
@ -3,16 +3,32 @@
 apt-get update
 ## Install the `dig` command
 apt-get install dnsutils -y
+apt-get install wget -y

-peer_IP=$(dig +short nwaku_sqlite)
+bootstrap_IP=$(dig +short bootstrap)

 apt-get install libpq5 -y
-
 chmod +x /usr/bin/wakunode

+RETRIES=${RETRIES:=10}
+
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://${bootstrap_IP}:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready in ${bootstrap_IP}, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+IP=$(hostname -I)
+
+echo "I am postgres ubuntu. Listening on: ${IP}"
+
 ./usr/bin/wakunode\
-  --nodekey=1d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\
-  --staticnode=/ip4/${peer_IP}/tcp/30304/p2p/16Uiu2HAkxj3WzLiqBximSaHc8wV9Co87GyRGRYLVGsHZrzi3TL5W\
  --relay=true\
  --topic=/waku/2/default-waku/proto\
  --topic=/waku/2/dev-waku/proto\
@ -21,10 +37,15 @@ chmod +x /usr/bin/wakunode
  --log-level=DEBUG\
  --rpc-port=8545\
  --rpc-address=0.0.0.0\
-  --tcp-port=30303\
  --metrics-server=True\
  --metrics-server-port=8003\
  --metrics-server-address=0.0.0.0\
+  --max-connections=250\
+  --dns-discovery=true\
+  --discv5-discovery=true\
+  --discv5-enr-auto-update=True\
+  --discv5-bootstrap-node=${BOOTSTRAP_ENR}\
+  --nat=extip:${IP}\
  --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres"\
  --store=true\
  --store-message-retention-policy=capacity:4000000
--- a/docker/run_nwaku_store_sqlite_ubuntu.sh
+++ b/docker/run_nwaku_store_sqlite_ubuntu.sh
@ -3,16 +3,32 @@
 apt-get update
 ## Install the `dig` command
 apt-get install dnsutils -y
+apt-get install wget -y

-peer_IP=$(dig +short nwaku_postgres)
+bootstrap_IP=$(dig +short bootstrap)

 apt-get install libpq5 -y
-
 chmod +x /usr/bin/wakunode

+RETRIES=${RETRIES:=10}
+
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://${bootstrap_IP}:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready in ${bootstrap_IP}, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+IP=$(hostname -I)
+
+echo "I am sqlite ubuntu. Listening on: ${IP}"
+
 ./usr/bin/wakunode\
-  --nodekey=2d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\
-  --staticnode=/ip4/${peer_IP}/tcp/30303/p2p/16Uiu2HAmJyLCRhiErTRFcW5GKPrpoMjGbbWdFMx4GCUnnhmxeYhd\
  --relay=true\
  --topic=/waku/2/default-waku/proto\
  --topic=/waku/2/dev-waku/proto\
@ -21,10 +37,15 @@ chmod +x /usr/bin/wakunode
  --log-level=DEBUG\
  --rpc-port=8546\
  --rpc-address=0.0.0.0\
-  --tcp-port=30304\
  --metrics-server=True\
  --metrics-server-port=8004\
  --metrics-server-address=0.0.0.0\
+  --max-connections=250\
+  --dns-discovery=true\
+  --discv5-discovery=true\
+  --discv5-enr-auto-update=True\
+  --discv5-bootstrap-node=${BOOTSTRAP_ENR}\
+  --nat=extip:${IP}\
  --store-message-db-url="sqlite:///data/store.sqlite3"\
  --store=true\
  --store-message-retention-policy=capacity:4000000
--- a/docker/run_waku_store_query_maker.sh
+++ b/docker/run_waku_store_query_maker.sh
@ -0,0 +1,58 @@
+#!/bin/sh
+
+IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
+
+echo "I am a waku store query generator"
+
+## Getting the address of the Postgres node
+RETRIES=10
+while [ -z "${POSTGRES_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do
+  POSTGRES_ADDR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-postgres:8545/ 2> /dev/null | sed 's/.*"listenAddresses":\["\(.*\)"].*/\1/');
+  wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-postgres:8545/
+  echo "Store Postgres node node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${POSTGRES_ADDR}" ]; then
+   echo "Could not get POSTGRES_ADDR and none provided. Failing"
+   exit 1
+fi
+
+## Getting the address of the SQLite node
+RETRIES=10
+while [ -z "${SQLITE_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do
+  SQLITE_ADDR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-sqlite:8546/ 2> /dev/null | sed 's/.*"listenAddresses":\["\(.*\)"].*/\1/');
+  echo "Store SQLite node node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${SQLITE_ADDR}" ]; then
+   echo "Could not get SQLITE_ADDR and none provided. Failing"
+   exit 1
+fi
+
+## Getting the bootstrap node ENR
+RETRIES=10
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://bootstrap:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
+exec /main\
+    --pubsub-topic="/waku/2/default-waku/proto"\
+    --content-topic="my-ctopic"\
+    --queries-per-second=${STORE_QUERIES_PER_SECOND}\
+    --bootstrap-node=${BOOTSTRAP_ENR}\
+    --peer-store-postgres-addr="${POSTGRES_ADDR}"\
+    --peer-store-sqlite-addr="${SQLITE_ADDR}"\
+    --num-minutes-query=60
--- a/docker/run_wakupublisher.sh
+++ b/docker/run_wakupublisher.sh
@ -0,0 +1,28 @@
+#!/bin/sh
+
+IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
+
+echo "I am a traffic generator"
+
+RETRIES=${RETRIES:=10}
+
+while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
+  BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://bootstrap:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
+  echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})"
+  sleep 1
+  RETRIES=$(( $RETRIES - 1 ))
+done
+
+if [ -z "${BOOTSTRAP_ENR}" ]; then
+   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
+   exit 1
+fi
+
+echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
+exec /main\
+    --pubsub-topic="/waku/2/default-waku/proto"\
+    --content-topic="my-ctopic"\
+    --msg-per-second=${MSG_PER_SECOND}\
+    --msg-size-kb=${MSG_SIZE_KBYTES}\
+    --bootstrap-node=${BOOTSTRAP_ENR}\
+    --max-peers=50