From 2859cae961b7fadb33be4040117dc06cf8ed4e54 Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Sun, 15 Oct 2023 20:26:40 +0200 Subject: [PATCH] Connecting nodes with bootstrap node and adding go-waku store clients One "go-waku" relay node is responsible for generating traffic, which in turns creates "insert" events. And another go-waku node that acts as a store client to generate "store" requests to both "sqlite" and "postgres" nwaku store nodes. --- docker/docker-compose-manual-binaries.yml | 123 ++++++++++++++-------- docker/monitoring/prometheus-config.yml | 4 +- docker/run_bootstrap.sh | 21 ++++ docker/run_nwaku_store_postgres_ubuntu.sh | 31 +++++- docker/run_nwaku_store_sqlite_ubuntu.sh | 31 +++++- docker/run_waku_store_query_maker.sh | 58 ++++++++++ docker/run_wakupublisher.sh | 28 +++++ 7 files changed, 238 insertions(+), 58 deletions(-) create mode 100755 docker/run_bootstrap.sh create mode 100755 docker/run_waku_store_query_maker.sh create mode 100755 docker/run_wakupublisher.sh diff --git a/docker/docker-compose-manual-binaries.yml b/docker/docker-compose-manual-binaries.yml index 23695bf..ba8199b 100644 --- a/docker/docker-compose-manual-binaries.yml +++ b/docker/docker-compose-manual-binaries.yml @@ -3,6 +3,15 @@ ## folder as this file. version: "3.7" + +networks: + simulation: + driver: bridge + ipam: + driver: default + config: + - subnet: "10.1.0.0/22" + x-logging: &logging logging: driver: json-file @@ -29,25 +38,28 @@ x-pg-exporter-env: &pg_exp_env services: - nwaku_client_postgres: - ## Waku node that acts as a Store client to the `nwaku_postgres` and listens - ## to REST-Store requests + bootstrap: image: statusteam/nim-waku:v0.20.0 restart: on-failure - <<: - - *logging - volumes: - - ./run_nwaku_store_client_postgres.sh:/opt/run_nwaku_store_client_postgres.sh:Z + ports: + - 127.0.0.1:60000:60000 + - 127.0.0.1:8008:8008 + - 127.0.0.1:9000:9000 + - 127.0.0.1:8544:8544 entrypoint: sh command: - - /opt/run_nwaku_store_client_postgres.sh - depends_on: - - nwaku_postgres + - '/opt/run_bootstrap.sh' + volumes: + - ./run_bootstrap.sh:/opt/run_bootstrap.sh:Z + networks: + - simulation - nwaku_sqlite: + nwaku-sqlite: ## Waku node with Store mounted & SQLite image: ubuntu restart: on-failure + ports: + - 0.0.0.0:8546:8546 <<: - *logging volumes: @@ -57,53 +69,60 @@ services: entrypoint: sh command: - /opt/run_nwaku_store_sqlite.sh + networks: + - simulation - nwaku_client_sqlite: - ## Waku node that acts as a Store client to the `nwaku_sqlite` and listens - ## to REST-Store requests - image: statusteam/nim-waku:v0.20.0 - restart: on-failure - <<: - - *logging - volumes: - - ./run_nwaku_store_client_sqlite.sh:/opt/run_nwaku_store_client_sqlite.sh:Z - entrypoint: sh - command: - - /opt/run_nwaku_store_client_sqlite.sh - depends_on: - - nwaku_sqlite - - nwaku_postgres: + nwaku-postgres: ## Waku node with Store mounted & Postgres image: ubuntu restart: on-failure + ports: + - 0.0.0.0:8545:8545 <<: - *logging - *pg_env volumes: # This is expected to be run from metal-01.he-eu-hel1.wakudev.misc.statusim.net - - /home/shared/nwaku/build/wakunode2:/usr/bin/wakunode:Z - - ./run_nwaku_store_postgres_ubuntu.sh:/opt/run_nwaku_store_postgres.sh:Z + - /home/ivansete/workspace/status/nwaku/build/wakunode2:/usr/bin/wakunode:Z + - ./run_nwaku_store_postgres_ubuntu.sh:/opt/run_nwaku_store_postgres_ubuntu.sh:Z entrypoint: bash command: - - /opt/run_nwaku_store_postgres.sh + - /opt/run_nwaku_store_postgres_ubuntu.sh depends_on: - postgres + networks: + - simulation - # msg_publisher: - # ## Sends json-rpc 'post_waku_v2_relay_v1_message' messages infinitely - # image: alpine:3.16 - # restart: on-failure - # logging: - # driver: "none" - # volumes: - # - ./msg_publisher.sh:/opt/msg_publisher.sh:Z - # entrypoint: sh - # command: - # - /opt/msg_publisher.sh - # depends_on: - # - nwaku_postgres - # - nwaku_sqlite + waku-store-query-generator: + image: ivansete/waku-store-request-maker:91bb3ffd + restart: on-failure + entrypoint: sh + - 'opt/run_waku_store_query_maker.sh' + deploy: + replicas: 1 + volumes: + - ./run_waku_store_query_maker.sh:/opt/run_waku_store_query_maker.sh:Z + environment: + STORE_QUERIES_PER_SECOND: 10 + networks: + - simulation + + waku-publisher: + image: alrevuelta/waku-publisher:9fb206c + entrypoint: sh + - 'opt/run_wakupublisher.sh' + deploy: + replicas: 1 + volumes: + - ./run_wakupublisher.sh:/opt/run_wakupublisher.sh:Z + environment: + MSG_PER_SECOND: 10 + MSG_SIZE_KBYTES: 10 + depends_on: + - nwaku-postgres + - nwaku-sqlite + networks: + - simulation prometheus: image: docker.io/prom/prometheus:latest @@ -114,11 +133,15 @@ services: restart: on-failure depends_on: - postgres-exporter - - nwaku_postgres - - nwaku_sqlite + - nwaku-postgres + - nwaku-sqlite + networks: + - simulation grafana: image: docker.io/grafana/grafana:latest + ports: + - 0.0.0.0:3000:3000 env_file: - ./monitoring/configuration/grafana-plugins.env volumes: @@ -132,12 +155,16 @@ services: restart: on-failure depends_on: - prometheus + networks: + - simulation postgres: # This service is used when the Waku node has the 'store' protocol enabled # and the store-message-db-url is set to use Postgres image: postgres:15.4-alpine3.18 restart: on-failure + ports: + - 0.0.0.0:5432:5432 <<: *pg_env volumes: - ./data/postgres-data:/var/lib/postgresql/data @@ -150,6 +177,8 @@ services: timeout: 60s retries: 5 start_period: 80s + networks: + - simulation postgres-exporter: # Service aimed to scrape information from Postgres and post it to Prometeus @@ -164,3 +193,5 @@ services: - --config.file=/etc/pgexporter/postgres-exporter.yml depends_on: - postgres + networks: + - simulation diff --git a/docker/monitoring/prometheus-config.yml b/docker/monitoring/prometheus-config.yml index e4398cc..2ec0ca7 100644 --- a/docker/monitoring/prometheus-config.yml +++ b/docker/monitoring/prometheus-config.yml @@ -7,11 +7,11 @@ global: scrape_configs: - job_name: "nwaku_postgres" static_configs: - - targets: ['nwaku_postgres:8003'] + - targets: ['nwaku-postgres:8003'] - job_name: "nwaku_sqlite" static_configs: - - targets: ['nwaku_sqlite:8004'] + - targets: ['nwaku-sqlite:8004'] - job_name: postgres-exporter static_configs: diff --git a/docker/run_bootstrap.sh b/docker/run_bootstrap.sh new file mode 100755 index 0000000..2531c44 --- /dev/null +++ b/docker/run_bootstrap.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') + +echo "I am a bootstrap node. Listening to: ${IP}" + +exec /usr/bin/wakunode\ + --relay=true\ + --rpc-admin=true\ + --keep-alive=true\ + --max-connections=300\ + --dns-discovery=true\ + --discv5-discovery=true\ + --discv5-enr-auto-update=True\ + --log-level=INFO\ + --rpc-port=8544\ + --rpc-address=0.0.0.0\ + --metrics-server=True\ + --metrics-server-address=0.0.0.0\ + --nodekey=30348dd51465150e04a5d9d932c72864c8967f806cce60b5d26afeca1e77eb68\ + --nat=extip:${IP} \ No newline at end of file diff --git a/docker/run_nwaku_store_postgres_ubuntu.sh b/docker/run_nwaku_store_postgres_ubuntu.sh index c3ddc18..7035941 100644 --- a/docker/run_nwaku_store_postgres_ubuntu.sh +++ b/docker/run_nwaku_store_postgres_ubuntu.sh @@ -3,16 +3,32 @@ apt-get update ## Install the `dig` command apt-get install dnsutils -y +apt-get install wget -y -peer_IP=$(dig +short nwaku_sqlite) +bootstrap_IP=$(dig +short bootstrap) apt-get install libpq5 -y - chmod +x /usr/bin/wakunode +RETRIES=${RETRIES:=10} + +while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do + BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://${bootstrap_IP}:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); + echo "Bootstrap node not ready in ${bootstrap_IP}, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${BOOTSTRAP_ENR}" ]; then + echo "Could not get BOOTSTRAP_ENR and none provided. Failing" + exit 1 +fi + +IP=$(hostname -I) + +echo "I am postgres ubuntu. Listening on: ${IP}" + ./usr/bin/wakunode\ - --nodekey=1d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\ - --staticnode=/ip4/${peer_IP}/tcp/30304/p2p/16Uiu2HAkxj3WzLiqBximSaHc8wV9Co87GyRGRYLVGsHZrzi3TL5W\ --relay=true\ --topic=/waku/2/default-waku/proto\ --topic=/waku/2/dev-waku/proto\ @@ -21,10 +37,15 @@ chmod +x /usr/bin/wakunode --log-level=DEBUG\ --rpc-port=8545\ --rpc-address=0.0.0.0\ - --tcp-port=30303\ --metrics-server=True\ --metrics-server-port=8003\ --metrics-server-address=0.0.0.0\ + --max-connections=250\ + --dns-discovery=true\ + --discv5-discovery=true\ + --discv5-enr-auto-update=True\ + --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ + --nat=extip:${IP}\ --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres"\ --store=true\ --store-message-retention-policy=capacity:4000000 diff --git a/docker/run_nwaku_store_sqlite_ubuntu.sh b/docker/run_nwaku_store_sqlite_ubuntu.sh index aab7699..30651ac 100644 --- a/docker/run_nwaku_store_sqlite_ubuntu.sh +++ b/docker/run_nwaku_store_sqlite_ubuntu.sh @@ -3,16 +3,32 @@ apt-get update ## Install the `dig` command apt-get install dnsutils -y +apt-get install wget -y -peer_IP=$(dig +short nwaku_postgres) +bootstrap_IP=$(dig +short bootstrap) apt-get install libpq5 -y - chmod +x /usr/bin/wakunode +RETRIES=${RETRIES:=10} + +while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do + BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://${bootstrap_IP}:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); + echo "Bootstrap node not ready in ${bootstrap_IP}, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${BOOTSTRAP_ENR}" ]; then + echo "Could not get BOOTSTRAP_ENR and none provided. Failing" + exit 1 +fi + +IP=$(hostname -I) + +echo "I am sqlite ubuntu. Listening on: ${IP}" + ./usr/bin/wakunode\ - --nodekey=2d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\ - --staticnode=/ip4/${peer_IP}/tcp/30303/p2p/16Uiu2HAmJyLCRhiErTRFcW5GKPrpoMjGbbWdFMx4GCUnnhmxeYhd\ --relay=true\ --topic=/waku/2/default-waku/proto\ --topic=/waku/2/dev-waku/proto\ @@ -21,10 +37,15 @@ chmod +x /usr/bin/wakunode --log-level=DEBUG\ --rpc-port=8546\ --rpc-address=0.0.0.0\ - --tcp-port=30304\ --metrics-server=True\ --metrics-server-port=8004\ --metrics-server-address=0.0.0.0\ + --max-connections=250\ + --dns-discovery=true\ + --discv5-discovery=true\ + --discv5-enr-auto-update=True\ + --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ + --nat=extip:${IP}\ --store-message-db-url="sqlite:///data/store.sqlite3"\ --store=true\ --store-message-retention-policy=capacity:4000000 diff --git a/docker/run_waku_store_query_maker.sh b/docker/run_waku_store_query_maker.sh new file mode 100755 index 0000000..dc468dc --- /dev/null +++ b/docker/run_waku_store_query_maker.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') + +echo "I am a waku store query generator" + +## Getting the address of the Postgres node +RETRIES=10 +while [ -z "${POSTGRES_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do + POSTGRES_ADDR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-postgres:8545/ 2> /dev/null | sed 's/.*"listenAddresses":\["\(.*\)"].*/\1/'); + wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-postgres:8545/ + echo "Store Postgres node node not ready, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${POSTGRES_ADDR}" ]; then + echo "Could not get POSTGRES_ADDR and none provided. Failing" + exit 1 +fi + +## Getting the address of the SQLite node +RETRIES=10 +while [ -z "${SQLITE_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do + SQLITE_ADDR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://nwaku-sqlite:8546/ 2> /dev/null | sed 's/.*"listenAddresses":\["\(.*\)"].*/\1/'); + echo "Store SQLite node node not ready, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${SQLITE_ADDR}" ]; then + echo "Could not get SQLITE_ADDR and none provided. Failing" + exit 1 +fi + +## Getting the bootstrap node ENR +RETRIES=10 +while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do + BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://bootstrap:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); + echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${BOOTSTRAP_ENR}" ]; then + echo "Could not get BOOTSTRAP_ENR and none provided. Failing" + exit 1 +fi + +echo "Using bootstrap node: ${BOOTSTRAP_ENR}" +exec /main\ + --pubsub-topic="/waku/2/default-waku/proto"\ + --content-topic="my-ctopic"\ + --queries-per-second=${STORE_QUERIES_PER_SECOND}\ + --bootstrap-node=${BOOTSTRAP_ENR}\ + --peer-store-postgres-addr="${POSTGRES_ADDR}"\ + --peer-store-sqlite-addr="${SQLITE_ADDR}"\ + --num-minutes-query=60 diff --git a/docker/run_wakupublisher.sh b/docker/run_wakupublisher.sh new file mode 100755 index 0000000..dd75753 --- /dev/null +++ b/docker/run_wakupublisher.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') + +echo "I am a traffic generator" + +RETRIES=${RETRIES:=10} + +while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do + BOOTSTRAP_ENR=$(wget -O - --post-data='{"jsonrpc":"2.0","method":"get_waku_v2_debug_v1_info","params":[],"id":1}' --header='Content-Type:application/json' http://bootstrap:8544/ 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); + echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${BOOTSTRAP_ENR}" ]; then + echo "Could not get BOOTSTRAP_ENR and none provided. Failing" + exit 1 +fi + +echo "Using bootstrap node: ${BOOTSTRAP_ENR}" +exec /main\ + --pubsub-topic="/waku/2/default-waku/proto"\ + --content-topic="my-ctopic"\ + --msg-per-second=${MSG_PER_SECOND}\ + --msg-size-kb=${MSG_SIZE_KBYTES}\ + --bootstrap-node=${BOOTSTRAP_ENR}\ + --max-peers=50