From fd8475f4ca8e5e6864e19aca1370a1cc5af45d3a Mon Sep 17 00:00:00 2001 From: stubbsta Date: Tue, 1 Oct 2024 10:19:35 +0200 Subject: [PATCH] store sync config WIP --- docker-compose.yml | 344 ++++++++++-------- monitoring/prometheus-config.yml | 3 +- run_bootstrap.sh | 8 +- run_nwaku_noRLN.sh | 6 +- ...u_store.sh => run_nwaku_store_postgres1.sh | 18 +- run_nwaku_store_postgres2.sh | 44 +++ 6 files changed, 252 insertions(+), 171 deletions(-) rename run_nwaku_store.sh => run_nwaku_store_postgres1.sh (80%) create mode 100644 run_nwaku_store_postgres2.sh diff --git a/docker-compose.yml b/docker-compose.yml index efcf015..7746526 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,45 +12,45 @@ services: # Accounts are hardcoded to 520 with the idea that nwaku nodes use up to 500 for membership registration and the last 20 are used for ad-hoc testing. # The account number and private key pairs of the last 20 accounts can be found in the Register memberships section of the Waku-simulator book. - foundry: - image: ghcr.io/foundry-rs/foundry:nightly-9b73e06e1fe376738b92ae081107620291d50188 - labels: - com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - ports: - - 0.0.0.0:8545:8545 - command: - - anvil - --port=8545 - --host=0.0.0.0 - --accounts=520 - --allow-origin=* - --block-time=3 - --chain-id=1234 - --silent - --config-out=/shared/anvil-config.txt - volumes: - - privatekeys-volume:/shared - networks: - - simulation + # foundry: + # image: ghcr.io/foundry-rs/foundry:nightly-9b73e06e1fe376738b92ae081107620291d50188 + # labels: + # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + # ports: + # - 0.0.0.0:8545:8545 + # command: + # - anvil + # --port=8545 + # --host=0.0.0.0 + # --accounts=520 + # --allow-origin=* + # --block-time=3 + # --chain-id=1234 + # --silent + # --config-out=/shared/anvil-config.txt + # volumes: + # - privatekeys-volume:/shared + # networks: + # - simulation - contract-repo-deployer: - image: node:hydrogen-bullseye - labels: - com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - environment: - - PRIVATE_KEY=${PRIVATE_KEY} - - RPC_URL=${RPC_URL:-http://foundry:8545} - - ETH_FROM=${ETH_FROM} - - MAX_MESSAGE_LIMIT=${MAX_MESSAGE_LIMIT:-20} - entrypoint: sh - command: - - '/opt/deploy_rln_contract.sh' - volumes: - - ./deploy_rln_contract.sh:/opt/deploy_rln_contract.sh - depends_on: - - foundry - networks: - - simulation + # contract-repo-deployer: + # image: node:hydrogen-bullseye + # labels: + # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + # environment: + # - PRIVATE_KEY=${PRIVATE_KEY} + # - RPC_URL=${RPC_URL:-http://foundry:8545} + # - ETH_FROM=${ETH_FROM} + # - MAX_MESSAGE_LIMIT=${MAX_MESSAGE_LIMIT:-20} + # entrypoint: sh + # command: + # - '/opt/deploy_rln_contract.sh' + # volumes: + # - ./deploy_rln_contract.sh:/opt/deploy_rln_contract.sh + # depends_on: + # - foundry + # networks: + # - simulation bootstrap: image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} @@ -79,57 +79,67 @@ services: deploy: replicas: ${NUM_NWAKU_NODES:-5} entrypoint: sh - environment: - - RPC_URL=${RPC_URL:-http://foundry:8545} - - RLN_CONTRACT_ADDRESS=0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9 - - RLN_CREDENTIAL_PATH=/keystore.json - - RLN_CREDENTIAL_PASSWORD=passw123 - - RLN_RELAY_MSG_LIMIT=${RLN_RELAY_MSG_LIMIT:-10} - - RLN_RELAY_EPOCH_SEC=${RLN_RELAY_EPOCH_SEC:-60} command: - - '/opt/run_nwaku.sh' + - '/opt/run_nwaku_noRLN.sh' volumes: - - ./run_nwaku.sh:/opt/run_nwaku.sh:Z - - privatekeys-volume:/shared + - ./run_nwaku_noRLN.sh:/opt/run_nwaku_noRLN.sh:Z depends_on: - contract-repo-deployer: - condition: service_completed_successfully + - bootstrap networks: - simulation - # nwaku_store: - # image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} - # restart: on-failure - # labels: - # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - # ports: - # - 127.0.0.1:60001:60001 - # entrypoint: sh - # environment: - # - POSTGRES_USER=${POSTGRES_USER:-postgres} - # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} - # command: - # - '/opt/run_nwaku_store.sh' - # volumes: - # - ./run_nwaku_store.sh:/opt/run_nwaku_store.sh:Z - # depends_on: - # - postgres - # networks: - # simulation: - # ipv4_address: 10.2.0.99 - - rest-traffic: - image: alrevuelta/rest-traffic:6992bb5 + nwaku_store_pg1: + image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} + restart: on-failure + labels: + com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + entrypoint: sh + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} command: - --multiple-nodes=http://waku-simulator_nwaku_[1..${NUM_NWAKU_NODES:-5}]:8645 - --msg-size-kbytes=${MSG_SIZE_KBYTES:-10} - --delay-seconds=${TRAFFIC_DELAY_SECONDS:-15} + - '/opt/run_nwaku_store_postgres1.sh' + volumes: + - ./run_nwaku_store_postgres1.sh:/opt/run_nwaku_store_postgres1.sh:Z + depends_on: + postgres1: + condition: service_healthy networks: - simulation - depends_on: - - nwaku - postgres: + nwaku_store_pg2: + image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} + restart: on-failure + labels: + com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + entrypoint: sh + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + command: + - '/opt/run_nwaku_store_postgres2.sh' + volumes: + - ./run_nwaku_store_postgres2.sh:/opt/run_nwaku_store_postgres2.sh:Z + depends_on: + postgres2: + condition: service_healthy + networks: + - simulation + + # rest-traffic: + # image: alrevuelta/rest-traffic:6992bb5 + # command: +# --content-topic= +# --pubsub-topic="/waku/2/rs/16/32" + # --multiple-nodes=http://waku-simulator_nwaku_[1..${NUM_NWAKU_NODES:-5}]:8645 + # --msg-size-kbytes=${MSG_SIZE_KBYTES:-10} + # --delay-seconds=${TRAFFIC_DELAY_SECONDS:-15} + # networks: + # - simulation + # depends_on: + # - nwaku + + postgres1: # This service is used when the Waku node has the 'store' protocol enabled # and the store-message-db-url is set to use Postgres image: postgres:15.4-alpine3.18 @@ -140,7 +150,7 @@ services: volumes: - ./postgres_cfg/postgresql.conf:/etc/postgresql/postgresql.conf:Z - ./postgres_cfg/db.sql:/docker-entrypoint-initdb.d/db.sql:Z - - ${PG_DATA_DIR:-./postgresql}:/var/lib/postgresql/data:Z + - ${PG_DATA_DIR:-./postgresql1}:/var/lib/postgresql/data:Z command: postgres -c config_file=/etc/postgresql/postgresql.conf ports: - 127.0.0.1:5432:5432 @@ -153,26 +163,50 @@ services: retries: 5 start_period: 80s - postgres-exporter: - # Service aimed to scrape information from Postgres and post it to Prometeus - image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 + postgres2: + # This service is used when the Waku node has the 'store' protocol enabled + # and the store-message-db-url is set to use Postgres + image: postgres:15.4-alpine3.18 restart: on-failure:5 environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} - - DATA_SOURCE_URI=postgres?sslmode=disable - - DATA_SOURCE_USER=${POSTGRES_USER:-postgres} - - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123} - - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml volumes: - - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z - - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z - command: - # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info - - --config.file=/etc/pgexporter/postgres-exporter.yml - depends_on: - - postgres + - ./postgres_cfg/postgresql.conf:/etc/postgresql/postgresql.conf:Z + - ./postgres_cfg/db.sql:/docker-entrypoint-initdb.d/db.sql:Z + - ${PG_DATA_DIR:-./postgresql2}:/var/lib/postgresql/data:Z + command: postgres -c config_file=/etc/postgresql/postgresql.conf + ports: + - 127.0.0.1:5433:5432 networks: - simulation + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 30s + timeout: 60s + retries: 5 + start_period: 80s + + # postgres-exporter: + # # Service aimed to scrape information from Postgres and post it to Prometeus + # image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 + # restart: on-failure:5 + # environment: + # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + # - DATA_SOURCE_URI=postgres?sslmode=disable + # - DATA_SOURCE_USER=${POSTGRES_USER:-postgres} + # - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123} + # - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml + # volumes: + # - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z + # - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z + # command: + # # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info + # - --config.file=/etc/pgexporter/postgres-exporter.yml + # depends_on: + # - postgres + # networks: + # - simulation prometheus: image: prom/prometheus:latest @@ -233,71 +267,71 @@ services: networks: - simulation - api: - image: web3labs/epirus-free-api:latest - ports: - - 127.0.0.1:8090:8090 - environment: - - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} - - MONGO_CLIENT_URI=mongodb://mongodb:27017 - - REINDEX_ENDPOINT=http://ingestion/reindex/ - - MONGO_DB_NAME=epirus - - MONGO_CREATE_INDICES=true - - REDIS_HOST=redis - - REDIS_PORT=6379 - depends_on: - - redis - - mongodb - - foundry - networks: - - simulation + # api: + # image: web3labs/epirus-free-api:latest + # ports: + # - 127.0.0.1:8090:8090 + # environment: + # - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} + # - MONGO_CLIENT_URI=mongodb://mongodb:27017 + # - REINDEX_ENDPOINT=http://ingestion/reindex/ + # - MONGO_DB_NAME=epirus + # - MONGO_CREATE_INDICES=true + # - REDIS_HOST=redis + # - REDIS_PORT=6379 + # depends_on: + # - redis + # - mongodb + # - foundry + # networks: + # - simulation - mongodb: - image: mongo:5.0.8 - environment: - - COMPOSE_HTTP_TIMEOUT=900 - - DOCKER_CLIENT_TIMEOUT=900 - entrypoint: mongod --bind_ip "0.0.0.0" - networks: - - simulation + # mongodb: + # image: mongo:5.0.8 + # environment: + # - COMPOSE_HTTP_TIMEOUT=900 + # - DOCKER_CLIENT_TIMEOUT=900 + # entrypoint: mongod --bind_ip "0.0.0.0" + # networks: + # - simulation - web: - image: web3labs/epirus-free-web:latest - environment: - - API_URL=${EPIRUS_WEB_API_URL:-/api} - - WS_API_URL=${EPIRUS_WEB_WS_API_URL:-ws://localhost:8090} - - DISPLAY_NETWORK_TAB=disabled - depends_on: - - api - networks: - - simulation + # web: + # image: web3labs/epirus-free-web:latest + # environment: + # - API_URL=${EPIRUS_WEB_API_URL:-/api} + # - WS_API_URL=${EPIRUS_WEB_WS_API_URL:-ws://localhost:8090} + # - DISPLAY_NETWORK_TAB=disabled + # depends_on: + # - api + # networks: + # - simulation - ingestion: - image: web3labs/epirus-free-ingestion:latest - environment: - - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} - - MONGO_CLIENT_URI=mongodb://mongodb:27017 - - MONGO_DB_NAME=epirus - - LIST_OF_METRICS_TO_CALCULATE_PER_MINUTE=hourly,daily,monthly,yearly - depends_on: - - mongodb - - redis - - foundry - networks: - - simulation + # ingestion: + # image: web3labs/epirus-free-ingestion:latest + # environment: + # - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} + # - MONGO_CLIENT_URI=mongodb://mongodb:27017 + # - MONGO_DB_NAME=epirus + # - LIST_OF_METRICS_TO_CALCULATE_PER_MINUTE=hourly,daily,monthly,yearly + # depends_on: + # - mongodb + # - redis + # - foundry + # networks: + # - simulation - nginx: - image: nginx:latest - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf - - ./5xx.html:/www/error_pages/5xx.html - ports: - - 0.0.0.0:3000:80 - depends_on: - - api - - web - networks: - - simulation + # nginx: + # image: nginx:latest + # volumes: + # - ./nginx.conf:/etc/nginx/nginx.conf + # - ./5xx.html:/www/error_pages/5xx.html + # ports: + # - 0.0.0.0:3000:80 + # depends_on: + # - api + # - web + # networks: + # - simulation volumes: privatekeys-volume: \ No newline at end of file diff --git a/monitoring/prometheus-config.yml b/monitoring/prometheus-config.yml index 7baacb2..dca9592 100644 --- a/monitoring/prometheus-config.yml +++ b/monitoring/prometheus-config.yml @@ -17,7 +17,8 @@ scrape_configs: static_configs: - targets: - bootstrap:8008 - - waku-simulator_nwaku_store_1:8008 + - waku-simulator_nwaku_store_pg1_1:8008 + - waku-simulator_nwaku_store_pg2_1:8008 - waku-simulator_nwaku_1:8008 - waku-simulator_nwaku_2:8008 - waku-simulator_nwaku_3:8008 diff --git a/run_bootstrap.sh b/run_bootstrap.sh index cab3d90..bb181a8 100755 --- a/run_bootstrap.sh +++ b/run_bootstrap.sh @@ -17,5 +17,9 @@ exec /usr/bin/wakunode\ --metrics-server-address=0.0.0.0\ --nodekey=30348dd51465150e04a5d9d932c72864c8967f806cce60b5d26afeca1e77eb68\ --nat=extip:${IP}\ - --pubsub-topic=/waku/2/rs/66/0\ - --cluster-id=66 \ No newline at end of file + --pubsub-topic=/waku/2/rs/16/1\ + --pubsub-topic=/waku/2/rs/16/32\ + --pubsub-topic=/waku/2/rs/16/64\ + --pubsub-topic=/waku/2/rs/16/128\ + --pubsub-topic=/waku/2/rs/16/256\ + --cluster-id=16 \ No newline at end of file diff --git a/run_nwaku_noRLN.sh b/run_nwaku_noRLN.sh index 424d08d..04b80fd 100644 --- a/run_nwaku_noRLN.sh +++ b/run_nwaku_noRLN.sh @@ -26,7 +26,6 @@ fi echo "Using bootstrap node: ${BOOTSTRAP_ENR}" exec /usr/bin/wakunode\ --relay=true\ - --lightpush=true\ --max-connections=250\ --rest=true\ --rest-address=0.0.0.0\ @@ -39,6 +38,5 @@ exec /usr/bin/wakunode\ --metrics-server-address=0.0.0.0\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ --nat=extip:${IP}\ - --pubsub-topic=/waku/2/rs/66/0\ - --cluster-id=66\ - --storenode=/ip4/10.2.0.99/tcp/60000/p2p/16Uiu2HAmTVafvweaXrXKmFFkUo4qWYP7wTa2H6PXee8iMyQw4eHm \ No newline at end of file + --pubsub-topic=/waku/2/rs/16/32\ + --cluster-id=16 \ No newline at end of file diff --git a/run_nwaku_store.sh b/run_nwaku_store_postgres1.sh similarity index 80% rename from run_nwaku_store.sh rename to run_nwaku_store_postgres1.sh index e779c09..cf5b4ff 100644 --- a/run_nwaku_store.sh +++ b/run_nwaku_store_postgres1.sh @@ -21,10 +21,8 @@ fi echo "Using bootstrap node: ${BOOTSTRAP_ENR}" exec /usr/bin/wakunode\ --relay=true\ - --max-connections=20\ + --max-connections=50\ --rest=true\ - --rest-admin=true\ - --rest-private=true\ --rest-address=0.0.0.0\ --rest-port=8645\ --dns-discovery=true\ @@ -34,11 +32,13 @@ exec /usr/bin/wakunode\ --metrics-server=True\ --metrics-server-address=0.0.0.0\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ - --nat=extip:${IP}\ - --pubsub-topic=/waku/2/rs/66/0\ - --cluster-id=66\ + --pubsub-topic=/waku/2/rs/16/32\ + --cluster-id=16\ --ports-shift=1\ - --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942df25\ --store=true\ - --store-message-retention-policy=time:90000\ - --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres" \ No newline at end of file + --store-message-retention-policy=size:1GB\ + --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres1:5432/postgres"\ + --store-sync=true\ + --store-sync-interval=120\ + --store-sync-range=3600\ + --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa1 \ No newline at end of file diff --git a/run_nwaku_store_postgres2.sh b/run_nwaku_store_postgres2.sh new file mode 100644 index 0000000..c0566f2 --- /dev/null +++ b/run_nwaku_store_postgres2.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') + +echo "I am a nwaku store node" + +RETRIES=${RETRIES:=10} + +while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do + BOOTSTRAP_ENR=$(wget -qO- http://bootstrap:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); + echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${BOOTSTRAP_ENR}" ]; then + echo "Could not get BOOTSTRAP_ENR and none provided. Failing" + exit 1 +fi + +echo "Using bootstrap node: ${BOOTSTRAP_ENR}" +exec /usr/bin/wakunode\ + --relay=true\ + --max-connections=50\ + --rest=true\ + --rest-address=0.0.0.0\ + --rest-port=8645\ + --dns-discovery=true\ + --discv5-discovery=true\ + --discv5-enr-auto-update=True\ + --log-level=DEBUG\ + --metrics-server=True\ + --metrics-server-address=0.0.0.0\ + --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ + --pubsub-topic=/waku/2/rs/16/32\ + --cluster-id=16\ + --ports-shift=1\ + --store=true\ + --store-message-retention-policy=size:1GB\ + --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres2:5432/postgres"\ + --store-sync=true\ + --store-sync-interval=120\ + --store-sync-range=3600\ + --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa3 \ No newline at end of file