From 5a1e3974ae15cabf4e2fdd0e98e848ea80e6cf80 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Fri, 13 Sep 2024 12:17:50 +0200 Subject: [PATCH] connect external store node --- docker-compose.yml | 354 ++++++++++++++++++++++--------------------- run_bootstrap.sh | 2 - run_nwaku.sh | 7 +- run_nwaku_noRLN.sh | 2 - run_nwaku_static1.sh | 113 ++++++++++++++ 5 files changed, 293 insertions(+), 185 deletions(-) create mode 100644 run_nwaku_static1.sh diff --git a/docker-compose.yml b/docker-compose.yml index f2a2b45..efcf015 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,72 +12,72 @@ services: # Accounts are hardcoded to 520 with the idea that nwaku nodes use up to 500 for membership registration and the last 20 are used for ad-hoc testing. # The account number and private key pairs of the last 20 accounts can be found in the Register memberships section of the Waku-simulator book. - # foundry: - # image: ghcr.io/foundry-rs/foundry:nightly-9b73e06e1fe376738b92ae081107620291d50188 - # labels: - # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - # ports: - # - 0.0.0.0:8545:8545 - # command: - # - anvil - # --port=8545 - # --host=0.0.0.0 - # --accounts=520 - # --allow-origin=* - # --block-time=3 - # --chain-id=1234 - # --silent - # --config-out=/shared/anvil-config.txt - # volumes: - # - privatekeys-volume:/shared - # networks: - # - simulation + foundry: + image: ghcr.io/foundry-rs/foundry:nightly-9b73e06e1fe376738b92ae081107620291d50188 + labels: + com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + ports: + - 0.0.0.0:8545:8545 + command: + - anvil + --port=8545 + --host=0.0.0.0 + --accounts=520 + --allow-origin=* + --block-time=3 + --chain-id=1234 + --silent + --config-out=/shared/anvil-config.txt + volumes: + - privatekeys-volume:/shared + networks: + - simulation - # contract-repo-deployer: - # image: node:hydrogen-bullseye - # labels: - # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - # environment: - # - PRIVATE_KEY=${PRIVATE_KEY} - # - RPC_URL=${RPC_URL:-http://foundry:8545} - # - ETH_FROM=${ETH_FROM} - # - MAX_MESSAGE_LIMIT=${MAX_MESSAGE_LIMIT:-20} - # entrypoint: sh - # command: - # - '/opt/deploy_rln_contract.sh' - # volumes: - # - ./deploy_rln_contract.sh:/opt/deploy_rln_contract.sh - # depends_on: - # - foundry - # networks: - # - simulation + contract-repo-deployer: + image: node:hydrogen-bullseye + labels: + com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + environment: + - PRIVATE_KEY=${PRIVATE_KEY} + - RPC_URL=${RPC_URL:-http://foundry:8545} + - ETH_FROM=${ETH_FROM} + - MAX_MESSAGE_LIMIT=${MAX_MESSAGE_LIMIT:-20} + entrypoint: sh + command: + - '/opt/deploy_rln_contract.sh' + volumes: + - ./deploy_rln_contract.sh:/opt/deploy_rln_contract.sh + depends_on: + - foundry + networks: + - simulation - # bootstrap: - # image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} - # restart: on-failure - # labels: - # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - # # TODO: expose some ports to inject traffic - # ports: - # - 127.0.0.1:60000:60000 - # - 127.0.0.1:8008:8008 - # - 127.0.0.1:9000:9000 - # - 127.0.0.1:8645:8645 - # entrypoint: sh - # command: - # - '/opt/run_bootstrap.sh' - # volumes: - # - ./run_bootstrap.sh:/opt/run_bootstrap.sh:Z - # networks: - # - simulation + bootstrap: + image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} + restart: on-failure + labels: + com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + # TODO: expose some ports to inject traffic + ports: + - 127.0.0.1:60000:60000 + - 127.0.0.1:8008:8008 + - 127.0.0.1:9000:9000 + - 127.0.0.1:8645:8645 + entrypoint: sh + command: + - '/opt/run_bootstrap.sh' + volumes: + - ./run_bootstrap.sh:/opt/run_bootstrap.sh:Z + networks: + - simulation nwaku: image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} restart: on-failure labels: com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - # deploy: - # replicas: ${NUM_NWAKU_NODES:-5} + deploy: + replicas: ${NUM_NWAKU_NODES:-5} entrypoint: sh environment: - RPC_URL=${RPC_URL:-http://foundry:8545} @@ -87,13 +87,15 @@ services: - RLN_RELAY_MSG_LIMIT=${RLN_RELAY_MSG_LIMIT:-10} - RLN_RELAY_EPOCH_SEC=${RLN_RELAY_EPOCH_SEC:-60} command: - - '/opt/run_nwaku_noRLN.sh' + - '/opt/run_nwaku.sh' volumes: - - ./run_nwaku_noRLN.sh:/opt/run_nwaku_noRLN.sh:Z + - ./run_nwaku.sh:/opt/run_nwaku.sh:Z - privatekeys-volume:/shared + depends_on: + contract-repo-deployer: + condition: service_completed_successfully networks: - simulation: - ipv4_address: 10.2.0.50 + - simulation # nwaku_store: # image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} @@ -116,61 +118,61 @@ services: # simulation: # ipv4_address: 10.2.0.99 - # rest-traffic: - # image: alrevuelta/rest-traffic:6992bb5 - # command: - # --multiple-nodes=http://waku-simulator_nwaku_[1..${NUM_NWAKU_NODES:-5}]:8645 - # --msg-size-kbytes=${MSG_SIZE_KBYTES:-10} - # --delay-seconds=${TRAFFIC_DELAY_SECONDS:-15} - # networks: - # - simulation - # depends_on: - # - nwaku + rest-traffic: + image: alrevuelta/rest-traffic:6992bb5 + command: + --multiple-nodes=http://waku-simulator_nwaku_[1..${NUM_NWAKU_NODES:-5}]:8645 + --msg-size-kbytes=${MSG_SIZE_KBYTES:-10} + --delay-seconds=${TRAFFIC_DELAY_SECONDS:-15} + networks: + - simulation + depends_on: + - nwaku - # postgres: - # # This service is used when the Waku node has the 'store' protocol enabled - # # and the store-message-db-url is set to use Postgres - # image: postgres:15.4-alpine3.18 - # restart: on-failure:5 - # environment: - # - POSTGRES_USER=${POSTGRES_USER:-postgres} - # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} - # volumes: - # - ./postgres_cfg/postgresql.conf:/etc/postgresql/postgresql.conf:Z - # - ./postgres_cfg/db.sql:/docker-entrypoint-initdb.d/db.sql:Z - # - ${PG_DATA_DIR:-./postgresql}:/var/lib/postgresql/data:Z - # command: postgres -c config_file=/etc/postgresql/postgresql.conf - # ports: - # - 127.0.0.1:5432:5432 - # networks: - # - simulation - # healthcheck: - # test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] - # interval: 30s - # timeout: 60s - # retries: 5 - # start_period: 80s + postgres: + # This service is used when the Waku node has the 'store' protocol enabled + # and the store-message-db-url is set to use Postgres + image: postgres:15.4-alpine3.18 + restart: on-failure:5 + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + volumes: + - ./postgres_cfg/postgresql.conf:/etc/postgresql/postgresql.conf:Z + - ./postgres_cfg/db.sql:/docker-entrypoint-initdb.d/db.sql:Z + - ${PG_DATA_DIR:-./postgresql}:/var/lib/postgresql/data:Z + command: postgres -c config_file=/etc/postgresql/postgresql.conf + ports: + - 127.0.0.1:5432:5432 + networks: + - simulation + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 30s + timeout: 60s + retries: 5 + start_period: 80s - # postgres-exporter: - # # Service aimed to scrape information from Postgres and post it to Prometeus - # image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 - # restart: on-failure:5 - # environment: - # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} - # - DATA_SOURCE_URI=postgres?sslmode=disable - # - DATA_SOURCE_USER=${POSTGRES_USER:-postgres} - # - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123} - # - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml - # volumes: - # - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z - # - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z - # command: - # # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info - # - --config.file=/etc/pgexporter/postgres-exporter.yml - # depends_on: - # - postgres - # networks: - # - simulation + postgres-exporter: + # Service aimed to scrape information from Postgres and post it to Prometeus + image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 + restart: on-failure:5 + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + - DATA_SOURCE_URI=postgres?sslmode=disable + - DATA_SOURCE_USER=${POSTGRES_USER:-postgres} + - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123} + - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml + volumes: + - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z + - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z + command: + # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info + - --config.file=/etc/pgexporter/postgres-exporter.yml + depends_on: + - postgres + networks: + - simulation prometheus: image: prom/prometheus:latest @@ -231,71 +233,71 @@ services: networks: - simulation - # api: - # image: web3labs/epirus-free-api:latest - # ports: - # - 127.0.0.1:8090:8090 - # environment: - # - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} - # - MONGO_CLIENT_URI=mongodb://mongodb:27017 - # - REINDEX_ENDPOINT=http://ingestion/reindex/ - # - MONGO_DB_NAME=epirus - # - MONGO_CREATE_INDICES=true - # - REDIS_HOST=redis - # - REDIS_PORT=6379 - # depends_on: - # - redis - # - mongodb - # - foundry - # networks: - # - simulation + api: + image: web3labs/epirus-free-api:latest + ports: + - 127.0.0.1:8090:8090 + environment: + - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} + - MONGO_CLIENT_URI=mongodb://mongodb:27017 + - REINDEX_ENDPOINT=http://ingestion/reindex/ + - MONGO_DB_NAME=epirus + - MONGO_CREATE_INDICES=true + - REDIS_HOST=redis + - REDIS_PORT=6379 + depends_on: + - redis + - mongodb + - foundry + networks: + - simulation - # mongodb: - # image: mongo:5.0.8 - # environment: - # - COMPOSE_HTTP_TIMEOUT=900 - # - DOCKER_CLIENT_TIMEOUT=900 - # entrypoint: mongod --bind_ip "0.0.0.0" - # networks: - # - simulation + mongodb: + image: mongo:5.0.8 + environment: + - COMPOSE_HTTP_TIMEOUT=900 + - DOCKER_CLIENT_TIMEOUT=900 + entrypoint: mongod --bind_ip "0.0.0.0" + networks: + - simulation - # web: - # image: web3labs/epirus-free-web:latest - # environment: - # - API_URL=${EPIRUS_WEB_API_URL:-/api} - # - WS_API_URL=${EPIRUS_WEB_WS_API_URL:-ws://localhost:8090} - # - DISPLAY_NETWORK_TAB=disabled - # depends_on: - # - api - # networks: - # - simulation + web: + image: web3labs/epirus-free-web:latest + environment: + - API_URL=${EPIRUS_WEB_API_URL:-/api} + - WS_API_URL=${EPIRUS_WEB_WS_API_URL:-ws://localhost:8090} + - DISPLAY_NETWORK_TAB=disabled + depends_on: + - api + networks: + - simulation - # ingestion: - # image: web3labs/epirus-free-ingestion:latest - # environment: - # - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} - # - MONGO_CLIENT_URI=mongodb://mongodb:27017 - # - MONGO_DB_NAME=epirus - # - LIST_OF_METRICS_TO_CALCULATE_PER_MINUTE=hourly,daily,monthly,yearly - # depends_on: - # - mongodb - # - redis - # - foundry - # networks: - # - simulation + ingestion: + image: web3labs/epirus-free-ingestion:latest + environment: + - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545} + - MONGO_CLIENT_URI=mongodb://mongodb:27017 + - MONGO_DB_NAME=epirus + - LIST_OF_METRICS_TO_CALCULATE_PER_MINUTE=hourly,daily,monthly,yearly + depends_on: + - mongodb + - redis + - foundry + networks: + - simulation - # nginx: - # image: nginx:latest - # volumes: - # - ./nginx.conf:/etc/nginx/nginx.conf - # - ./5xx.html:/www/error_pages/5xx.html - # ports: - # - 0.0.0.0:3000:80 - # depends_on: - # - api - # - web - # networks: - # - simulation + nginx: + image: nginx:latest + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + - ./5xx.html:/www/error_pages/5xx.html + ports: + - 0.0.0.0:3000:80 + depends_on: + - api + - web + networks: + - simulation volumes: privatekeys-volume: \ No newline at end of file diff --git a/run_bootstrap.sh b/run_bootstrap.sh index aeb7fa0..cab3d90 100755 --- a/run_bootstrap.sh +++ b/run_bootstrap.sh @@ -7,8 +7,6 @@ echo "I am a bootstrap node" exec /usr/bin/wakunode\ --relay=false\ --rest=true\ - --rest-admin=true\ - --rest-private=true\ --rest-address=0.0.0.0\ --max-connections=300\ --dns-discovery=true\ diff --git a/run_nwaku.sh b/run_nwaku.sh index dd98831..8f6fecf 100755 --- a/run_nwaku.sh +++ b/run_nwaku.sh @@ -136,8 +136,6 @@ exec /usr/bin/wakunode\ --lightpush=true\ --max-connections=250\ --rest=true\ - --rest-admin=true\ - --rest-private=true\ --rest-address=0.0.0.0\ --rest-port=8645\ --rln-relay=true\ @@ -152,11 +150,10 @@ exec /usr/bin/wakunode\ --dns-discovery=true\ --discv5-discovery=true\ --discv5-enr-auto-update=True\ - --log-level=TRACE\ + --log-level=DEBUG\ --metrics-server=True\ --metrics-server-address=0.0.0.0\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ --nat=extip:${IP}\ --pubsub-topic=/waku/2/rs/66/0\ - --cluster-id=66\ - --storenode=/ip4/10.2.0.99/tcp/60001/p2p/16Uiu2HAmTVafvweaXrXKmFFkUo4qWYP7wTa2H6PXee8iMyQw4eHm \ No newline at end of file + --cluster-id=66 \ No newline at end of file diff --git a/run_nwaku_noRLN.sh b/run_nwaku_noRLN.sh index cf6f1ed..424d08d 100644 --- a/run_nwaku_noRLN.sh +++ b/run_nwaku_noRLN.sh @@ -29,8 +29,6 @@ exec /usr/bin/wakunode\ --lightpush=true\ --max-connections=250\ --rest=true\ - --rest-admin=true\ - --rest-private=true\ --rest-address=0.0.0.0\ --rest-port=8645\ --dns-discovery=true\ diff --git a/run_nwaku_static1.sh b/run_nwaku_static1.sh new file mode 100644 index 0000000..035acc0 --- /dev/null +++ b/run_nwaku_static1.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +# Check Linux Distro Version - it can differ depending on the nwaku image used +# Install bind-tools/dnsutils package used for domain name resolution +OS=$(cat /etc/os-release) +if echo $OS | grep -q "Debian"; then + echo "The operating system is Debian." + apt update + apt install -y dnsutils +elif echo $OS | grep -q "Alpine"; then + echo "The operating system is Alpine." + apk add bind-tools +fi + +if test -f .env; then + echo "Using .env file" + . $(pwd)/.env +fi + +# Function to extract IP address from URL, resolve the IP and replace it in the original URL +get_ip_address_and_replace() { + local url=$1 + local domain_name=$(echo $RPC_URL | awk -F[/:] '{print $4}') + local ip_address=$(dig +short $domain_name) + valid_rpc_url="$(echo "$url" | sed "s/$domain_name/$ip_address/g")" + echo $valid_rpc_url +} + +# the format of the RPC URL is checked in the generateRlnKeystore command and hostnames are not valid +pattern="^(https?):\/\/((localhost)|([\w_-]+(?:(?:\.[\w_-]+)+)))(:[0-9]{1,5})?([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])*" +# Perform regex matching +if echo "$RPC_URL" | grep -q "$pattern"; then + echo "RPC URL is valid" +else + echo "RPC URL is invalid: $RPC_URL. Attempting to resolve hostname." + resolved_rpc_url="$(get_ip_address_and_replace $RPC_URL)" + if [ -z "$resolved_rpc_url" ]; then + echo -e "Failed to retrieve IP address for $RPC_URL\n" + else + echo -e "Resolved RPC URL for $RPC_URL: $resolved_rpc_url" + RPC_URL="$resolved_rpc_url" + fi +fi + +if test -f .$RLN_CREDENTIAL_PATH; then + echo "$RLN_CREDENTIAL_PATH already exists. Use it instead of creating a new one." +else + echo "Generating RLN keystore..." + /usr/bin/wakunode generateRlnKeystore \ + --rln-relay-eth-client-address="$RPC_URL" \ + --rln-relay-eth-private-key=$PRIVATE_KEY \ + --rln-relay-eth-contract-address=$RLN_CONTRACT_ADDRESS \ + --rln-relay-cred-path=$RLN_CREDENTIAL_PATH \ + --rln-relay-cred-password=$RLN_CREDENTIAL_PASSWORD \ + --rln-relay-user-message-limit=$RLN_RELAY_MSG_LIMIT \ + --log-level=DEBUG \ + --execute +fi + +IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') + +echo "I am a nwaku node" + +# Get an unique node index based on the container's IP +FOURTH_OCTET=${IP##*.} +THIRD_OCTET="${IP%.*}"; THIRD_OCTET="${THIRD_OCTET##*.}" +NODE_INDEX=$((FOURTH_OCTET + 256 * THIRD_OCTET)) + +echo "NODE_INDEX $NODE_INDEX" + +RETRIES=${RETRIES:=10} + +while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do + BOOTSTRAP_ENR=$(wget -qO- http://bootstrap:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); + echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})" + sleep 1 + RETRIES=$(( $RETRIES - 1 )) +done + +if [ -z "${BOOTSTRAP_ENR}" ]; then + echo "Could not get BOOTSTRAP_ENR and none provided. Failing" + exit 1 +fi + +echo "Using bootstrap node: ${BOOTSTRAP_ENR}" +exec /usr/bin/wakunode\ + --relay=true\ + --max-connections=250\ + --rest=true\ + --rest-admin=true\ + --rest-private=true\ + --rest-address=0.0.0.0\ + --cluster-id=0\ + --pubsub-topic=/waku/2/default-waku/proto\ + --rest-port=8645\ + --rln-relay=true\ + --rln-relay-dynamic=true\ + --rln-relay-eth-client-address="$RPC_URL"\ + --rln-relay-eth-contract-address=$RLN_CONTRACT_ADDRESS\ + --rln-relay-cred-path=$RLN_CREDENTIAL_PATH\ + --rln-relay-cred-password=$RLN_CREDENTIAL_PASSWORD\ + --rln-relay-tree-path="rlnv2_tree1"\ + --rln-relay-epoch-sec=$RLN_RELAY_EPOCH_SEC\ + --rln-relay-user-message-limit=$RLN_RELAY_MSG_LIMIT\ + --dns-discovery=true\ + --discv5-discovery=true\ + --discv5-enr-auto-update=True\ + --log-level=DEBUG\ + --metrics-server=True\ + --metrics-server-address=0.0.0.0\ + --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ + --nat=extip:${IP}\ + --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942df25 \ No newline at end of file