This commit is contained in:
stubbsta 2024-07-09 15:52:51 +02:00
parent fa4792e101
commit ce272dbbf1
No known key found for this signature in database
7 changed files with 468 additions and 127 deletions

View File

@ -12,45 +12,45 @@ services:
# Accounts are hardcoded to 520 with the idea that nwaku nodes use up to 500 for membership registration and the last 20 are used for ad-hoc testing.
# The account number and private key pairs of the last 20 accounts can be found in the Register memberships section of the Waku-simulator book.
foundry:
image: ghcr.io/foundry-rs/foundry:nightly-9b73e06e1fe376738b92ae081107620291d50188
labels:
com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}'
ports:
- 0.0.0.0:8545:8545
command:
- anvil
--port=8545
--host=0.0.0.0
--accounts=520
--allow-origin=*
--block-time=3
--chain-id=1234
--silent
--config-out=/shared/anvil-config.txt
volumes:
- privatekeys-volume:/shared
networks:
- simulation
# foundry:
# image: ghcr.io/foundry-rs/foundry:nightly-9b73e06e1fe376738b92ae081107620291d50188
# labels:
# com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}'
# ports:
# - 0.0.0.0:8545:8545
# command:
# - anvil
# --port=8545
# --host=0.0.0.0
# --accounts=520
# --allow-origin=*
# --block-time=3
# --chain-id=1234
# --silent
# --config-out=/shared/anvil-config.txt
# volumes:
# - privatekeys-volume:/shared
# networks:
# - simulation
contract-repo-deployer:
image: node:hydrogen-bullseye
labels:
com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}'
environment:
- PRIVATE_KEY=${PRIVATE_KEY}
- RPC_URL=${RPC_URL:-http://foundry:8545}
- ETH_FROM=${ETH_FROM}
- MAX_MESSAGE_LIMIT=${MAX_MESSAGE_LIMIT:-20}
entrypoint: sh
command:
- '/opt/deploy_rln_contract.sh'
volumes:
- ./deploy_rln_contract.sh:/opt/deploy_rln_contract.sh
depends_on:
- foundry
networks:
- simulation
# contract-repo-deployer:
# image: node:hydrogen-bullseye
# labels:
# com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}'
# environment:
# - PRIVATE_KEY=${PRIVATE_KEY}
# - RPC_URL=${RPC_URL:-http://foundry:8545}
# - ETH_FROM=${ETH_FROM}
# - MAX_MESSAGE_LIMIT=${MAX_MESSAGE_LIMIT:-20}
# entrypoint: sh
# command:
# - '/opt/deploy_rln_contract.sh'
# volumes:
# - ./deploy_rln_contract.sh:/opt/deploy_rln_contract.sh
# depends_on:
# - foundry
# networks:
# - simulation
bootstrap:
image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest}
@ -87,13 +87,12 @@ services:
- RLN_RELAY_MSG_LIMIT=${RLN_RELAY_MSG_LIMIT:-10}
- RLN_RELAY_EPOCH_SEC=${RLN_RELAY_EPOCH_SEC:-60}
command:
- '/opt/run_nwaku.sh'
- '/opt/run_nwaku_noRLN.sh'
volumes:
- ./run_nwaku.sh:/opt/run_nwaku.sh:Z
- ./run_nwaku_noRLN.sh:/opt/run_nwaku_noRLN.sh:Z
- privatekeys-volume:/shared
depends_on:
contract-repo-deployer:
condition: service_completed_successfully
- bootstrap
networks:
- simulation
@ -151,26 +150,26 @@ services:
retries: 5
start_period: 80s
# postgres-exporter:
# # Service aimed to scrape information from Postgres and post it to Prometeus
# image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0
# restart: on-failure:5
# environment:
# - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123}
# - DATA_SOURCE_URI=postgres?sslmode=disable
# - DATA_SOURCE_USER=${POSTGRES_USER:-postgres}
# - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123}
# - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml
# volumes:
# - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z
# - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z
# command:
# # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info
# - --config.file=/etc/pgexporter/postgres-exporter.yml
# depends_on:
# - postgres
# networks:
# - simulation
postgres-exporter:
# Service aimed to scrape information from Postgres and post it to Prometeus
image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0
restart: on-failure:5
environment:
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123}
- DATA_SOURCE_URI=postgres?sslmode=disable
- DATA_SOURCE_USER=${POSTGRES_USER:-postgres}
- DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123}
- PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml
volumes:
- ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z
- ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z
command:
# Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info
- --config.file=/etc/pgexporter/postgres-exporter.yml
depends_on:
- postgres
networks:
- simulation
prometheus:
image: prom/prometheus:latest
@ -231,71 +230,71 @@ services:
networks:
- simulation
api:
image: web3labs/epirus-free-api:latest
ports:
- 127.0.0.1:8090:8090
environment:
- NODE_ENDPOINT=${RPC_URL:-http://foundry:8545}
- MONGO_CLIENT_URI=mongodb://mongodb:27017
- REINDEX_ENDPOINT=http://ingestion/reindex/
- MONGO_DB_NAME=epirus
- MONGO_CREATE_INDICES=true
- REDIS_HOST=redis
- REDIS_PORT=6379
depends_on:
- redis
- mongodb
- foundry
networks:
- simulation
# api:
# image: web3labs/epirus-free-api:latest
# ports:
# - 127.0.0.1:8090:8090
# environment:
# - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545}
# - MONGO_CLIENT_URI=mongodb://mongodb:27017
# - REINDEX_ENDPOINT=http://ingestion/reindex/
# - MONGO_DB_NAME=epirus
# - MONGO_CREATE_INDICES=true
# - REDIS_HOST=redis
# - REDIS_PORT=6379
# depends_on:
# - redis
# - mongodb
# - foundry
# networks:
# - simulation
mongodb:
image: mongo:5.0.8
environment:
- COMPOSE_HTTP_TIMEOUT=900
- DOCKER_CLIENT_TIMEOUT=900
entrypoint: mongod --bind_ip "0.0.0.0"
networks:
- simulation
# mongodb:
# image: mongo:5.0.8
# environment:
# - COMPOSE_HTTP_TIMEOUT=900
# - DOCKER_CLIENT_TIMEOUT=900
# entrypoint: mongod --bind_ip "0.0.0.0"
# networks:
# - simulation
web:
image: web3labs/epirus-free-web:latest
environment:
- API_URL=${EPIRUS_WEB_API_URL:-/api}
- WS_API_URL=${EPIRUS_WEB_WS_API_URL:-ws://localhost:8090}
- DISPLAY_NETWORK_TAB=disabled
depends_on:
- api
networks:
- simulation
# web:
# image: web3labs/epirus-free-web:latest
# environment:
# - API_URL=${EPIRUS_WEB_API_URL:-/api}
# - WS_API_URL=${EPIRUS_WEB_WS_API_URL:-ws://localhost:8090}
# - DISPLAY_NETWORK_TAB=disabled
# depends_on:
# - api
# networks:
# - simulation
ingestion:
image: web3labs/epirus-free-ingestion:latest
environment:
- NODE_ENDPOINT=${RPC_URL:-http://foundry:8545}
- MONGO_CLIENT_URI=mongodb://mongodb:27017
- MONGO_DB_NAME=epirus
- LIST_OF_METRICS_TO_CALCULATE_PER_MINUTE=hourly,daily,monthly,yearly
depends_on:
- mongodb
- redis
- foundry
networks:
- simulation
# ingestion:
# image: web3labs/epirus-free-ingestion:latest
# environment:
# - NODE_ENDPOINT=${RPC_URL:-http://foundry:8545}
# - MONGO_CLIENT_URI=mongodb://mongodb:27017
# - MONGO_DB_NAME=epirus
# - LIST_OF_METRICS_TO_CALCULATE_PER_MINUTE=hourly,daily,monthly,yearly
# depends_on:
# - mongodb
# - redis
# - foundry
# networks:
# - simulation
nginx:
image: nginx:latest
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf
- ./5xx.html:/www/error_pages/5xx.html
ports:
- 0.0.0.0:3000:80
depends_on:
- api
- web
networks:
- simulation
# nginx:
# image: nginx:latest
# volumes:
# - ./nginx.conf:/etc/nginx/nginx.conf
# - ./5xx.html:/www/error_pages/5xx.html
# ports:
# - 0.0.0.0:3000:80
# depends_on:
# - api
# - web
# networks:
# - simulation
volumes:
privatekeys-volume:

View File

@ -0,0 +1,284 @@
pg_replication:
query: "SELECT CASE WHEN NOT pg_is_in_recovery() THEN 0 ELSE GREATEST (0, EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))) END AS lag"
master: true
metrics:
- lag:
usage: "GAUGE"
description: "Replication lag behind master in seconds"
pg_postmaster:
query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()"
master: true
metrics:
- start_time_seconds:
usage: "GAUGE"
description: "Time at which postmaster started"
pg_stat_user_tables:
query: |
SELECT
current_database() datname,
schemaname,
relname,
seq_scan,
seq_tup_read,
idx_scan,
idx_tup_fetch,
n_tup_ins,
n_tup_upd,
n_tup_del,
n_tup_hot_upd,
n_live_tup,
n_dead_tup,
n_mod_since_analyze,
COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum,
COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum,
COALESCE(last_analyze, '1970-01-01Z') as last_analyze,
COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze,
vacuum_count,
autovacuum_count,
analyze_count,
autoanalyze_count
FROM
pg_stat_user_tables
metrics:
- datname:
usage: "LABEL"
description: "Name of current database"
- schemaname:
usage: "LABEL"
description: "Name of the schema that this table is in"
- relname:
usage: "LABEL"
description: "Name of this table"
- seq_scan:
usage: "COUNTER"
description: "Number of sequential scans initiated on this table"
- seq_tup_read:
usage: "COUNTER"
description: "Number of live rows fetched by sequential scans"
- idx_scan:
usage: "COUNTER"
description: "Number of index scans initiated on this table"
- idx_tup_fetch:
usage: "COUNTER"
description: "Number of live rows fetched by index scans"
- n_tup_ins:
usage: "COUNTER"
description: "Number of rows inserted"
- n_tup_upd:
usage: "COUNTER"
description: "Number of rows updated"
- n_tup_del:
usage: "COUNTER"
description: "Number of rows deleted"
- n_tup_hot_upd:
usage: "COUNTER"
description: "Number of rows HOT updated (i.e., with no separate index update required)"
- n_live_tup:
usage: "GAUGE"
description: "Estimated number of live rows"
- n_dead_tup:
usage: "GAUGE"
description: "Estimated number of dead rows"
- n_mod_since_analyze:
usage: "GAUGE"
description: "Estimated number of rows changed since last analyze"
- last_vacuum:
usage: "GAUGE"
description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)"
- last_autovacuum:
usage: "GAUGE"
description: "Last time at which this table was vacuumed by the autovacuum daemon"
- last_analyze:
usage: "GAUGE"
description: "Last time at which this table was manually analyzed"
- last_autoanalyze:
usage: "GAUGE"
description: "Last time at which this table was analyzed by the autovacuum daemon"
- vacuum_count:
usage: "COUNTER"
description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)"
- autovacuum_count:
usage: "COUNTER"
description: "Number of times this table has been vacuumed by the autovacuum daemon"
- analyze_count:
usage: "COUNTER"
description: "Number of times this table has been manually analyzed"
- autoanalyze_count:
usage: "COUNTER"
description: "Number of times this table has been analyzed by the autovacuum daemon"
pg_statio_user_tables:
query: "SELECT current_database() datname, schemaname, relname, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables"
metrics:
- datname:
usage: "LABEL"
description: "Name of current database"
- schemaname:
usage: "LABEL"
description: "Name of the schema that this table is in"
- relname:
usage: "LABEL"
description: "Name of this table"
- heap_blks_read:
usage: "COUNTER"
description: "Number of disk blocks read from this table"
- heap_blks_hit:
usage: "COUNTER"
description: "Number of buffer hits in this table"
- idx_blks_read:
usage: "COUNTER"
description: "Number of disk blocks read from all indexes on this table"
- idx_blks_hit:
usage: "COUNTER"
description: "Number of buffer hits in all indexes on this table"
- toast_blks_read:
usage: "COUNTER"
description: "Number of disk blocks read from this table's TOAST table (if any)"
- toast_blks_hit:
usage: "COUNTER"
description: "Number of buffer hits in this table's TOAST table (if any)"
- tidx_blks_read:
usage: "COUNTER"
description: "Number of disk blocks read from this table's TOAST table indexes (if any)"
- tidx_blks_hit:
usage: "COUNTER"
description: "Number of buffer hits in this table's TOAST table indexes (if any)"
# WARNING: This set of metrics can be very expensive on a busy server as every unique query executed will create an additional time series
pg_stat_statements:
query: "SELECT t2.rolname, t3.datname, queryid, calls, ( total_plan_time + total_exec_time ) / 1000 as total_time_seconds, ( min_plan_time + min_exec_time ) / 1000 as min_time_seconds, ( max_plan_time + max_exec_time ) / 1000 as max_time_seconds, ( mean_plan_time + mean_exec_time ) / 1000 as mean_time_seconds, ( stddev_plan_time + stddev_exec_time ) / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin' AND queryid IS NOT NULL"
master: true
metrics:
- rolname:
usage: "LABEL"
description: "Name of user"
- datname:
usage: "LABEL"
description: "Name of database"
- queryid:
usage: "LABEL"
description: "Query ID"
- calls:
usage: "COUNTER"
description: "Number of times executed"
- total_time_seconds:
usage: "COUNTER"
description: "Total time spent in the statement, in milliseconds"
- min_time_seconds:
usage: "GAUGE"
description: "Minimum time spent in the statement, in milliseconds"
- max_time_seconds:
usage: "GAUGE"
description: "Maximum time spent in the statement, in milliseconds"
- mean_time_seconds:
usage: "GAUGE"
description: "Mean time spent in the statement, in milliseconds"
- stddev_time_seconds:
usage: "GAUGE"
description: "Population standard deviation of time spent in the statement, in milliseconds"
- rows:
usage: "COUNTER"
description: "Total number of rows retrieved or affected by the statement"
- shared_blks_hit:
usage: "COUNTER"
description: "Total number of shared block cache hits by the statement"
- shared_blks_read:
usage: "COUNTER"
description: "Total number of shared blocks read by the statement"
- shared_blks_dirtied:
usage: "COUNTER"
description: "Total number of shared blocks dirtied by the statement"
- shared_blks_written:
usage: "COUNTER"
description: "Total number of shared blocks written by the statement"
- local_blks_hit:
usage: "COUNTER"
description: "Total number of local block cache hits by the statement"
- local_blks_read:
usage: "COUNTER"
description: "Total number of local blocks read by the statement"
- local_blks_dirtied:
usage: "COUNTER"
description: "Total number of local blocks dirtied by the statement"
- local_blks_written:
usage: "COUNTER"
description: "Total number of local blocks written by the statement"
- temp_blks_read:
usage: "COUNTER"
description: "Total number of temp blocks read by the statement"
- temp_blks_written:
usage: "COUNTER"
description: "Total number of temp blocks written by the statement"
- blk_read_time_seconds:
usage: "COUNTER"
description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)"
- blk_write_time_seconds:
usage: "COUNTER"
description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)"
pg_process_idle:
query: |
WITH
metrics AS (
SELECT
application_name,
SUM(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change))::bigint)::float AS process_idle_seconds_sum,
COUNT(*) AS process_idle_seconds_count
FROM pg_stat_activity
WHERE state = 'idle'
GROUP BY application_name
),
buckets AS (
SELECT
application_name,
le,
SUM(
CASE WHEN EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change)) <= le
THEN 1
ELSE 0
END
)::bigint AS bucket
FROM
pg_stat_activity,
UNNEST(ARRAY[1, 2, 5, 15, 30, 60, 90, 120, 300]) AS le
GROUP BY application_name, le
ORDER BY application_name, le
)
SELECT
application_name,
process_idle_seconds_sum as seconds_sum,
process_idle_seconds_count as seconds_count,
ARRAY_AGG(le) AS seconds,
ARRAY_AGG(bucket) AS seconds_bucket
FROM metrics JOIN buckets USING (application_name)
GROUP BY 1, 2, 3
metrics:
- application_name:
usage: "LABEL"
description: "Application Name"
- seconds:
usage: "HISTOGRAM"
description: "Idle time of server processes"
pg_tb_stats:
query: |
select pubsubtopic, count(*) AS messages FROM (SELECT id, array_agg(pubsubtopic ORDER BY pubsubtopic) AS pubsubtopic FROM messages GROUP BY id) sub GROUP BY pubsubtopic ORDER BY pubsubtopic;
metrics:
- pubsubtopic:
usage: "LABEL"
description: "pubsubtopic"
- messages:
usage: "GAUGE"
description: "Number of messages for the given pubsub topic"
pg_tb_messages:
query: |
SELECT
COUNT(ID)
FROM messages
metrics:
- count:
usage: "GAUGE"
description: "Row count in `messages` table"

View File

@ -0,0 +1,9 @@
auth_modules:
mypostgres:
type: userpass
userpass:
username: postgres
password: ${POSTGRES_PASSWORD}
options:
# options become key=value parameters of the DSN
sslmode: disable

View File

@ -10,6 +10,9 @@ scrape_configs:
static_configs:
- targets:
- cadvisor:8080
- job_name: postgres-exporter
static_configs:
- targets: ['postgres-exporter:9187']
- job_name: "nwaku"
static_configs:
- targets:

View File

@ -14,7 +14,7 @@ exec /usr/bin/wakunode\
--dns-discovery=true\
--discv5-discovery=true\
--discv5-enr-auto-update=True\
--log-level=INFO\
--log-level=DEBUG\
--metrics-server=True\
--metrics-server-address=0.0.0.0\
--nodekey=30348dd51465150e04a5d9d932c72864c8967f806cce60b5d26afeca1e77eb68\

46
run_nwaku_noRLN.sh Normal file
View File

@ -0,0 +1,46 @@
#!/bin/sh
if test -f .env; then
echo "Using .env file"
. $(pwd)/.env
fi
IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
echo "I am a nwaku node - no RLN"
RETRIES=${RETRIES:=10}
while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
BOOTSTRAP_ENR=$(wget -qO- http://bootstrap:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})"
sleep 1
RETRIES=$(( $RETRIES - 1 ))
done
if [ -z "${BOOTSTRAP_ENR}" ]; then
echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
exit 1
fi
echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
exec /usr/bin/wakunode\
--relay=true\
--lightpush=true\
--max-connections=250\
--rest=true\
--rest-admin=true\
--rest-private=true\
--rest-address=0.0.0.0\
--rest-port=8645\
--dns-discovery=true\
--discv5-discovery=true\
--discv5-enr-auto-update=True\
--log-level=DEBUG\
--metrics-server=True\
--metrics-server-address=0.0.0.0\
--discv5-bootstrap-node=${BOOTSTRAP_ENR}\
--nat=extip:${IP}\
--pubsub-topic=/waku/2/rs/66/0\
--cluster-id=66\
--storenode=/ip4/10.2.0.99/tcp/60000/p2p/16Uiu2HAmTVafvweaXrXKmFFkUo4qWYP7wTa2H6PXee8iMyQw4eHm

View File

@ -20,8 +20,8 @@ fi
echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
exec /usr/bin/wakunode\
--relay=false\
--max-connections=10\
--relay=true\
--max-connections=20\
--rest=true\
--rest-admin=true\
--rest-private=true\
@ -30,7 +30,7 @@ exec /usr/bin/wakunode\
--dns-discovery=true\
--discv5-discovery=true\
--discv5-enr-auto-update=True\
--log-level=TRACE\
--log-level=DEBUG\
--metrics-server=True\
--metrics-server-address=0.0.0.0\
--discv5-bootstrap-node=${BOOTSTRAP_ENR}\