diff --git a/docker-compose.yml b/docker-compose.yml index fa72a84..8c00960 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -195,26 +195,27 @@ services: retries: 5 start_period: 80s - # postgres-exporter: - # # Service aimed to scrape information from Postgres and post it to Prometeus - # image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 - # restart: on-failure:5 - # environment: - # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} - # - DATA_SOURCE_URI=postgres?sslmode=disable - # - DATA_SOURCE_USER=${POSTGRES_USER:-postgres} - # - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123} - # - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml - # volumes: - # - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z - # - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z - # command: - # # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info - # - --config.file=/etc/pgexporter/postgres-exporter.yml - # depends_on: - # - postgres - # networks: - # - simulation + postgres-exporter: + # Service aimed to scrape information from Postgres and post it to Prometeus + image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 + restart: on-failure:5 + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + - DATA_SOURCE_URI=postgres1:5432/postgres?sslmode=disable + - DATA_SOURCE_USER=${POSTGRES_USER:-postgres} + - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123} + - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml + volumes: + - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z + - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z + command: + # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info + - --config.file=/etc/pgexporter/postgres-exporter.yml + depends_on: + postgres1: + condition: service_healthy + networks: + - simulation prometheus: image: prom/prometheus:latest diff --git a/monitoring/configuration/dashboards/nwaku-store.json b/monitoring/configuration/dashboards/nwaku-store.json index 0213bf5..efb60fd 100644 --- a/monitoring/configuration/dashboards/nwaku-store.json +++ b/monitoring/configuration/dashboards/nwaku-store.json @@ -270,12 +270,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 12, - "x": 0, - "y": 8 + "x": 12, + "y": 60 }, - "id": 4, + "id": 38, "options": { "legend": { "calcs": [], @@ -295,15 +295,247 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "waku_legacy_store_queries", - "instant": false, - "legendFormat": "{{instance}}_{{__name__}}", + "exemplar": true, + "expr": "waku_store_messages", + "interval": "", + "legendFormat": "{{type}}", "range": true, "refId": "A" } ], - "title": "Legacy store queries", + "title": "Waku Store Messages", "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 77, + "legend": { + "show": false + }, + "maxDataPoints": 120, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.2", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "sum by (le)(rate(waku_store_query_duration_seconds_bucket[$__rate_interval]))", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Store Query Duration", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 75, + "legend": { + "show": false + }, + "maxDataPoints": 120, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.2", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": false, + "expr": "sum by (le)(rate(waku_store_insert_duration_seconds_bucket[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "Store Insert Duration", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" } ], "refresh": "", diff --git a/monitoring/prometheus-config.yml b/monitoring/prometheus-config.yml index dca9592..c058c4b 100644 --- a/monitoring/prometheus-config.yml +++ b/monitoring/prometheus-config.yml @@ -17,8 +17,8 @@ scrape_configs: static_configs: - targets: - bootstrap:8008 - - waku-simulator_nwaku_store_pg1_1:8008 - - waku-simulator_nwaku_store_pg2_1:8008 + - waku-simulator_nwaku_store_pg1_1:8009 + - waku-simulator_nwaku_store_pg2_1:8010 - waku-simulator_nwaku_1:8008 - waku-simulator_nwaku_2:8008 - waku-simulator_nwaku_3:8008 diff --git a/run_nwaku_static1.sh b/run_nwaku_static1.sh deleted file mode 100644 index 035acc0..0000000 --- a/run_nwaku_static1.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/sh - -# Check Linux Distro Version - it can differ depending on the nwaku image used -# Install bind-tools/dnsutils package used for domain name resolution -OS=$(cat /etc/os-release) -if echo $OS | grep -q "Debian"; then - echo "The operating system is Debian." - apt update - apt install -y dnsutils -elif echo $OS | grep -q "Alpine"; then - echo "The operating system is Alpine." - apk add bind-tools -fi - -if test -f .env; then - echo "Using .env file" - . $(pwd)/.env -fi - -# Function to extract IP address from URL, resolve the IP and replace it in the original URL -get_ip_address_and_replace() { - local url=$1 - local domain_name=$(echo $RPC_URL | awk -F[/:] '{print $4}') - local ip_address=$(dig +short $domain_name) - valid_rpc_url="$(echo "$url" | sed "s/$domain_name/$ip_address/g")" - echo $valid_rpc_url -} - -# the format of the RPC URL is checked in the generateRlnKeystore command and hostnames are not valid -pattern="^(https?):\/\/((localhost)|([\w_-]+(?:(?:\.[\w_-]+)+)))(:[0-9]{1,5})?([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])*" -# Perform regex matching -if echo "$RPC_URL" | grep -q "$pattern"; then - echo "RPC URL is valid" -else - echo "RPC URL is invalid: $RPC_URL. Attempting to resolve hostname." - resolved_rpc_url="$(get_ip_address_and_replace $RPC_URL)" - if [ -z "$resolved_rpc_url" ]; then - echo -e "Failed to retrieve IP address for $RPC_URL\n" - else - echo -e "Resolved RPC URL for $RPC_URL: $resolved_rpc_url" - RPC_URL="$resolved_rpc_url" - fi -fi - -if test -f .$RLN_CREDENTIAL_PATH; then - echo "$RLN_CREDENTIAL_PATH already exists. Use it instead of creating a new one." -else - echo "Generating RLN keystore..." - /usr/bin/wakunode generateRlnKeystore \ - --rln-relay-eth-client-address="$RPC_URL" \ - --rln-relay-eth-private-key=$PRIVATE_KEY \ - --rln-relay-eth-contract-address=$RLN_CONTRACT_ADDRESS \ - --rln-relay-cred-path=$RLN_CREDENTIAL_PATH \ - --rln-relay-cred-password=$RLN_CREDENTIAL_PASSWORD \ - --rln-relay-user-message-limit=$RLN_RELAY_MSG_LIMIT \ - --log-level=DEBUG \ - --execute -fi - -IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') - -echo "I am a nwaku node" - -# Get an unique node index based on the container's IP -FOURTH_OCTET=${IP##*.} -THIRD_OCTET="${IP%.*}"; THIRD_OCTET="${THIRD_OCTET##*.}" -NODE_INDEX=$((FOURTH_OCTET + 256 * THIRD_OCTET)) - -echo "NODE_INDEX $NODE_INDEX" - -RETRIES=${RETRIES:=10} - -while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do - BOOTSTRAP_ENR=$(wget -qO- http://bootstrap:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); - echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})" - sleep 1 - RETRIES=$(( $RETRIES - 1 )) -done - -if [ -z "${BOOTSTRAP_ENR}" ]; then - echo "Could not get BOOTSTRAP_ENR and none provided. Failing" - exit 1 -fi - -echo "Using bootstrap node: ${BOOTSTRAP_ENR}" -exec /usr/bin/wakunode\ - --relay=true\ - --max-connections=250\ - --rest=true\ - --rest-admin=true\ - --rest-private=true\ - --rest-address=0.0.0.0\ - --cluster-id=0\ - --pubsub-topic=/waku/2/default-waku/proto\ - --rest-port=8645\ - --rln-relay=true\ - --rln-relay-dynamic=true\ - --rln-relay-eth-client-address="$RPC_URL"\ - --rln-relay-eth-contract-address=$RLN_CONTRACT_ADDRESS\ - --rln-relay-cred-path=$RLN_CREDENTIAL_PATH\ - --rln-relay-cred-password=$RLN_CREDENTIAL_PASSWORD\ - --rln-relay-tree-path="rlnv2_tree1"\ - --rln-relay-epoch-sec=$RLN_RELAY_EPOCH_SEC\ - --rln-relay-user-message-limit=$RLN_RELAY_MSG_LIMIT\ - --dns-discovery=true\ - --discv5-discovery=true\ - --discv5-enr-auto-update=True\ - --log-level=DEBUG\ - --metrics-server=True\ - --metrics-server-address=0.0.0.0\ - --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ - --nat=extip:${IP}\ - --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942df25 \ No newline at end of file diff --git a/run_nwaku_store_postgres1.sh b/run_nwaku_store_postgres1.sh index 129655a..3a18c85 100644 --- a/run_nwaku_store_postgres1.sh +++ b/run_nwaku_store_postgres1.sh @@ -39,6 +39,6 @@ exec /usr/bin/wakunode\ --store-message-retention-policy=size:20MB\ --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres1:5432/postgres"\ --store-sync=true\ - --store-sync-interval=120\ + --store-sync-interval=200\ --store-sync-range=3600\ --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa1 \ No newline at end of file diff --git a/run_nwaku_store_postgres2.sh b/run_nwaku_store_postgres2.sh index 562c94a..66dfe82 100644 --- a/run_nwaku_store_postgres2.sh +++ b/run_nwaku_store_postgres2.sh @@ -28,7 +28,7 @@ exec /usr/bin/wakunode\ --dns-discovery=true\ --discv5-discovery=true\ --discv5-enr-auto-update=True\ - --log-level=TRACE\ + --log-level=DEBUG\ --metrics-server=True\ --metrics-server-address=0.0.0.0\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ @@ -37,9 +37,9 @@ exec /usr/bin/wakunode\ --ports-shift=2\ --store=true\ --store-message-retention-policy=size:40MB\ + --nat=extip:${IP}\ --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres2:5432/postgres"\ --store-sync=true\ - --store-sync-interval=80\ + --store-sync-interval=60\ --store-sync-range=3600\ - --staticnode=/ip4/10.2.0.100/tcp/60001/p2p/16Uiu2HAm3iFcvPDcYge7JFev3ZRdsiUoYFwxtzGZW1BSoV4b4Gc6\ --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa3 \ No newline at end of file