From 0a599c186d5c01924b3b5f5c39030fc1ba721e40 Mon Sep 17 00:00:00 2001 From: stubbsta Date: Tue, 22 Oct 2024 12:45:05 +0200 Subject: [PATCH] expanding --- docker-compose.yml | 64 ++- .../configuration/dashboards/nwaku-store.json | 390 +++++++----------- monitoring/prometheus-config.yml | 8 +- run_nwaku_store_postgres1.sh | 11 +- run_nwaku_store_postgres2.sh | 12 +- 5 files changed, 220 insertions(+), 265 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8c00960..747345f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,12 +93,14 @@ services: restart: on-failure labels: com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - ports: - - 127.0.0.1:60001:60001 + # ports: + # - 127.0.0.1:60001:60001 entrypoint: sh environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + - WAKU_SYNC_INTERVAL=${WAKU_SYNC_INTERVAL:-300} + - WAKU_SYNC_RANGE=${WAKU_SYNC_RANGE:-3600} command: - '/opt/run_nwaku_store_postgres1.sh' volumes: @@ -115,12 +117,14 @@ services: restart: on-failure labels: com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' - ports: - - 127.0.0.1:60002:60002 + # ports: + # - 127.0.0.1:60002:60002 entrypoint: sh environment: - POSTGRES_USER=${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + - WAKU_SYNC_INTERVAL=${WAKU_SYNC_INTERVAL:-300} + - WAKU_SYNC_RANGE=${WAKU_SYNC_RANGE:-3600} command: - '/opt/run_nwaku_store_postgres2.sh' volumes: @@ -134,6 +138,30 @@ services: simulation: ipv4_address: 10.2.0.101 + # nwaku_store_pg3: + # image: ${NWAKU_IMAGE:-wakuorg/nwaku:latest} + # restart: on-failure + # labels: + # com.centurylinklabs.watchtower.enable: '${WATCHTOWER_ENABLED:-false}' + # ports: + # - 127.0.0.1:60003:60003 + # entrypoint: sh + # environment: + # - POSTGRES_USER=${POSTGRES_USER:-postgres} + # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + # command: + # - '/opt/run_nwaku_store_postgres3.sh' + # volumes: + # - ./run_nwaku_store_postgres3.sh:/opt/run_nwaku_store_postgres3.sh:Z + # depends_on: + # nwaku_store_pg1: + # condition: service_started + # postgres3: + # condition: service_healthy + # networks: + # simulation: + # ipv4_address: 10.2.0.102 + # rest-traffic: # image: alrevuelta/rest-traffic:6992bb5 # command: @@ -161,7 +189,7 @@ services: - ${PG_DATA_DIR:-./postgresql1}:/var/lib/postgresql/data:Z command: postgres -c config_file=/etc/postgresql/postgresql.conf ports: - - 127.0.0.1:5432:5432 + - 0.0.0.0:5432:5432 networks: - simulation healthcheck: @@ -185,7 +213,31 @@ services: - ${PG_DATA_DIR:-./postgresql2}:/var/lib/postgresql/data:Z command: postgres -c config_file=/etc/postgresql/postgresql.conf ports: - - 127.0.0.1:5433:5432 + - 0.0.0.0:5433:5432 + networks: + - simulation + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 30s + timeout: 60s + retries: 5 + start_period: 80s + + postgres3: + # This service is used when the Waku node has the 'store' protocol enabled + # and the store-message-db-url is set to use Postgres + image: postgres:15.4-alpine3.18 + restart: on-failure:5 + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123} + volumes: + - ./postgres_cfg/postgresql.conf:/etc/postgresql/postgresql.conf:Z + - ./postgres_cfg/db.sql:/docker-entrypoint-initdb.d/db.sql:Z + - ${PG_DATA_DIR:-./postgresql3}:/var/lib/postgresql/data:Z + command: postgres -c config_file=/etc/postgresql/postgresql.conf + ports: + - 0.0.0.0:5434:5432 networks: - simulation healthcheck: diff --git a/monitoring/configuration/dashboards/nwaku-store.json b/monitoring/configuration/dashboards/nwaku-store.json index efb60fd..1ad7124 100644 --- a/monitoring/configuration/dashboards/nwaku-store.json +++ b/monitoring/configuration/dashboards/nwaku-store.json @@ -85,101 +85,6 @@ "x": 0, "y": 0 }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "waku_store_peers", - "instant": false, - "legendFormat": "{{instance}}_{{__name__}}", - "range": true, - "refId": "A" - } - ], - "title": "Store peers", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, "id": 3, "options": { "legend": { @@ -221,7 +126,6 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -235,7 +139,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -270,12 +173,12 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 0 }, - "id": 38, + "id": 40, "options": { "legend": { "calcs": [], @@ -296,42 +199,84 @@ }, "editorMode": "code", "exemplar": true, - "expr": "waku_store_messages", + "expr": "sum by (type)(increase(waku_archive_errors{instance=~\".*store.*\"}[1m]))", + "hide": false, "interval": "", "legendFormat": "{{type}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_store_errors{instance=~\".*store.*\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "B" } ], - "title": "Waku Store Messages", + "title": "Waku Archive Errors (1m rate)", "type": "timeseries" }, { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 10, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" }, "overrides": [] }, @@ -341,111 +286,97 @@ "x": 0, "y": 8 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 77, - "legend": { - "show": false - }, - "maxDataPoints": 120, + "id": 148, "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, "legend": { - "show": false + "calcs": [ + "mean", + "max", + "min", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", "tooltip": { "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" + "sort": "none" } }, - "pluginVersion": "10.4.2", - "reverseYBuckets": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "exemplar": false, - "expr": "sum by (le)(rate(waku_store_query_duration_seconds_bucket[$__rate_interval]))", - "format": "heatmap", + "editorMode": "code", + "exemplar": true, + "expr": "waku_store_time_seconds{instance=~\".*store.*\"} and deriv(waku_store_time_seconds{instance=~\".*store.*\"}[45s]) != 0", "interval": "", - "legendFormat": "{{le}}", + "legendFormat": "{{phase}}", + "range": true, "refId": "A" } ], - "title": "Store Query Duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "title": "Store V3 Times (sec)", + "type": "timeseries" }, { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 2, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -456,88 +387,51 @@ "x": 12, "y": 8 }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 75, - "legend": { - "show": false - }, - "maxDataPoints": 120, + "id": 38, "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, "legend": { - "show": false + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "10.4.2", - "reverseYBuckets": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "exemplar": false, - "expr": "sum by (le)(rate(waku_store_insert_duration_seconds_bucket[$__rate_interval]))", - "format": "heatmap", + "editorMode": "code", + "exemplar": true, + "expr": "waku_store_messages{instance=~\".*store.*\"}", "hide": false, "interval": "", - "legendFormat": "{{le}}", + "legendFormat": "{{type}}: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "waku_archive_messages{instance=~\".*store.*\"}", + "hide": false, + "legendFormat": "{{type}}: {{instance}}", + "range": true, "refId": "B" } ], - "title": "Store Insert Duration", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" + "title": "Waku Archive Messages", + "type": "timeseries" } - ], + ], "refresh": "", "schemaVersion": 39, "tags": [], diff --git a/monitoring/prometheus-config.yml b/monitoring/prometheus-config.yml index c058c4b..ccf106f 100644 --- a/monitoring/prometheus-config.yml +++ b/monitoring/prometheus-config.yml @@ -17,8 +17,12 @@ scrape_configs: static_configs: - targets: - bootstrap:8008 - - waku-simulator_nwaku_store_pg1_1:8009 - - waku-simulator_nwaku_store_pg2_1:8010 + - waku-simulator_nwaku_store_pg1_1:8008 + - waku-simulator_nwaku_store_pg2_1:8008 + - waku-simulator_nwaku_store_pg3_1:8011 + - waku-simulator_postgres1_1:8008 + - waku-simulator_postgres2_1:8008 + - waku-simulator_postgres3_1:8008 - waku-simulator_nwaku_1:8008 - waku-simulator_nwaku_2:8008 - waku-simulator_nwaku_3:8008 diff --git a/run_nwaku_store_postgres1.sh b/run_nwaku_store_postgres1.sh index 3a18c85..e63c14f 100644 --- a/run_nwaku_store_postgres1.sh +++ b/run_nwaku_store_postgres1.sh @@ -21,6 +21,7 @@ fi echo "Using bootstrap node: ${BOOTSTRAP_ENR}" exec /usr/bin/wakunode\ --relay=true\ + --lightpush=true\ --max-connections=50\ --rest=true\ --rest-address=0.0.0.0\ @@ -28,17 +29,19 @@ exec /usr/bin/wakunode\ --dns-discovery=true\ --discv5-discovery=true\ --discv5-enr-auto-update=True\ - --log-level=DEBUG\ + --log-level=TRACE\ --metrics-server=True\ --metrics-server-address=0.0.0.0\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ --pubsub-topic=/waku/2/rs/16/32\ --cluster-id=16\ - --ports-shift=1\ + --ports-shift=0\ + --nat=extip:${IP}\ --store=true\ - --store-message-retention-policy=size:20MB\ + --store-message-retention-policy=size:1Gb\ --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres1:5432/postgres"\ --store-sync=true\ - --store-sync-interval=200\ + --store-sync-interval=300\ --store-sync-range=3600\ + --store-sync-relay-jitter=0\ --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa1 \ No newline at end of file diff --git a/run_nwaku_store_postgres2.sh b/run_nwaku_store_postgres2.sh index 66dfe82..b53acb3 100644 --- a/run_nwaku_store_postgres2.sh +++ b/run_nwaku_store_postgres2.sh @@ -20,7 +20,8 @@ fi echo "Using bootstrap node: ${BOOTSTRAP_ENR}" exec /usr/bin/wakunode\ - --relay=true\ + --relay=false\ + --lightpush=true\ --max-connections=50\ --rest=true\ --rest-address=0.0.0.0\ @@ -28,18 +29,19 @@ exec /usr/bin/wakunode\ --dns-discovery=true\ --discv5-discovery=true\ --discv5-enr-auto-update=True\ - --log-level=DEBUG\ + --log-level=TRACE\ --metrics-server=True\ --metrics-server-address=0.0.0.0\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ --pubsub-topic=/waku/2/rs/16/32\ --cluster-id=16\ - --ports-shift=2\ + --ports-shift=0\ --store=true\ - --store-message-retention-policy=size:40MB\ + --store-message-retention-policy=size:1GB\ --nat=extip:${IP}\ --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres2:5432/postgres"\ --store-sync=true\ - --store-sync-interval=60\ + --store-sync-interval=300\ --store-sync-range=3600\ + --store-sync-relay-jitter=0\ --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa3 \ No newline at end of file