diff --git a/.github/workflows/fleet_tests.yml b/.github/workflows/fleet_tests.yml new file mode 100644 index 000000000..fda82f69e --- /dev/null +++ b/.github/workflows/fleet_tests.yml @@ -0,0 +1,37 @@ +name: Waku Fleet Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + inputs: + node1: + required: true + description: "Node that usually publishes messages. Used for all tests" + type: string + default: "wakuorg/nwaku:latest" + node2: + required: true + description: "Node that usually queries for published messages. Used for all tests" + type: string + default: "wakuorg/nwaku:latest" + additional_nodes: + required: false + description: "Additional optional nodes used in e2e tests, separated by ," + type: string + default: "wakuorg/nwaku:latest,wakuorg/nwaku:latest,wakuorg/nwaku:latest" + +jobs: + test-common: + uses: ./.github/workflows/test_common.yml + secrets: inherit + with: + node1: ${{ inputs.node1 || 'wakuorg/nwaku:latest' }} + node2: ${{ inputs.node2 || 'wakuorg/nwaku:latest' }} + additional_nodes: ${{ inputs.additional_nodes || 'wakuorg/nwaku:latest,wakuorg/nwaku:latest,wakuorg/nwaku:latest' }} + fleet_tests: true + caller: "fleet" \ No newline at end of file diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 41a4c3f6b..079be507c 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -22,6 +22,11 @@ on: required: false description: "Workflow caller. Used in reporting" type: string + fleet_tests: + required: false + description: "Run fleet tests only" + type: boolean + default: false deploy_allure: required: false description: "Deploy allure report to gh-pages" @@ -47,7 +52,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] + shard: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17] # total number of shards =18 means tests will split into 18 thread and run in parallel to increase execution speed # command for sharding : # pytest --shard-id= --num-shards= @@ -152,6 +157,10 @@ jobs: echo "Built library:" ls -l "$BINDINGS_DIR/lib/liblogosdelivery.so" + + # Cleanup build artifacts + rm -rf .gh-pages/nim + - name: Verify wrapper library run: | @@ -180,7 +189,15 @@ jobs: export PATH="$HOME/.nimble/bin:$PATH" export PYTHONPATH="$(pwd)/vendor/logos-delivery-python-bindings/waku:$PYTHONPATH" - if [ "${{ matrix.shard }}" == "16" ]; then + if [ "${{ inputs.fleet_tests }}" == "true" ]; then + if [ "${{ matrix.shard }}" == "0" ]; then + pytest --fleet -m waku_test_fleet \ + --ignore=vendor/logos-delivery-python-bindings/tests \ + --alluredir=allure-results-${{ matrix.shard }} + else + echo "Skipping shard ${{ matrix.shard }}: fleet tests run on shard 0 only" + fi + elif [ "${{ matrix.shard }}" == "16" ]; then pytest tests/relay/test_rln.py \ --ignore=vendor/logos-delivery-python-bindings/tests \ --alluredir=allure-results-${{ matrix.shard }} @@ -338,7 +355,7 @@ jobs: for key in $(jq -r 'keys[]' results.json); do result=$(jq -r --arg key "$key" '.[$key]' results.json) echo "Key: $key, Value: $result" - if [ "$result" != "success" ]; then + if [ -n "$result" ] && [ "$result" != "success" ]; then echo "Value 'success' not found at key: $key" TESTS_RESULT="failure" break diff --git a/pytest.ini b/pytest.ini index 2406bfb06..9c02caf43 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,3 +13,5 @@ timeout = 300 markers = smoke: marks tests as smoke test (deselect with '-m "not smoke"') docker_required: test requires Docker nodes (WakuNode) + waku_test_fleet: marks tests that run against a live Waku test fleet + store2000: marks tests that use 2000 store messages diff --git a/scripts/Lite_protocol_scripts/store/Store_multinodes.sh b/scripts/Lite_protocol_scripts/store/Store_multinodes.sh deleted file mode 100644 index ce65c241b..000000000 --- a/scripts/Lite_protocol_scripts/store/Store_multinodes.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -set -e -# Stress Store with multiple Store nodes while running phased LPT publishers/receivers. - -STORE_NODES="/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n,/ip4/10.2.0.102/tcp/60001/p2p/16Uiu2HA7abcDEF451tGkbzz4Mjcg6DRnmAHxNeWyF4zp23RbpXYZ2,/ip4/10.2.0.103/tcp/60001/p2p/16Uiu2HA9LMNoPQ451tGkbww4Mjcg6DRnmAHxNeWyF4zp23Rbppppp" -RELAY_NODE_REST_ADDRESS="http://127.0.0.1:8645" -STORE_NODE_REST_ADDRESS="http://127.0.0.1:8644" -PUBSUB_TOPIC="/waku/2/default-waku/proto" -CONTENT_TOPIC="/sonda/2/polls/proto" -PHASE_SLEEP=300 - -echo "Running test..." -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Bringing up simulator at $current_time" - -cd ./waku-simulator -export SERVICENODE_CPU_CORES=0 -export POSTGRES_CPU_CORES=1-3 -export GF_SECURITY_ADMIN_USER=admin -export GF_SECURITY_ADMIN_PASSWORD=admin -docker compose up -d -while [ "$(docker inspect --format "{{.State.Status}}" $(docker compose ps -q servicenode))" != "running" ]; do - sleep 1 -done -cd .. - -cd ./sonda -docker build -t local-perf-sonda -f ./Dockerfile.sonda . -cat > ./perf-test.env </dev/null 2>&1; then + wn() { docker compose "$@"; } +fi +# - waku-simulator: 15-node network, 1 service, 1 edge +# - Service node limited by env below (CPU cores 0-3; memory 512 MiB) +# - Phases driven by LPT; Sonda runs throughout to exercise Store +# - This variant stresses CPU more and uses 120s observe windows + +echo " Running test..." + +# -------------------- Bring up simulator ------------------------ +cd ./waku-simulator + +export NWAKU_IMAGE=wakuorg/nwaku:latest +export NUM_NWAKU_NODES=15 +export RLN_ENABLED=false + +export SERVICENODE_CPU_CORES="0-3" +export SERVICENODE_MEM_LIMIT=2g +export POSTGRES_CPU_CORES="0-3" +export POSTGRES_MEM_LIMIT=2g +export POSTGRES_SHM=1g + +docker compose up -d + +# Wait until service node is running +while true; do + sid="$(docker compose ps -q servicenode || true)" + if [[ -n "$sid" ]]; then + state="$(docker inspect --format '{{.State.Status}}' "$sid" 2>/dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd ../lpt + +# -------------------- LPT common knobs (same exports) ------------------------- +export LPT_IMAGE=harbor.status.im/wakuorg/liteprotocoltester:latest +export START_PUBLISHING_AFTER=15 +export NUM_MESSAGES=0 +export MESSAGE_INTERVAL_MILLIS=100 + +export MIN_MESSAGE_SIZE=120Kb +export MAX_MESSAGE_SIZE=145Kb + +export LIGHTPUSH_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export FILTER_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n + +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim +export CLUSTER_ID=66 + +# wait time before starting traffic +sleep 60 + + +# -------------------- Sonda (Store monitor) ----------------------------------- +cd ../sonda + +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +# perf-test.env +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +QUERY_DELAY=0.5 +STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +CLUSTER_ID=66 +SHARD=0 +EOF + +sleep 5 + +docker rm -f sonda >/dev/null 2>&1 || true +docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda + +cd ../lpt + +# -------------------- Phase 1: 6 pub / 6 recv -------------------------------- +export NUM_PUBLISHER_NODES=6 +export NUM_RECEIVER_NODES=6 +docker compose down -v >/dev/null 2>&1 || true +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] LPT is running with 6 publishers and 6 receivers + sonda from now: $current_time" + +sleep 120 + +# -------------------- Phase 2: 3 pub / 12 recv ------------------------------- +docker compose down -v +export NUM_PUBLISHER_NODES=3 +export NUM_RECEIVER_NODES=12 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] LPT is running with 3 publishers and 12 receivers from now: $current_time" + +sleep 120 + +# -------------------- Phase 3: 12 pub / 3 recv ------------------------------- +docker compose down -v +export NUM_PUBLISHER_NODES=12 +export NUM_RECEIVER_NODES=3 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] LPT is running with 12 publishers and 3 receivers from now: $current_time" + +sleep 120 + +# -------------------- Phase 4: receivers down; keep publisher + sonda -------- +docker compose down -v +export NUM_PUBLISHER_NODES=12 +export NUM_RECEIVER_NODES=0 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] LPT receivers are down; sonda and lightpush publisher running from now: $current_time" + +sleep 120 + +# -------------------- Phase 5: LPT down; only sonda -------------------------- +docker compose down -v + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] LPT down; only sonda is working from now: $current_time" + +sleep 120 + +# -------------------- Phase 6: final high-load burst ----------------- + +export NUM_PUBLISHER_NODES=12 +export NUM_RECEIVER_NODES=12 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] Final burst: LPT running with 12 publishers and 12 receivers from now: $current_time" + +sleep 120 + +cd .. + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[test] Test finished at $current_time" + +# finish +# exec ./stop_test.sh diff --git a/scripts/Lite_protocol_scripts/store/nodes_15_service_1.sh b/scripts/Lite_protocol_scripts/store/nodes_15_service_1.sh new file mode 100755 index 000000000..60b30633d --- /dev/null +++ b/scripts/Lite_protocol_scripts/store/nodes_15_service_1.sh @@ -0,0 +1,140 @@ +#!/bin/bash +set -e + +# - waku-simulator: 15-node network, 1 service, 1 edge +# - Service node limited by env below (CPU cores 0-3; memory 512 MiB) +# - Phases driven by LPT; Sonda runs throughout to exercise Store +# - This variant stresses CPU more and uses 120s observe windows + +echo "Running test..." + +# -------------------- Bring up simulator ------------------------ +cd ./waku-simulator + +export NWAKU_IMAGE=wakuorg/nwaku:latest +export NUM_NWAKU_NODES=15 +export RLN_ENABLED=false + +export SERVICENODE_CPU_CORES="0-3" +export SERVICENODE_MEM_LIMIT=512m +export POSTGRES_CPU_CORES="0-3" +export POSTGRES_MEM_LIMIT=2g +export POSTGRES_SHM=1g + +docker compose up -d + +# Wait until service node is running +while true; do + sid="$(docker compose ps -q servicenode || true)" + if [[ -n "$sid" ]]; then + state="$(docker inspect --format '{{.State.Status}}' "$sid" 2>/dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd ../lpt + +# -------------------- LPT common knobs ------------------------- +export LPT_IMAGE=harbor.status.im/wakuorg/liteprotocoltester:latest +export START_PUBLISHING_AFTER=15 +export NUM_MESSAGES=0 +export MESSAGE_INTERVAL_MILLIS=100 + +export MIN_MESSAGE_SIZE=120Kb +export MAX_MESSAGE_SIZE=145Kb + +export LIGHTPUSH_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export FILTER_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n + +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim +export CLUSTER_ID=66 + +# wait time before starting traffic +sleep 60 + +# Start LPT for Phase 1 will happen after Sonda is up + +# -------------------- Sonda (Store monitor) ----------------------------------- +cd ../sonda + +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +# perf-test.env +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +QUERY_DELAY=0.5 +STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +CLUSTER_ID=66 +SHARD=0 +EOF + +sleep 5 + +docker rm -f sonda >/dev/null 2>&1 || true +docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda + +cd ../lpt + +# -------------------- Phase 1: 6 pub / 6 recv -------------------------------- +export NUM_PUBLISHER_NODES=6 +export NUM_RECEIVER_NODES=6 +docker compose down -v >/dev/null 2>&1 || true +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "LPT is running with 6 publishers and 6 receivers + sonda from now: $current_time" + +sleep 120 + +# -------------------- Phase 2: 3 pub / 12 recv ------------------------------- +docker compose down -v +export NUM_PUBLISHER_NODES=3 +export NUM_RECEIVER_NODES=12 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "LPT is running with 3 publishers and 12 receivers from now: $current_time" + +sleep 120 + +# -------------------- Phase 3: 12 pub / 3 recv ------------------------------- +docker compose down -v +export NUM_PUBLISHER_NODES=12 +export NUM_RECEIVER_NODES=3 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "LPT is running with 12 publishers and 3 receivers from now: $current_time" + +sleep 120 + +# -------------------- Phase 4: receivers down; keep publisher + sonda -------- +docker compose down -v +export NUM_PUBLISHER_NODES=12 +export NUM_RECEIVER_NODES=0 +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "LPT receivers are down; sonda and lightpush publisher running from now: $current_time" + +sleep 120 + +# -------------------- Phase 5: LPT down; only sonda -------------------------- +docker compose down -v + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "LPT down; only sonda is working from now: $current_time" + +sleep 120 + +cd .. + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "Test finished at $current_time" + +# finish +# exec ./stop_test.sh +s \ No newline at end of file diff --git a/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-26-55.png b/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-26-55.png new file mode 100644 index 000000000..ad54f6867 Binary files /dev/null and b/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-26-55.png differ diff --git a/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-27-31.png b/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-27-31.png new file mode 100644 index 000000000..8fa9b90d1 Binary files /dev/null and b/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-27-31.png differ diff --git a/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-28-14.png b/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-28-14.png new file mode 100644 index 000000000..eb7a6e199 Binary files /dev/null and b/scripts/Lite_protocol_scripts/store/reports/store_kpi_simple/Screenshot from 2025-10-01 15-28-14.png differ diff --git a/scripts/Lite_protocol_scripts/store/store_15nodes_scenario1.sh b/scripts/Lite_protocol_scripts/store/store_15nodes_scenario1.sh deleted file mode 100644 index b3360cb81..000000000 --- a/scripts/Lite_protocol_scripts/store/store_15nodes_scenario1.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -set -e - -# This implements a waku-simulator with 15 nodes network, 1 service, 1 edge node added -# service node is limited to 1 cpu core with only 512 MB -# Runs different test phases with different load from sonda (STORE) side -# 1. 2 sonda instances, query every 500ms -# 2. 5 sonda instances, query every 200ms -# 3. 10 sonda instances, query every 100ms - -echo "Running test..." - -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Bringing up simulator at $current_time" - -cd ./waku-simulator - -# simulator exports -export NUM_NWAKU_NODES=15 -export TRAFFIC_DELAY_SECONDS=15 -export MSG_SIZE_KBYTES=10 -export SERVICENODE_CPU_CORES="0-3" -export POSTGRES_CPU_CORES="0-3" - -docker compose up -d -cd .. - -echo "Waiting 30s for service node to be ready..." -sleep 30 - -cd ./sonda - -# build sonda image -docker build -t local-perf-sonda -f Dockerfile.sonda . - -# 2 sondas 500ms - -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.5 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 1 running 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -#5 sondas 200ms - -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.2 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 2 running 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -#10 sondas 100ms -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.1 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 3 running 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -cd .. - -cd ./waku-simulator -docker compose down -cd .. - -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Test finished at $current_time" diff --git a/scripts/Lite_protocol_scripts/store/store_15nodes_scenario2.sh b/scripts/Lite_protocol_scripts/store/store_15nodes_scenario2.sh deleted file mode 100644 index ae7a59892..000000000 --- a/scripts/Lite_protocol_scripts/store/store_15nodes_scenario2.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -set -e - -echo "Running test..." - -# This implements a short version of 1st store scenario. -# waku-simulator with 15 nodes network, 1 service, 1 edge node added -# service node is limited to 1 cpu core with only 512 MB -# Runs 1 phase with sonda load: -# 1. 10 sonda instances, each queries in every 100ms - -cd ./waku-simulator - -export NUM_NWAKU_NODES=15 -export TRAFFIC_DELAY_SECONDS=10 -export MSG_SIZE_KBYTES=8 - -docker compose up -d -cd .. - -echo "Waiting 20s for service node..." -sleep 20 - -cd ./sonda - -docker build -t local-perf-sonda -f Dockerfile.sonda . - -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.1 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase running 300s..." -sleep 300 - -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -cd .. - -cd ./waku-simulator -docker compose down -cd .. diff --git a/scripts/Lite_protocol_scripts/store/store_15nodes_stress_scenario.sh b/scripts/Lite_protocol_scripts/store/store_15nodes_stress_scenario.sh deleted file mode 100644 index f2a685213..000000000 --- a/scripts/Lite_protocol_scripts/store/store_15nodes_stress_scenario.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash - - -# waku-simulator with 15 nodes network, 1 service, 1 edge node added -# service node is limited to 1 cpu core with only 512 MB -# This scenario intended to stress test service node STORE queries with high request frequency -# Increasing number of sonda instances and decreasing query delay ms -# 16 sondas 100ms -# 16 sondas 50ms -# 24 sondas 50ms -# 24 sondas 20ms - -set -e - -echo "Running test..." - -cd ./waku-simulator - -export NUM_NWAKU_NODES=15 -export TRAFFIC_DELAY_SECONDS=10 -export MSG_SIZE_KBYTES=12 - -docker compose up -d -cd .. - -echo "Waiting 30s for service node..." -sleep 30 - -cd ./sonda - -docker build -t local-perf-sonda -f Dockerfile.sonda . - -#16 sondas 100ms -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.1 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 1 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -# 16 sondas 50ms -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.05 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 2 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -# 24 sondas 50ms -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.05 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 3 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -#24 sondas 20ms -cat < perf-test.env -RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 -STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 -QUERY_DELAY=0.02 -STORE_NODES=/ip4/127.0.0.1/tcp/60001/p2p/ -CLUSTER_ID=66 -SHARD=0 -HEALTH_THRESHOLD=0.95 -EOF - -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -# repeat 23 more times -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda -docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda - -echo "Phase 4 300s..." -sleep 300 -docker kill $(docker ps -q -f "label=sonda") >/dev/null 2>&1 || true - -cd .. - -cd ./waku-simulator -docker compose down -cd .. diff --git a/scripts/Lite_protocol_scripts/store/store_KPI_simple.sh b/scripts/Lite_protocol_scripts/store/store_KPI_simple.sh new file mode 100755 index 000000000..5d53e3f01 --- /dev/null +++ b/scripts/Lite_protocol_scripts/store/store_KPI_simple.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -e + +echo " Running test (store_kpi with LPT publishers)…" + +# -------- Bring up simulator -------- +cd ./waku-simulator + +export NWAKU_IMAGE=wakuorg/nwaku:latest +export NUM_NWAKU_NODES=15 +export RLN_ENABLED=false + +export SERVICENODE_CPU_CORES="0-3" +export SERVICENODE_MEM_LIMIT=2g +export POSTGRES_CPU_CORES="0-3" +export POSTGRES_MEM_LIMIT=2g +export POSTGRES_SHM=1g + +docker compose up -d + +# wait for servicenode +while true; do + sid="$(docker compose ps -q servicenode || true)" + if [[ -n "$sid" ]]; then + state="$(docker inspect --format '{{.State.Status}}' "$sid" 2>/dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd .. + +# -------- Shared exports -------- +export RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +export STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +export STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export QUERY_DELAY=0.25 +export CLUSTER_ID=66 +export SHARD=0 +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim + +cd ./lpt + +export LPT_IMAGE=harbor.status.im/wakuorg/liteprotocoltester:latest + +export NUM_PUBLISHER_NODES=5 +export NUM_RECEIVER_NODES=5 +export START_PUBLISHING_AFTER=15 +export NUM_MESSAGES=0 +export MESSAGE_INTERVAL_MILLIS=100 +export MIN_MESSAGE_SIZE=120Kb +export MAX_MESSAGE_SIZE=145Kb + + +export LIGHTPUSH_SERVICE_PEER="$STORE_NODES" +export FILTER_SERVICE_PEER="$STORE_NODES" +export PUBSUB +export CONTENT_TOPIC +export CLUSTER_ID + +docker compose up -d + +cd .. + +cd ./sonda +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=${RELAY_NODE_REST_ADDRESS} +STORE_NODE_REST_ADDRESS=${STORE_NODE_REST_ADDRESS} +QUERY_DELAY=${QUERY_DELAY} +STORE_NODES=${STORE_NODES} +CLUSTER_ID=${CLUSTER_ID} +SHARD=${SHARD} +EOF + +sleep 5 +docker rm -f sonda >/dev/null 2>&1 || true +docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda +cd .. + +echo "[store_kpi] warmup 60s to build history…" +sleep 60 + +# -------- Store KPI (10-minute query loop, in parallel with LPT) -------- +URL="${STORE_NODE_REST_ADDRESS}/store/v3/messages" +PEERADDR="${STORE_NODES%%,*}" + +burst_once() { + local now start par + now=$(( $(date +%s) * 1000 )) + start=$(( now - 600 * 1000 )) # 10-minute window + par=20 + echo "[burst] 10m window, ${par} parallel → ${URL}" + for i in $(seq 1 $par); do + curl -s --get "$URL" \ + --data-urlencode "peerAddr=$PEERADDR" \ + --data-urlencode "pubsubTopic=$PUBSUB" \ + --data-urlencode "contentTopics=$CONTENT_TOPIC" \ + --data-urlencode "includeData=true" \ + --data-urlencode "startTime=$start" \ + > /dev/null & + done + wait +} + +RUN_MINUTES=10 +END_TS=$(( $(date +%s) + RUN_MINUTES*60 )) +iter=0 +echo "[store_kpi] querying for ${RUN_MINUTES} minutes while LPT publishes…" +while (( $(date +%s) < END_TS )); do + iter=$((iter+1)) + burst_once + echo "[burst] iter=$iter done; sleeping 10s" + sleep 10 +done + +# -------- Tidy -------- +echo "[store_kpi] 10min run complete. Stopping LPT stack…" +( cd ./lpt && docker compose down ) diff --git a/scripts/Lite_protocol_scripts/store/store_cpu_stress.sh b/scripts/Lite_protocol_scripts/store/store_cpu_stress.sh new file mode 100755 index 000000000..299a92633 --- /dev/null +++ b/scripts/Lite_protocol_scripts/store/store_cpu_stress.sh @@ -0,0 +1,107 @@ +#!/bin/bash +set -e + +export SCENARIO_ID="cpu_store_stress" +export TEST_NAME="${SCENARIO_ID}_$(date +%Y%m%d_%H%M%S)" +echo "[${TEST_NAME}] start" + +export GF_SECURITY_ADMIN_USER="admin" +export GF_SECURITY_ADMIN_PASSWORD="admin" +export NODEKEY="${NODEKEY:-}" +export STORAGE_SIZE="${STORAGE_SIZE:-}" + +export NWAKU_IMAGE="wakuorg/nwaku:latest" +export NUM_NWAKU_NODES=15 +export RLN_ENABLED=false + +# Service node +export SERVICENODE_METRICS_PORT=8008 +export SERVICENODE_HTTP_PORT=8644 +export SERVICENODE_REST_PORT=8645 + +export POSTGRES_EXPORTER_PORT=9187 +export PROMETHEUS_PORT=9090 +export GRAFANA_PORT=3001 + +export SERVICENODE_CPU_CORES="0-1" +export SERVICENODE_MEM_LIMIT=512m +export POSTGRES_CPU_CORES="0-3" +export POSTGRES_MEM_LIMIT=2g +export POSTGRES_SHM=1g + +export LPT_IMAGE="harbor.status.im/wakuorg/liteprotocoltester:latest" + +# Topic / shard +export CLUSTER_ID=66 +export SHARD=0 +export PUBSUB_TOPIC="/waku/2/rs/${CLUSTER_ID}/${SHARD}" + +export RELAY_NODE_REST_ADDRESS="http://127.0.0.1:${SERVICENODE_REST_PORT}" +export STORE_NODE_REST_ADDRESS="http://127.0.0.1:${SERVICENODE_HTTP_PORT}" + +# Phase 0 — bring up simulator +echo "[${TEST_NAME}] Phase 0: bring up simulator" +cd ./waku-simulator +docker compose up -d +cd .. +echo "[${TEST_NAME}] wait 30s" +sleep 30 + +# ========================= +# Phase 1 — CPU-heavy writers (small msgs, high rate) +# ========================= +export NUM_PUBLISHER_NODES=24 +export NUM_RECEIVER_NODES=8 +export MESSAGE_INTERVAL_MILLIS=10 +export MIN_MESSAGE_SIZE=256 +export MAX_MESSAGE_SIZE=1024 +export START_PUBLISHING_AFTER=10 +export NUM_MESSAGES=0 + +echo "[${TEST_NAME}] Phase 1: start writers" +docker run -d --rm --name lpt_cpu \ + -e PUB_NODES=${NUM_PUBLISHER_NODES} \ + -e RCV_NODES=${NUM_RECEIVER_NODES} \ + -e MSG_INTERVAL_MS=${MESSAGE_INTERVAL_MILLIS} \ + -e MIN_MSG=${MIN_MESSAGE_SIZE} \ + -e MAX_MSG=${MAX_MESSAGE_SIZE} \ + -e PUBSUB_TOPIC=${PUBSUB_TOPIC} \ + --network host ${LPT_IMAGE} + +# ========================= +# Phase 2 +# ========================= +echo "[${TEST_NAME}] Phase 2: build and start Sonda" +cd ./sonda +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=${RELAY_NODE_REST_ADDRESS} +STORE_NODE_REST_ADDRESS=${STORE_NODE_REST_ADDRESS} +QUERY_DELAY=0.5 +STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +CLUSTER_ID=${CLUSTER_ID} +SHARD=${SHARD} +PUBSUB_TOPIC=/waku/2/rs/${CLUSTER_ID}/${SHARD} +EOF + +docker run --env-file perf-test.env -l sonda -d --network host local-perf-sonda +cd .. + +echo "[${TEST_NAME}] hold 240s" +sleep 240 + +# ========================= +# Phase 3 — recovery +# ========================= +echo "[${TEST_NAME}] Phase 3: stop writers; observe 120s" +docker kill lpt_cpu || true +sleep 120 + +# ========================= +# Cleanup +# ========================= +echo "[${TEST_NAME}] cleanup" +docker kill $(docker ps -q --filter "label=sonda") || true +cd ./waku-simulator +docker compose down -v diff --git a/scripts/Lite_protocol_scripts/store/store_different_payloads.sh b/scripts/Lite_protocol_scripts/store/store_different_payloads.sh deleted file mode 100644 index 6b2e5985d..000000000 --- a/scripts/Lite_protocol_scripts/store/store_different_payloads.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Scenario 6: Alternate tiny and huge payloads at fast rates to stress allocators and CPU. -set -e - -STORE_NODES="/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n" -RELAY_NODE_REST_ADDRESS="http://127.0.0.1:8645" -STORE_NODE_REST_ADDRESS="http://127.0.0.1:8644" -PUBSUB_TOPIC="/waku/2/default-waku/proto" -CONTENT_TOPIC="/sonda/2/polls/proto" -PHASE_SLEEP=240 - -echo "Running test..." -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Bringing up simulator at $current_time" - -cd ./waku-simulator -export SERVICENODE_CPU_CORES=0 -export POSTGRES_CPU_CORES=1-3 -export GF_SECURITY_ADMIN_USER=admin -export GF_SECURITY_ADMIN_PASSWORD=admin -docker compose up -d -while [ "$(docker inspect --format "{{.State.Status}}" $(docker compose ps -q servicenode))" != "running" ]; do - sleep 1 -done -cd .. - -cd ./sonda -docker build -t local-perf-sonda -f ./Dockerfile.sonda . -cat > ./perf-test.env </dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd .. + +export RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +export STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +export STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export QUERY_DELAY=0.5 +export CLUSTER_ID=66 +export SHARD=0 +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim + +cd ./sonda +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=${RELAY_NODE_REST_ADDRESS} +STORE_NODE_REST_ADDRESS=${STORE_NODE_REST_ADDRESS} +QUERY_DELAY=${QUERY_DELAY} +STORE_NODES=${STORE_NODES} +CLUSTER_ID=${CLUSTER_ID} +SHARD=${SHARD} +EOF + +sleep 5 +docker rm -f sonda >/dev/null 2>&1 || true +docker run --env-file ./perf-test.env -l sonda -d --network host local-perf-sonda +cd .. + +echo "[store_kpi_smallpage] warmup 60s to build history…" +sleep 60 + +URL="${STORE_NODE_REST_ADDRESS}/store/v3/messages" +PEERADDR="${STORE_NODES%%,*}" # fixed to first entry + +burst_once() { + local now start + now=$(( $(date +%s) * 1000 )) + start=$(( now - 600 * 1000 )) # 10-minute window + local par=20 + echo "[burst-smallpage] 10m window, ${par} parallel → ${URL}" + for i in $(seq 1 $par); do + curl -s --get "$URL" \ + --data-urlencode "peerAddr=$PEERADDR" \ + --data-urlencode "pubsubTopic=$PUBSUB" \ + --data-urlencode "contentTopics=$CONTENT_TOPIC" \ + --data-urlencode "includeData=true" \ + --data-urlencode "pageSize=5" \ + --data-urlencode "startTime=$start" \ + > /dev/null & + done + wait +} + +# run bursts for a full 10 minutes +RUN_MINUTES=10 +END_TS=$(( $(date +%s) + RUN_MINUTES*60 )) +iter=0 +echo "[store_kpi_smallpage] querying for ${RUN_MINUTES} minutes…" +while (( $(date +%s) < END_TS )); do + iter=$((iter+1)) + burst_once + echo "[burst-smallpage] iter=$iter done; sleeping 10s" + sleep 10 +done + diff --git a/scripts/Lite_protocol_scripts/store/store_multi-topic.sh b/scripts/Lite_protocol_scripts/store/store_multi-topic.sh deleted file mode 100644 index 693680b89..000000000 --- a/scripts/Lite_protocol_scripts/store/store_multi-topic.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -# Scenario 4: Run Sonda on two pubsub topics (and shards) in parallel while LPT publishes/receives in phases. -set -e - -STORE_NODES="/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n" -RELAY_NODE_REST_ADDRESS="http://127.0.0.1:8645" -STORE_NODE_REST_ADDRESS="http://127.0.0.1:8644" -PUBSUB_TOPIC_A="/waku/2/default-waku/proto" -PUBSUB_TOPIC_B="/waku/2/extra-waku/proto" -CONTENT_TOPIC="/sonda/2/polls/proto" -PHASE_SLEEP=300 - -echo "Running test..." -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Bringing up simulator at $current_time" - -cd ./waku-simulator -export SERVICENODE_CPU_CORES=0 -export POSTGRES_CPU_CORES=1-3 -export GF_SECURITY_ADMIN_USER=admin -export GF_SECURITY_ADMIN_PASSWORD=admin -docker compose up -d -while [ "$(docker inspect --format "{{.State.Status}}" $(docker compose ps -q servicenode))" != "running" ]; do - sleep 1 -done -cd .. - -cd ./sonda -docker build -t local-perf-sonda -f ./Dockerfile.sonda . - -cat > ./perf-topic-a.env < ./perf-topic-b.env < 1x5 -> 5x1 -> receivers down -> lpt down -# ------------------------------------------------------------------------------ - -# >>> EDIT THESE IF YOUR SETUP DIFFERS <<< -STORE_NODES="/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n" -RELAY_NODE_REST_ADDRESS="http://127.0.0.1:8645" -STORE_NODE_REST_ADDRESS="http://127.0.0.1:8644" -PUBSUB_TOPIC="/waku/2/default-waku/proto" -CONTENT_TOPIC="/sonda/2/polls/proto" -# <<< EDIT ABOVE >>> - -echo "Running test..." - -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Bringing up simulator at $current_time" - -cd ./waku-simulator - -# same style: exports before compose -export SERVICENODE_CPU_CORES=0 # 1 core for service node -export POSTGRES_CPU_CORES=1-3 # keep DB off core 0 -export GF_SECURITY_ADMIN_USER=admin -export GF_SECURITY_ADMIN_PASSWORD=admin -export NWAKU_IMAGE=wakuorg/nwaku:latest -export NUM_NWAKU_NODES=15 -export RLN_ENABLED=false - -docker compose up -d - -# Wait for servicenode to be running (scenario_1.sh style) -while [ "$(docker inspect --format "{{.State.Status}}" $(docker compose ps -q servicenode))" != "running" ]; do - sleep 1 -done - -cd .. - -# ------------------------------------------------------------------------------ -# Start Sonda (like scenario_1.sh: build & run with env-file, host network) -# ------------------------------------------------------------------------------ -cd ./sonda - -docker build -t local-perf-sonda -f ./Dockerfile.sonda . - -cat > ./perf-test.env < per-message CPU (encode/verify/route) dominates. +export NUM_PUBLISHER_NODES=24 +export NUM_RECEIVER_NODES=8 +export MESSAGE_INTERVAL_MILLIS=8 +export MIN_MESSAGE_SIZE=256 +export MAX_MESSAGE_SIZE=1024 +export START_PUBLISHING_AFTER=10 +export NUM_MESSAGES=0 + +echo "[${TEST_NAME}] phase1 writers: ${NUM_PUBLISHER_NODES} pubs @ ${MESSAGE_INTERVAL_MILLIS}ms, 256-1024B" +docker run -d --rm --name lpt_cpu \ + -e PUB_NODES=${NUM_PUBLISHER_NODES} \ + -e RCV_NODES=${NUM_RECEIVER_NODES} \ + -e MSG_INTERVAL_MS=${MESSAGE_INTERVAL_MILLIS} \ + -e MIN_MSG=${MIN_MESSAGE_SIZE} \ + -e MAX_MSG=${MAX_MESSAGE_SIZE} \ + -e PUBSUB_TOPIC=${PUBSUB_TOPIC} \ + --network host ${LPT_IMAGE} + +# ---------- Phase 2: Store read hammer (concurrent readers) ---------- +# Mix includeData true/false and page sizes to exercise CPU (serialization/JSON) & DB. +echo "[${TEST_NAME}] phase2 store readers" +docker run -d --rm --name sonda_idx \ + --network host ${SONDA_IMAGE} \ + --relay-node-rest-address "${RELAY_NODE_REST_ADDRESS}" \ + --store-node-rest-address "${STORE_NODE_REST_ADDRESS}" \ + --pubsub-topic "${PUBSUB_TOPIC}" \ + --store-nodes "${STORE_NODES}" \ + --delay-seconds 0.07 --health-threshold ${HEALTH_THRESHOLD} \ + --metrics-port 8004 --include-data=false --page-size 150 + +docker run -d --rm --name sonda_smallpages \ + --network host ${SONDA_IMAGE} \ + --relay-node-rest-address "${RELAY_NODE_REST_ADDRESS}" \ + --store-node-rest-address "${STORE_NODE_REST_ADDRESS}" \ + --pubsub-topic "${PUBSUB_TOPIC}" \ + --store-nodes "${STORE_NODES}" \ + --delay-seconds 0.05 --health-threshold ${HEALTH_THRESHOLD} \ + --metrics-port 8005 --include-data=true --page-size 5 + +docker run -d --rm --name sonda_bigpages \ + --network host ${SONDA_IMAGE} \ + --relay-node-rest-address "${RELAY_NODE_REST_ADDRESS}" \ + --store-node-rest-address "${STORE_NODE_REST_ADDRESS}" \ + --pubsub-topic "${PUBSUB_TOPIC}" \ + --store-nodes "${STORE_NODES}" \ + --delay-seconds 0.05 --health-threshold ${HEALTH_THRESHOLD} \ + --metrics-port 8006 --include-data=true --page-size 50 + +# Extra readers to push CPU on the service process: +docker run -d --rm --name sonda_mix1 \ + --network host ${SONDA_IMAGE} \ + --relay-node-rest-address "${RELAY_NODE_REST_ADDRESS}" \ + --store-node-rest-address "${STORE_NODE_REST_ADDRESS}" \ + --pubsub-topic "${PUBSUB_TOPIC}" \ + --store-nodes "${STORE_NODES}" \ + --delay-seconds 0.03 --health-threshold ${HEALTH_THRESHOLD} \ + --metrics-port 8007 --include-data=true --page-size 25 + +docker run -d --rm --name sonda_mix2 \ + --network host ${SONDA_IMAGE} \ + --relay-node-rest-address "${RELAY_NODE_REST_ADDRESS}" \ + --store-node-rest-address "${STORE_NODE_REST_ADDRESS}" \ + --pubsub-topic "${PUBSUB_TOPIC}" \ + --store-nodes "${STORE_NODES}" \ + --delay-seconds 0.03 --health-threshold ${HEALTH_THRESHOLD} \ + --metrics-port 8008 --include-data=false --page-size 200 + +echo "[${TEST_NAME}] hold 12m" +sleep 720 + +# ---------- Phase 3: plateau & recovery ---------- +echo "[${TEST_NAME}] phase3 stop writers; keep readers 2m (recovery CPU/GC)" +docker kill lpt_cpu || true +sleep 120 + +# ---------- Cleanup ---------- +echo "[${TEST_NAME}] cleanup" +docker kill sonda_idx sonda_smallpages sonda_bigpages sonda_mix1 sonda_mix2 || true +cd ./waku-simulator +docker compose down -v diff --git a/scripts/Lite_protocol_scripts/store/store_stress1.sh b/scripts/Lite_protocol_scripts/store/store_stress1.sh deleted file mode 100644 index ddee3d1e3..000000000 --- a/scripts/Lite_protocol_scripts/store/store_stress1.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -set -e - -STORE_NODES="/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n,/ip4/10.2.0.102/tcp/60001/p2p/16Uiu2HA7abcDEF451tGkbzz4Mjcg6DRnmAHxNeWyF4zp23RbpXYZ2" -RELAY_NODE_REST_ADDRESS="http://127.0.0.1:8645" -STORE_NODE_REST_ADDRESS="http://127.0.0.1:8644" -PUBSUB_TOPIC="/waku/2/default-waku/proto" -CONTENT_TOPIC="/sonda/2/polls/proto" -PHASE_SLEEP=300 - -echo "Running test..." -current_time=$(date +"%Y-%m-%d %H:%M:%S") -echo "Bringing up simulator at $current_time" - -cd ./waku-simulator -export SERVICENODE_CPU_CORES=0 -export POSTGRES_CPU_CORES=1-3 -export GF_SECURITY_ADMIN_USER=admin -export GF_SECURITY_ADMIN_PASSWORD=admin -docker compose up -d -while [ "$(docker inspect --format "{{.State.Status}}" $(docker compose ps -q servicenode))" != "running" ]; do - sleep 1 -done -cd .. - -cd ./sonda -docker build -t local-perf-sonda -f ./Dockerfile.sonda . -cat > ./perf-test.env < ./perf-test.env < ./perf-test.env < ./perf-test.env </dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd ../lpt + +# -------------------- LPT config --------------------------- +export LPT_IMAGE=harbor.status.im/wakuorg/liteprotocoltester:latest +export START_PUBLISHING_AFTER=15 +export NUM_MESSAGES=0 +export MESSAGE_INTERVAL_MILLIS=100 + +# Service peers +export LIGHTPUSH_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export FILTER_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n + +# Topics +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim +export CLUSTER_ID=66 + +# sleep before traffic +sleep 60 + +cd ../sonda +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +QUERY_DELAY=0.5 +STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +CLUSTER_ID=66 +SHARD=0 +EOF + +sleep 5 + +# Clean up +docker rm -f sonda1 sonda2 sonda3 sonda4 sonda5 >/dev/null 2>&1 || true + +# Start a baseline Sonda (0.5s) +docker run -d --name sonda1 --network host -l sonda \ + --env-file ./perf-test.env local-perf-sonda + +cd ../lpt + +# ============================================================================ # +# PHASE 1: Moderate traffic +# LPT 6/6, 50–90 KB; Sonda x1 time 0.5s +# ============================================================================ # +export NUM_PUBLISHER_NODES=6 +export NUM_RECEIVER_NODES=6 +export MIN_MESSAGE_SIZE=50Kb +export MAX_MESSAGE_SIZE=90Kb + +docker compose down -v >/dev/null 2>&1 || true +docker compose up -d + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[scenario_6_store_stress] Phase 1: LPT 6/6, 50–90KB; Sonda x1 @0.5s — $current_time" +sleep 120 + +# ============================================================================ # +# PHASE 2: Same traffic but triple query rate +# Keep LPT, add Sonda x2 more time 0.1s +# ============================================================================ # +docker run -d --name sonda2 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.1 local-perf-sonda +docker run -d --name sonda3 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.1 local-perf-sonda + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[scenario_6_store_stress] Phase 2: add Sonda x2 @0.1s (total 3) — $current_time" +sleep 120 + +# ============================================================================ # +# PHASE 3: Heavier traffic + peak query rate +# LPT 10/10, 130–149 KB; add Sonda x2 more time 0.05s +# ============================================================================ # +docker compose down -v +export NUM_PUBLISHER_NODES=10 +export NUM_RECEIVER_NODES=10 +export MIN_MESSAGE_SIZE=130Kb +export MAX_MESSAGE_SIZE=149Kb +docker compose up -d + +docker run -d --name sonda4 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.05 local-perf-sonda +docker run -d --name sonda5 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.05 local-perf-sonda + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[scenario_6_store_stress] Phase 3: LPT 10/10, 130–149KB; Sonda total x5 (0.5s + 0.1s + 0.05s) — $current_time" +sleep 120 + +# ============================================================================ # +# PHASE 4: Store-only +# Stop LPT; keep all Sonda x5 times +# ============================================================================ # +docker compose down -v + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[scenario_6_store_stress] Phase 4: LPT down; Store-only flood via Sonda x5 — $current_time" +sleep 120 + +# -------------------- Cleanup ------------------------------------------------- +docker rm -f sonda1 sonda2 sonda3 sonda4 sonda5 >/dev/null 2>&1 || true + +cd .. + +current_time=$(date +"%Y-%m-%d %H:%M:%S") +echo "[scenario_6_store_stress] Test finished at $current_time" + +# finish +# exec ./stop_test.sh diff --git a/scripts/Lite_protocol_scripts/store/stress_alternate_high_concurrency_scenario3 b/scripts/Lite_protocol_scripts/store/stress_alternate_high_concurrency_scenario3 new file mode 100755 index 000000000..51712c906 --- /dev/null +++ b/scripts/Lite_protocol_scripts/store/stress_alternate_high_concurrency_scenario3 @@ -0,0 +1,208 @@ +#!/bin/bash +set -e + +cd ./waku-simulator + +export NWAKU_IMAGE=wakuorg/nwaku:latest +export NUM_NWAKU_NODES=15 +export RLN_ENABLED=false + +# Service node config +export SERVICENODE_CPU_CORES="0-3" +export SERVICENODE_MEM_LIMIT=2g +export POSTGRES_CPU_CORES="0-3" +export POSTGRES_MEM_LIMIT=2g +export POSTGRES_SHM=1g + +docker compose up -d + +# Wait until service node is running +while true; do + sid="$(docker compose ps -q servicenode || true)" + if [[ -n "$sid" ]]; then + state="$(docker inspect --format '{{.State.Status}}' "$sid" 2>/dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd ../lpt + +# -------------------- LPT config --------------------------- +export LPT_IMAGE=harbor.status.im/wakuorg/liteprotocoltester:latest +export START_PUBLISHING_AFTER=15 +export NUM_MESSAGES=0 +export MESSAGE_INTERVAL_MILLIS=100 + +# Service peers +export LIGHTPUSH_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export FILTER_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n + +# Topics +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim +export CLUSTER_ID=66 + +# Allow services to settle +sleep 60 + +cd ../sonda +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +QUERY_DELAY=0.5 +STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +CLUSTER_ID=66 +SHARD=0 +EOF + +sleep 5 + +# Clean old +docker rm -f sonda1 sonda2 sonda3 sonda4 sonda5 sonda6 sonda7 sonda8 sonda9 sonda10 sonda11 sonda12 >/dev/null 2>&1 || true + +# Baseline Sonda (0.5s) +docker run -d --name sonda1 --network host -l sonda \ + --env-file ./perf-test.env local-perf-sonda + +cd ../lpt + +# Helper echo +ts() { date '+%F %T'; } + +# ============================================================================ # +# PHASE A: Wave burst #1 (short & intense), then cooldown +# Idea: quick history fill + heavy query spike +# ============================================================================ # +export NUM_PUBLISHER_NODES=8 +export NUM_RECEIVER_NODES=8 +export MIN_MESSAGE_SIZE=80Kb +export MAX_MESSAGE_SIZE=120Kb + +docker compose down -v >/dev/null 2>&1 || true +docker compose up -d +echo " Phase A1: LPT 8/8, 80–120KB; Sonda x1 @0.5s — $(ts)" + +# Add two fast Sonda for burst @0.05s +docker run -d --name sonda2 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.05 local-perf-sonda +docker run -d --name sonda3 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.05 local-perf-sonda + +sleep 90 + +# Cooldown: remove the two fast ones +docker rm -f sonda2 sonda3 >/dev/null 2>&1 || true +echo "Phase A2: cooldown to Sonda x1 @0.5s — $(ts)" +sleep 60 + +# ============================================================================ # +# PHASE B: Wave burst #2 (bigger messages, more pubs/recvs), then cooldown +# ============================================================================ # +export NUM_PUBLISHER_NODES=10 +export NUM_RECEIVER_NODES=10 +export MIN_MESSAGE_SIZE=120Kb +export MAX_MESSAGE_SIZE=160Kb + +docker compose down -v +docker compose up -d +echo " Phase B1: LPT 10/10, 120–160KB; Sonda x1 — $(ts)" + +# Heavier spike: three Sonda @0.02s +docker run -d --name sonda4 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +docker run -d --name sonda5 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +docker run -d --name sonda6 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda + +sleep 120 + +# Cooldown again: keep only baseline +docker rm -f sonda4 sonda5 sonda6 >/dev/null 2>&1 || true +echo "[scenario_7_store_wave] Phase B2: cooldown to Sonda x1 @0.5s — $(ts)" +sleep 60 + +# ============================================================================ # +# PHASE C: Microburst trains (while LPT steady) +# Start/stop pairs rapidly to create sawtooth Store pressure. +# ============================================================================ # +export NUM_PUBLISHER_NODES=10 +export NUM_RECEIVER_NODES=10 +export MIN_MESSAGE_SIZE=60Kb +export MAX_MESSAGE_SIZE=100Kb + +docker compose down -v +docker compose up -d +echo " Phase C: microburst trains begin — $(ts)" + +# Train 1 (two @0.01s for 45s) +docker run -d --name sonda7 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +docker run -d --name sonda8 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +sleep 45 +docker rm -f sonda7 sonda8 >/dev/null 2>&1 || true + +# Train 2 (two @0.02s for 60s) +docker run -d --name sonda9 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +docker run -d --name sonda10 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +sleep 60 +docker rm -f sonda9 sonda10 >/dev/null 2>&1 || true + +# Train 3 (three @0.01s for 60s) +docker run -d --name sonda11 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +docker run -d --name sonda12 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +docker run -d --name sonda3 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +sleep 60 +docker rm -f sonda11 sonda12 sonda3 >/dev/null 2>&1 || true + +echo "[scenario_7_store_wave] Phase C: microburst trains done — $(ts)" +sleep 30 + +# ============================================================================ # +# PHASE D: Store-only oscillation (LPT down), cycling query delays +# Observe Store under alternating light/heavy read pressure. +# ============================================================================ # +docker compose down -v +echo "Phase D: Store-only oscillation — $(ts)" + +# Round 1: fast pair @0.02s +docker run -d --name sonda4 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +docker run -d --name sonda5 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +sleep 60 +docker rm -f sonda4 sonda5 >/dev/null 2>&1 || true + +# Round 2: very fast pair @0.01s +docker run -d --name sonda6 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +docker run -d --name sonda7 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +sleep 60 +docker rm -f sonda6 sonda7 >/dev/null 2>&1 || true + +# Round 3: light @0.1s for recovery view +docker run -d --name sonda8 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.1 local-perf-sonda +sleep 60 +docker rm -f sonda8 >/dev/null 2>&1 || true + +echo "Final observe window — $(ts)" +sleep 60 + +# -------------------- Cleanup ------------------------------------------------- +docker rm -f sonda1 sonda2 sonda3 sonda4 sonda5 sonda6 sonda7 sonda8 sonda9 sonda10 sonda11 sonda12 >/dev/null 2>&1 || true + +cd .. + +echo "Test finished at $(ts)" +# exec ./stop_test.sh diff --git a/scripts/Lite_protocol_scripts/store/stress_high_concurrency_scenario2.sh b/scripts/Lite_protocol_scripts/store/stress_high_concurrency_scenario2.sh new file mode 100755 index 000000000..760dc4004 --- /dev/null +++ b/scripts/Lite_protocol_scripts/store/stress_high_concurrency_scenario2.sh @@ -0,0 +1,183 @@ +#!/bin/bash +set -e + +# Heavier Store stress by stacking more concurrent Sonda readers +# and ramping LPT publishers/receivers & message sizes. + +cd ./waku-simulator + +export NWAKU_IMAGE=wakuorg/nwaku:latest +export NUM_NWAKU_NODES=15 +export RLN_ENABLED=false + +# Service node config +export SERVICENODE_CPU_CORES="0-3" +export SERVICENODE_MEM_LIMIT=2g +export POSTGRES_CPU_CORES="0-3" +export POSTGRES_MEM_LIMIT=2g +export POSTGRES_SHM=1g + +docker compose up -d + +# Wait until service node is running +while true; do + sid="$(docker compose ps -q servicenode || true)" + if [[ -n "$sid" ]]; then + state="$(docker inspect --format '{{.State.Status}}' "$sid" 2>/dev/null || true)" + [[ "$state" == "running" ]] && break + fi + sleep 1 +done + +cd ../lpt + +# -------------------- LPT config --------------------------- +export LPT_IMAGE=harbor.status.im/wakuorg/liteprotocoltester:latest +export START_PUBLISHING_AFTER=15 +export NUM_MESSAGES=0 +export MESSAGE_INTERVAL_MILLIS=100 + +# Service peers +export LIGHTPUSH_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +export FILTER_SERVICE_PEER=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n + +# Topics +export PUBSUB=/waku/2/rs/66/0 +export CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim +export CLUSTER_ID=66 + +# settle before traffic +sleep 60 + +cd ../sonda +docker build -t local-perf-sonda -f ./Dockerfile.sonda . + +# Keep perf-test.env location and content +cat < perf-test.env +RELAY_NODE_REST_ADDRESS=http://127.0.0.1:8645 +STORE_NODE_REST_ADDRESS=http://127.0.0.1:8644 +QUERY_DELAY=0.5 +STORE_NODES=/ip4/10.2.0.101/tcp/60001/p2p/16Uiu2HAkyte8uj451tGkbww4Mjcg6DRnmAHxNeWyF4zp23RbpG3n +CLUSTER_ID=66 +SHARD=0 +EOF + +sleep 5 + +# Clean old sonda +docker rm -f sonda1 sonda2 sonda3 sonda4 sonda5 sonda6 sonda7 sonda8 sonda9 sonda10 >/dev/null 2>&1 || true + +# Baseline Sonda (0.5s) +docker run -d --name sonda1 --network host -l sonda \ + --env-file ./perf-test.env local-perf-sonda + +cd ../lpt + +# ============================================================================ # +# PHASE 0: Warmup, small messages, quick history fill +# LPT 8/8, 20–40 KB; Sonda x1 @0.5s +# ============================================================================ # +export NUM_PUBLISHER_NODES=8 +export NUM_RECEIVER_NODES=8 +export MIN_MESSAGE_SIZE=20Kb +export MAX_MESSAGE_SIZE=40Kb + +docker compose down -v >/dev/null 2>&1 || true +docker compose up -d + +echo "Phase 0: LPT 8/8, 20–40KB; Sonda x1 @0.5s — $(date '+%F %T')" +sleep 30 + +# ============================================================================ # +# PHASE 1: Moderate traffic +# LPT 6/6, 50–90 KB; Sonda x1 @0.5s +# ============================================================================ # +export NUM_PUBLISHER_NODES=6 +export NUM_RECEIVER_NODES=6 +export MIN_MESSAGE_SIZE=50Kb +export MAX_MESSAGE_SIZE=90Kb + +docker compose down -v >/dev/null 2>&1 || true +docker compose up -d + +echo "Phase 1: LPT 6/6, 50–90KB; Sonda x1 @0.5s — $(date '+%F %T')" +sleep 30 + +# ============================================================================ # +# PHASE 2: Triple query rate +# Keep LPT; add Sonda x2 @0.1s (total 3) +# ============================================================================ # +docker run -d --name sonda2 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.1 local-perf-sonda +docker run -d --name sonda3 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.1 local-perf-sonda + +echo "Phase 2: +Sonda x2 @0.1s (total 3) — $(date '+%F %T')" +sleep 30 + +# ============================================================================ # +# PHASE 3: Heavier traffic + peak query rate +# LPT 10/10, 130–149 KB; +Sonda x2 @0.05s (total 5) +# ============================================================================ # +docker compose down -v +export NUM_PUBLISHER_NODES=10 +export NUM_RECEIVER_NODES=10 +export MIN_MESSAGE_SIZE=130Kb +export MAX_MESSAGE_SIZE=149Kb +docker compose up -d + +docker run -d --name sonda4 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.05 local-perf-sonda +docker run -d --name sonda5 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.05 local-perf-sonda + +echo "Phase 3: LPT 10/10, 130–149KB; Sonda total x5 — $(date '+%F %T')" +sleep 30 + +# ============================================================================ # +# PHASE 3.5: Max Store pressure spike (Store-heavy while LPT still running) +# Add Sonda x3 @0.02s (total 8) +# ============================================================================ # +docker run -d --name sonda6 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +docker run -d --name sonda7 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda +docker run -d --name sonda8 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.02 local-perf-sonda + +echo "Phase 3.5: +Sonda x3 @0.02s (total 8) — $(date '+%F %T')" +sleep 30 + +# ============================================================================ # +# PHASE 4: Store-only flood +# Stop LPT; observe pure Store query saturation +# ============================================================================ # +docker compose down -v + +echo "Phase 4: LPT down; Store-only with Sonda x8 — $(date '+%F %T')" +sleep 30 + +# ============================================================================ # +# PHASE 5: Final squeeze +# Add Sonda x2 more @0.01s (total 10) for short burst +# ============================================================================ # +docker run -d --name sonda9 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda +docker run -d --name sonda10 --network host -l sonda \ + --env-file ../sonda/perf-test.env -e QUERY_DELAY=0.01 local-perf-sonda + +echo "Phase 5: +Sonda x2 @0.01s (total 10) — $(date '+%F %T')" +sleep 30 + +# brief post-run observation +echo "Final observe window — $(date '+%F %T')" +sleep 30 + +# -------------------- Cleanup ------------------------------------------------- +docker rm -f sonda1 sonda2 sonda3 sonda4 sonda5 sonda6 sonda7 sonda8 sonda9 sonda10 >/dev/null 2>&1 || true + +cd .. + +echo "Test finished at $(date '+%F %T')" +# exec ./stop_test.sh + diff --git a/scripts/build_logos_delivery_lib.sh b/scripts/build_logos_delivery_lib.sh new file mode 100755 index 000000000..be3c1d6b4 --- /dev/null +++ b/scripts/build_logos_delivery_lib.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Prerequisites: +# - Nim / nimble / choosenim installed (e.g. via https://nim-lang.org/choosenim) +# - make, gcc / g++ (or clang on macOS) +# - python3 available on PATH +# - git submodules initialised: +# git submodule update --init --recursive + +set -euo pipefail + +# ── Resolve repository root ─────────────────────────────────────────────────── +# The script lives in /scripts/, so go one level up. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# ── Resolve Python interpreter ──────────────────────────────────────────────── +# Prefer the project's own virtual environment (cffi + all requirements). +# Checks both the hidden (.venv) and non-hidden (venv) conventional names. +# Falls back to whatever python3 is on PATH (e.g. in CI after pip install). +if [ -x "$REPO_ROOT/.venv/bin/python" ]; then + PYTHON="$REPO_ROOT/.venv/bin/python" + echo "Using venv Python: $PYTHON" +elif [ -x "$REPO_ROOT/venv/bin/python" ]; then + PYTHON="$REPO_ROOT/venv/bin/python" + echo "Using venv Python: $PYTHON" +else + PYTHON="$(command -v python3 || command -v python)" + echo "No local venv found, falling back to: $PYTHON" +fi + +# ── 1. Build liblogosdelivery shared library for Python bindings ────────────── +echo "──────────────────────────────────────────────────────────────────────────" +echo "Step 1 – Build liblogosdelivery shared library for Python bindings" +echo "──────────────────────────────────────────────────────────────────────────" + +# Make sure nimble / choosenim binaries are on PATH (same as CI) +export PATH="$HOME/.nimble/bin:$PATH" + +BINDINGS_DIR="$REPO_ROOT/vendor/logos-delivery-python-bindings" +DELIVERY_DIR="$BINDINGS_DIR/vendor/logos-delivery" + +export "PYTHONPATH=$BINDINGS_DIR/waku${PYTHONPATH:+:$PYTHONPATH}" + +echo "--> Creating lib output directory: $BINDINGS_DIR/lib" +mkdir -p "$BINDINGS_DIR/lib" + +echo "--> Entering: $DELIVERY_DIR" +cd "$DELIVERY_DIR" + +echo "--> Creating waku.nims symlink (waku.nimble -> waku.nims)" +ln -sf waku.nimble waku.nims + +echo "--> Installing Nim dependencies (nimble install -y)" +nimble install -y + +echo "--> Running: make setup" +make setup + +echo "--> Running: make liblogosdelivery" +make liblogosdelivery + +# On Linux the library is .so; on macOS it may be .dylib +SO_PATH="$(find . -type f \( -name 'liblogosdelivery.so' -o -name 'liblogosdelivery.dylib' \) | head -n 1)" + +if [ -z "$SO_PATH" ]; then + echo "ERROR: liblogosdelivery shared library was not built (neither .so nor .dylib found)" + exit 1 +fi + +# Preserve the platform-native extension in the destination +case "$SO_PATH" in + *.dylib) DEST_LIB="$BINDINGS_DIR/lib/liblogosdelivery.dylib" ;; + *) DEST_LIB="$BINDINGS_DIR/lib/liblogosdelivery.so" ;; +esac + +cp "$SO_PATH" "$DEST_LIB" +echo "Built library:" +ls -l "$DEST_LIB" + +# ── 2. Verify wrapper library ───────────────────────────────────────────────── +echo "" +echo "──────────────────────────────────────────────────────────────────────────" +echo "Step 2 – Verify wrapper library" +echo "──────────────────────────────────────────────────────────────────────────" + +if test -f "$BINDINGS_DIR/lib/liblogosdelivery.so" \ + || test -f "$BINDINGS_DIR/lib/liblogosdelivery.dylib"; then + echo "OK: wrapper library is present in $BINDINGS_DIR/lib/" +else + echo "ERROR: wrapper library not found in $BINDINGS_DIR/lib/" + exit 1 +fi + +# ── 3. Debug Python import paths ────────────────────────────────────────────── +echo "" +echo "──────────────────────────────────────────────────────────────────────────" +echo "Step 3 – Debug Python import paths" +echo "──────────────────────────────────────────────────────────────────────────" + +cd "$REPO_ROOT" + +# Prepend the waku bindings directory to PYTHONPATH (mirrors the CI env step) +export PYTHONPATH="$REPO_ROOT/vendor/logos-delivery-python-bindings/waku${PYTHONPATH:+:$PYTHONPATH}" + +pwd +echo "PYTHONPATH=$PYTHONPATH" +find . -maxdepth 5 | grep wrapper || true + +"$PYTHON" - <<'PY' +import sys +print("sys.path:") +for p in sys.path: + print(p) +try: + import wrapper + print("wrapper import OK:", wrapper) +except Exception as e: + print("wrapper import failed:", e) + raise +PY diff --git a/src/env_vars.py b/src/env_vars.py index 32e96a7e1..4985a6f17 100644 --- a/src/env_vars.py +++ b/src/env_vars.py @@ -32,5 +32,19 @@ RLN_CREDENTIALS = get_env_var("RLN_CREDENTIALS") PG_USER = get_env_var("POSTGRES_USER", "postgres") PG_PASS = get_env_var("POSTGRES_PASSWORD", "test123") +FLEET_NODES = [ + # Amsterdam + "/dns4/node-01.do-ams3.waku.test.status.im/tcp/30303/p2p/16Uiu2HAkykgaECHswi3YKJ5dMLbq2kPVCo89fcyTd38UcQD6ej5W", + # US Central + "/dns4/node-01.gc-us-central1-a.waku.test.status.im/tcp/30303/p2p/16Uiu2HAmDCp8XJ9z1ev18zuv8NHekAsjNyezAvmMfFEJkiharitG", + # Hong Kong + "/dns4/node-01.ac-cn-hongkong-c.waku.test.status.im/tcp/30303/p2p/16Uiu2HAkzHaTP5JsUwfR9NR8Rj9HC24puS6ocaU8wze4QrXr9iXp", +] +FLEET_PRIMARY_MULTIADDR = FLEET_NODES[0] +FLEET_DNS_DISCOVERY_URL = "enrtree://AOGYWMBYOUIMOENHXCHILPKY3ZRFEULMFI4DOM442QSZ73TT2A7VI@test.waku.nodes.status.im" + +FLEET_N1_MULTIADDR = FLEET_NODES[0] # node-01.do-ams3 – used by NODE1 in --fleet mode +FLEET_N2_MULTIADDR = FLEET_NODES[1] # node-01.gc-us-central1-a – used by NODE2 in --fleet mode + # example for .env file # RLN_CREDENTIALS = {"rln-relay-cred-password": "password", "rln-relay-eth-client-address": "https://rpc.sepolia.linea.build", "rln-relay-eth-contract-address": "0xB9cd878C90E49F797B4431fBF4fb333108CB90e6", "rln-relay-eth-private-key-1": "", "rln-relay-eth-private-key-2": "", "rln-relay-eth-private-key-3": "", "rln-relay-eth-private-key-4": "", "rln-relay-eth-private-key-5": ""} diff --git a/src/node/docker_mananger.py b/src/node/docker_mananger.py index 44329d098..d124b2981 100644 --- a/src/node/docker_mananger.py +++ b/src/node/docker_mananger.py @@ -46,6 +46,7 @@ class DockerManager: port_bindings = {f"{port}/tcp": ("", port) for port in ports} port_bindings_for_log = " ".join(f"-p {port}:{port}" for port in ports) cli_args_str_for_log = " ".join(cli_args) + cli_args_str_for_log = re.sub(r"(--rln-relay-eth-private-key=)\S+", r"\1REDACTED", cli_args_str_for_log) logger.debug(f"docker run -i -t {port_bindings_for_log} {image_name} {cli_args_str_for_log}") container = self._client.containers.run( image_name, command=cli_args, ports=port_bindings, detach=True, remove=remove_container, auto_remove=remove_container, volumes=volumes diff --git a/src/node/fleet_waku_node.py b/src/node/fleet_waku_node.py new file mode 100644 index 000000000..6a604340e --- /dev/null +++ b/src/node/fleet_waku_node.py @@ -0,0 +1,138 @@ +"""Fleet bootstrap configuration for WakuNode. + +When fleet bootstrap is active (``--fleet`` CLI flag or ``FLEET_BOOTSTRAP=true`` +env var), an instance of :class:`FleetBootstrapConfig` is assigned to +``WakuNode._pre_start_hook``. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from src.libs.custom_logger import get_custom_logger +from src.data_storage import DS +from src.env_vars import ( + FLEET_N1_MULTIADDR, + FLEET_N2_MULTIADDR, + FLEET_PRIMARY_MULTIADDR, + FLEET_DNS_DISCOVERY_URL, + RLN_CREDENTIALS, +) +from src.test_data import FLEET_CLUSTER_ID + +if TYPE_CHECKING: + from src.node.waku_node import WakuNode + +logger = get_custom_logger(__name__) + + +def _append_fleet_kwarg(kwargs: dict, key: str, value: Any) -> None: + """Append *value* to the kwargs entry *key*, creating a list when needed.""" + existing = kwargs.get(key) + if existing is None: + kwargs[key] = value + elif isinstance(existing, list): + if value not in existing: + kwargs[key] = existing + [value] + else: + if existing != value: + kwargs[key] = [existing, value] + + +@dataclass +class FleetBootstrapConfig: + """Holds fleet session state and implements the pre-start kwargs injection. + + One instance is created per pytest session when fleet bootstrap is active. + It is registered as ``WakuNode._pre_start_hook`` so that every call to + ``WakuNode.start()`` automatically receives fleet bootstrap arguments. + + Bootstrap assignment by node creation order (mirrors config-n*.toml files): + - NODE1 (1st started) → FLEET_N1_MULTIADDR (node-01.do-ams3) + - NODE2 (2nd started) → FLEET_N2_MULTIADDR (node-01.gc-us-central1-a) + - additional nodes → FLEET_PRIMARY_MULTIADDR (Amsterdam, same as NODE1) + """ + + fleet_rln_state: dict + + def prepare_start_kwargs(self, node: "WakuNode", kwargs: dict) -> dict: + """Inject fleet bootstrap arguments into *kwargs* before node start.""" + + logger.debug("FleetBootstrapConfig.prepare_start_kwargs: injecting waku.test bootstrap args") + + if kwargs.pop("skip_fleet_peering", False): + kwargs.pop("discv5_bootstrap_node", None) + logger.debug( + "FleetBootstrapConfig: skip_fleet_peering=True – " "bypassing fleet bootstrap for this node (no fleet staticnode / shard injected)" + ) + return kwargs + + # Determine which fleet peer to connect to based on node creation order + # within the current test (DS.waku_nodes is reset to [] before each test). + node_index = len(DS.waku_nodes) + if node_index == 0: + fleet_multiaddr = FLEET_N1_MULTIADDR + logger.debug( + "FleetBootstrapConfig: NODE1 – bootstrapping from config-n1.toml (%s)", + fleet_multiaddr, + ) + elif node_index == 1: + fleet_multiaddr = FLEET_N2_MULTIADDR + logger.debug( + "FleetBootstrapConfig: NODE2 – bootstrapping from config-n2.toml (%s)", + fleet_multiaddr, + ) + else: + fleet_multiaddr = FLEET_PRIMARY_MULTIADDR + logger.debug( + "FleetBootstrapConfig: additional node %d – bootstrapping from primary (%s)", + node_index, + fleet_multiaddr, + ) + + _append_fleet_kwarg(kwargs, "staticnode", fleet_multiaddr) + kwargs.setdefault("dns_discovery", "true") + kwargs.setdefault("dns_discovery_url", FLEET_DNS_DISCOVERY_URL) + kwargs.setdefault("cluster_id", FLEET_CLUSTER_ID) + kwargs.setdefault("shard", list(range(8))) + + # Inject session-level RLN credentials into relay enabled nodes that + # don't already carry explicit RLN args. + rln_prefixes = self.fleet_rln_state.get("keystore_prefixes", []) + if rln_prefixes and kwargs.get("rln_creds_source") is None: + if str(kwargs.get("relay", "")).lower() == "true": + if node_index < len(rln_prefixes): + kwargs["rln_creds_source"] = RLN_CREDENTIALS + kwargs["rln_creds_id"] = str(node_index + 1) + kwargs["rln_keystore_prefix"] = rln_prefixes[node_index] + kwargs["rln_relay_membership_index"] = str(self.fleet_rln_state["rln_membership_indexes"][node_index]) + kwargs.setdefault("rln_relay_user_message_limit", "300") + logger.debug( + "FleetBootstrapConfig: injected session RLN for node %d – " "prefix=%s index=%s user_message_limit=300", + node_index, + kwargs["rln_keystore_prefix"], + kwargs["rln_relay_membership_index"], + ) + else: + logger.debug( + "FleetBootstrapConfig: skipping RLN injection for node %d – " "relay not enabled (relay=%r)", + node_index, + kwargs.get("relay"), + ) + + # Strip any local-node discv5 bootstrap ENR so that each node bootstraps + # independently from its assigned fleet peer rather than from another + # local container. The fleet DNS tree (dns_discovery_url) replaces it. + if "discv5_bootstrap_node" in kwargs: + logger.debug( + "FleetBootstrapConfig: dropping local discv5_bootstrap_node=%s " "(fleet DNS discovery replaces it)", + kwargs["discv5_bootstrap_node"], + ) + del kwargs["discv5_bootstrap_node"] + + logger.debug( + "FleetBootstrapConfig: staticnode=%s dns_discovery_url=%s", + kwargs.get("staticnode"), + kwargs.get("dns_discovery_url"), + ) + return kwargs diff --git a/src/node/waku_node.py b/src/node/waku_node.py index 8e73fe4cd..7afe881c6 100644 --- a/src/node/waku_node.py +++ b/src/node/waku_node.py @@ -10,7 +10,7 @@ import pytest import requests from src.libs.common import delay from src.libs.custom_logger import get_custom_logger -from tenacity import retry, stop_after_delay, wait_fixed, sleep +from tenacity import retry, stop_after_attempt, stop_after_delay, wait_fixed, sleep from docker.errors import NotFound as DockerNotFound from src.node.api_clients.rest import REST from src.node.docker_mananger import DockerManager @@ -95,6 +95,9 @@ def resolve_sharding_flags(kwargs): class WakuNode: + # Optional pre-start hook to allow modifications for fleet tests + _pre_start_hook = None + def __init__(self, docker_image, docker_log_prefix=""): self._image_name = docker_image self._log_path = os.path.join(DOCKER_LOG_DIR, f"{docker_log_prefix}__{self._image_name.replace('/', '_')}.log") @@ -106,7 +109,9 @@ class WakuNode: logger.debug(f"WakuNode instance initialized with log path {self._log_path}") @retry(stop=stop_after_delay(60), wait=wait_fixed(0.1), reraise=True) - def start(self, wait_for_node_sec=20, **kwargs): + def start(self, wait_for_node_sec=20, use_wrapper=False, **kwargs): + if WakuNode._pre_start_hook is not None: + kwargs = WakuNode._pre_start_hook(self, kwargs) logger.debug("Starting Node...") default_args, remove_container = self._prepare_start_context(**kwargs) self._start_docker(default_args, remove_container, wait_for_node_sec) @@ -224,6 +229,52 @@ class WakuNode: logger.error(f"REST service did not become ready in time: {ex}") raise + def _start_wrapper(self, default_args, wait_for_node_sec): + logger.debug("Starting node using wrappers") + wrapper_config = self._default_args_to_wrapper_config(default_args) + + result = WrapperManager.create_and_start(config=wrapper_config, timeout_s=wait_for_node_sec) + if result.is_err(): + raise RuntimeError(f"Failed to start wrapper node: {result.err()}") + self._wrapper_node = result.ok_value + + logger.debug(f"Started wrapper node. REST: {self._rest_port}") + DS.waku_nodes.append(self) + delay(1) + try: + self.ensure_ready(timeout_duration=wait_for_node_sec) + except Exception as ex: + logger.error(f"REST service did not become ready in time: {ex}") + raise + + def _default_args_to_wrapper_config(self, default_args): + def _bool(key, default="false"): + return default_args.get(key, default).lower() == "true" + + bootstrap = default_args.get("discv5-bootstrap-node") + + return { + "logLevel": default_args.get("log-level", "DEBUG"), + "mode": "Core", + "networkingConfig": { + "listenIpv4": default_args.get("listen-address", "0.0.0.0"), + "p2pTcpPort": int(default_args["tcp-port"]), + "discv5UdpPort": int(default_args["discv5-udp-port"]), + "restPort": int(default_args["rest-port"]), + "restAddress": default_args.get("rest-address", "0.0.0.0"), + }, + "protocolsConfig": { + "clusterId": int(default_args.get("cluster-id", DEFAULT_CLUSTER_ID)), + "relay": _bool("relay"), + "store": _bool("store"), + "filter": _bool("filter"), + "lightpush": _bool("lightpush"), + "peerExchange": _bool("peer-exchange"), + "discv5Discovery": _bool("discv5-discovery", "true"), + "discv5BootstrapNodes": [bootstrap] if bootstrap else [], + }, + } + @retry(stop=stop_after_delay(250), wait=wait_fixed(0.1), reraise=True) def register_rln(self, **kwargs): logger.debug("Registering RLN credentials...") diff --git a/src/steps/filter.py b/src/steps/filter.py index 8a5d90c41..39429770b 100644 --- a/src/steps/filter.py +++ b/src/steps/filter.py @@ -95,9 +95,15 @@ class StepsFilter(StepsCommon): for index, peer in enumerate(peer_list): logger.debug(f"Checking that peer NODE_{index + 2}:{peer.image} can find the published message") get_messages_response = self.get_filter_messages(message["contentTopic"], pubsub_topic=pubsub_topic, node=peer) - assert get_messages_response, f"Peer NODE_{index + 2}:{peer.image} couldn't find any messages" - assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" - waku_message = WakuMessage(get_messages_response) + # get_filter_messages already scopes to the requested content topic; the + # additional filter guards against any residual or fleet messages that may + # have been queued under the same topic before this call. + test_messages = [m for m in get_messages_response if m.get("contentTopic") == message["contentTopic"]] + assert test_messages, f"Peer NODE_{index + 2}:{peer.image} couldn't find any messages" + assert len(test_messages) == 1, ( + f"Expected 1 test message but got {len(test_messages)} " f"(total messages returned: {len(get_messages_response)})" + ) + waku_message = WakuMessage(test_messages) waku_message.assert_received_message(message) @allure.step diff --git a/src/steps/light_push.py b/src/steps/light_push.py index fd7da42d0..3980513cf 100644 --- a/src/steps/light_push.py +++ b/src/steps/light_push.py @@ -22,6 +22,7 @@ class StepsLightPush(StepsCommon): test_content_topic = "/myapp/1/latest/proto" test_pubsub_topic = VALID_PUBSUB_TOPICS[0] test_payload = "Light push works!!" + default_message_propagation_delay = 0.1 @pytest.fixture(scope="function", autouse=True) def light_push_setup(self): @@ -109,7 +110,7 @@ class StepsLightPush(StepsCommon): @allure.step def check_light_pushed_message_reaches_receiving_peer( - self, pubsub_topic=None, message=None, message_propagation_delay=0.1, sender=None, peer_list=None + self, pubsub_topic=None, message=None, message_propagation_delay=None, sender=None, peer_list=None ): if pubsub_topic is None: pubsub_topic = self.test_pubsub_topic @@ -117,6 +118,8 @@ class StepsLightPush(StepsCommon): sender = self.light_push_node1 if not peer_list: peer_list = self.main_receiving_nodes + self.optional_nodes + if message_propagation_delay is None: + message_propagation_delay = self.default_message_propagation_delay payload = self.create_payload(pubsub_topic, message) logger.debug("Lightpushing message") sender.send_light_push_message(payload) @@ -124,9 +127,12 @@ class StepsLightPush(StepsCommon): for index, peer in enumerate(peer_list): logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the lightpushed message") get_messages_response = peer.get_relay_messages(pubsub_topic) - assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" - assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" - waku_message = WakuMessage(get_messages_response) + test_messages = [m for m in get_messages_response if m.get("contentTopic") == payload["message"]["contentTopic"]] + assert test_messages, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" + assert len(test_messages) == 1, ( + f"Expected 1 test message but got {len(test_messages)} " f"(total messages in cache: {len(get_messages_response)})" + ) + waku_message = WakuMessage(test_messages) waku_message.assert_received_message(payload["message"]) @allure.step diff --git a/src/steps/relay.py b/src/steps/relay.py index 840461bbe..b801f848b 100644 --- a/src/steps/relay.py +++ b/src/steps/relay.py @@ -122,9 +122,15 @@ class StepsRelay(StepsCommon): for index, peer in enumerate(peer_list): logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the published message") get_messages_response = peer.get_relay_messages(pubsub_topic) - assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" - assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" - waku_message = WakuMessage(get_messages_response) + # In fleet mode the relay cache may contain background messages from other + # fleet participants. Filter to only the message whose contentTopic matches + # what the test sent so that fleet noise does not break the count assertion. + test_messages = [m for m in get_messages_response if m.get("contentTopic") == message["contentTopic"]] + assert test_messages, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" + assert len(test_messages) == 1, ( + f"Expected 1 test message but got {len(test_messages)} " f"(total messages in cache: {len(get_messages_response)})" + ) + waku_message = WakuMessage(test_messages) waku_message.assert_received_message(message) @allure.step diff --git a/src/test_config.py b/src/test_config.py new file mode 100644 index 000000000..11f098d48 --- /dev/null +++ b/src/test_config.py @@ -0,0 +1,27 @@ +"""Test session configuration objects. + +These dataclasses carry configuration that varies between fleet and +non-fleet test runs. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import List + + +@dataclass(frozen=True) +class PubsubConfig: + """Named pubsub-topic slots for a test session. + + A *default* instance uses ``VALID_PUBSUB_TOPICS`` / ``PUBSUB_TOPICS_RLN`` + (cluster-id 198). A *fleet* instance uses ``FLEET_PUBSUB_TOPICS`` + (cluster-id 1, shards 0-7). + """ + + relay_test_topic: str + filter_test_topic: str + filter_second_topic: str + lightpush_test_topic: str + store_test_topic: str + rln_test_topic: str + all_topics: List[str] diff --git a/src/test_data.py b/src/test_data.py index 659dd39ae..2c1c6946b 100644 --- a/src/test_data.py +++ b/src/test_data.py @@ -1,3 +1,4 @@ +import os from time import time from datetime import datetime, timedelta @@ -105,6 +106,9 @@ VALID_PUBSUB_TOPICS = [ f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1000", ] +FLEET_CLUSTER_ID = "1" +FLEET_PUBSUB_TOPICS = [f"/waku/2/rs/{FLEET_CLUSTER_ID}/{i}" for i in range(8)] + PUBSUB_TOPICS_STORE = [ f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0", f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1", @@ -147,26 +151,45 @@ PUBSUB_TOPICS_WRONG_FORMAT = [ {"description": "A bool", "value": True}, ] -SAMPLE_TIMESTAMPS = [ - {"description": "Now", "value": int(time() * 1e9), "valid_for": ["nwaku"]}, - { - "description": "Far future", - "value": int((NOW + timedelta(days=365 * 10)).timestamp() * 1e9), - "valid_for": ["nwaku"], - }, # 10 years from now - {"description": "Recent past", "value": int((NOW - timedelta(hours=1)).timestamp() * 1e9), "valid_for": ["nwaku"]}, # 1 hour ago - {"description": "Near future", "value": int((NOW + timedelta(hours=1)).timestamp() * 1e9), "valid_for": ["nwaku"]}, # 1 hour ahead - {"description": "Positive number", "value": 1, "valid_for": ["nwaku"]}, - {"description": "Negative number", "value": -1, "valid_for": ["nwaku"]}, - {"description": "DST change", "value": int(datetime(2020, 3, 8, 2, 0, 0).timestamp() * 1e9), "valid_for": ["nwaku"]}, # DST starts - {"description": "Timestamp as string number", "value": str(int(time() * 1e9)), "valid_for": []}, - {"description": "Invalid large number", "value": 2**63, "valid_for": []}, - {"description": "Float number", "value": float(time() * 1e9), "valid_for": []}, - {"description": "Array instead of timestamp", "value": [int(time() * 1e9)], "valid_for": []}, - {"description": "Object instead of timestamp", "value": {"time": int(time() * 1e9)}, "valid_for": []}, - {"description": "ISO 8601 timestamp", "value": "2023-12-26T10:58:51", "valid_for": []}, - {"description": "Missing", "value": None, "valid_for": []}, -] + +def get_sample_timestamps(): + """Return timestamp test-cases with values evaluated fresh at call time. + + This factory function MUST be called from inside each test (never at module + import time) so that the "Now" value reflects the actual time when the + message is published. + + Valid_for semantics: + ``["nwaku"]`` – accepted by nwaku in the current run mode. + ``[]`` – rejected by nwaku in the current run mode (either + structurally invalid type, or out-of-epoch in fleet/RLN + mode). + """ + now_ns = int(time() * 1e9) + now_dt = datetime.now() + fleet_mode = os.getenv("FLEET_BOOTSTRAP", "false").lower() == "true" + standalone_valid = [] if fleet_mode else ["nwaku"] + return [ + {"description": "Now", "value": now_ns, "valid_for": ["nwaku"]}, + # 10 years from now + {"description": "Far future", "value": int((now_dt + timedelta(days=365 * 10)).timestamp() * 1e9), "valid_for": standalone_valid}, + # 1 hour ago + {"description": "Recent past", "value": int((now_dt - timedelta(hours=1)).timestamp() * 1e9), "valid_for": standalone_valid}, + # 1 hour ahead + {"description": "Near future", "value": int((now_dt + timedelta(hours=1)).timestamp() * 1e9), "valid_for": standalone_valid}, + {"description": "Positive number", "value": 1, "valid_for": standalone_valid}, + {"description": "Negative number", "value": -1, "valid_for": standalone_valid}, + # DST starts + {"description": "DST change", "value": int(datetime(2020, 3, 8, 2, 0, 0).timestamp() * 1e9), "valid_for": standalone_valid}, + {"description": "Timestamp as string number", "value": str(now_ns), "valid_for": []}, + {"description": "Invalid large number", "value": 2**63, "valid_for": []}, + {"description": "Float number", "value": float(now_ns), "valid_for": []}, + {"description": "Array instead of timestamp", "value": [now_ns], "valid_for": []}, + {"description": "Object instead of timestamp", "value": {"time": now_ns}, "valid_for": []}, + {"description": "ISO 8601 timestamp", "value": "2023-12-26T10:58:51", "valid_for": []}, + {"description": "Missing", "value": None, "valid_for": []}, + ] + PUBSUB_TOPICS_RLN = [f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0"] diff --git a/tests/conftest.py b/tests/conftest.py index 258c81c66..28f87dd73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,20 +1,217 @@ # -*- coding: utf-8 -*- import inspect import glob +import random +import string from src.libs.custom_logger import get_custom_logger import os import pytest from datetime import datetime from time import time from uuid import uuid4 -from src.libs.common import attach_allure_file +from src.libs.common import attach_allure_file, gen_step_id import src.env_vars as env_vars +from src.env_vars import FLEET_PRIMARY_MULTIADDR, FLEET_DNS_DISCOVERY_URL, FLEET_N1_MULTIADDR, FLEET_N2_MULTIADDR from src.data_storage import DS from src.postgres_setup import start_postgres, stop_postgres +from src.test_data import FLEET_CLUSTER_ID, FLEET_PUBSUB_TOPICS, PUBSUB_TOPICS_RLN, VALID_PUBSUB_TOPICS +from src.test_config import PubsubConfig logger = get_custom_logger(__name__) +def pytest_addoption(parser): + """Register the --fleet command-line option.""" + parser.addoption( + "--fleet", + action="store_true", + default=False, + help=( + "Bootstrap every local nwaku Docker node against the live waku.test " + "fleet (node-01.do-ams3 / gc-us-central1-a / ac-cn-hongkong-c). " + "Also activatable via FLEET_BOOTSTRAP=true env var." + ), + ) + + +def _fleet_bootstrap_enabled(config) -> bool: + """Return True when fleet bootstrap should be activated.""" + + if config.getoption("--fleet", default=False): + return True + return os.getenv("FLEET_BOOTSTRAP", "false").lower() == "true" + + +@pytest.fixture(scope="session") +def fleet_rln_state(request): + """Register 2 RLN memberships once per test session when ``--fleet`` is active.""" + if not _fleet_bootstrap_enabled(request.config): + yield {"keystore_prefixes": [], "rln_membership_indexes": []} + return + + from src.node.waku_node import WakuNode + from src.env_vars import RLN_CREDENTIALS, DEFAULT_NWAKU + + if not RLN_CREDENTIALS: + logger.info("Fleet RLN: RLN_CREDENTIALS not set – nodes will start without RLN") + yield {"keystore_prefixes": [], "rln_membership_indexes": []} + return + + state: dict = {"keystore_prefixes": [], "rln_membership_indexes": []} + try: + for i in range(2): + prefix = "".join(random.choices(string.ascii_lowercase, k=4)) + node = WakuNode(DEFAULT_NWAKU, f"rln_reg_{i + 1}_{gen_step_id()}") + membership_index = node.register_rln( + rln_keystore_prefix=prefix, + rln_creds_source=RLN_CREDENTIALS, + rln_creds_id=str(i + 1), + ) + state["keystore_prefixes"].append(prefix) + state["rln_membership_indexes"].append(membership_index) + logger.info( + "Fleet RLN: registered %d memberships – indexes=%s prefixes=%s", + len(state["rln_membership_indexes"]), + state["rln_membership_indexes"], + state["keystore_prefixes"], + ) + except BaseException as ex: + logger.error("Fleet RLN: registration failed – aborting test session: %s", ex) + pytest.exit(f"Fleet RLN registration failed – aborting session: {ex}", returncode=1) + + yield state + + +@pytest.fixture(scope="session") +def pubsub_cfg(request) -> PubsubConfig: + """Return the pubsub-topic configuration for the current session.""" + + if _fleet_bootstrap_enabled(request.config): + return PubsubConfig( + relay_test_topic=FLEET_PUBSUB_TOPICS[1], + filter_test_topic=FLEET_PUBSUB_TOPICS[1], + filter_second_topic=FLEET_PUBSUB_TOPICS[2], + lightpush_test_topic=FLEET_PUBSUB_TOPICS[0], + store_test_topic=FLEET_PUBSUB_TOPICS[0], + rln_test_topic=FLEET_PUBSUB_TOPICS[0], + all_topics=FLEET_PUBSUB_TOPICS, + ) + return PubsubConfig( + relay_test_topic=VALID_PUBSUB_TOPICS[1], + filter_test_topic=VALID_PUBSUB_TOPICS[1], + filter_second_topic=VALID_PUBSUB_TOPICS[2], + lightpush_test_topic=VALID_PUBSUB_TOPICS[0], + store_test_topic=VALID_PUBSUB_TOPICS[0], + rln_test_topic=PUBSUB_TOPICS_RLN[0], + all_topics=VALID_PUBSUB_TOPICS, + ) + + +@pytest.fixture(scope="session", autouse=True) +def configure_fleet_bootstrap(request, fleet_rln_state): + """Register ``FleetBootstrapConfig`` as ``WakuNode._pre_start_hook`` for the session.""" + + if not _fleet_bootstrap_enabled(request.config): + logger.info("Fleet bootstrap inactive – pass --fleet (or set FLEET_BOOTSTRAP=true) " "to connect local nodes to the waku.test fleet") + yield + return + + os.environ["FLEET_BOOTSTRAP"] = "true" + + from src.node.fleet_waku_node import FleetBootstrapConfig + from src.node.waku_node import WakuNode + + cfg = FleetBootstrapConfig(fleet_rln_state=fleet_rln_state) + WakuNode._pre_start_hook = cfg.prepare_start_kwargs + logger.info( + "Fleet bootstrap active – NODE1→%s NODE2→%s (additional nodes→%s) dns_discovery_url=%s", + FLEET_N1_MULTIADDR, + FLEET_N2_MULTIADDR, + FLEET_PRIMARY_MULTIADDR, + FLEET_DNS_DISCOVERY_URL, + ) + yield + WakuNode._pre_start_hook = None + + +@pytest.fixture(scope="function", autouse=True) +def skip_fleet_test_without_rln(request, fleet_rln_state): + """Skip tests marked @pytest.mark.waku_test_fleet when no RLN keystore is + available for the current session. + """ + if not _fleet_bootstrap_enabled(request.config): + return + if not request.node.get_closest_marker("waku_test_fleet"): + return + if not fleet_rln_state.get("keystore_prefixes"): + pytest.fail("Failing fleet tests: RLN keystore not available " "(RLN_CREDENTIALS not set or on-chain registration failed)") + + +@pytest.fixture(scope="session", autouse=True) +def configure_fleet_cluster(request, pubsub_cfg): + """Apply fleet cluster configuration to step classes when ``--fleet`` is active.""" + + if not _fleet_bootstrap_enabled(request.config): + yield + return + + from src.steps.relay import StepsRelay + from src.steps.filter import StepsFilter + from src.steps.light_push import StepsLightPush + from src.steps.store import StepsStore + from src.steps.rln import StepsRLN + import tests.relay.test_publish as _relay_publish_mod + + # Override step-class topic attributes with fleet cluster-1 topics. + StepsRelay.test_pubsub_topic = pubsub_cfg.relay_test_topic + StepsFilter.test_pubsub_topic = pubsub_cfg.filter_test_topic + StepsFilter.second_pubsub_topic = pubsub_cfg.filter_second_topic + StepsLightPush.test_pubsub_topic = pubsub_cfg.lightpush_test_topic + StepsStore.test_pubsub_topic = pubsub_cfg.store_test_topic + StepsRLN.test_pubsub_topic = pubsub_cfg.rln_test_topic + + # tests/relay/test_publish.py::test_publish_on_multiple_pubsub_topics iterates + # over the module-level VALID_PUBSUB_TOPICS import directly; rebind it. + _relay_publish_mod.VALID_PUBSUB_TOPICS = pubsub_cfg.all_topics + + def _fleet_setup_lightpush_node(self, image, node_index, **kwargs): + from src.node.waku_node import WakuNode + + node = WakuNode(image, f"lightpush_node{node_index}_{self.test_id}") + fleet_kwargs = dict(kwargs) + fleet_kwargs["relay"] = "false" + fleet_kwargs["lightpush"] = "false" + fleet_kwargs["skip_fleet_peering"] = True + fleet_kwargs.setdefault("cluster_id", FLEET_CLUSTER_ID) + fleet_kwargs.setdefault("shard", list(range(8))) + + lightpush_service_addr = self.multiaddr_list[0] + node.start(lightpushnode=lightpush_service_addr, **fleet_kwargs) + self.add_node_peer(node, self.multiaddr_list) + logger.debug( + "fleet _fleet_setup_lightpush_node: node %d started with relay=false, " "skip_fleet_peering=True, lightpushnode=%s", + node_index, + lightpush_service_addr, + ) + return node + + StepsLightPush.setup_lightpush_node = _fleet_setup_lightpush_node + StepsLightPush.default_message_propagation_delay = 0.5 + + logger.info( + "Fleet cluster config active – pubsub topics overridden to cluster-id=%s " + "(shards 0-7, e.g. relay_test_topic=%s rln_test_topic=%s); " + "StepsLightPush.setup_lightpush_node overridden to use receiving_node1 as " + "lightpush service (fleet-peered with RLN membership #1; messages relay " + "through fleet mesh to receiving_node2 peered with %s)", + FLEET_CLUSTER_ID, + pubsub_cfg.relay_test_topic, + pubsub_cfg.rln_test_topic, + FLEET_N2_MULTIADDR, + ) + yield + + # See https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_runtest_makereport(item): diff --git a/tests/filter/test_get_messages.py b/tests/filter/test_get_messages.py index 4c86394be..89df2ff7e 100644 --- a/tests/filter/test_get_messages.py +++ b/tests/filter/test_get_messages.py @@ -2,7 +2,7 @@ import pytest from src.env_vars import NODE_1, NODE_2 from src.libs.common import delay, to_base64 from src.libs.custom_logger import get_custom_logger -from src.test_data import SAMPLE_INPUTS, SAMPLE_TIMESTAMPS +from src.test_data import SAMPLE_INPUTS, get_sample_timestamps from src.steps.filter import StepsFilter logger = get_custom_logger(__name__) @@ -12,6 +12,7 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node", "subscribe_main_nodes") class TestFilterGetMessages(StepsFilter): @pytest.mark.smoke + @pytest.mark.waku_test_fleet def test_filter_get_message_with_valid_payloads(self): failed_payloads = [] for payload in SAMPLE_INPUTS: @@ -24,9 +25,10 @@ class TestFilterGetMessages(StepsFilter): failed_payloads.append(payload["description"]) assert not failed_payloads, f"Payloads failed: {failed_payloads}" + @pytest.mark.waku_test_fleet def test_filter_get_message_with_valid_timestamps(self): failed_timestamps = [] - for timestamp in SAMPLE_TIMESTAMPS: + for timestamp in get_sample_timestamps(): if self.node1.type() in timestamp["valid_for"]: logger.debug(f'Running test with timestamp {timestamp["description"]}') message = self.create_message(timestamp=timestamp["value"]) @@ -37,12 +39,15 @@ class TestFilterGetMessages(StepsFilter): failed_timestamps.append(timestamp) assert not failed_timestamps, f"Timestamps failed: {failed_timestamps}" + @pytest.mark.waku_test_fleet def test_filter_get_message_with_version(self): self.check_published_message_reaches_filter_peer(self.create_message(version=10)) + @pytest.mark.waku_test_fleet def test_filter_get_message_with_meta(self): self.check_published_message_reaches_filter_peer(self.create_message(meta=to_base64(self.test_payload))) + @pytest.mark.waku_test_fleet def test_filter_get_message_with_ephemeral(self): failed_ephemeral = [] for ephemeral in [True, False]: @@ -54,6 +59,7 @@ class TestFilterGetMessages(StepsFilter): failed_ephemeral.append(ephemeral) assert not failed_ephemeral, f"Ephemeral that failed: {failed_ephemeral}" + @pytest.mark.waku_test_fleet def test_filter_get_message_with_extra_field(self): try: self.check_published_message_reaches_filter_peer(self.create_message(extraField="extraValue")) diff --git a/tests/filter/test_subscribe_create.py b/tests/filter/test_subscribe_create.py index 5a57b277d..4dcac4f2d 100644 --- a/tests/filter/test_subscribe_create.py +++ b/tests/filter/test_subscribe_create.py @@ -10,6 +10,7 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node") class TestFilterSubscribeCreate(StepsFilter): + @pytest.mark.waku_test_fleet def test_filter_subscribe_to_single_topics(self): self.wait_for_subscriptions_on_main_nodes([self.test_content_topic]) self.check_published_message_reaches_filter_peer() @@ -47,6 +48,7 @@ class TestFilterSubscribeCreate(StepsFilter): failed_pubsub_topics.append(pubsub_topic) assert not failed_pubsub_topics, f"PubsubTopics failed: {failed_pubsub_topics}" + @pytest.mark.waku_test_fleet def test_filter_subscribe_to_100_content_topics_in_one_call(self): failed_content_topics = [] _100_content_topics = [str(i) for i in range(100)] @@ -98,6 +100,7 @@ class TestFilterSubscribeCreate(StepsFilter): except Exception as ex: assert "Bad Request" in str(ex) + @pytest.mark.waku_test_fleet def test_filter_subscribe_refresh(self): for _ in range(2): self.wait_for_subscriptions_on_main_nodes([self.test_content_topic]) @@ -159,6 +162,7 @@ class TestFilterSubscribeCreate(StepsFilter): except Exception as ex: assert "Bad Request" in str(ex) + @pytest.mark.waku_test_fleet def test_filter_subscribe_with_extra_field(self, subscribe_main_nodes): try: self.create_filter_subscription( diff --git a/tests/filter/test_subscribe_update.py b/tests/filter/test_subscribe_update.py index fed6aef9b..c5f48da8e 100644 --- a/tests/filter/test_subscribe_update.py +++ b/tests/filter/test_subscribe_update.py @@ -8,6 +8,7 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node") class TestFilterSubscribeUpdate(StepsFilter): + @pytest.mark.waku_test_fleet def test_filter_update_subscription_add_a_new_content_topic(self): self.wait_for_subscriptions_on_main_nodes([self.test_content_topic], pubsub_topic=self.test_pubsub_topic) self.update_filter_subscription({"requestId": "1", "contentFilters": [self.second_content_topic], "pubsubTopic": self.test_pubsub_topic}) @@ -39,6 +40,7 @@ class TestFilterSubscribeUpdate(StepsFilter): except Exception as ex: assert "Bad Request" in str(ex) + @pytest.mark.waku_test_fleet def test_filter_update_subscription_refresh_existing(self): self.wait_for_subscriptions_on_main_nodes([self.test_content_topic], pubsub_topic=self.test_pubsub_topic) self.update_filter_subscription({"requestId": "1", "contentFilters": [self.test_content_topic], "pubsubTopic": self.test_pubsub_topic}) diff --git a/tests/filter/test_unsubscribe.py b/tests/filter/test_unsubscribe.py index ded2623e3..b4fc9ed93 100644 --- a/tests/filter/test_unsubscribe.py +++ b/tests/filter/test_unsubscribe.py @@ -5,11 +5,13 @@ from src.steps.filter import StepsFilter @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node", "subscribe_main_nodes") class TestFilterUnSubscribe(StepsFilter): + @pytest.mark.waku_test_fleet def test_filter_unsubscribe_from_single_content_topic(self): self.check_published_message_reaches_filter_peer() self.delete_filter_subscription({"requestId": "1", "contentFilters": [self.test_content_topic], "pubsubTopic": self.test_pubsub_topic}) self.check_publish_without_filter_subscription() + @pytest.mark.waku_test_fleet def test_filter_unsubscribe_from_all_subscribed_content_topics(self): content_topics = [input["value"] for input in SAMPLE_INPUTS[:2]] self.wait_for_subscriptions_on_main_nodes(content_topics) @@ -25,6 +27,7 @@ class TestFilterUnSubscribe(StepsFilter): self.check_published_message_reaches_filter_peer(self.create_message(contentTopic=content_topics[1])) self.check_publish_without_filter_subscription(self.create_message(contentTopic=content_topics[0])) + @pytest.mark.waku_test_fleet def test_filter_unsubscribe_from_pubsub_topics(self): self.wait_for_subscriptions_on_main_nodes([self.test_content_topic], self.test_pubsub_topic) self.wait_for_subscriptions_on_main_nodes([self.second_content_topic], self.second_pubsub_topic) diff --git a/tests/filter/test_unsubscribe_all.py b/tests/filter/test_unsubscribe_all.py index f80a10a33..c46b70960 100644 --- a/tests/filter/test_unsubscribe_all.py +++ b/tests/filter/test_unsubscribe_all.py @@ -6,6 +6,7 @@ from random import choice @pytest.mark.usefixtures("setup_main_relay_node", "setup_main_filter_node") class TestFilterUnSubscribeAll(StepsFilter): + @pytest.mark.waku_test_fleet def test_filter_unsubscribe_all_from_few_content_topics(self): content_topics = [input["value"] for input in SAMPLE_INPUTS[:5]] self.wait_for_subscriptions_on_main_nodes(content_topics) @@ -30,6 +31,7 @@ class TestFilterUnSubscribeAll(StepsFilter): self.check_publish_without_filter_subscription(self.create_message(contentTopic=choice(second_list))) self.check_publish_without_filter_subscription(self.create_message(contentTopic=choice(third_list))) + @pytest.mark.waku_test_fleet def test_filter_unsubscribe_all_from_multiple_pubsub_topics(self): for pubsub_topic in VALID_PUBSUB_TOPICS: content_topic = pubsub_topic diff --git a/tests/light_push/test_publish.py b/tests/light_push/test_publish.py index 1e39ac0bb..5e90b4edc 100644 --- a/tests/light_push/test_publish.py +++ b/tests/light_push/test_publish.py @@ -4,7 +4,14 @@ from src.libs.custom_logger import get_custom_logger from time import time from src.libs.common import delay, to_base64 from src.steps.light_push import StepsLightPush -from src.test_data import INVALID_CONTENT_TOPICS, INVALID_PAYLOADS, PUBSUB_TOPICS_WRONG_FORMAT, SAMPLE_INPUTS, SAMPLE_TIMESTAMPS, VALID_PUBSUB_TOPICS +from src.test_data import ( + INVALID_CONTENT_TOPICS, + INVALID_PAYLOADS, + PUBSUB_TOPICS_WRONG_FORMAT, + SAMPLE_INPUTS, + VALID_PUBSUB_TOPICS, + get_sample_timestamps, +) logger = get_custom_logger(__name__) @@ -17,6 +24,7 @@ class TestLightPushPublish(StepsLightPush): self.setup_first_lightpush_node() self.subscribe_to_pubsub_topics_via_relay() + @pytest.mark.waku_test_fleet def test_light_push_with_valid_payloads(self): failed_payloads = [] for payload in SAMPLE_INPUTS: @@ -75,6 +83,7 @@ class TestLightPushPublish(StepsLightPush): except Exception as ex: assert "Message size exceeded maximum of 153600 bytes" in str(ex) + @pytest.mark.waku_test_fleet def test_light_push_with_valid_content_topics(self): failed_content_topics = [] for content_topic in SAMPLE_INPUTS: @@ -151,9 +160,10 @@ class TestLightPushPublish(StepsLightPush): except Exception as ex: assert "not_published_to_any_peer" in str(ex) or "timeout" in str(ex) + @pytest.mark.waku_test_fleet def test_light_push_with_valid_timestamps(self): failed_timestamps = [] - for timestamp in SAMPLE_TIMESTAMPS: + for timestamp in get_sample_timestamps(): if self.light_push_node1.type() in timestamp["valid_for"]: logger.debug(f'Running test with timestamp {timestamp["description"]}') message = self.create_message(timestamp=timestamp["value"]) @@ -166,7 +176,7 @@ class TestLightPushPublish(StepsLightPush): def test_light_push_with_invalid_timestamps(self): success_timestamps = [] - for timestamp in SAMPLE_TIMESTAMPS: + for timestamp in get_sample_timestamps(): if self.light_push_node1.type() not in timestamp["valid_for"]: logger.debug(f'Running test with timestamp {timestamp["description"]}') message = self.create_message(timestamp=timestamp["value"]) @@ -191,6 +201,7 @@ class TestLightPushPublish(StepsLightPush): except Exception as ex: assert "Bad Request" in str(ex) + @pytest.mark.waku_test_fleet def test_light_push_with_valid_meta(self): self.check_light_pushed_message_reaches_receiving_peer(message=self.create_message(meta=to_base64(self.test_payload))) @@ -208,6 +219,7 @@ class TestLightPushPublish(StepsLightPush): except Exception as ex: assert '(kind: InvalidLengthField, field: "meta")' in str(ex) or "invalid length for Meta field" in str(ex) + @pytest.mark.waku_test_fleet def test_light_push_with_ephemeral(self): failed_ephemeral = [] for ephemeral in [True, False]: @@ -219,6 +231,7 @@ class TestLightPushPublish(StepsLightPush): failed_ephemeral.append(ephemeral) assert not failed_ephemeral, f"Ephemeral that failed: {failed_ephemeral}" + @pytest.mark.waku_test_fleet def test_light_push_with_extra_field(self): try: self.check_light_pushed_message_reaches_receiving_peer(message=self.create_message(extraField="extraValue")) diff --git a/tests/relay/test_publish.py b/tests/relay/test_publish.py index 5e2879045..d61ea1dbd 100644 --- a/tests/relay/test_publish.py +++ b/tests/relay/test_publish.py @@ -4,7 +4,7 @@ from src.libs.custom_logger import get_custom_logger from time import time from src.libs.common import delay, to_base64 from src.steps.relay import StepsRelay -from src.test_data import INVALID_CONTENT_TOPICS, INVALID_PAYLOADS, SAMPLE_INPUTS, SAMPLE_TIMESTAMPS, VALID_PUBSUB_TOPICS +from src.test_data import INVALID_CONTENT_TOPICS, INVALID_PAYLOADS, SAMPLE_INPUTS, VALID_PUBSUB_TOPICS, get_sample_timestamps from src.node.waku_message import WakuMessage logger = get_custom_logger(__name__) @@ -12,6 +12,7 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("setup_main_relay_nodes", "subscribe_main_relay_nodes", "relay_warm_up") class TestRelayPublish(StepsRelay): + @pytest.mark.waku_test_fleet def test_publish_with_valid_payloads(self): failed_payloads = [] for payload in SAMPLE_INPUTS: @@ -44,6 +45,7 @@ class TestRelayPublish(StepsRelay): except Exception as ex: assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex) + @pytest.mark.waku_test_fleet def test_publish_with_payload_less_than_150_kb(self): payload_length = 1024 * 100 # after encoding to base64 this will be close to 150KB logger.debug(f"Running test with payload length of {payload_length} bytes") @@ -60,6 +62,7 @@ class TestRelayPublish(StepsRelay): except Exception as ex: assert "couldn't find any messages" in str(ex) or "Bad Request" in str(ex) or "Internal Server Error" in str(ex) + @pytest.mark.waku_test_fleet def test_publish_with_valid_content_topics(self): failed_content_topics = [] for content_topic in SAMPLE_INPUTS: @@ -92,6 +95,7 @@ class TestRelayPublish(StepsRelay): except Exception as ex: assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex) + @pytest.mark.waku_test_fleet def test_publish_on_multiple_pubsub_topics(self): self.ensure_relay_subscriptions_on_nodes(self.main_nodes, VALID_PUBSUB_TOPICS) failed_pubsub_topics = [] @@ -118,9 +122,10 @@ class TestRelayPublish(StepsRelay): except Exception as ex: assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex) + @pytest.mark.waku_test_fleet def test_publish_with_valid_timestamps(self): failed_timestamps = [] - for timestamp in SAMPLE_TIMESTAMPS: + for timestamp in get_sample_timestamps(): if self.node1.type() in timestamp["valid_for"]: logger.debug(f'Running test with timestamp {timestamp["description"]}') message = self.create_message(timestamp=timestamp["value"]) @@ -133,7 +138,7 @@ class TestRelayPublish(StepsRelay): def test_publish_with_invalid_timestamps(self): success_timestamps = [] - for timestamp in SAMPLE_TIMESTAMPS: + for timestamp in get_sample_timestamps(): if self.node1.type() not in timestamp["valid_for"]: logger.debug(f'Running test with timestamp {timestamp["description"]}') message = self.create_message(timestamp=timestamp["value"]) @@ -144,10 +149,12 @@ class TestRelayPublish(StepsRelay): pass assert not success_timestamps, f"Invalid Timestamps that didn't failed: {success_timestamps}" + @pytest.mark.waku_test_fleet def test_publish_with_no_timestamp(self): message = {"payload": to_base64(self.test_payload), "contentTopic": self.test_content_topic} self.check_published_message_reaches_relay_peer(message) + @pytest.mark.waku_test_fleet def test_publish_with_valid_version(self): self.check_published_message_reaches_relay_peer(self.create_message(version=10)) @@ -168,6 +175,7 @@ class TestRelayPublish(StepsRelay): except Exception as ex: assert "Bad Request" in str(ex) + @pytest.mark.waku_test_fleet def test_publish_with_ephemeral(self): failed_ephemeral = [] for ephemeral in [True, False]: diff --git a/tests/store/test_get_messages.py b/tests/store/test_get_messages.py index 1af7f85ab..70e7d715f 100644 --- a/tests/store/test_get_messages.py +++ b/tests/store/test_get_messages.py @@ -10,6 +10,7 @@ logger = get_custom_logger(__name__) @pytest.mark.usefixtures("node_setup") class TestGetMessages(StepsStore): + @pytest.mark.waku_test_fleet def test_get_store_messages_with_different_payloads(self): failed_payloads = [] for payload in SAMPLE_INPUTS: @@ -22,8 +23,19 @@ class TestGetMessages(StepsStore): logger.error(f'Payload {payload["description"]} failed: {str(e)}') failed_payloads.append(payload["description"]) assert not failed_payloads, f"Payloads failed: {failed_payloads}" - assert len(self.store_response.messages) == len(SAMPLE_INPUTS) + # Content-topic-scoped query to accommodate fleet tests + for node in self.store_nodes: + scoped = self.get_messages_from_store( + node, + page_size=len(SAMPLE_INPUTS) + 10, + content_topics=self.test_content_topic, + ascending="true", + ) + assert len(scoped.messages) == len(SAMPLE_INPUTS), ( + f"Expected {len(SAMPLE_INPUTS)} test messages on {node.image} " f"but found {len(scoped.messages)}" + ) + @pytest.mark.waku_test_fleet def test_get_store_messages_with_different_content_topics(self): failed_content_topics = [] for content_topic in CONTENT_TOPICS_DIFFERENT_SHARDS: @@ -50,11 +62,13 @@ class TestGetMessages(StepsStore): failed_pubsub_topics.append(pubsub_topic) assert not failed_pubsub_topics, f"PubsubTopics failed: {failed_pubsub_topics}" + @pytest.mark.waku_test_fleet def test_get_store_message_with_meta(self): message = self.create_message(meta=to_base64(self.test_payload)) self.publish_message(message=message) self.check_published_message_is_stored(page_size=5, ascending="true") + @pytest.mark.waku_test_fleet def test_get_store_message_with_version(self): message = self.create_message(version=10) self.publish_message(message=message) @@ -68,6 +82,7 @@ class TestGetMessages(StepsStore): # only one message is stored assert len(self.store_response.messages) == 1 + @pytest.mark.waku_test_fleet def test_get_multiple_store_messages(self): message_hash_list = {"nwaku": []} for payload in SAMPLE_INPUTS: @@ -75,8 +90,12 @@ class TestGetMessages(StepsStore): self.publish_message(message=message) message_hash_list["nwaku"].append(self.compute_message_hash(self.test_pubsub_topic, message, hash_type="hex")) for node in self.store_nodes: - store_response = self.get_messages_from_store(node, page_size=50) - assert len(store_response.messages) == len(SAMPLE_INPUTS) + # Scope the store query to the test content topic so that background fleet + # messages archived on the same shard do not inflate the expected count. + store_response = self.get_messages_from_store(node, page_size=50, content_topics=self.test_content_topic) + assert len(store_response.messages) == len( + SAMPLE_INPUTS + ), f"Expected {len(SAMPLE_INPUTS)} messages but got {len(store_response.messages)}" for index in range(len(store_response.messages)): assert store_response.message_hash(index) == message_hash_list[node.type()][index], f"Message hash at index {index} doesn't match"