180 lines
32 KiB
Plaintext

2025-12-11 04:14:25.710 DEBUG [tests.conftest] Running fixture setup: test_id
2025-12-11 04:14:25.710 DEBUG [tests.conftest] Running test: test_static_shard_relay with id: 2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc
2025-12-11 04:14:25.711 DEBUG [src.steps.common] Running fixture setup: common_setup
2025-12-11 04:14:25.711 DEBUG [src.steps.relay] Running fixture setup: relay_setup
2025-12-11 04:14:25.711 DEBUG [src.steps.sharding] Running fixture setup: sharding_setup
2025-12-11 04:14:25.718 DEBUG [src.node.docker_mananger] Docker client initialized with image wakuorg/nwaku:latest
2025-12-11 04:14:25.718 DEBUG [src.node.waku_node] WakuNode instance initialized with log path ./log/docker/node1_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:25.718 DEBUG [src.node.waku_node] Starting Node...
2025-12-11 04:14:25.718 DEBUG [src.node.docker_mananger] Attempting to create or retrieve network waku
2025-12-11 04:14:25.719 DEBUG [src.node.docker_mananger] Network waku already exists
2025-12-11 04:14:25.719 DEBUG [src.node.docker_mananger] Generated random external IP 172.18.34.59
2025-12-11 04:14:25.720 DEBUG [src.node.docker_mananger] Generated ports ['55070', '55071', '55072', '55073', '55074']
2025-12-11 04:14:25.720 DEBUG [src.node.waku_node] Cluster id was resolved to: 2
2025-12-11 04:14:25.720 DEBUG [src.node.waku_node] Removing pubsub-topic from nwaku args
2025-12-11 04:14:25.720 DEBUG [src.node.waku_node] RLN credentials were not set
2025-12-11 04:14:25.720 INFO [src.node.waku_node] RLN credentials not set or credential store not available, starting without RLN
2025-12-11 04:14:25.720 DEBUG [src.node.waku_node] Using volumes []
2025-12-11 04:14:25.720 DEBUG [src.node.docker_mananger] docker run -i -t -p 55070:55070 -p 55071:55071 -p 55072:55072 -p 55073:55073 -p 55074:55074 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=55072 --rest-port=55070 --tcp-port=55071 --discv5-udp-port=55073 --rest-address=0.0.0.0 --nat=extip:172.18.34.59 --peer-exchange=true --discv5-discovery=true --cluster-id=2 --nodekey=aa61140fba90dcfbccccf68a9afe78e9cc9c4a39eb5e0adf0aa9adf11411d9e0 --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=55074 --metrics-logging=true --relay=true --filter=true
2025-12-11 04:14:25.897 DEBUG [src.node.docker_mananger] docker network connect --ip 172.18.34.59 waku c95d8509815cb8e314d294ef189b92951c59caa07a3abfcf4d303ab93c5de880
2025-12-11 04:14:25.926 DEBUG [src.node.docker_mananger] Container started with ID c95d8509815c. Setting up logs at ./log/docker/node1_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:25.927 DEBUG [src.node.waku_node] Started container from image wakuorg/nwaku:latest. REST: 55070
2025-12-11 04:14:25.927 DEBUG [src.libs.common] Sleeping for 1 seconds
2025-12-11 04:14:25.985 ERROR [src.node.docker_mananger] Max retries reached for container cb68b3fa7665. Exiting log stream.
2025-12-11 04:14:26.506 ERROR [src.node.docker_mananger] Max retries reached for container 39645ae0367d. Exiting log stream.
2025-12-11 04:14:26.928 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:55070/health" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:26.931 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"nodeHealth":"READY","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Rln Relay":"NOT_MOUNTED"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_READY","desc":"Relay is not ready, filter will not be able to sort out messages"},{"Store":"NOT_MOUNTED"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"NOT_READY","desc":"No Store service peer available yet, neither Store service set up for the node"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"NOT_READY","desc":"No Filter service peer available yet"}]}'
2025-12-11 04:14:26.931 INFO [src.node.waku_node] Node protocols are initialized !!
2025-12-11 04:14:26.931 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:55070/debug/v1/info" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:26.933 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.34.59/tcp/55071/p2p/16Uiu2HAmNR6qAKhZVFEGmCBtmULLGc35v77ra3Er1gZZfJkW7Lwz","/ip4/172.18.34.59/tcp/55072/ws/p2p/16Uiu2HAmNR6qAKhZVFEGmCBtmULLGc35v77ra3Er1gZZfJkW7Lwz"],"enrUri":"enr:-L24QHAfTVHFxgYZN7JkB7tpc4BfZ-rsaCxIc5Slmvpn13OmWi327YSTXpmFCOC3YD7Jx3dywlRnE2j8SJTfhiPMtrICgmlkgnY0gmlwhKwSIjuKbXVsdGlhZGRyc5YACASsEiI7BtcfAAoErBIiOwbXIN0DgnJzhQACAQAAiXNlY3AyNTZrMaEDkQYQxXophz1PllNghVEX1jugrt4a_I7_YraDHlYa5QGDdGNwgtcfg3VkcILXIYV3YWt1MgU"}'
2025-12-11 04:14:26.933 INFO [src.node.waku_node] REST service is ready !!
2025-12-11 04:14:26.940 DEBUG [src.node.docker_mananger] Docker client initialized with image wakuorg/nwaku:latest
2025-12-11 04:14:26.940 DEBUG [src.node.waku_node] WakuNode instance initialized with log path ./log/docker/node2_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:26.940 DEBUG [src.node.waku_node] Starting Node...
2025-12-11 04:14:26.940 DEBUG [src.node.docker_mananger] Attempting to create or retrieve network waku
2025-12-11 04:14:26.941 DEBUG [src.node.docker_mananger] Network waku already exists
2025-12-11 04:14:26.941 DEBUG [src.node.docker_mananger] Generated random external IP 172.18.115.143
2025-12-11 04:14:26.941 DEBUG [src.node.docker_mananger] Generated ports ['47668', '47669', '47670', '47671', '47672']
2025-12-11 04:14:26.942 DEBUG [src.node.waku_node] Cluster id was resolved to: 2
2025-12-11 04:14:26.942 DEBUG [src.node.waku_node] Removing pubsub-topic from nwaku args
2025-12-11 04:14:26.942 DEBUG [src.node.waku_node] RLN credentials were not set
2025-12-11 04:14:26.942 INFO [src.node.waku_node] RLN credentials not set or credential store not available, starting without RLN
2025-12-11 04:14:26.942 DEBUG [src.node.waku_node] Using volumes []
2025-12-11 04:14:26.942 DEBUG [src.node.docker_mananger] docker run -i -t -p 47668:47668 -p 47669:47669 -p 47670:47670 -p 47671:47671 -p 47672:47672 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=47670 --rest-port=47668 --tcp-port=47669 --discv5-udp-port=47671 --rest-address=0.0.0.0 --nat=extip:172.18.115.143 --peer-exchange=true --discv5-discovery=true --cluster-id=2 --nodekey=53b1d6996bffbced2d18bb8bdaa0745aefef2beb5b1b774ed7282b6e09d64419 --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=47672 --metrics-logging=true --relay=true --discv5-bootstrap-node=enr:-L24QHAfTVHFxgYZN7JkB7tpc4BfZ-rsaCxIc5Slmvpn13OmWi327YSTXpmFCOC3YD7Jx3dywlRnE2j8SJTfhiPMtrICgmlkgnY0gmlwhKwSIjuKbXVsdGlhZGRyc5YACASsEiI7BtcfAAoErBIiOwbXIN0DgnJzhQACAQAAiXNlY3AyNTZrMaEDkQYQxXophz1PllNghVEX1jugrt4a_I7_YraDHlYa5QGDdGNwgtcfg3VkcILXIYV3YWt1MgU
2025-12-11 04:14:27.113 DEBUG [src.node.docker_mananger] docker network connect --ip 172.18.115.143 waku faff220cc164aa8f96c606d2bfba6867c23fe2487bc765a5c6b52562cbe2e733
2025-12-11 04:14:27.145 DEBUG [src.node.docker_mananger] Container started with ID faff220cc164. Setting up logs at ./log/docker/node2_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:27.145 DEBUG [src.node.waku_node] Started container from image wakuorg/nwaku:latest. REST: 47668
2025-12-11 04:14:27.146 DEBUG [src.libs.common] Sleeping for 1 seconds
2025-12-11 04:14:28.146 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:47668/health" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:28.157 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"nodeHealth":"READY","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Rln Relay":"NOT_MOUNTED"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"NOT_MOUNTED"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"NOT_READY","desc":"No Store service peer available yet, neither Store service set up for the node"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"READY"}]}'
2025-12-11 04:14:28.158 INFO [src.node.waku_node] Node protocols are initialized !!
2025-12-11 04:14:28.159 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:47668/debug/v1/info" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:28.165 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.115.143/tcp/47669/p2p/16Uiu2HAm3wRyvsdjSNMJxhz2B7twFh93rU7ZBE9pyind9vkUs7mf","/ip4/172.18.115.143/tcp/47670/ws/p2p/16Uiu2HAm3wRyvsdjSNMJxhz2B7twFh93rU7ZBE9pyind9vkUs7mf"],"enrUri":"enr:-L24QHWkik4KPJ6g6D29pN9MCYTOkUksp3POhcBhbWnRh_J0fSHRLg9CVgGyNDYnLpdsopDGZifK6KYzmBAyn6TvLJ0CgmlkgnY0gmlwhKwSc4-KbXVsdGlhZGRyc5YACASsEnOPBro1AAoErBJzjwa6Nt0DgnJzhQACAQAAiXNlY3AyNTZrMaECfn0XrTWcdzfMoGmggodvlKCZITuOmA_ozXoc_Nt-AJaDdGNwgro1g3VkcIK6N4V3YWt1MgE"}'
2025-12-11 04:14:28.166 INFO [src.node.waku_node] REST service is ready !!
2025-12-11 04:14:28.167 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:47668/admin/v1/peers" -H "Content-Type: application/json" -d '["/ip4/172.18.34.59/tcp/55071/p2p/16Uiu2HAmNR6qAKhZVFEGmCBtmULLGc35v77ra3Er1gZZfJkW7Lwz"]'
2025-12-11 04:14:28.170 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:28.177 DEBUG [src.node.docker_mananger] Docker client initialized with image wakuorg/nwaku:latest
2025-12-11 04:14:28.177 DEBUG [src.node.waku_node] WakuNode instance initialized with log path ./log/docker/node3_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:28.177 DEBUG [src.node.waku_node] Starting Node...
2025-12-11 04:14:28.177 DEBUG [src.node.docker_mananger] Attempting to create or retrieve network waku
2025-12-11 04:14:28.179 DEBUG [src.node.docker_mananger] Network waku already exists
2025-12-11 04:14:28.179 DEBUG [src.node.docker_mananger] Generated random external IP 172.18.2.249
2025-12-11 04:14:28.179 DEBUG [src.node.docker_mananger] Generated ports ['3530', '3531', '3532', '3533', '3534']
2025-12-11 04:14:28.179 DEBUG [src.node.waku_node] Cluster id was resolved to: 2
2025-12-11 04:14:28.179 DEBUG [src.node.waku_node] Removing pubsub-topic from nwaku args
2025-12-11 04:14:28.179 DEBUG [src.node.waku_node] RLN credentials were not set
2025-12-11 04:14:28.179 INFO [src.node.waku_node] RLN credentials not set or credential store not available, starting without RLN
2025-12-11 04:14:28.180 DEBUG [src.node.waku_node] Using volumes []
2025-12-11 04:14:28.180 DEBUG [src.node.docker_mananger] docker run -i -t -p 3530:3530 -p 3531:3531 -p 3532:3532 -p 3533:3533 -p 3534:3534 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=3532 --rest-port=3530 --tcp-port=3531 --discv5-udp-port=3533 --rest-address=0.0.0.0 --nat=extip:172.18.2.249 --peer-exchange=true --discv5-discovery=true --cluster-id=2 --nodekey=d65a6cc66da3c9cdbf632dcbba41bae8ae07aaffcb32d999ffdca3bddcffacdc --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=3534 --metrics-logging=true --relay=true --discv5-bootstrap-node=enr:-L24QHAfTVHFxgYZN7JkB7tpc4BfZ-rsaCxIc5Slmvpn13OmWi327YSTXpmFCOC3YD7Jx3dywlRnE2j8SJTfhiPMtrICgmlkgnY0gmlwhKwSIjuKbXVsdGlhZGRyc5YACASsEiI7BtcfAAoErBIiOwbXIN0DgnJzhQACAQAAiXNlY3AyNTZrMaEDkQYQxXophz1PllNghVEX1jugrt4a_I7_YraDHlYa5QGDdGNwgtcfg3VkcILXIYV3YWt1MgU
2025-12-11 04:14:28.356 DEBUG [src.node.docker_mananger] docker network connect --ip 172.18.2.249 waku cd1e09f0f5b4db4a9cfd50a1b910003d7d80ed56f421638ef5d65cd61c59b972
2025-12-11 04:14:28.388 DEBUG [src.node.docker_mananger] Container started with ID cd1e09f0f5b4. Setting up logs at ./log/docker/node3_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:28.389 DEBUG [src.node.waku_node] Started container from image wakuorg/nwaku:latest. REST: 3530
2025-12-11 04:14:28.389 DEBUG [src.libs.common] Sleeping for 1 seconds
2025-12-11 04:14:29.389 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:3530/health" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:29.398 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"nodeHealth":"READY","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Rln Relay":"NOT_MOUNTED"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"NOT_MOUNTED"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"NOT_READY","desc":"No Store service peer available yet, neither Store service set up for the node"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"READY"}]}'
2025-12-11 04:14:29.398 INFO [src.node.waku_node] Node protocols are initialized !!
2025-12-11 04:14:29.398 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:3530/debug/v1/info" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:29.404 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.2.249/tcp/3531/p2p/16Uiu2HAmDyLMG4Sz89b62EYE6Y7SrYBtxaRAsHmQM6ssxLkKqQsq","/ip4/172.18.2.249/tcp/3532/ws/p2p/16Uiu2HAmDyLMG4Sz89b62EYE6Y7SrYBtxaRAsHmQM6ssxLkKqQsq"],"enrUri":"enr:-L24QDMyhrXiwBDK9mGBvBz0584QrE0pa7C7t5fj7Awd5wjlEQUcfM37RBaz6Vqd77mFumo704KY9cOHf1ZkI8jHJ_wCgmlkgnY0gmlwhKwSAvmKbXVsdGlhZGRyc5YACASsEgL5Bg3LAAoErBIC-QYNzN0DgnJzhQACAQAAiXNlY3AyNTZrMaEDE47BRusOkEvW-xQjhk0vZgz4WsJj5EuXAQQ4oAIgUICDdGNwgg3Lg3VkcIINzYV3YWt1MgE"}'
2025-12-11 04:14:29.405 INFO [src.node.waku_node] REST service is ready !!
2025-12-11 04:14:29.405 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:3530/admin/v1/peers" -H "Content-Type: application/json" -d '["/ip4/172.18.34.59/tcp/55071/p2p/16Uiu2HAmNR6qAKhZVFEGmCBtmULLGc35v77ra3Er1gZZfJkW7Lwz"]'
2025-12-11 04:14:29.408 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:29.414 DEBUG [src.node.docker_mananger] Docker client initialized with image wakuorg/nwaku:latest
2025-12-11 04:14:29.414 DEBUG [src.node.waku_node] WakuNode instance initialized with log path ./log/docker/node4_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:29.414 DEBUG [src.node.waku_node] Starting Node...
2025-12-11 04:14:29.415 DEBUG [src.node.docker_mananger] Attempting to create or retrieve network waku
2025-12-11 04:14:29.416 DEBUG [src.node.docker_mananger] Network waku already exists
2025-12-11 04:14:29.416 DEBUG [src.node.docker_mananger] Generated random external IP 172.18.161.150
2025-12-11 04:14:29.416 DEBUG [src.node.docker_mananger] Generated ports ['59456', '59457', '59458', '59459', '59460']
2025-12-11 04:14:29.416 DEBUG [src.node.waku_node] Cluster id was resolved to: 2
2025-12-11 04:14:29.416 DEBUG [src.node.waku_node] Removing pubsub-topic from nwaku args
2025-12-11 04:14:29.416 DEBUG [src.node.waku_node] RLN credentials were not set
2025-12-11 04:14:29.417 INFO [src.node.waku_node] RLN credentials not set or credential store not available, starting without RLN
2025-12-11 04:14:29.417 DEBUG [src.node.waku_node] Using volumes []
2025-12-11 04:14:29.417 DEBUG [src.node.docker_mananger] docker run -i -t -p 59456:59456 -p 59457:59457 -p 59458:59458 -p 59459:59459 -p 59460:59460 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=59458 --rest-port=59456 --tcp-port=59457 --discv5-udp-port=59459 --rest-address=0.0.0.0 --nat=extip:172.18.161.150 --peer-exchange=true --discv5-discovery=true --cluster-id=2 --nodekey=5f6ed0ab35c7d4144f661d5bde9eeb7fe061c0fb42e094ce13ace8b5cacacbcf --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=59460 --metrics-logging=true --relay=true --discv5-bootstrap-node=enr:-L24QHAfTVHFxgYZN7JkB7tpc4BfZ-rsaCxIc5Slmvpn13OmWi327YSTXpmFCOC3YD7Jx3dywlRnE2j8SJTfhiPMtrICgmlkgnY0gmlwhKwSIjuKbXVsdGlhZGRyc5YACASsEiI7BtcfAAoErBIiOwbXIN0DgnJzhQACAQAAiXNlY3AyNTZrMaEDkQYQxXophz1PllNghVEX1jugrt4a_I7_YraDHlYa5QGDdGNwgtcfg3VkcILXIYV3YWt1MgU
2025-12-11 04:14:29.594 DEBUG [src.node.docker_mananger] docker network connect --ip 172.18.161.150 waku 5516d16ee35a41e94cafe3c061f49df72831327a5547ba1e6a9757a6a616f02d
2025-12-11 04:14:29.627 DEBUG [src.node.docker_mananger] Container started with ID 5516d16ee35a. Setting up logs at ./log/docker/node4_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:29.627 DEBUG [src.node.waku_node] Started container from image wakuorg/nwaku:latest. REST: 59456
2025-12-11 04:14:29.628 DEBUG [src.libs.common] Sleeping for 1 seconds
2025-12-11 04:14:30.628 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:59456/health" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:30.636 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"nodeHealth":"READY","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Rln Relay":"NOT_MOUNTED"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"NOT_MOUNTED"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"NOT_READY","desc":"No Store service peer available yet, neither Store service set up for the node"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"READY"}]}'
2025-12-11 04:14:30.638 INFO [src.node.waku_node] Node protocols are initialized !!
2025-12-11 04:14:30.638 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:59456/debug/v1/info" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:30.644 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.161.150/tcp/59457/p2p/16Uiu2HAmDMjkWXLNJwRczcjWvSQuk9xfd3a6y4pAsKFSb2wxqVTT","/ip4/172.18.161.150/tcp/59458/ws/p2p/16Uiu2HAmDMjkWXLNJwRczcjWvSQuk9xfd3a6y4pAsKFSb2wxqVTT"],"enrUri":"enr:-L24QFU_vw0BLk91awhsgDixzSIWLvG86biQDm_Tvb2z7LkWNygMZY7yoFqS6I00rOSK1fCiW96XZ3TJlht1yjhSiN4CgmlkgnY0gmlwhKwSoZaKbXVsdGlhZGRyc5YACASsEqGWBuhBAAoErBKhlgboQt0DgnJzhQACAQAAiXNlY3AyNTZrMaEDCnBOJ2NpfpWYUu8xwHPgruaAxZ_Z34zUOtF8Hn4VE96DdGNwguhBg3VkcILoQ4V3YWt1MgE"}'
2025-12-11 04:14:30.645 INFO [src.node.waku_node] REST service is ready !!
2025-12-11 04:14:30.645 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:59456/admin/v1/peers" -H "Content-Type: application/json" -d '["/ip4/172.18.34.59/tcp/55071/p2p/16Uiu2HAmNR6qAKhZVFEGmCBtmULLGc35v77ra3Er1gZZfJkW7Lwz"]'
2025-12-11 04:14:30.647 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:30.654 DEBUG [src.node.docker_mananger] Docker client initialized with image wakuorg/nwaku:latest
2025-12-11 04:14:30.654 DEBUG [src.node.waku_node] WakuNode instance initialized with log path ./log/docker/node5_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:30.654 DEBUG [src.node.waku_node] Starting Node...
2025-12-11 04:14:30.654 DEBUG [src.node.docker_mananger] Attempting to create or retrieve network waku
2025-12-11 04:14:30.655 DEBUG [src.node.docker_mananger] Network waku already exists
2025-12-11 04:14:30.656 DEBUG [src.node.docker_mananger] Generated random external IP 172.18.225.166
2025-12-11 04:14:30.656 DEBUG [src.node.docker_mananger] Generated ports ['48103', '48104', '48105', '48106', '48107']
2025-12-11 04:14:30.656 DEBUG [src.node.waku_node] Cluster id was resolved to: 2
2025-12-11 04:14:30.656 DEBUG [src.node.waku_node] Removing pubsub-topic from nwaku args
2025-12-11 04:14:30.656 DEBUG [src.node.waku_node] RLN credentials were not set
2025-12-11 04:14:30.656 INFO [src.node.waku_node] RLN credentials not set or credential store not available, starting without RLN
2025-12-11 04:14:30.656 DEBUG [src.node.waku_node] Using volumes []
2025-12-11 04:14:30.657 DEBUG [src.node.docker_mananger] docker run -i -t -p 48103:48103 -p 48104:48104 -p 48105:48105 -p 48106:48106 -p 48107:48107 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=48105 --rest-port=48103 --tcp-port=48104 --discv5-udp-port=48106 --rest-address=0.0.0.0 --nat=extip:172.18.225.166 --peer-exchange=true --discv5-discovery=true --cluster-id=2 --nodekey=e6bd0f65e9e7f6884cb3bfabfe4d7f67cda2efcaccbd2956ed6c3cbcbcefe5e0 --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=48107 --metrics-logging=true --relay=true --discv5-bootstrap-node=enr:-L24QHAfTVHFxgYZN7JkB7tpc4BfZ-rsaCxIc5Slmvpn13OmWi327YSTXpmFCOC3YD7Jx3dywlRnE2j8SJTfhiPMtrICgmlkgnY0gmlwhKwSIjuKbXVsdGlhZGRyc5YACASsEiI7BtcfAAoErBIiOwbXIN0DgnJzhQACAQAAiXNlY3AyNTZrMaEDkQYQxXophz1PllNghVEX1jugrt4a_I7_YraDHlYa5QGDdGNwgtcfg3VkcILXIYV3YWt1MgU
2025-12-11 04:14:30.836 DEBUG [src.node.docker_mananger] docker network connect --ip 172.18.225.166 waku 0c542fe9962d3228ba8e935ade1a29f76ae4596646837d8c8212cf21ef62c5a1
2025-12-11 04:14:30.868 DEBUG [src.node.docker_mananger] Container started with ID 0c542fe9962d. Setting up logs at ./log/docker/node5_2025-12-11_04-14-25__5721d7df-74f8-4e8f-97f7-525cef5b55dc__wakuorg_nwaku:latest.log
2025-12-11 04:14:30.869 DEBUG [src.node.waku_node] Started container from image wakuorg/nwaku:latest. REST: 48103
2025-12-11 04:14:30.869 DEBUG [src.libs.common] Sleeping for 1 seconds
2025-12-11 04:14:31.869 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:48103/health" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:31.875 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"nodeHealth":"READY","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Rln Relay":"NOT_MOUNTED"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"NOT_MOUNTED"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"NOT_READY","desc":"No Store service peer available yet, neither Store service set up for the node"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"READY"}]}'
2025-12-11 04:14:31.876 INFO [src.node.waku_node] Node protocols are initialized !!
2025-12-11 04:14:31.876 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:48103/debug/v1/info" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:31.880 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.225.166/tcp/48104/p2p/16Uiu2HAkwAE4pxF5NC1KJ9QyZqVwFsAZrsEBwHPuD95nZ2s9ExER","/ip4/172.18.225.166/tcp/48105/ws/p2p/16Uiu2HAkwAE4pxF5NC1KJ9QyZqVwFsAZrsEBwHPuD95nZ2s9ExER"],"enrUri":"enr:-L24QJKA4eBCeBij-Ehh58-ZcB9PcbSvA0O1WzziPnkQT3KMeuIU-SKZ8XRHE6lLzUjaFEhWSevyQ2djh3kvv5LtZAkCgmlkgnY0gmlwhKwS4aaKbXVsdGlhZGRyc5YACASsEuGmBrvoAAoErBLhpga76d0DgnJzhQACAQAAiXNlY3AyNTZrMaECGcJcj0n4-NuB8Mqm0XX6yhFCshzryKu8xDMT6isAFs6DdGNwgrvog3VkcIK76oV3YWt1MgE"}'
2025-12-11 04:14:31.880 INFO [src.node.waku_node] REST service is ready !!
2025-12-11 04:14:31.881 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:48103/admin/v1/peers" -H "Content-Type: application/json" -d '["/ip4/172.18.34.59/tcp/55071/p2p/16Uiu2HAmNR6qAKhZVFEGmCBtmULLGc35v77ra3Er1gZZfJkW7Lwz"]'
2025-12-11 04:14:31.885 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.885 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:55070/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/2/0"]'
2025-12-11 04:14:31.887 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.888 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:47668/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/2/0"]'
2025-12-11 04:14:31.890 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.890 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:3530/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/2/0"]'
2025-12-11 04:14:31.893 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.893 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:59456/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/2/0"]'
2025-12-11 04:14:31.895 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.896 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:48103/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/2/0"]'
2025-12-11 04:14:31.898 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.898 INFO [src.node.api_clients.base_client] curl -v -X POST "http://127.0.0.1:55070/relay/v1/messages/%2Fwaku%2F2%2Frs%2F2%2F0" -H "Content-Type: application/json" -d '{"payload": "U2hhcmRpbmcgd29ya3MhIQ==", "contentTopic": "/myapp/1/latest/proto", "timestamp": '$(date +%s%N)'}'
2025-12-11 04:14:31.907 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'OK'
2025-12-11 04:14:31.907 DEBUG [src.libs.common] Sleeping for 0.1 seconds
2025-12-11 04:14:32.007 DEBUG [src.steps.sharding] Checking that peer NODE_1:wakuorg/nwaku:latest can find the published message
2025-12-11 04:14:32.007 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:55070/relay/v1/messages/%2Fwaku%2F2%2Frs%2F2%2F0" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:32.010 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'[{"payload":"U2hhcmRpbmcgd29ya3MhIQ==","contentTopic":"/myapp/1/latest/proto","version":0,"timestamp":1765426471898811253,"ephemeral":false,"proof":""}]'
2025-12-11 04:14:32.011 DEBUG [src.steps.sharding] Checking that peer NODE_2:wakuorg/nwaku:latest can find the published message
2025-12-11 04:14:32.011 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:47668/relay/v1/messages/%2Fwaku%2F2%2Frs%2F2%2F0" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:32.014 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'[{"payload":"U2hhcmRpbmcgd29ya3MhIQ==","contentTopic":"/myapp/1/latest/proto","version":0,"timestamp":1765426471898811253,"ephemeral":false,"proof":""}]'
2025-12-11 04:14:32.015 DEBUG [src.steps.sharding] Checking that peer NODE_3:wakuorg/nwaku:latest can find the published message
2025-12-11 04:14:32.015 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:3530/relay/v1/messages/%2Fwaku%2F2%2Frs%2F2%2F0" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:32.017 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'[{"payload":"U2hhcmRpbmcgd29ya3MhIQ==","contentTopic":"/myapp/1/latest/proto","version":0,"timestamp":1765426471898811253,"ephemeral":false,"proof":""}]'
2025-12-11 04:14:32.019 DEBUG [src.steps.sharding] Checking that peer NODE_4:wakuorg/nwaku:latest can find the published message
2025-12-11 04:14:32.019 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:59456/relay/v1/messages/%2Fwaku%2F2%2Frs%2F2%2F0" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:32.021 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'[{"payload":"U2hhcmRpbmcgd29ya3MhIQ==","contentTopic":"/myapp/1/latest/proto","version":0,"timestamp":1765426471898811253,"ephemeral":false,"proof":""}]'
2025-12-11 04:14:32.022 DEBUG [src.steps.sharding] Checking that peer NODE_5:wakuorg/nwaku:latest can find the published message
2025-12-11 04:14:32.022 INFO [src.node.api_clients.base_client] curl -v -X GET "http://127.0.0.1:48103/relay/v1/messages/%2Fwaku%2F2%2Frs%2F2%2F0" -H "Content-Type: application/json" -d 'None'
2025-12-11 04:14:32.024 INFO [src.node.api_clients.base_client] Response status code: 200. Response content: b'[{"payload":"U2hhcmRpbmcgd29ya3MhIQ==","contentTopic":"/myapp/1/latest/proto","version":0,"timestamp":1765426471898811253,"ephemeral":false,"proof":""}]'
2025-12-11 04:14:32.027 DEBUG [tests.conftest] Running fixture teardown: test_setup
2025-12-11 04:14:32.028 DEBUG [tests.conftest] Running fixture teardown: close_open_nodes
2025-12-11 04:14:32.028 DEBUG [src.node.waku_node] Stopping container with id c95d8509815c
2025-12-11 04:14:32.571 DEBUG [src.node.waku_node] Container stopped.
2025-12-11 04:14:32.571 DEBUG [src.node.waku_node] Stopping container with id faff220cc164
2025-12-11 04:14:33.109 DEBUG [src.node.waku_node] Container stopped.
2025-12-11 04:14:33.110 DEBUG [src.node.waku_node] Stopping container with id cd1e09f0f5b4
2025-12-11 04:14:33.383 ERROR [src.node.docker_mananger] Max retries reached for container c95d8509815c. Exiting log stream.
2025-12-11 04:14:33.655 DEBUG [src.node.waku_node] Container stopped.
2025-12-11 04:14:33.656 DEBUG [src.node.waku_node] Stopping container with id 5516d16ee35a
2025-12-11 04:14:33.938 ERROR [src.node.docker_mananger] Max retries reached for container faff220cc164. Exiting log stream.
2025-12-11 04:14:34.195 DEBUG [src.node.waku_node] Container stopped.
2025-12-11 04:14:34.195 DEBUG [src.node.waku_node] Stopping container with id 0c542fe9962d
2025-12-11 04:14:34.487 ERROR [src.node.docker_mananger] Max retries reached for container cd1e09f0f5b4. Exiting log stream.
2025-12-11 04:14:34.680 DEBUG [src.node.waku_node] Container stopped.
2025-12-11 04:14:34.682 DEBUG [tests.conftest] Running fixture teardown: check_waku_log_errors
2025-12-11 04:14:34.696 DEBUG [src.node.docker_mananger] No errors found in the waku logs.
2025-12-11 04:14:34.704 DEBUG [src.node.docker_mananger] No errors found in the waku logs.
2025-12-11 04:14:34.713 DEBUG [src.node.docker_mananger] No errors found in the waku logs.
2025-12-11 04:14:34.718 DEBUG [src.node.docker_mananger] No errors found in the waku logs.
2025-12-11 04:14:34.722 DEBUG [src.node.docker_mananger] No errors found in the waku logs.