DEBUG    tests.conftest:conftest.py:51 Running fixture setup: test_id
DEBUG    tests.conftest:conftest.py:57 Running test: test_publishing_node_restarts with id: 2026-02-25_19-36-40__60d44777-7e7f-45a9-a785-eac6751abc1f
DEBUG    src.steps.common:common.py:19 Running fixture setup: common_setup
DEBUG    src.steps.store:store.py:31 Running fixture setup: store_setup
DEBUG    src.steps.store:store.py:39 Running fixture setup: node_setup
DEBUG    src.node.docker_mananger:docker_mananger.py:19 Docker client initialized with image wakuorg/nwaku:latest
DEBUG    src.node.waku_node:waku_node.py:86 WakuNode instance initialized with log path ./log/docker/publishing_node1_2026-02-25_19-36-40__60d44777-7e7f-45a9-a785-eac6751abc1f__wakuorg_nwaku:latest.log
DEBUG    src.node.waku_node:waku_node.py:90 Starting Node...
DEBUG    src.node.docker_mananger:docker_mananger.py:22 Attempting to create or retrieve network waku
DEBUG    src.node.docker_mananger:docker_mananger.py:25 Network waku already exists
DEBUG    src.node.docker_mananger:docker_mananger.py:108 Generated random external IP 172.18.154.11
DEBUG    src.node.docker_mananger:docker_mananger.py:101 Generated ports ['10538', '10539', '10540', '10541', '10542']
DEBUG    src.node.waku_node:waku_node.py:439 RLN credentials were not set
INFO     src.node.waku_node:waku_node.py:176 RLN credentials not set or credential store not available, starting without RLN
DEBUG    src.node.waku_node:waku_node.py:178 Using volumes []
DEBUG    src.node.docker_mananger:docker_mananger.py:49 docker run -i -t -p 10538:10538 -p 10539:10539 -p 10540:10540 -p 10541:10541 -p 10542:10542 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=10540 --rest-port=10538 --tcp-port=10539 --discv5-udp-port=10541 --rest-address=0.0.0.0 --nat=extip:172.18.154.11 --peer-exchange=true --discv5-discovery=true --cluster-id=3 --nodekey=e28eaeebe2cb71f43fa0252fb19df6ad997fd1c0ccfdc90112d80a241b00d5f2 --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=10542 --metrics-logging=true --store=true --relay=true
ERROR    src.node.docker_mananger:docker_mananger.py:89 Max retries reached for container 498b77a06745. Exiting log stream.
DEBUG    src.node.docker_mananger:docker_mananger.py:55 docker network connect --ip 172.18.154.11 waku 5abc3d387235db2f86508053c195c22c540d7d4d9c5ce6d1ed15fb881f567f03
DEBUG    src.node.docker_mananger:docker_mananger.py:58 Container started with ID 5abc3d387235. Setting up logs at ./log/docker/publishing_node1_2026-02-25_19-36-40__60d44777-7e7f-45a9-a785-eac6751abc1f__wakuorg_nwaku:latest.log
DEBUG    src.node.waku_node:waku_node.py:190 Started container from image wakuorg/nwaku:latest. REST: 10538
DEBUG    src.libs.common:common.py:47 Sleeping for 1 seconds
ERROR    src.node.docker_mananger:docker_mananger.py:89 Max retries reached for container 6186a38553c5. Exiting log stream.
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/health" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"nodeHealth":"READY","connectionStatus":"Disconnected","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"READY"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"READY"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"NOT_READY","desc":"No Filter service peer available yet"},{"Rln Relay":"NOT_MOUNTED"}]}'
INFO     src.node.waku_node:waku_node.py:287 Node protocols are initialized !!
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/debug/v1/info" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.154.11/tcp/10539/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG","/ip4/172.18.154.11/tcp/10540/ws/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG"],"enrUri":"enr:-L24QKFJjy0MA32__SMesS-bjA96H3ePt5yj5qAYs7BMw4ftFEHWZTXuETDBE7l7woh_iviVpnOyC2bCxvaipPIWZpwCgmlkgnY0gmlwhKwSmguKbXVsdGlhZGRyc5YACASsEpoLBikrAAoErBKaCwYpLN0DgnJzhQADAQAAiXNlY3AyNTZrMaEDacl7AxNn6rpcnbDdEj_bxzoP31V0MEECgGlxMTP89VeDdGNwgikrg3VkcIIpLYV3YWt1MgM"}'
INFO     src.node.waku_node:waku_node.py:292 REST service is ready !!
DEBUG    src.node.docker_mananger:docker_mananger.py:19 Docker client initialized with image wakuorg/nwaku:latest
DEBUG    src.node.waku_node:waku_node.py:86 WakuNode instance initialized with log path ./log/docker/store_node1_2026-02-25_19-36-40__60d44777-7e7f-45a9-a785-eac6751abc1f__wakuorg_nwaku:latest.log
DEBUG    src.node.waku_node:waku_node.py:90 Starting Node...
DEBUG    src.node.docker_mananger:docker_mananger.py:22 Attempting to create or retrieve network waku
DEBUG    src.node.docker_mananger:docker_mananger.py:25 Network waku already exists
DEBUG    src.node.docker_mananger:docker_mananger.py:108 Generated random external IP 172.18.242.137
DEBUG    src.node.docker_mananger:docker_mananger.py:101 Generated ports ['18532', '18533', '18534', '18535', '18536']
DEBUG    src.node.waku_node:waku_node.py:439 RLN credentials were not set
INFO     src.node.waku_node:waku_node.py:176 RLN credentials not set or credential store not available, starting without RLN
DEBUG    src.node.waku_node:waku_node.py:178 Using volumes []
DEBUG    src.node.docker_mananger:docker_mananger.py:49 docker run -i -t -p 18532:18532 -p 18533:18533 -p 18534:18534 -p 18535:18535 -p 18536:18536 wakuorg/nwaku:latest --listen-address=0.0.0.0 --rest=true --rest-admin=true --websocket-support=true --log-level=TRACE --rest-relay-cache-capacity=100 --websocket-port=18534 --rest-port=18532 --tcp-port=18533 --discv5-udp-port=18535 --rest-address=0.0.0.0 --nat=extip:172.18.242.137 --peer-exchange=true --discv5-discovery=true --cluster-id=3 --nodekey=a5db6e18ef7ebda5c992dafdff87ef5ffc3f7b595d7bfc04da7efd3ed8c83d19 --shard=0 --metrics-server=true --metrics-server-address=0.0.0.0 --metrics-server-port=18536 --metrics-logging=true --discv5-bootstrap-node=enr:-L24QKFJjy0MA32__SMesS-bjA96H3ePt5yj5qAYs7BMw4ftFEHWZTXuETDBE7l7woh_iviVpnOyC2bCxvaipPIWZpwCgmlkgnY0gmlwhKwSmguKbXVsdGlhZGRyc5YACASsEpoLBikrAAoErBKaCwYpLN0DgnJzhQADAQAAiXNlY3AyNTZrMaEDacl7AxNn6rpcnbDdEj_bxzoP31V0MEECgGlxMTP89VeDdGNwgikrg3VkcIIpLYV3YWt1MgM --storenode=/ip4/172.18.154.11/tcp/10539/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG --store=true --relay=true
DEBUG    src.node.docker_mananger:docker_mananger.py:55 docker network connect --ip 172.18.242.137 waku 04943d819f202d9b37b3dbb015eb36f8f05be3d8a77fdf8bdf29811d94347e43
DEBUG    src.node.docker_mananger:docker_mananger.py:58 Container started with ID 04943d819f20. Setting up logs at ./log/docker/store_node1_2026-02-25_19-36-40__60d44777-7e7f-45a9-a785-eac6751abc1f__wakuorg_nwaku:latest.log
DEBUG    src.node.waku_node:waku_node.py:190 Started container from image wakuorg/nwaku:latest. REST: 18532
DEBUG    src.libs.common:common.py:47 Sleeping for 1 seconds
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:18532/health" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"nodeHealth":"READY","connectionStatus":"PartiallyConnected","protocolsHealth":[{"Relay":"READY"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"READY"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"READY"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"NOT_READY","desc":"No Filter service peer available yet"},{"Rln Relay":"NOT_MOUNTED"}]}'
INFO     src.node.waku_node:waku_node.py:287 Node protocols are initialized !!
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:18532/debug/v1/info" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.242.137/tcp/18533/p2p/16Uiu2HAmRUD3r1Wobb9XV9hMmYZHmCqhBxKjRfNcb8j5sb4nfs2W","/ip4/172.18.242.137/tcp/18534/ws/p2p/16Uiu2HAmRUD3r1Wobb9XV9hMmYZHmCqhBxKjRfNcb8j5sb4nfs2W"],"enrUri":"enr:-L24QC_1LhPEvCGH1XIP-tluWdouYcJ59B-aFmSkwdfoB-ulO6SYdObR7sXYrUK50Biwn3EjPti-UehRXZhIKieM7ysCgmlkgnY0gmlwhKwS8omKbXVsdGlhZGRyc5YACASsEvKJBkhlAAoErBLyiQZIZt0DgnJzhQADAQAAiXNlY3AyNTZrMaEDvmTmdMQ1X1euw3qaJ2A7o4LGcT5_Ot2z4UcI6oZORX-DdGNwgkhlg3VkcIJIZ4V3YWt1MgM"}'
INFO     src.node.waku_node:waku_node.py:292 REST service is ready !!
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:18532/admin/v1/peers" -H "Content-Type: application/json" -d '["/ip4/172.18.154.11/tcp/10539/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG"]'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:10538/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/3/0"]'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:18532/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/3/0"]'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
DEBUG    src.steps.store:store.py:132 Relaying message
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:10538/relay/v1/messages/%2Fwaku%2F2%2Frs%2F3%2F0" -H "Content-Type: application/json" -d '{"payload": "U3RvcmUgd29ya3MhIQ==", "contentTopic": "/myapp/1/latest/proto", "timestamp": '$(date +%s%N)'}'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
DEBUG    src.libs.common:common.py:47 Sleeping for 0.2 seconds
DEBUG    src.steps.store:store.py:211 Checking that peer wakuorg/nwaku:latest can find the stored messages
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/store/v3/messages?pubsubTopic=%2Fwaku%2F2%2Frs%2F3%2F0&pageSize=5" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"requestId":"","statusCode":200,"statusDesc":"OK","messages":[{"messageHash":"0x9265751a493c9ca955dfb2ac6762b117375ee0dd4317198e5fdd8da34d30230d"}]}'
DEBUG    src.steps.store:store.py:228 messages length is 1
DEBUG    src.steps.store:store.py:211 Checking that peer wakuorg/nwaku:latest can find the stored messages
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:18532/store/v3/messages?pubsubTopic=%2Fwaku%2F2%2Frs%2F3%2F0&pageSize=5" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"requestId":"","statusCode":200,"statusDesc":"OK","messages":[{"messageHash":"0x9265751a493c9ca955dfb2ac6762b117375ee0dd4317198e5fdd8da34d30230d"}]}'
DEBUG    src.steps.store:store.py:228 messages length is 1
DEBUG    src.node.waku_node:waku_node.py:257 Restarting container with id 5abc3d387235
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/health" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/health" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"nodeHealth":"READY","connectionStatus":"Disconnected","protocolsHealth":[{"Relay":"NOT_READY","desc":"No connected peers"},{"Lightpush":"NOT_MOUNTED"},{"Legacy Lightpush":"NOT_MOUNTED"},{"Filter":"NOT_MOUNTED"},{"Store":"READY"},{"Legacy Store":"NOT_MOUNTED"},{"Peer Exchange":"READY"},{"Rendezvous":"NOT_READY","desc":"No Rendezvous peers are available yet"},{"Mix":"NOT_MOUNTED"},{"Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Legacy Lightpush Client":"NOT_READY","desc":"No Lightpush service peer available yet"},{"Store Client":"READY"},{"Legacy Store Client":"NOT_READY","desc":"No Legacy Store service peers are available yet, neither Store service set up for the node"},{"Filter Client":"NOT_READY","desc":"No Filter service peer available yet"},{"Rln Relay":"NOT_MOUNTED"}]}'
INFO     src.node.waku_node:waku_node.py:287 Node protocols are initialized !!
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/debug/v1/info" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"listenAddresses":["/ip4/172.18.154.11/tcp/10539/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG","/ip4/172.18.154.11/tcp/10540/ws/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG"],"enrUri":"enr:-L24QKFJjy0MA32__SMesS-bjA96H3ePt5yj5qAYs7BMw4ftFEHWZTXuETDBE7l7woh_iviVpnOyC2bCxvaipPIWZpwCgmlkgnY0gmlwhKwSmguKbXVsdGlhZGRyc5YACASsEpoLBikrAAoErBKaCwYpLN0DgnJzhQADAQAAiXNlY3AyNTZrMaEDacl7AxNn6rpcnbDdEj_bxzoP31V0MEECgGlxMTP89VeDdGNwgikrg3VkcIIpLYV3YWt1MgM"}'
INFO     src.node.waku_node:waku_node.py:292 REST service is ready !!
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:18532/admin/v1/peers" -H "Content-Type: application/json" -d '["/ip4/172.18.154.11/tcp/10539/p2p/16Uiu2HAmKmwMDErL5KFkmJ5v8wRdQXrXAdHScqv28En4dAEH6bVG"]'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:10538/relay/v1/subscriptions" -H "Content-Type: application/json" -d '["/waku/2/rs/3/0"]'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
DEBUG    src.steps.store:store.py:132 Relaying message
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X POST "http://127.0.0.1:10538/relay/v1/messages/%2Fwaku%2F2%2Frs%2F3%2F0" -H "Content-Type: application/json" -d '{"payload": "U3RvcmUgd29ya3MhIQ==", "contentTopic": "/myapp/1/latest/proto", "timestamp": '$(date +%s%N)'}'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'OK'
DEBUG    src.libs.common:common.py:47 Sleeping for 0.2 seconds
DEBUG    src.steps.store:store.py:211 Checking that peer wakuorg/nwaku:latest can find the stored messages
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/store/v3/messages?pubsubTopic=%2Fwaku%2F2%2Frs%2F3%2F0&pageSize=5" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"requestId":"","statusCode":200,"statusDesc":"OK","messages":[{"messageHash":"0x9265751a493c9ca955dfb2ac6762b117375ee0dd4317198e5fdd8da34d30230d"},{"messageHash":"0x35620d19ce481249d9c8eb1ed92cd4fb6a6e034e99b057dd6121699fc0b79123"}]}'
DEBUG    src.steps.store:store.py:228 messages length is 2
DEBUG    src.steps.store:store.py:211 Checking that peer wakuorg/nwaku:latest can find the stored messages
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:18532/store/v3/messages?pubsubTopic=%2Fwaku%2F2%2Frs%2F3%2F0&pageSize=5" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"requestId":"","statusCode":200,"statusDesc":"OK","messages":[{"messageHash":"0x9265751a493c9ca955dfb2ac6762b117375ee0dd4317198e5fdd8da34d30230d"},{"messageHash":"0x35620d19ce481249d9c8eb1ed92cd4fb6a6e034e99b057dd6121699fc0b79123"}]}'
DEBUG    src.steps.store:store.py:228 messages length is 2
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:10538/store/v3/messages?pubsubTopic=%2Fwaku%2F2%2Frs%2F3%2F0&pageSize=5&ascending=true" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"requestId":"","statusCode":200,"statusDesc":"OK","messages":[{"messageHash":"0x9265751a493c9ca955dfb2ac6762b117375ee0dd4317198e5fdd8da34d30230d"},{"messageHash":"0x35620d19ce481249d9c8eb1ed92cd4fb6a6e034e99b057dd6121699fc0b79123"}]}'
INFO     src.node.api_clients.base_client:base_client.py:37 curl -v -X GET "http://127.0.0.1:18532/store/v3/messages?pubsubTopic=%2Fwaku%2F2%2Frs%2F3%2F0&pageSize=5&ascending=true" -H "Content-Type: application/json" -d 'None'
INFO     src.node.api_clients.base_client:base_client.py:22 Response status code: 200. Response content: b'{"requestId":"","statusCode":200,"statusDesc":"OK","messages":[{"messageHash":"0x9265751a493c9ca955dfb2ac6762b117375ee0dd4317198e5fdd8da34d30230d"},{"messageHash":"0x35620d19ce481249d9c8eb1ed92cd4fb6a6e034e99b057dd6121699fc0b79123"}]}'
DEBUG    tests.conftest:conftest.py:59 Running fixture teardown: test_setup
DEBUG    tests.conftest:conftest.py:83 Running fixture teardown: close_open_nodes
DEBUG    src.node.waku_node:waku_node.py:234 Stopping container with id 5abc3d387235
DEBUG    src.node.waku_node:waku_node.py:241 Container stopped.
DEBUG    src.node.waku_node:waku_node.py:234 Stopping container with id 04943d819f20
DEBUG    src.node.waku_node:waku_node.py:241 Container stopped.
DEBUG    tests.conftest:conftest.py:98 Running fixture teardown: check_waku_log_errors
DEBUG    src.node.docker_mananger:docker_mananger.py:144 No errors found in the waku logs.
DEBUG    src.node.docker_mananger:docker_mananger.py:144 No errors found in the waku logs.