status-im
/
status-cli-tests
mirror of https://github.com/status-im/status-cli-tests.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15 from status-im/fix/flaky 

flaki test:
This commit is contained in:
Pablo Lopez 2024-10-16 11:10:39 +03:00 committed by GitHub
parent 1cd824fe81 b43fa65a55
commit e1fa7861de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 110 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
tests/test_community_messages.py
View File

@ -48,23 +48,24 @@ class TestCommunityMessages(StepsCommon):
f"{len(missing_messages)} messages out of {NUM_MESSAGES} were not received: " + "\n".join(formatted_missing_messages)
)
def test_community_messages_with_latency(self):
self.setup_community_nodes(node_limit=1)
self.join_created_communities()
with self.add_latency():
self.test_community_messages_baseline()
# skipping these low-latency, packet loss, and low-bandwidth tests since we don't have an E2E solution for them yet (https://forum.vac.dev/t/end-to-end-reliability-for-scalable-distributed-logs/293)
# def test_community_messages_with_latency(self):
# self.setup_community_nodes(node_limit=1)
# self.join_created_communities()
# with self.add_latency():
# self.test_community_messages_baseline()
def test_community_messages_with_packet_loss(self):
self.setup_community_nodes(node_limit=1)
self.join_created_communities()
with self.add_packet_loss():
self.test_community_messages_baseline()
# def test_community_messages_with_packet_loss(self):
# self.setup_community_nodes(node_limit=1)
# self.join_created_communities()
# with self.add_packet_loss():
# self.test_community_messages_baseline()
def test_community_messages_with_low_bandwith(self):
self.setup_community_nodes(node_limit=1)
self.join_created_communities()
with self.add_low_bandwith():
self.test_community_messages_baseline()
# def test_community_messages_with_low_bandwith(self):
# self.setup_community_nodes(node_limit=1)
# self.join_created_communities()
# with self.add_low_bandwith():
# self.test_community_messages_baseline()
@pytest.mark.flaky(reruns=2)
def test_community_messages_with_node_pause_10_seconds(self):
@ -76,8 +77,7 @@ class TestCommunityMessages(StepsCommon):
with self.node_pause(community_node):
message = str(uuid4())
self.first_node.send_community_chat_message(message_chat_id, message)
delay(10)
assert community_node.wait_for_logs([message])
assert community_node.wait_for_logs([message], 60)
@pytest.mark.flaky(reruns=2)
def test_community_messages_with_node_pause_30_seconds(self):
@ -89,5 +89,4 @@ class TestCommunityMessages(StepsCommon):
with self.node_pause(community_node):
message = str(uuid4())
self.first_node.send_community_chat_message(message_chat_id, message)
delay(30)
assert community_node.wait_for_logs([message])
assert community_node.wait_for_logs([message], 60)

3
tests/test_contact_request.py
View File

@ -94,5 +94,4 @@ class TestContacRequest(StepsCommon):
with self.node_pause(self.second_node):
message = str(uuid4())
self.first_node.send_contact_request(self.second_node_pubkey, message)
delay(10)
assert self.second_node.wait_for_logs([message])
assert self.second_node.wait_for_logs([message], 60)

19
tests/test_create_private_groups.py
View File

@ -45,26 +45,11 @@ class TestCreatePrivateGroups(StepsCommon):
f"{len(missing_private_groups)} private groups out of {num_private_groups} were not created: " + "\n".join(formatted_missing_groups)
)
def test_create_group_chat_with_latency(self):
self.accept_contact_request()
# we want to set latency only on the group creation requests
with self.add_latency():
self.test_create_group_chat_baseline()
def test_create_group_chat_with_packet_loss(self):
self.accept_contact_request()
with self.add_packet_loss():
self.test_create_group_chat_baseline()
def test_create_group_chat_with_low_bandwith(self):
self.accept_contact_request()
with self.add_low_bandwith():
self.test_create_group_chat_baseline()
# for creating private group we don't have realiability protocol, therefore skipping the tests for latency, packet loss and low bandwith
def test_create_group_with_node_pause(self):
self.accept_contact_request()
with self.node_pause(self.second_node):
group_name = str(uuid4())
self.first_node.create_group_chat_with_members([self.second_node_pubkey], group_name)
delay(10)
assert self.second_node.wait_for_logs([group_name])
assert self.second_node.wait_for_logs([group_name], 60)

4
tests/test_join_community.py
View File

@ -25,7 +25,7 @@ class TestJoinCommunity(StepsCommon):
request_to_join_id = response_to_join["result"]["requestsToJoinCommunity"][0]["id"]
community_join_requests.append((community_id, request_to_join_id, timestamp, community_node["status_node"], initial_members))
delay(4)
delay(10)
failed_community_joins = []
for community_id, request_to_join_id, join_req_ts, status_node, initial_members in community_join_requests:
@ -77,7 +77,7 @@ class TestJoinCommunity(StepsCommon):
][0]
initial_members = len(target_community["members"])
request_to_join_id = response_to_join["result"]["requestsToJoinCommunity"][0]["id"]
delay(10)
delay(40)
response_accept_to_join = community_node.accept_request_to_join_community(request_to_join_id)
target_community = [
existing_community for existing_community in response_accept_to_join["result"]["communities"] if existing_community["id"] == community_id

31
tests/test_leave_community.py
View File

@ -34,20 +34,21 @@ class TestLeaveCommunity(StepsCommon):
f"{len(failed_community_leave)} community joins out of {len(self.community_nodes)}: " + "\n".join(formatted_missing_requests)
)
def test_leave_community_with_latency(self):
self.setup_community_nodes()
self.join_created_communities()
with self.add_latency():
self.test_leave_community_baseline()
# for leaving community we don't have realiability protocol
# def test_leave_community_with_latency(self):
# self.setup_community_nodes()
# self.join_created_communities()
# with self.add_latency():
# self.test_leave_community_baseline()
def test_leave_community_with_packet_loss(self):
self.setup_community_nodes()
self.join_created_communities()
with self.add_packet_loss():
self.test_leave_community_baseline()
# def test_leave_community_with_packet_loss(self):
# self.setup_community_nodes()
# self.join_created_communities()
# with self.add_packet_loss():
# self.test_leave_community_baseline()
def test_leave_community_with_low_bandwith(self):
self.setup_community_nodes()
self.join_created_communities()
with self.add_low_bandwith():
self.test_leave_community_baseline()
# def test_leave_community_with_low_bandwith(self):
# self.setup_community_nodes()
# self.join_created_communities()
# with self.add_low_bandwith():
# self.test_leave_community_baseline()

76
tests/test_one_to_one_messages.py
View File

@ -1,20 +1,23 @@
from time import sleep
import asyncio
from uuid import uuid4
import pytest
from src.env_vars import DELAY_BETWEEN_MESSAGES, NUM_MESSAGES
from src.libs.common import delay
from src.node.status_node import StatusNode
from src.steps.common import StepsCommon
@pytest.mark.usefixtures("start_2_nodes")
class TestOneToOneMessages(StepsCommon):
def test_one_to_one_message_baseline(self):
@pytest.mark.asyncio
async def test_one_to_one_message_baseline(self, recover_network_fn=None):
timeout_secs = 180
reset_network_in_secs = 10
num_messages = NUM_MESSAGES # Set the number of messages to send
self.accept_contact_request()
messages = []
for i in range(num_messages):
# Alternating which node sends the message
if i % 2 == 0:
@ -29,16 +32,32 @@ class TestOneToOneMessages(StepsCommon):
messages.append((timestamp, message, message_id, sending_node.name))
delay(DELAY_BETWEEN_MESSAGES)
# Wait for 10 seconds to give all messages time to be received
delay(10)
# Validate that all messages were received
missing_messages = []
tasks = []
for msg in messages:
search_node = self.first_node if msg[3] == self.second_node.name else self.second_node
tasks.append(asyncio.create_task(self.wait_for_message_async(search_node, msg, timeout_secs)))
for timestamp, message, message_id, sender in messages:
search_node = self.first_node if sender == self.second_node.name else self.second_node
if not search_node.search_logs(f"message received: {message}"):
missing_messages.append((timestamp, message, message_id, sender))
done, pending = await asyncio.wait(tasks, timeout=reset_network_in_secs)
if pending:
if recover_network_fn is not None:
# after `reset_network_in_secs` the network will recover and MVDS will eventually deliver the messages
recover_network_fn()
print("waiting for pending tasks")
done2, _ = await asyncio.wait(pending)
done.update(done2)
else:
print("no pending tasks")
missing_messages = []
for task in done:
if task.exception():
print(f"Task raised an exception: {task.exception()}")
raise task.exception()
else:
res = task.result()
if res is not None:
missing_messages.append(res)
if missing_messages:
formatted_missing_messages = [f"Timestamp: {ts}, Message: {msg}, ID: {mid}, Sender: {snd}" for ts, msg, mid, snd in missing_messages]
@ -46,34 +65,43 @@ class TestOneToOneMessages(StepsCommon):
f"{len(missing_messages)} messages out of {num_messages} were not received: " + "\n".join(formatted_missing_messages)
)
def test_one_to_one_message_with_latency(self):
@pytest.mark.asyncio
async def test_one_to_one_message_with_latency(self):
self.accept_contact_request()
# we want to set latency only on the message sending requests
with self.add_latency():
self.test_one_to_one_message_baseline()
with self.add_latency() as recover_network_fn:
await self.test_one_to_one_message_baseline(recover_network_fn)
def test_one_to_one_message_with_packet_loss(self):
@pytest.mark.asyncio
async def test_one_to_one_message_with_packet_loss(self):
self.accept_contact_request()
with self.add_packet_loss():
self.test_one_to_one_message_baseline()
with self.add_packet_loss() as recover_network_fn:
await self.test_one_to_one_message_baseline(recover_network_fn)
def test_one_to_one_message_with_low_bandwith(self):
@pytest.mark.asyncio
async def test_one_to_one_message_with_low_bandwith(self):
self.accept_contact_request()
with self.add_low_bandwith():
self.test_one_to_one_message_baseline()
with self.add_low_bandwith() as recover_network_fn:
await self.test_one_to_one_message_baseline(recover_network_fn)
def test_one_to_one_message_with_node_pause_5_seconds(self):
self.accept_contact_request()
with self.node_pause(self.first_node):
message = str(uuid4())
self.second_node.send_message(self.first_node_pubkey, message)
delay(5)
assert self.first_node.wait_for_logs([message])
assert self.first_node.wait_for_logs([message], 60)
def test_one_to_one_message_with_node_pause_30_seconds(self):
self.accept_contact_request()
with self.node_pause(self.first_node):
message = str(uuid4())
self.second_node.send_message(self.first_node_pubkey, message)
delay(30)
assert self.first_node.wait_for_logs([message])
assert self.first_node.wait_for_logs([message], 60)
async def wait_for_message_async(self, node: StatusNode, msg: tuple[int, str, str, str], timeout_secs: int = 45):
res = await node.wait_for_logs_async([f"message received: {msg[1]}"], timeout_secs)
if res:
return None
else:
# return missing
return msg

39
tests/test_private_group_messages.py
View File

@ -47,24 +47,25 @@ class TestPrivateGroupMessages(StepsCommon):
f"{len(missing_messages)} messages out of {num_private_groups} were not received: " + "\n".join(formatted_missing_messages)
)
def test_group_chat_messages_with_latency(self):
self.accept_contact_request()
self.join_private_group()
# we want to set latency only on the group creation requests
with self.add_latency():
self.test_group_chat_messages_baseline()
# skipping these low-latency, packet loss, and low-bandwidth tests since we don't have an E2E solution for them yet (https://forum.vac.dev/t/end-to-end-reliability-for-scalable-distributed-logs/293)
# def test_group_chat_messages_with_latency(self):
# self.accept_contact_request()
# self.join_private_group()
# # we want to set latency only on the group creation requests
# with self.add_latency():
# self.test_group_chat_messages_baseline()
def test_group_chat_messages_with_packet_loss(self):
self.accept_contact_request()
self.join_private_group()
with self.add_packet_loss():
self.test_group_chat_messages_baseline()
# def test_group_chat_messages_with_packet_loss(self):
# self.accept_contact_request()
# self.join_private_group()
# with self.add_packet_loss():
# self.test_group_chat_messages_baseline()
def test_group_chat_messages_with_low_bandwith(self):
self.accept_contact_request()
self.join_private_group()
with self.add_low_bandwith():
self.test_group_chat_messages_baseline()
# def test_group_chat_messages_with_low_bandwith(self):
# self.accept_contact_request()
# self.join_private_group()
# with self.add_low_bandwith():
# self.test_group_chat_messages_baseline()
def test_group_chat_messages_with_node_pause_10_seconds(self):
self.accept_contact_request()
@ -72,8 +73,7 @@ class TestPrivateGroupMessages(StepsCommon):
with self.node_pause(self.first_node):
message = str(uuid4())
self.second_node.send_group_chat_message(self.private_group_id, message)
delay(10)
assert self.first_node.wait_for_logs([message])
assert self.first_node.wait_for_logs([message], 60)
def test_group_chat_messages_with_node_pause_40_seconds(self):
self.accept_contact_request()
@ -81,5 +81,4 @@ class TestPrivateGroupMessages(StepsCommon):
with self.node_pause(self.first_node):
message = str(uuid4())
self.second_node.send_group_chat_message(self.first_node_pubkey, message)
delay(40)
assert self.first_node.wait_for_logs([message])
assert self.first_node.wait_for_logs([message], 60)