diff --git a/test/appium/requirements.txt b/test/appium/requirements.txt index 03c848b3ef..56d49969ba 100644 --- a/test/appium/requirements.txt +++ b/test/appium/requirements.txt @@ -1,3 +1,4 @@ +matplotlib emoji aiohttp==2.2.3 allpairspy==2.3.0 diff --git a/test/appium/support/message_reliability_chart.py b/test/appium/support/message_reliability_chart.py new file mode 100644 index 0000000000..450ccd2e4b --- /dev/null +++ b/test/appium/support/message_reliability_chart.py @@ -0,0 +1,48 @@ +def create_chart_one_to_one_chat(one_to_one_chat_data: dict): + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + user_a = one_to_one_chat_data['user_a'] + user_b = one_to_one_chat_data['user_b'] + + fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15, 7)) + time_1 = sorted(user_a['message_time']) + ax.plot([i / 60 for i in time_1], [user_a['message_time'][i] for i in time_1], + 'o-', color='#0c0fea', label='user_a') + time_2 = sorted(user_b['message_time']) + ax.plot([i / 60 for i in time_2], [user_b['message_time'][i] for i in time_2], + 'o-', color='#f61e06', label='user_b') + sent_messages = user_a['sent_messages'] + user_b['sent_messages'] + title = "User A: sent messages: {}, received messages: {}" \ + "\nUser B: sent messages: {}, received messages: {}".format(user_a['sent_messages'], + len(user_a['message_time']), + user_b['sent_messages'], + len(user_b['message_time'])) + if sent_messages: + title += "\nReceived messages: {}%".format( + round((len(user_a['message_time']) + len(user_b['message_time'])) / sent_messages * 100, ndigits=2)) + plt.title(title) + plt.xlabel('chat session duration, minutes') + plt.ylabel('time to receive a message, seconds') + plt.legend() + fig.savefig('chart.png') + + +def create_chart_public_chat(public_chat_data: dict): + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + sent_messages = public_chat_data['sent_messages'] + message_time = public_chat_data['message_time'] + + fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(15, 7)) + sorted_time = sorted(message_time) + ax.plot([i / 60 for i in sorted_time], [message_time[i] for i in sorted_time], 'o-', color='#0c0fea') + title = "Sent messages: {}\nReceived messages: {}".format(sent_messages, len(message_time)) + plt.title(title) + plt.xlabel('chat session duration, minutes') + plt.ylabel('time to receive a message, seconds') + plt.legend() + fig.savefig('chart.png') diff --git a/test/appium/support/network_api.py b/test/appium/support/network_api.py index 0f2282736e..0c33dfa196 100644 --- a/test/appium/support/network_api.py +++ b/test/appium/support/network_api.py @@ -87,3 +87,8 @@ class NetworkApi: def get_ethereum_price_in_usd(self) -> float: url = 'https://min-api.cryptocompare.com/data/price?fsym=ETH&tsyms=USD' return float(requests.request('GET', url).json()['USD']) + + def start_chat_bot(self, chat_name: str, messages_number: int, interval: int = 1) -> list: + url = 'http://offsite.chat:8099/ping/%s?count=%s&interval=%s' % (chat_name, messages_number, interval) + text = requests.request('GET', url).text + return [i.split(maxsplit=5)[-1].strip('*') for i in text.splitlines()] diff --git a/test/appium/tests/base_test_case.py b/test/appium/tests/base_test_case.py index 8441326bd0..91b80bab77 100644 --- a/test/appium/tests/base_test_case.py +++ b/test/appium/tests/base_test_case.py @@ -3,6 +3,8 @@ import sys import re import subprocess import asyncio + +from support.message_reliability_chart import create_chart_one_to_one_chat, create_chart_public_chat from support.network_api import NetworkApi from os import environ from appium import webdriver @@ -13,7 +15,6 @@ from views.base_view import BaseView class AbstractTestCase: - __metaclass__ = ABCMeta @property @@ -68,9 +69,9 @@ class AbstractTestCase: desired_caps['ignoreUnimportantViews'] = False return desired_caps - def update_capabilities_sauce_lab(self, key, value): + def update_capabilities_sauce_lab(self, new_capabilities: dict): caps = self.capabilities_sauce_lab.copy() - caps[key] = value + caps.update(new_capabilities) return caps @property @@ -173,17 +174,18 @@ class SauceMultipleDeviceTestCase(AbstractTestCase): def setup_method(self, method): self.drivers = dict() - def create_drivers(self, quantity=2): - if self.__class__.__name__ == 'TestOfflineMessages': - capabilities = self.update_capabilities_sauce_lab('platformVersion', '6.0') - else: - capabilities = self.capabilities_sauce_lab - self.drivers = self.loop.run_until_complete(start_threads(quantity, webdriver.Remote, - self.drivers, - self.executor_sauce_lab, - capabilities)) + def create_drivers(self, quantity=2, max_duration=1800, custom_implicitly_wait=None, offline_mode=False): + capabilities = {'maxDuration': max_duration} + if offline_mode: + capabilities['platformVersion'] = '6.0' + self.drivers = self.loop.run_until_complete(start_threads(quantity, + webdriver.Remote, + self.drivers, + self.executor_sauce_lab, + self.update_capabilities_sauce_lab(capabilities))) for driver in range(quantity): - self.drivers[driver].implicitly_wait(self.implicitly_wait) + self.drivers[driver].implicitly_wait( + custom_implicitly_wait if custom_implicitly_wait else self.implicitly_wait) BaseView(self.drivers[driver]).accept_agreements() test_suite_data.current_test.testruns[-1].jobs.append(self.drivers[driver].session_id) @@ -215,3 +217,16 @@ class MultipleDeviceTestCase(environments[pytest.config.getoption('env')]): self.network_api.faucet(address=self.senders[user]['address']) super(MultipleDeviceTestCase, self).teardown_method(method) + +class MessageReliabilityTestCase(MultipleDeviceTestCase): + + def setup_method(self, method): + super(MessageReliabilityTestCase, self).setup_method(method) + self.one_to_one_chat_data = dict() + self.public_chat_data = dict() + + def teardown_method(self, method): + if self.one_to_one_chat_data: + create_chart_one_to_one_chat(self.one_to_one_chat_data) + if self.public_chat_data: + create_chart_public_chat(self.public_chat_data) diff --git a/test/appium/tests/conftest.py b/test/appium/tests/conftest.py index 39b43cfe4d..08fb670e74 100644 --- a/test/appium/tests/conftest.py +++ b/test/appium/tests/conftest.py @@ -1,9 +1,8 @@ from _pytest.runner import runtestprotocol from support.test_rerun import should_rerun_test -from tests import test_suite_data, debug +from tests import test_suite_data import requests -import re import pytest from datetime import datetime from os import environ @@ -50,6 +49,18 @@ def pytest_addoption(parser): action='store', default=0, help='How many times tests should be re-run if failed') + parser.addoption('--messages_number', + action='store', + default=20, + help='Messages number') + parser.addoption('--message_wait_time', + action='store', + default=20, + help='Max time to wait for a message to be received') + parser.addoption('--participants_number', + action='store', + default=5, + help='Public chat participants number') def get_rerun_count(): @@ -140,3 +151,18 @@ def pytest_runtest_protocol(item, nextitem): break # rerun else: return True # no need to rerun + + +@pytest.fixture +def messages_number(): + return int(pytest.config.getoption('messages_number')) + + +@pytest.fixture +def message_wait_time(): + return int(pytest.config.getoption('message_wait_time')) + + +@pytest.fixture +def participants_number(): + return int(pytest.config.getoption('participants_number')) diff --git a/test/appium/tests/marks.py b/test/appium/tests/marks.py index 7e0d83570d..2216bec1f8 100644 --- a/test/appium/tests/marks.py +++ b/test/appium/tests/marks.py @@ -6,6 +6,7 @@ testrail_case_id = pytest.mark.testrail_case_id all = pytest.mark.all chat = pytest.mark.chat chat_management = pytest.mark.chat_management +message_reliability = pytest.mark.message_reliability transaction = pytest.mark.transaction wallet = pytest.mark.wallet skip = pytest.mark.skip diff --git a/test/appium/tests/test_message_reliability.py b/test/appium/tests/test_message_reliability.py new file mode 100644 index 0000000000..e0c5b4d723 --- /dev/null +++ b/test/appium/tests/test_message_reliability.py @@ -0,0 +1,147 @@ +import random +import string +import time + +from itertools import cycle +from timeit import timeit +from selenium.common.exceptions import TimeoutException + +from tests import info, marks +from tests.base_test_case import MessageReliabilityTestCase +from views.sign_in_view import SignInView + + +def wrapper(func, *args, **kwargs): + def wrapped(): + return func(*args, **kwargs) + + return wrapped + + +@marks.message_reliability +class TestMessageReliability(MessageReliabilityTestCase): + + def test_message_reliability_1_1_chat(self, messages_number, message_wait_time): + user_a_sent_messages = 0 + user_a_received_messages = 0 + user_b_sent_messages = 0 + user_b_received_messages = 0 + user_a_message_time = dict() + user_b_message_time = dict() + try: + self.create_drivers(2, max_duration=10800, custom_implicitly_wait=2) + device_1, device_2 = SignInView(self.drivers[0]), SignInView(self.drivers[1]) + device_1.create_user(username='user_a') + device_2.create_user(username='user_b') + device_1_home, device_2_home = device_1.get_home_view(), device_2.get_home_view() + device_2_public_key = device_2_home.get_public_key() + device_2_home.home_button.click() + device_1_home.add_contact(device_2_public_key) + device_1_chat = device_1_home.get_chat_view() + device_1_chat.chat_message_input.send_keys('hello') + device_1_chat.send_message_button.click() + device_2_home.element_by_text('hello').click() + device_2_chat = device_2_home.get_chat_view() + device_2_chat.add_to_contacts.click() + + start_time = time.time() + for i in range(int(messages_number / 2)): + message_1 = ''.join(random.sample(string.ascii_lowercase, k=10)) + device_1_chat.chat_message_input.send_keys(message_1) + device_1_chat.send_message_button.click() + user_a_sent_messages += 1 + try: + user_b_receive_time = timeit(wrapper(device_2_chat.wait_for_element_starts_with_text, + message_1, message_wait_time), + number=1) + duration_time = round(time.time() - start_time, ndigits=2) + user_b_message_time[duration_time] = user_b_receive_time + user_b_received_messages += 1 + except TimeoutException: + info("Message with text '%s' was not received by user_b" % message_1) + message_2 = ''.join(random.sample(string.ascii_lowercase, k=10)) + device_2_chat.chat_message_input.send_keys(message_2) + device_2_chat.send_message_button.click() + user_b_sent_messages += 1 + try: + user_a_receive_time = timeit(wrapper(device_1_chat.wait_for_element_starts_with_text, + message_2, message_wait_time), + number=1) + duration_time = round(time.time() - start_time, ndigits=2) + user_a_message_time[duration_time] = user_a_receive_time + user_a_received_messages += 1 + except TimeoutException: + info("Message with text '%s' was not received by user_a" % message_2) + finally: + self.one_to_one_chat_data['user_a'] = {'sent_messages': user_a_sent_messages, + 'message_time': user_a_message_time} + self.one_to_one_chat_data['user_b'] = {'sent_messages': user_b_sent_messages, + 'message_time': user_b_message_time} + + def test_message_reliability_public_chat(self, messages_number, message_wait_time, participants_number): + self.public_chat_data['sent_messages'] = int() + self.public_chat_data['message_time'] = dict() + + self.create_drivers(participants_number, max_duration=10800, custom_implicitly_wait=2) + users = list() + chat_views = list() + chat_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(7)) + for i in range(participants_number): + device = SignInView(self.drivers[i]) + users.append(device.create_user()) + home_view = device.get_home_view() + home_view.join_public_chat(chat_name) + chat_views.append(home_view.get_chat_view()) + + start_time = time.time() + repeat = cycle(range(participants_number)) + for i in repeat: + message_text = ''.join(random.sample(string.ascii_lowercase, k=10)) + chat_views[i].chat_message_input.send_keys(message_text) + chat_views[i].send_message_button.click() + self.public_chat_data['sent_messages'] += 1 + try: + user_b_receive_time = timeit(wrapper(chat_views[next(repeat)].wait_for_element_starts_with_text, + message_text, message_wait_time), + number=1) + duration_time = round(time.time() - start_time, ndigits=2) + self.public_chat_data['message_time'][duration_time] = user_b_receive_time + except TimeoutException: + pass + if self.public_chat_data['sent_messages'] == messages_number: + break + + def test_message_reliability_offline_public_chat(self, messages_number, message_wait_time): + self.public_chat_data['sent_messages'] = int() + self.public_chat_data['message_time'] = dict() + + self.create_drivers(1, max_duration=10800, custom_implicitly_wait=2, offline_mode=True) + driver = self.drivers[0] + sign_in_view = SignInView(driver) + sign_in_view.create_user() + home_view = sign_in_view.get_home_view() + chat_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(7)) + home_view.join_public_chat(chat_name) + + start_time = time.time() + iterations = int(messages_number / 10 if messages_number > 10 else messages_number) + for _ in range(iterations): + home_view.get_back_to_home_view() + driver.set_network_connection(1) # airplane mode + + sent_messages_texts = self.network_api.start_chat_bot(chat_name=chat_name, messages_number=10) + self.public_chat_data['sent_messages'] += 10 + + driver.set_network_connection(2) # turning on WiFi connection + + home_view.get_chat_with_user('#' + chat_name).click() + chat_view = home_view.get_chat_view() + for message in sent_messages_texts: + try: + user_b_receive_time = timeit(wrapper(chat_view.wait_for_element_starts_with_text, + message, message_wait_time), + number=1) + duration_time = round(time.time() - start_time, ndigits=2) + self.public_chat_data['message_time'][duration_time] = user_b_receive_time + except TimeoutException: + pass diff --git a/test/appium/tests/test_messaging.py b/test/appium/tests/test_messaging.py index 882483af71..ee895af3de 100644 --- a/test/appium/tests/test_messaging.py +++ b/test/appium/tests/test_messaging.py @@ -268,7 +268,7 @@ class TestOfflineMessages(MultipleDeviceTestCase): @marks.testrail_case_id(3420) def test_offline_messaging_1_1_chat(self): - self.create_drivers(2) + self.create_drivers(2, offline_mode=True) device_1, device_2 = self.drivers[0], self.drivers[1] sign_in_1, sign_in_2 = SignInView(device_1), SignInView(device_2) username_1 = sign_in_1.create_user() @@ -302,7 +302,7 @@ class TestOfflineMessages(MultipleDeviceTestCase): @marks.testrail_case_id(3430) @marks.skip def test_offline_messaging_group_chat(self): - self.create_drivers(2) + self.create_drivers(2, offline_mode=True) device_1, device_2 = self.drivers[0], self.drivers[1] sign_in_1, sign_in_2 = SignInView(device_1), SignInView(device_2) username_1 = sign_in_1.create_user() diff --git a/test/appium/views/chat_view.py b/test/appium/views/chat_view.py index 4ad1e178c2..5a81d2886c 100644 --- a/test/appium/views/chat_view.py +++ b/test/appium/views/chat_view.py @@ -1,5 +1,5 @@ import time -from selenium.common.exceptions import TimeoutException, NoSuchElementException +from selenium.common.exceptions import TimeoutException from tests import info from views.base_element import BaseButton, BaseEditBox, BaseText from views.base_view import BaseView diff --git a/test/appium/views/sign_in_view.py b/test/appium/views/sign_in_view.py index 12287280fc..2deb40128e 100644 --- a/test/appium/views/sign_in_view.py +++ b/test/appium/views/sign_in_view.py @@ -1,7 +1,7 @@ from tests import get_current_time from views.base_element import BaseButton, BaseEditBox from views.base_view import BaseView -import time + class AccountButton(BaseButton): @@ -98,15 +98,15 @@ class SignInView(BaseView): self.name_input = NameInput(self.driver) self.do_not_share = DonNotShare(self.driver) - def create_user(self, password: str = 'qwerty1234'): + def create_user(self, username: str = '', password='qwerty'): self.create_account_button.click() self.password_input.set_value(password) self.next_button.click() self.confirm_password_input.set_value(password) self.next_button.click() - self.element_by_text_part('Display name').wait_for_element(10) - username = 'user_%s' % get_current_time() + self.element_by_text_part('Display name').wait_for_element(30) + username = username if username else 'user_%s' % get_current_time() self.name_input.send_keys(username) self.next_button.click() @@ -129,4 +129,4 @@ class SignInView(BaseView): self.sign_in_button.click() def click_account_by_position(self, position: int): - self.account_button.find_elements()[position].click() \ No newline at end of file + self.account_button.find_elements()[position].click()