From d9e207fb1dfe7b0e7d69b4c9309cc03fcd826b81 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 5 Mar 2025 06:50:20 +0000 Subject: [PATCH] test: remote API call - parallel container stop --- src/api_clients/rest.py | 15 +++++------ src/cli/nomos_cli.py | 6 +++++ src/node/nomos_node.py | 3 +++ src/steps/da.py | 10 +++++-- tests/conftest.py | 31 ++++++++++++---------- tests/dos_robustness/test_high_load_dos.py | 14 +++++----- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/api_clients/rest.py b/src/api_clients/rest.py index 52bc741..2da68e2 100644 --- a/src/api_clients/rest.py +++ b/src/api_clients/rest.py @@ -6,20 +6,17 @@ logger = get_custom_logger(__name__) class REST(BaseClient): - def __init__(self, rest_port): + def __init__(self, rest_port, rest_host="127.0.0.1"): self._rest_port = rest_port + self._rest_host = rest_host - def rest_call(self, method, endpoint, payload=None, host="127.0.0.1", port=None): - if port is None: - port = self._rest_port - url = f"http://{host}:{port}/{endpoint}" + def rest_call(self, method, endpoint, payload=None): + url = f"http://{self._rest_host}:{self._rest_port}/{endpoint}" headers = {"Content-Type": "application/json", "Connection": "close"} return self.make_request(method, url, headers=headers, data=payload) - def rest_call_text(self, method, endpoint, payload=None, host="127.0.0.1", port=None): - if port is None: - port = self._rest_port - url = f"http://{host}:{port}/{endpoint}" + def rest_call_text(self, method, endpoint, payload=None): + url = f"http://{self._rest_host}:{self._rest_port}/{endpoint}" headers = {"accept": "text/plain", "Connection": "close"} return self.make_request(method, url, headers=headers, data=payload) diff --git a/src/cli/nomos_cli.py b/src/cli/nomos_cli.py index df9dab9..d051ded 100644 --- a/src/cli/nomos_cli.py +++ b/src/cli/nomos_cli.py @@ -2,6 +2,7 @@ import json import os import re +from src.api_clients.rest import REST from src.data_storage import DS from src.libs.common import generate_log_prefix, delay, remove_padding from src.libs.custom_logger import get_custom_logger @@ -35,6 +36,7 @@ class NomosCli: self._docker_manager = DockerManager(self._image_name) self._container_name = container_name self._container = None + self._api = None cwd = os.getcwd() self._volumes = [cwd + "/" + volume for volume in self._volumes] @@ -106,6 +108,10 @@ class NomosCli: return result + def set_rest_api(self, host, port): + logger.debug(f"Setting rest API object to host {host} port {port}") + self._api = REST(port, host) + @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) def stop(self): self._container = stop(self._container) diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 86c5658..2c867ae 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -126,6 +126,9 @@ class NomosNode: def name(self): return self._container_name + def api_port(self): + return self._tcp_port + def check_nomos_log_errors(self, whitelist=None): keywords = LOG_ERROR_KEYWORDS diff --git a/src/steps/da.py b/src/steps/da.py index 720f11e..41575e6 100644 --- a/src/steps/da.py +++ b/src/steps/da.py @@ -1,4 +1,5 @@ import allure +from requests.packages import target from tenacity import retry, stop_after_delay, wait_fixed from src.env_vars import NOMOS_EXECUTOR @@ -45,14 +46,19 @@ class StepsDataAvailability(StepsCommon): return executor @allure.step - def disperse_data(self, data, app_id, index, timeout_duration=65, utf8=True, padding=True): + def disperse_data(self, data, app_id, index, client_node=None, timeout_duration=65, utf8=True, padding=True): @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(1), reraise=True) def disperse(my_self=self): response = [] request = prepare_dispersal_request(data, app_id, index, utf8=utf8, padding=padding) executor = my_self.find_executor_node() + try: - response = executor.send_dispersal_request(request) + if client_node is None: + response = executor.send_dispersal_request(request) + else: + response = client_node.set_rest_api(executor.name(), executor.api_port()) + response = client_node.send_dispersal_request(request) except Exception as ex: assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex) diff --git a/tests/conftest.py b/tests/conftest.py index 3145fb2..f70f72a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import inspect import glob +from concurrent.futures import ThreadPoolExecutor, as_completed + from src.libs.custom_logger import get_custom_logger import os import pytest @@ -68,6 +70,14 @@ def attach_logs_on_fail(request): attach_allure_file(file) +def stop_node(node): + try: + node.stop() + except Exception as ex: + if "No such container" in str(ex): + logger.error(f"Failed to stop node container because of error {ex}") + + @pytest.fixture(scope="function", autouse=True) def close_open_nodes(attach_logs_on_fail): DS.nomos_nodes = [] @@ -75,20 +85,13 @@ def close_open_nodes(attach_logs_on_fail): yield logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") crashed_containers = [] - for node in DS.nomos_nodes: - try: - node.stop() - except Exception as ex: - if "No such container" in str(ex): - crashed_containers.append(node.image) - logger.error(f"Failed to stop node container because of error {ex}") - for node in DS.client_nodes: - try: - node.stop() - except Exception as ex: - if "No such container" in str(ex): - crashed_containers.append(node.name()) - logger.error(f"Failed to stop client node container because of error {ex}") + with ThreadPoolExecutor(max_workers=30) as executor: + node_cleanups = [executor.submit(stop_node, node) for node in DS.nomos_nodes] + [executor.submit(stop_node, node) for node in DS.client_nodes] + for cleanup in as_completed(node_cleanups): + result = cleanup.result() + if result is not None: + crashed_containers.append(result) + assert not crashed_containers, f"Containers {crashed_containers} crashed during the test!!!" diff --git a/tests/dos_robustness/test_high_load_dos.py b/tests/dos_robustness/test_high_load_dos.py index 5e73dd6..858f847 100644 --- a/tests/dos_robustness/test_high_load_dos.py +++ b/tests/dos_robustness/test_high_load_dos.py @@ -114,7 +114,7 @@ class TestHighLoadDos(StepsDataAvailability): @pytest.mark.usefixtures("setup_2_node_cluster", "setup_client_nodes") def test_sustained_high_rate_multiple_clients(self): - timeout = 60 + timeout = 10 start_time = time.time() successful_dispersals = 0 unsuccessful_dispersals = 0 @@ -127,7 +127,7 @@ class TestHighLoadDos(StepsDataAvailability): delay(0.01) try: - response = self.disperse_data(DATA_TO_DISPERSE[6], to_app_id(1), to_index(0), timeout_duration=0) + response = self.disperse_data(DATA_TO_DISPERSE[6], to_app_id(1), to_index(0), client_node=self.client_nodes[0], timeout_duration=0) if response.status_code == 200: successful_dispersals += 1 else: @@ -135,11 +135,11 @@ class TestHighLoadDos(StepsDataAvailability): except Exception: unsuccessful_dispersals += 1 - try: - self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5), timeout_duration=0) - successful_downloads += 1 - except Exception: - unsuccessful_downloads += 1 + # try: + # self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5), timeout_duration=0) + # successful_downloads += 1 + # except Exception: + # unsuccessful_downloads += 1 assert successful_dispersals > 0, "No successful dispersals" assert successful_downloads > 0, "No successful downloads"