test: remote API call

- parallel container stop

src/api_clients/rest.py

@@ -6,20 +6,17 @@ logger = get_custom_logger(__name__)
 
 
 class REST(BaseClient):
-    def __init__(self, rest_port):
+    def __init__(self, rest_port, rest_host="127.0.0.1"):
         self._rest_port = rest_port
+        self._rest_host = rest_host
 
-    def rest_call(self, method, endpoint, payload=None, host="127.0.0.1", port=None):
-        if port is None:
-            port = self._rest_port
-        url = f"http://{host}:{port}/{endpoint}"
+    def rest_call(self, method, endpoint, payload=None):
+        url = f"http://{self._rest_host}:{self._rest_port}/{endpoint}"
         headers = {"Content-Type": "application/json", "Connection": "close"}
         return self.make_request(method, url, headers=headers, data=payload)
 
-    def rest_call_text(self, method, endpoint, payload=None, host="127.0.0.1", port=None):
-        if port is None:
-            port = self._rest_port
-        url = f"http://{host}:{port}/{endpoint}"
+    def rest_call_text(self, method, endpoint, payload=None):
+        url = f"http://{self._rest_host}:{self._rest_port}/{endpoint}"
         headers = {"accept": "text/plain", "Connection": "close"}
         return self.make_request(method, url, headers=headers, data=payload)
 

src/cli/nomos_cli.py

@@ -2,6 +2,7 @@ import json
 import os
 import re
 
+from src.api_clients.rest import REST
 from src.data_storage import DS
 from src.libs.common import generate_log_prefix, delay, remove_padding
 from src.libs.custom_logger import get_custom_logger
@@ -35,6 +36,7 @@ class NomosCli:
         self._docker_manager = DockerManager(self._image_name)
         self._container_name = container_name
         self._container = None
+        self._api = None
 
         cwd = os.getcwd()
         self._volumes = [cwd + "/" + volume for volume in self._volumes]
@@ -106,6 +108,10 @@ class NomosCli:
 
         return result
 
+    def set_rest_api(self, host, port):
+        logger.debug(f"Setting rest API object to host {host} port {port}")
+        self._api = REST(port, host)
+
     @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True)
     def stop(self):
         self._container = stop(self._container)

src/node/nomos_node.py

@@ -126,6 +126,9 @@ class NomosNode:
     def name(self):
         return self._container_name
 
+    def api_port(self):
+        return self._tcp_port
+
     def check_nomos_log_errors(self, whitelist=None):
         keywords = LOG_ERROR_KEYWORDS
 

src/steps/da.py

@@ -1,4 +1,5 @@
 import allure
+from requests.packages import target
 from tenacity import retry, stop_after_delay, wait_fixed
 
 from src.env_vars import NOMOS_EXECUTOR
@@ -45,14 +46,19 @@ class StepsDataAvailability(StepsCommon):
         return executor
 
     @allure.step
-    def disperse_data(self, data, app_id, index, timeout_duration=65, utf8=True, padding=True):
+    def disperse_data(self, data, app_id, index, client_node=None, timeout_duration=65, utf8=True, padding=True):
         @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(1), reraise=True)
         def disperse(my_self=self):
             response = []
             request = prepare_dispersal_request(data, app_id, index, utf8=utf8, padding=padding)
             executor = my_self.find_executor_node()
+
             try:
-                response = executor.send_dispersal_request(request)
+                if client_node is None:
+                    response = executor.send_dispersal_request(request)
+                else:
+                    response = client_node.set_rest_api(executor.name(), executor.api_port())
+                    response = client_node.send_dispersal_request(request)
             except Exception as ex:
                 assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
 

tests/conftest.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 import inspect
 import glob
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 from src.libs.custom_logger import get_custom_logger
 import os
 import pytest
@@ -68,6 +70,14 @@ def attach_logs_on_fail(request):
             attach_allure_file(file)
 
 
+def stop_node(node):
+    try:
+        node.stop()
+    except Exception as ex:
+        if "No such container" in str(ex):
+            logger.error(f"Failed to stop node container because of error {ex}")
+
+
 @pytest.fixture(scope="function", autouse=True)
 def close_open_nodes(attach_logs_on_fail):
     DS.nomos_nodes = []
@@ -75,20 +85,13 @@ def close_open_nodes(attach_logs_on_fail):
     yield
     logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
     crashed_containers = []
-    for node in DS.nomos_nodes:
-        try:
-            node.stop()
-        except Exception as ex:
-            if "No such container" in str(ex):
-                crashed_containers.append(node.image)
-            logger.error(f"Failed to stop node container because of error {ex}")
-    for node in DS.client_nodes:
-        try:
-            node.stop()
-        except Exception as ex:
-            if "No such container" in str(ex):
-                crashed_containers.append(node.name())
-            logger.error(f"Failed to stop client node container because of error {ex}")
+    with ThreadPoolExecutor(max_workers=30) as executor:
+        node_cleanups = [executor.submit(stop_node, node) for node in DS.nomos_nodes] + [executor.submit(stop_node, node) for node in DS.client_nodes]
+        for cleanup in as_completed(node_cleanups):
+            result = cleanup.result()
+            if result is not None:
+                crashed_containers.append(result)
+
     assert not crashed_containers, f"Containers {crashed_containers} crashed during the test!!!"
 
 

tests/dos_robustness/test_high_load_dos.py

@@ -114,7 +114,7 @@ class TestHighLoadDos(StepsDataAvailability):
 
     @pytest.mark.usefixtures("setup_2_node_cluster", "setup_client_nodes")
     def test_sustained_high_rate_multiple_clients(self):
-        timeout = 60
+        timeout = 10
         start_time = time.time()
         successful_dispersals = 0
         unsuccessful_dispersals = 0
@@ -127,7 +127,7 @@ class TestHighLoadDos(StepsDataAvailability):
 
             delay(0.01)
             try:
-                response = self.disperse_data(DATA_TO_DISPERSE[6], to_app_id(1), to_index(0), timeout_duration=0)
+                response = self.disperse_data(DATA_TO_DISPERSE[6], to_app_id(1), to_index(0), client_node=self.client_nodes[0], timeout_duration=0)
                 if response.status_code == 200:
                     successful_dispersals += 1
                 else:
@@ -135,11 +135,11 @@ class TestHighLoadDos(StepsDataAvailability):
             except Exception:
                 unsuccessful_dispersals += 1
 
-            try:
-                self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5), timeout_duration=0)
-                successful_downloads += 1
-            except Exception:
-                unsuccessful_downloads += 1
+            # try:
+            #     self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5), timeout_duration=0)
+            #     successful_downloads += 1
+            # except Exception:
+            #     unsuccessful_downloads += 1
 
         assert successful_dispersals > 0, "No successful dispersals"
         assert successful_downloads > 0, "No successful downloads"