From d85c19269ae718d54a23ba5286306f52ecd3a5e2 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 10 Mar 2025 03:33:54 +0000 Subject: [PATCH] test: replace client node with proxy client --- README.md | 2 +- src/api_clients/base_client.py | 6 +- src/{cli => client}/__init__.py | 0 .../cli_vars.py => client/client_vars.py} | 19 ++-- src/{cli => client}/nomos_cli.py | 20 +--- src/client/proxy_client.py | 100 ++++++++++++++++++ src/env_vars.py | 4 +- src/steps/common.py | 2 +- 8 files changed, 124 insertions(+), 29 deletions(-) rename src/{cli => client}/__init__.py (100%) rename src/{cli/cli_vars.py => client/client_vars.py} (50%) rename src/{cli => client}/nomos_cli.py (86%) create mode 100644 src/client/proxy_client.py diff --git a/README.md b/README.md index 5e13480..410ba5a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ python -m venv .venv source .venv/bin/activate pip install -r requirements.txt pre-commit install -(optional) Overwrite default vars from src/env_vars.py via cli env vars or by adding a .env file +(optional) Overwrite default vars from src/env_vars.py via env vars or by adding a .env file pytest ``` diff --git a/src/api_clients/base_client.py b/src/api_clients/base_client.py index b731e0f..e8c9ddf 100644 --- a/src/api_clients/base_client.py +++ b/src/api_clients/base_client.py @@ -9,7 +9,11 @@ logger = get_custom_logger(__name__) class BaseClient: def make_request(self, method, url, headers=None, data=None): self.log_request_as_curl(method, url, headers, data) - response = requests.request(method.upper(), url, headers=headers, data=data, timeout=API_REQUEST_TIMEOUT) + try: + response = requests.request(method.upper(), url, headers=headers, data=data, timeout=API_REQUEST_TIMEOUT) + except Exception as ex: + logger.error(f"HERE An error occurred: {ex}. ") + try: response.raise_for_status() except requests.HTTPError as http_err: diff --git a/src/cli/__init__.py b/src/client/__init__.py similarity index 100% rename from src/cli/__init__.py rename to src/client/__init__.py diff --git a/src/cli/cli_vars.py b/src/client/client_vars.py similarity index 50% rename from src/cli/cli_vars.py rename to src/client/client_vars.py index 0dcb84c..2e31061 100644 --- a/src/cli/cli_vars.py +++ b/src/client/client_vars.py @@ -1,4 +1,4 @@ -from src.env_vars import NOMOS_IMAGE +from src.env_vars import NOMOS_IMAGE, HTTP_PROXY_IMAGE nomos_cli = { "reconstruct": { @@ -8,11 +8,14 @@ nomos_cli = { "ports": [], "entrypoint": "", }, - "client_node": { - "image": NOMOS_IMAGE, - "flags": [{"tail": [], "-f": [], "/dev/null": []}], - "volumes": [], - "ports": [], - "entrypoint": "", - }, +} + +http_proxy = { + "configurable-http-proxy": { + "image": HTTP_PROXY_IMAGE, + "flags": [{"--default-target": [0]}], # Value [] is a list of indexes into list of values required for the flag + "volumes": [], + "ports": ["8000/tcp"], + "entrypoint": "", + } } diff --git a/src/cli/nomos_cli.py b/src/client/nomos_cli.py similarity index 86% rename from src/cli/nomos_cli.py rename to src/client/nomos_cli.py index af9a053..b80db47 100644 --- a/src/cli/nomos_cli.py +++ b/src/client/nomos_cli.py @@ -9,7 +9,7 @@ from src.libs.common import generate_log_prefix, delay, remove_padding from src.libs.custom_logger import get_custom_logger from tenacity import retry, stop_after_delay, wait_fixed -from src.cli.cli_vars import nomos_cli +from src.client.client_vars import nomos_cli from src.docker_manager import DockerManager, stop, kill from src.env_vars import DOCKER_LOG_DIR, NOMOS_CLI @@ -25,7 +25,7 @@ class NomosCli: if command not in nomos_cli: raise ValueError("Unknown command provided") - logger.debug(f"Cli is going to be initialized with this config {nomos_cli[command]}") + logger.debug(f"NomosCli is going to be initialized with this config {nomos_cli[command]}") self._command = command self._image_name = nomos_cli[command]["image"] self._internal_ports = nomos_cli[command]["ports"] @@ -47,10 +47,7 @@ class NomosCli: self._port_map = {} - if self._command == "client_node": - cmd = [] - else: - cmd = [NOMOS_CLI, self._command] + cmd = [NOMOS_CLI, self._command] for flag in nomos_cli[self._command]["flags"]: for f, indexes in flag.items(): @@ -78,8 +75,6 @@ class NomosCli: case "reconstruct": decode_only = kwargs.get("decode_only", False) return self.reconstruct(decode_only=decode_only) - case "client_node": - return None case _: return None @@ -113,9 +108,6 @@ class NomosCli: def set_rest_api(self, host, port): self._api = REST(port, host) - def set_invalid_rest_api(self, host, port): - self._api = INVALID_REST(port, host) - @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) def stop(self): self._container = stop(self._container) @@ -126,9 +118,3 @@ class NomosCli: def name(self): return self._container_name - - def send_dispersal_request(self, data): - return self._api.send_dispersal_request(data) - - def send_get_data_range_request(self, data): - return self._api.send_get_range(data) diff --git a/src/client/proxy_client.py b/src/client/proxy_client.py new file mode 100644 index 0000000..cca35e0 --- /dev/null +++ b/src/client/proxy_client.py @@ -0,0 +1,100 @@ +import json +import os +import re + +from src.api_clients.invalid_rest import INVALID_REST +from src.api_clients.rest import REST +from src.data_storage import DS +from src.libs.common import generate_log_prefix, delay, remove_padding +from src.libs.custom_logger import get_custom_logger +from tenacity import retry, stop_after_delay, wait_fixed + +from src.client.client_vars import http_proxy +from src.docker_manager import DockerManager, stop, kill +from src.env_vars import DOCKER_LOG_DIR, NOMOS_CLI + +logger = get_custom_logger(__name__) + + +class ProxyClient: + def __init__(self): + command = "configurable-http-proxy" + + logger.debug(f"ProxyClient is going to be initialized with this config {http_proxy[command]}") + self._command = command + self._image_name = http_proxy[command]["image"] + self._internal_ports = http_proxy[command]["ports"] + self._volumes = http_proxy[command]["volumes"] + self._entrypoint = http_proxy[command]["entrypoint"] + + container_name = "proxy-client-" + generate_log_prefix() + self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}__{self._image_name.replace('/', '_')}.log") + self._docker_manager = DockerManager(self._image_name) + self._container_name = container_name + self._container = None + self._api = None + + cwd = os.getcwd() + self._volumes = [cwd + "/" + volume for volume in self._volumes] + + def run(self, input_values=None, **kwargs): + logger.debug(f"ProxyClient starting with log path {self._log_path}") + + self._port_map = {} + self._external_ports = self._docker_manager.generate_ports(count=1) + self._tcp_port = self._external_ports[0] + self._api = REST(self._tcp_port) + + logger.debug(f"Internal ports {self._internal_ports}") + + for i, port in enumerate(self._internal_ports): + self._port_map[port] = int(self._external_ports[i]) + + logger.debug(f"Port map {self._port_map}") + + cmd = [self._command] + + for flag in http_proxy[self._command]["flags"]: + for f, indexes in flag.items(): + cmd.append(f) + for j in indexes: + cmd.append(input_values[j]) + + logger.debug(f"ProxyCLient command to run {cmd}") + + self._container = self._docker_manager.start_container( + self._docker_manager.image, + port_bindings=self._port_map, + args=None, + log_path=self._log_path, + volumes=self._volumes, + entrypoint=self._entrypoint, + remove_container=True, + name=self._container_name, + command=cmd, + ) + + DS.client_nodes.append(self) + + def set_rest_api(self, host, port): + self._api = REST(port, host) + + def set_invalid_rest_api(self, host, port): + self._api = INVALID_REST(port, host) + + @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) + def stop(self): + self._container = stop(self._container) + + @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) + def kill(self): + self._container = kill(self._container) + + def name(self): + return self._container_name + + def send_dispersal_request(self, data): + return self._api.send_dispersal_request(data) + + def send_get_data_range_request(self, data): + return self._api.send_get_range(data) diff --git a/src/env_vars.py b/src/env_vars.py index 8dcef9b..46c149e 100644 --- a/src/env_vars.py +++ b/src/env_vars.py @@ -22,11 +22,13 @@ CFGSYNC = "cfgsync" DEFAULT_IMAGE = "ghcr.io/logos-co/nomos-node:testnet" NOMOS_IMAGE = get_env_var("NOMOS_IMAGE", DEFAULT_IMAGE) +HTTP_PROXY_IMAGE = get_env_var("HTTP_PROXY_IMAGE", "bitnami/configurable-http-proxy:latest") + NODE_1 = get_env_var("NODE_1", NOMOS) NODE_2 = get_env_var("NODE_2", NOMOS_EXECUTOR) NODE_3 = get_env_var("NODE_3", CFGSYNC) -NOMOS_CLI = "/usr/bin/nomos-cli" +NOMOS_CLI = "/usr/bin/nomos-client" ADDITIONAL_NODES = get_env_var("ADDITIONAL_NODES", f"{NOMOS},{NOMOS}") # more nodes need to follow the NODE_X pattern diff --git a/src/steps/common.py b/src/steps/common.py index 1e3cb60..e777b68 100644 --- a/src/steps/common.py +++ b/src/steps/common.py @@ -4,7 +4,7 @@ import shutil import pytest -from src.cli.nomos_cli import NomosCli +from src.client.nomos_cli import NomosCli from src.env_vars import CFGSYNC, NOMOS, NOMOS_EXECUTOR from src.libs.common import delay from src.libs.custom_logger import get_custom_logger