2024-12-13 15:58:25 +08:00
|
|
|
import os
|
|
|
|
|
|
2025-01-14 17:22:17 +08:00
|
|
|
from src.data_storage import DS
|
2024-12-13 15:58:25 +08:00
|
|
|
from src.libs.custom_logger import get_custom_logger
|
|
|
|
|
from tenacity import retry, stop_after_delay, wait_fixed
|
2024-12-19 16:02:57 +08:00
|
|
|
|
2024-12-13 15:58:25 +08:00
|
|
|
from src.node.api_clients.rest import REST
|
|
|
|
|
from src.node.docker_mananger import DockerManager
|
|
|
|
|
from src.env_vars import DOCKER_LOG_DIR
|
2024-12-19 16:02:57 +08:00
|
|
|
from src.node.node_vars import nomos_nodes
|
2025-01-14 17:22:17 +08:00
|
|
|
from src.test_data import LOG_ERROR_KEYWORDS
|
2024-12-13 15:58:25 +08:00
|
|
|
|
|
|
|
|
logger = get_custom_logger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sanitize_docker_flags(input_flags):
|
|
|
|
|
output_flags = {}
|
|
|
|
|
for key, value in input_flags.items():
|
|
|
|
|
key = key.replace("_", "-")
|
|
|
|
|
output_flags[key] = value
|
|
|
|
|
|
|
|
|
|
return output_flags
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class NomosNode:
|
2025-01-07 13:13:11 +08:00
|
|
|
def __init__(self, node_type, container_name=""):
|
2024-12-20 16:14:47 +08:00
|
|
|
logger.debug(f"Node is going to be initialized with this config {nomos_nodes[node_type]}")
|
2024-12-19 16:02:57 +08:00
|
|
|
self._image_name = nomos_nodes[node_type]["image"]
|
|
|
|
|
self._internal_ports = nomos_nodes[node_type]["ports"]
|
|
|
|
|
self._volumes = nomos_nodes[node_type]["volumes"]
|
|
|
|
|
self._entrypoint = nomos_nodes[node_type]["entrypoint"]
|
|
|
|
|
|
2025-01-07 13:13:11 +08:00
|
|
|
self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}__{self._image_name.replace('/', '_')}.log")
|
2024-12-13 15:58:25 +08:00
|
|
|
self._docker_manager = DockerManager(self._image_name)
|
2025-01-07 13:13:11 +08:00
|
|
|
self._container_name = container_name
|
2024-12-13 15:58:25 +08:00
|
|
|
self._container = None
|
2024-12-20 16:14:47 +08:00
|
|
|
|
2025-01-06 19:13:40 +08:00
|
|
|
cwd = os.getcwd()
|
2025-01-21 16:33:47 +08:00
|
|
|
updated_volumes = []
|
2025-01-06 19:13:40 +08:00
|
|
|
for i, volume in enumerate(self._volumes):
|
2025-01-21 16:33:47 +08:00
|
|
|
updated_volumes.append(cwd + "/" + volume)
|
|
|
|
|
self._volumes = updated_volumes
|
2025-01-06 19:13:40 +08:00
|
|
|
|
2025-01-21 16:33:47 +08:00
|
|
|
logger.debug(f"NomosNode instance initialized with volumes {self._volumes}")
|
2024-12-13 15:58:25 +08:00
|
|
|
logger.debug(f"NomosNode instance initialized with log path {self._log_path}")
|
|
|
|
|
|
2025-01-13 19:35:55 +08:00
|
|
|
@retry(stop=stop_after_delay(60), wait=wait_fixed(0.1), reraise=True)
|
2025-01-08 15:20:23 +08:00
|
|
|
def start(self, wait_for_node_sec=120, **kwargs):
|
2024-12-13 15:58:25 +08:00
|
|
|
logger.debug("Starting Node...")
|
2024-12-19 16:02:57 +08:00
|
|
|
self._docker_manager.create_network()
|
|
|
|
|
self._ext_ip = self._docker_manager.generate_random_ext_ip()
|
2024-12-20 16:14:47 +08:00
|
|
|
|
|
|
|
|
number_of_ports = len(self._internal_ports)
|
|
|
|
|
self._port_map = {}
|
|
|
|
|
|
|
|
|
|
if number_of_ports > 0:
|
|
|
|
|
self._external_ports = self._docker_manager.generate_ports(count=number_of_ports)
|
|
|
|
|
self._udp_port = self._external_ports[0]
|
|
|
|
|
self._tcp_port = self._external_ports[1]
|
2025-01-07 21:21:36 +08:00
|
|
|
self._api = REST(self._tcp_port)
|
2024-12-20 16:14:47 +08:00
|
|
|
|
|
|
|
|
logger.debug(f"Internal ports {self._internal_ports}")
|
|
|
|
|
|
|
|
|
|
for i, port in enumerate(self._internal_ports):
|
|
|
|
|
self._port_map[port] = int(self._external_ports[i])
|
2024-12-19 16:02:57 +08:00
|
|
|
|
2025-01-08 14:22:36 +08:00
|
|
|
default_args = {}
|
2024-12-19 16:02:57 +08:00
|
|
|
|
|
|
|
|
logger.debug(f"Using volumes {self._volumes}")
|
|
|
|
|
|
2024-12-20 16:14:47 +08:00
|
|
|
logger.debug(f"Port map {self._port_map}")
|
|
|
|
|
|
2024-12-19 16:02:57 +08:00
|
|
|
self._container = self._docker_manager.start_container(
|
|
|
|
|
self._docker_manager.image,
|
2024-12-20 18:14:43 +08:00
|
|
|
port_bindings=self._port_map,
|
2024-12-19 16:02:57 +08:00
|
|
|
args=default_args,
|
|
|
|
|
log_path=self._log_path,
|
|
|
|
|
volumes=self._volumes,
|
2024-12-20 18:33:59 +08:00
|
|
|
entrypoint=self._entrypoint,
|
2025-01-13 19:11:34 +08:00
|
|
|
remove_container=True,
|
2025-01-07 13:13:11 +08:00
|
|
|
name=self._container_name,
|
2024-12-19 16:02:57 +08:00
|
|
|
)
|
2025-01-07 14:34:24 +08:00
|
|
|
|
2025-01-07 15:04:52 +08:00
|
|
|
logger.debug(f"Started container from image {self._image_name}. " f"REST: {getattr(self, '_tcp_port', 'N/A')}")
|
2025-01-07 14:34:24 +08:00
|
|
|
|
2025-01-14 17:22:17 +08:00
|
|
|
DS.nomos_nodes.append(self)
|
|
|
|
|
|
2025-01-14 13:33:41 +08:00
|
|
|
@retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True)
|
|
|
|
|
def stop(self):
|
|
|
|
|
if self._container:
|
|
|
|
|
logger.debug(f"Stopping container with id {self._container.short_id}")
|
|
|
|
|
self._container.stop()
|
|
|
|
|
try:
|
|
|
|
|
self._container.remove()
|
|
|
|
|
except:
|
|
|
|
|
pass
|
|
|
|
|
self._container = None
|
|
|
|
|
logger.debug("Container stopped.")
|
|
|
|
|
|
|
|
|
|
@retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True)
|
|
|
|
|
def kill(self):
|
|
|
|
|
if self._container:
|
|
|
|
|
logger.debug(f"Killing container with id {self._container.short_id}")
|
|
|
|
|
self._container.kill()
|
|
|
|
|
try:
|
|
|
|
|
self._container.remove()
|
|
|
|
|
except:
|
|
|
|
|
pass
|
|
|
|
|
self._container = None
|
|
|
|
|
logger.debug("Container killed.")
|
|
|
|
|
|
|
|
|
|
def restart(self):
|
|
|
|
|
if self._container:
|
|
|
|
|
logger.debug(f"Restarting container with id {self._container.short_id}")
|
|
|
|
|
self._container.restart()
|
|
|
|
|
|
|
|
|
|
def pause(self):
|
|
|
|
|
if self._container:
|
|
|
|
|
logger.debug(f"Pausing container with id {self._container.short_id}")
|
|
|
|
|
self._container.pause()
|
|
|
|
|
|
|
|
|
|
def unpause(self):
|
|
|
|
|
if self._container:
|
|
|
|
|
logger.debug(f"Unpause container with id {self._container.short_id}")
|
|
|
|
|
self._container.unpause()
|
|
|
|
|
|
2025-01-07 14:34:24 +08:00
|
|
|
def ensure_ready(self, timeout_duration=10):
|
|
|
|
|
@retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(0.1), reraise=True)
|
|
|
|
|
def check_ready(node=self):
|
2025-01-07 21:08:41 +08:00
|
|
|
node.info_response = node.info()
|
2025-01-07 14:34:24 +08:00
|
|
|
logger.info("REST service is ready !!")
|
|
|
|
|
|
|
|
|
|
if self.is_nomos():
|
|
|
|
|
check_ready()
|
|
|
|
|
|
|
|
|
|
def is_nomos(self):
|
|
|
|
|
return "nomos" in self._container_name
|
|
|
|
|
|
2025-01-07 21:08:41 +08:00
|
|
|
def info(self):
|
|
|
|
|
return self._api.info()
|
2025-01-14 17:22:17 +08:00
|
|
|
|
|
|
|
|
def check_nomos_log_errors(self, whitelist=None):
|
|
|
|
|
keywords = LOG_ERROR_KEYWORDS
|
|
|
|
|
|
|
|
|
|
# If a whitelist is provided, remove those keywords from the keywords list
|
|
|
|
|
if whitelist:
|
|
|
|
|
keywords = [keyword for keyword in keywords if keyword not in whitelist]
|
|
|
|
|
|
|
|
|
|
matches = self._docker_manager.search_log_for_keywords(self._log_path, keywords, False)
|
|
|
|
|
assert not matches, f"Found errors {matches}"
|
2025-01-17 14:23:52 +08:00
|
|
|
|
|
|
|
|
def send_dispersal_request(self, data):
|
|
|
|
|
return self._api.send_dispersal_request(data)
|
2025-01-17 19:27:43 +08:00
|
|
|
|
|
|
|
|
def send_get_data_range_request(self, data):
|
|
|
|
|
return self._api.send_get_range(data)
|