94 lines
3.8 KiB
Python
Raw Normal View History

2023-11-01 14:02:29 +02:00
import os
from src.libs.custom_logger import get_custom_logger
2023-11-01 14:02:29 +02:00
import random
import threading
import docker
from src.env_vars import NETWORK_NAME, SUBNET, IP_RANGE, GATEWAY
from docker.types import IPAMConfig, IPAMPool
from docker.errors import NotFound
logger = get_custom_logger(__name__)
2023-11-01 14:02:29 +02:00
class DockerManager:
def __init__(self, image):
self._image = image
self._client = docker.from_env()
logger.debug(f"Docker client initialized with image {self._image}")
2023-11-01 14:02:29 +02:00
def create_network(self, network_name=NETWORK_NAME):
logger.debug(f"Attempting to create or retrieve network {network_name}")
2023-11-01 14:02:29 +02:00
networks = self._client.networks.list(names=[network_name])
if networks:
logger.debug(f"Network {network_name} already exists")
2023-11-01 14:02:29 +02:00
return networks[0]
network = self._client.networks.create(
network_name,
driver="bridge",
ipam=IPAMConfig(driver="default", pool_configs=[IPAMPool(subnet=SUBNET, iprange=IP_RANGE, gateway=GATEWAY)]),
)
logger.debug(f"Network {network_name} created")
2023-11-01 14:02:29 +02:00
return network
chore: RLN registration support (#16) * chore: parameters and volumes for RLN relay * chore: add startup test for RLN * fix: remove redundant rln_enabled from WakuNode * fix: - and _ magic in rln parameters * fix: key 'eth_testnet_key' -> 'eth_client_private_key' * fix: rln-register_only -> rln-register-only * fix: use extend instead of append for the volumes list * fix: use absolute path for the volumes names - mark volumes shared across containers * fix: remove :z attr from volumes * fix: remove filename from volume path * fix: remove request for ENR * test: plain RLN registration * fix: remove subscribe * fix: remove ENR related params * test: add run_container func to Docker Mananger * fix: remove run_container func from Docker Mananger - no need for exec * fix: pass exec commands instead of specialized docker exec func * fix: exclude RLN arguments from others * fix: separate RLN volumes by implementation * test: registration with nwaku * test: registration over existing credentials * test: add RLN Relay node startup * test: RLN credentials reuse for relay node startup * fix: clean up unnecessary commands * fix: clean up unnecessary commands for non RLN relay too * test: regression by sending one valid message * fix: add forgotten call to get enr_uri * fix: check RLN credentials set properly * fix: parenthesis in eval expression * fix: better check for RLN credentials * fix: update to new flags - gowaku - separate private key for go/nwaku * test: register RLN * fix: RLN credentials check * fix: remove enable rln-relay flag from registration * test: reorder commands to check go-waku registration * fix: restructure if statements for RLN registration * fix: different set of flag for RLN registration and operation * fix: forgotten "eth" in eth-contract-address * fix: remove redundant None from get_env_var call * fix: refactor rln credentials parsing from start function * fix: missed second return value * test: call to parse_rln_credentials * fix: remove redundant self in call to parse_rln_credentials * fix: remove rln related values if not valid * fix: refactored to accept multiple private keys - JSON source for RLN credentials - removed go-waku support for RLN * test: register RLN with 2 nwaku nodes * fix: missing open file, key errors * fix: return effective keystore_path * fix: cleanup unused env variables * fix: cleanup unused env variables from relay * fix: wait longer for credentials file to be written to disk * test: remove waiting for credentials file to be written to disk * fix: add select_private_key() * fix: merge parse_rln_credentials and parse_rln_registration_credentials * test: wait for filesystem cache * test: try with os.sync to flush cache * test: stop container to clear cache * fix: 15 sec wait + stop container to clear cache * test: RLN registration test with fixture and cred file check * test: added allure.step - RLN registration for single node - corresponding test to register all keys * fix: f-strings in the test * fix: sync naming for register_main_rln_relay_nodes * fix: add WARN message to log unset and expected RLN credentials * fix: pytest.skip added to exit tests when non nwaku image is used * fix: swap NODE2 for DEFAULT_NWAKU * fix: let rln_credential_store_ready to raise exception for retry * fix: let register_rln to raise exception too, when credential fine still not available * fix: remove container stop * fix: tune down retry timeouts * fix: remove unnecessary f-string * test: reduce unnecessary variables init * test: undo reduce unnecessary variables init * test: increase timeout for rln_credential_store_ready * test: refactor kwargs handling into sanitize_docker_flags * fix: created RLN registration check - changed rln_credential_store_ready to function * fix: delete unnecessary ports from register_rln init * fix: remove wait for registration entirely * test: RLN_CREDENTIALS env var example * fix: check_rln_registration to raise exception * fix: use f-string at check_rln_registration * fix: add gen_step_id function * fix: RLN_CREDENTIALS as loaded from .env * fix: RLN_CREDENTIALS example, skipping test if not set * test: RLN with actions * fix: tune up RLN timeouts for Github runners * test: filesystem write to / * fix: change RLN data to be stored at working directory * fix: catch exception instead of "if not" - print out container log * fix: wrap logs into debug msg * fix: print log file * fix: remove additional logging * fix: exit RLN cred parsing sooner when not used - delete proper keys from default_args * fix: Allure report * fix: Discord notifications * fix: remove f-string * fix: delete test workflow
2024-02-13 22:01:30 +08:00
def start_container(self, image_name, ports, args, log_path, container_ip, volumes):
2023-11-03 17:01:00 +02:00
cli_args = []
for key, value in args.items():
if isinstance(value, list): # Check if value is a list
cli_args.extend([f"--{key}={item}" for item in value]) # Add a command for each item in the list
chore: RLN registration support (#16) * chore: parameters and volumes for RLN relay * chore: add startup test for RLN * fix: remove redundant rln_enabled from WakuNode * fix: - and _ magic in rln parameters * fix: key 'eth_testnet_key' -> 'eth_client_private_key' * fix: rln-register_only -> rln-register-only * fix: use extend instead of append for the volumes list * fix: use absolute path for the volumes names - mark volumes shared across containers * fix: remove :z attr from volumes * fix: remove filename from volume path * fix: remove request for ENR * test: plain RLN registration * fix: remove subscribe * fix: remove ENR related params * test: add run_container func to Docker Mananger * fix: remove run_container func from Docker Mananger - no need for exec * fix: pass exec commands instead of specialized docker exec func * fix: exclude RLN arguments from others * fix: separate RLN volumes by implementation * test: registration with nwaku * test: registration over existing credentials * test: add RLN Relay node startup * test: RLN credentials reuse for relay node startup * fix: clean up unnecessary commands * fix: clean up unnecessary commands for non RLN relay too * test: regression by sending one valid message * fix: add forgotten call to get enr_uri * fix: check RLN credentials set properly * fix: parenthesis in eval expression * fix: better check for RLN credentials * fix: update to new flags - gowaku - separate private key for go/nwaku * test: register RLN * fix: RLN credentials check * fix: remove enable rln-relay flag from registration * test: reorder commands to check go-waku registration * fix: restructure if statements for RLN registration * fix: different set of flag for RLN registration and operation * fix: forgotten "eth" in eth-contract-address * fix: remove redundant None from get_env_var call * fix: refactor rln credentials parsing from start function * fix: missed second return value * test: call to parse_rln_credentials * fix: remove redundant self in call to parse_rln_credentials * fix: remove rln related values if not valid * fix: refactored to accept multiple private keys - JSON source for RLN credentials - removed go-waku support for RLN * test: register RLN with 2 nwaku nodes * fix: missing open file, key errors * fix: return effective keystore_path * fix: cleanup unused env variables * fix: cleanup unused env variables from relay * fix: wait longer for credentials file to be written to disk * test: remove waiting for credentials file to be written to disk * fix: add select_private_key() * fix: merge parse_rln_credentials and parse_rln_registration_credentials * test: wait for filesystem cache * test: try with os.sync to flush cache * test: stop container to clear cache * fix: 15 sec wait + stop container to clear cache * test: RLN registration test with fixture and cred file check * test: added allure.step - RLN registration for single node - corresponding test to register all keys * fix: f-strings in the test * fix: sync naming for register_main_rln_relay_nodes * fix: add WARN message to log unset and expected RLN credentials * fix: pytest.skip added to exit tests when non nwaku image is used * fix: swap NODE2 for DEFAULT_NWAKU * fix: let rln_credential_store_ready to raise exception for retry * fix: let register_rln to raise exception too, when credential fine still not available * fix: remove container stop * fix: tune down retry timeouts * fix: remove unnecessary f-string * test: reduce unnecessary variables init * test: undo reduce unnecessary variables init * test: increase timeout for rln_credential_store_ready * test: refactor kwargs handling into sanitize_docker_flags * fix: created RLN registration check - changed rln_credential_store_ready to function * fix: delete unnecessary ports from register_rln init * fix: remove wait for registration entirely * test: RLN_CREDENTIALS env var example * fix: check_rln_registration to raise exception * fix: use f-string at check_rln_registration * fix: add gen_step_id function * fix: RLN_CREDENTIALS as loaded from .env * fix: RLN_CREDENTIALS example, skipping test if not set * test: RLN with actions * fix: tune up RLN timeouts for Github runners * test: filesystem write to / * fix: change RLN data to be stored at working directory * fix: catch exception instead of "if not" - print out container log * fix: wrap logs into debug msg * fix: print log file * fix: remove additional logging * fix: exit RLN cred parsing sooner when not used - delete proper keys from default_args * fix: Allure report * fix: Discord notifications * fix: remove f-string * fix: delete test workflow
2024-02-13 22:01:30 +08:00
elif value is None:
cli_args.append(f"{key}") # Add simple command as it is passed in the key
2023-11-03 17:01:00 +02:00
else:
cli_args.append(f"--{key}={value}") # Add a single command
chore: RLN registration support (#16) * chore: parameters and volumes for RLN relay * chore: add startup test for RLN * fix: remove redundant rln_enabled from WakuNode * fix: - and _ magic in rln parameters * fix: key 'eth_testnet_key' -> 'eth_client_private_key' * fix: rln-register_only -> rln-register-only * fix: use extend instead of append for the volumes list * fix: use absolute path for the volumes names - mark volumes shared across containers * fix: remove :z attr from volumes * fix: remove filename from volume path * fix: remove request for ENR * test: plain RLN registration * fix: remove subscribe * fix: remove ENR related params * test: add run_container func to Docker Mananger * fix: remove run_container func from Docker Mananger - no need for exec * fix: pass exec commands instead of specialized docker exec func * fix: exclude RLN arguments from others * fix: separate RLN volumes by implementation * test: registration with nwaku * test: registration over existing credentials * test: add RLN Relay node startup * test: RLN credentials reuse for relay node startup * fix: clean up unnecessary commands * fix: clean up unnecessary commands for non RLN relay too * test: regression by sending one valid message * fix: add forgotten call to get enr_uri * fix: check RLN credentials set properly * fix: parenthesis in eval expression * fix: better check for RLN credentials * fix: update to new flags - gowaku - separate private key for go/nwaku * test: register RLN * fix: RLN credentials check * fix: remove enable rln-relay flag from registration * test: reorder commands to check go-waku registration * fix: restructure if statements for RLN registration * fix: different set of flag for RLN registration and operation * fix: forgotten "eth" in eth-contract-address * fix: remove redundant None from get_env_var call * fix: refactor rln credentials parsing from start function * fix: missed second return value * test: call to parse_rln_credentials * fix: remove redundant self in call to parse_rln_credentials * fix: remove rln related values if not valid * fix: refactored to accept multiple private keys - JSON source for RLN credentials - removed go-waku support for RLN * test: register RLN with 2 nwaku nodes * fix: missing open file, key errors * fix: return effective keystore_path * fix: cleanup unused env variables * fix: cleanup unused env variables from relay * fix: wait longer for credentials file to be written to disk * test: remove waiting for credentials file to be written to disk * fix: add select_private_key() * fix: merge parse_rln_credentials and parse_rln_registration_credentials * test: wait for filesystem cache * test: try with os.sync to flush cache * test: stop container to clear cache * fix: 15 sec wait + stop container to clear cache * test: RLN registration test with fixture and cred file check * test: added allure.step - RLN registration for single node - corresponding test to register all keys * fix: f-strings in the test * fix: sync naming for register_main_rln_relay_nodes * fix: add WARN message to log unset and expected RLN credentials * fix: pytest.skip added to exit tests when non nwaku image is used * fix: swap NODE2 for DEFAULT_NWAKU * fix: let rln_credential_store_ready to raise exception for retry * fix: let register_rln to raise exception too, when credential fine still not available * fix: remove container stop * fix: tune down retry timeouts * fix: remove unnecessary f-string * test: reduce unnecessary variables init * test: undo reduce unnecessary variables init * test: increase timeout for rln_credential_store_ready * test: refactor kwargs handling into sanitize_docker_flags * fix: created RLN registration check - changed rln_credential_store_ready to function * fix: delete unnecessary ports from register_rln init * fix: remove wait for registration entirely * test: RLN_CREDENTIALS env var example * fix: check_rln_registration to raise exception * fix: use f-string at check_rln_registration * fix: add gen_step_id function * fix: RLN_CREDENTIALS as loaded from .env * fix: RLN_CREDENTIALS example, skipping test if not set * test: RLN with actions * fix: tune up RLN timeouts for Github runners * test: filesystem write to / * fix: change RLN data to be stored at working directory * fix: catch exception instead of "if not" - print out container log * fix: wrap logs into debug msg * fix: print log file * fix: remove additional logging * fix: exit RLN cred parsing sooner when not used - delete proper keys from default_args * fix: Allure report * fix: Discord notifications * fix: remove f-string * fix: delete test workflow
2024-02-13 22:01:30 +08:00
2023-11-01 14:02:29 +02:00
port_bindings = {f"{port}/tcp": ("", port) for port in ports}
port_bindings_for_log = " ".join(f"-p {port}:{port}" for port in ports)
cli_args_str_for_log = " ".join(cli_args)
logger.debug(f"docker run -i -t {port_bindings_for_log} {image_name} {cli_args_str_for_log}")
chore: RLN registration support (#16) * chore: parameters and volumes for RLN relay * chore: add startup test for RLN * fix: remove redundant rln_enabled from WakuNode * fix: - and _ magic in rln parameters * fix: key 'eth_testnet_key' -> 'eth_client_private_key' * fix: rln-register_only -> rln-register-only * fix: use extend instead of append for the volumes list * fix: use absolute path for the volumes names - mark volumes shared across containers * fix: remove :z attr from volumes * fix: remove filename from volume path * fix: remove request for ENR * test: plain RLN registration * fix: remove subscribe * fix: remove ENR related params * test: add run_container func to Docker Mananger * fix: remove run_container func from Docker Mananger - no need for exec * fix: pass exec commands instead of specialized docker exec func * fix: exclude RLN arguments from others * fix: separate RLN volumes by implementation * test: registration with nwaku * test: registration over existing credentials * test: add RLN Relay node startup * test: RLN credentials reuse for relay node startup * fix: clean up unnecessary commands * fix: clean up unnecessary commands for non RLN relay too * test: regression by sending one valid message * fix: add forgotten call to get enr_uri * fix: check RLN credentials set properly * fix: parenthesis in eval expression * fix: better check for RLN credentials * fix: update to new flags - gowaku - separate private key for go/nwaku * test: register RLN * fix: RLN credentials check * fix: remove enable rln-relay flag from registration * test: reorder commands to check go-waku registration * fix: restructure if statements for RLN registration * fix: different set of flag for RLN registration and operation * fix: forgotten "eth" in eth-contract-address * fix: remove redundant None from get_env_var call * fix: refactor rln credentials parsing from start function * fix: missed second return value * test: call to parse_rln_credentials * fix: remove redundant self in call to parse_rln_credentials * fix: remove rln related values if not valid * fix: refactored to accept multiple private keys - JSON source for RLN credentials - removed go-waku support for RLN * test: register RLN with 2 nwaku nodes * fix: missing open file, key errors * fix: return effective keystore_path * fix: cleanup unused env variables * fix: cleanup unused env variables from relay * fix: wait longer for credentials file to be written to disk * test: remove waiting for credentials file to be written to disk * fix: add select_private_key() * fix: merge parse_rln_credentials and parse_rln_registration_credentials * test: wait for filesystem cache * test: try with os.sync to flush cache * test: stop container to clear cache * fix: 15 sec wait + stop container to clear cache * test: RLN registration test with fixture and cred file check * test: added allure.step - RLN registration for single node - corresponding test to register all keys * fix: f-strings in the test * fix: sync naming for register_main_rln_relay_nodes * fix: add WARN message to log unset and expected RLN credentials * fix: pytest.skip added to exit tests when non nwaku image is used * fix: swap NODE2 for DEFAULT_NWAKU * fix: let rln_credential_store_ready to raise exception for retry * fix: let register_rln to raise exception too, when credential fine still not available * fix: remove container stop * fix: tune down retry timeouts * fix: remove unnecessary f-string * test: reduce unnecessary variables init * test: undo reduce unnecessary variables init * test: increase timeout for rln_credential_store_ready * test: refactor kwargs handling into sanitize_docker_flags * fix: created RLN registration check - changed rln_credential_store_ready to function * fix: delete unnecessary ports from register_rln init * fix: remove wait for registration entirely * test: RLN_CREDENTIALS env var example * fix: check_rln_registration to raise exception * fix: use f-string at check_rln_registration * fix: add gen_step_id function * fix: RLN_CREDENTIALS as loaded from .env * fix: RLN_CREDENTIALS example, skipping test if not set * test: RLN with actions * fix: tune up RLN timeouts for Github runners * test: filesystem write to / * fix: change RLN data to be stored at working directory * fix: catch exception instead of "if not" - print out container log * fix: wrap logs into debug msg * fix: print log file * fix: remove additional logging * fix: exit RLN cred parsing sooner when not used - delete proper keys from default_args * fix: Allure report * fix: Discord notifications * fix: remove f-string * fix: delete test workflow
2024-02-13 22:01:30 +08:00
container = self._client.containers.run(
image_name, command=cli_args, ports=port_bindings, detach=True, remove=True, auto_remove=True, volumes=volumes
)
2023-11-01 14:02:29 +02:00
network = self._client.networks.get(NETWORK_NAME)
logger.debug(f"docker network connect --ip {container_ip} {NETWORK_NAME} {container.id}")
2023-11-01 14:02:29 +02:00
network.connect(container, ipv4_address=container_ip)
logger.debug(f"Container started with ID {container.short_id}. Setting up logs at {log_path}")
2023-11-01 14:02:29 +02:00
log_thread = threading.Thread(target=self._log_container_output, args=(container, log_path))
log_thread.daemon = True
log_thread.start()
return container
def _log_container_output(self, container, log_path):
os.makedirs(os.path.dirname(log_path), exist_ok=True)
with open(log_path, "wb+") as log_file:
for chunk in container.logs(stream=True):
log_file.write(chunk)
def generate_ports(self, base_port=None, count=5):
2023-11-01 14:02:29 +02:00
if base_port is None:
base_port = random.randint(1024, 65535 - count)
ports = [str(base_port + i) for i in range(count)]
logger.debug(f"Generated ports {ports}")
2023-11-01 14:02:29 +02:00
return ports
@staticmethod
def generate_random_ext_ip():
base_ip_fragments = ["172", "18"]
ext_ip = ".".join(base_ip_fragments + [str(random.randint(0, 255)) for _ in range(2)])
logger.debug(f"Generated random external IP {ext_ip}")
2023-11-01 14:02:29 +02:00
return ext_ip
def is_container_running(self, container):
try:
refreshed_container = self._client.containers.get(container.id)
return refreshed_container.status == "running"
except NotFound:
logger.error(f"Container with ID {container.id} not found")
2023-11-01 14:02:29 +02:00
return False
@property
def image(self):
return self._image