log fix

src/node/docker_mananger.py

@@ -1,12 +1,13 @@
 import os
 import re
+import time
 from src.libs.custom_logger import get_custom_logger
 import random
 import threading
 import docker
 from src.env_vars import NETWORK_NAME, SUBNET, IP_RANGE, GATEWAY
 from docker.types import IPAMConfig, IPAMPool
-from docker.errors import NotFound
+from docker.errors import NotFound, APIError
 
 logger = get_custom_logger(__name__)
 
@@ -63,9 +64,28 @@ class DockerManager:
 
     def _log_container_output(self, container, log_path):
         os.makedirs(os.path.dirname(log_path), exist_ok=True)
-        with open(log_path, "wb+") as log_file:
+        try:
-            for chunk in container.logs(stream=True):
+            with open(log_path, "wb+") as log_file:
-                log_file.write(chunk)
+                start_time = time.time()
+                while True:
+                    if container.status in ["exited", "dead"]:
+                        logger.info(f"Container {container.short_id} has stopped. Exiting log stream.")
+                        return
+                    try:
+                        for chunk in container.logs(stream=True):
+                            if chunk:
+                                log_file.write(chunk)
+                                log_file.flush()
+                                start_time = time.time()
+                            else:
+                                if time.time() - start_time > 10:
+                                    logger.error(f"Log stream timeout for container {container.short_id}")
+                                    return
+                    except (APIError, IOError) as e:
+                        logger.error(f"Error while reading logs from container {container.short_id}: {e}")
+                        time.sleep(1)
+        except Exception as e:
+            logger.error(f"Failed to set up logging for container {container.short_id}: {e}")
 
     def generate_ports(self, base_port=None, count=5):
         if base_port is None: