chore: check logs at teardown (#67)

* check logs at teardown

* fix

* fix

src/node/docker_mananger.py

@@ -1,4 +1,5 @@
 import os
+import re
 from src.libs.custom_logger import get_custom_logger
 import random
 import threading
@@ -91,3 +92,27 @@ class DockerManager:
     @property
     def image(self):
         return self._image
+
+    def search_log_for_keywords(self, log_path, keywords, use_regex=False):
+        matches = {keyword: [] for keyword in keywords}
+
+        # Open the log file and search line by line
+        with open(log_path, "r") as log_file:
+            for line in log_file:
+                for keyword in keywords:
+                    if use_regex:
+                        if re.search(keyword, line, re.IGNORECASE):
+                            matches[keyword].append(line.strip())
+                    else:
+                        if keyword.lower() in line.lower():
+                            matches[keyword].append(line.strip())
+
+        # Check if there were any matches
+        if any(matches[keyword] for keyword in keywords):
+            for keyword, lines in matches.items():
+                if lines:
+                    logger.debug(f"Found matches for keyword '{keyword}': {lines}")
+            return matches
+        else:
+            logger.debug("No errors found in the waku logs.")
+            return None

src/node/waku_node.py

@@ -2,6 +2,7 @@ import errno
 import json
 import os
 import random
+import re
 import shutil
 import string
 import pytest
@@ -13,7 +14,7 @@ from src.node.api_clients.rest import REST
 from src.node.docker_mananger import DockerManager
 from src.env_vars import DOCKER_LOG_DIR
 from src.data_storage import DS
-from src.test_data import DEFAULT_CLUSTER_ID
+from src.test_data import DEFAULT_CLUSTER_ID, LOG_ERROR_KEYWORDS
 
 logger = get_custom_logger(__name__)
 
@@ -517,3 +518,16 @@ class WakuNode:
         # Generate a random 64-character string from the hex characters
         random_key = "".join(random.choice(hex_chars) for _ in range(64))
         return random_key
+
+    def search_waku_log_for_string(self, search_pattern, use_regex=False):
+        return self._docker_manager.search_log_for_keywords(self._log_path, [search_pattern], use_regex)
+
+    def check_waku_log_errors(self, whitelist=None):
+        keywords = LOG_ERROR_KEYWORDS
+
+        # If a whitelist is provided, remove those keywords from the keywords list
+        if whitelist:
+            keywords = [keyword for keyword in keywords if keyword not in whitelist]
+
+        matches = self._docker_manager.search_log_for_keywords(self._log_path, keywords, False)
+        assert not matches, f"Found errors {matches}"

src/test_data.py

@@ -169,3 +169,29 @@ SAMPLE_TIMESTAMPS = [
 ]
 
 PUBSUB_TOPICS_RLN = ["/waku/2/rs/1/0"]
+
+LOG_ERROR_KEYWORDS = [
+    "crash",
+    "fatal",
+    "panic",
+    "abort",
+    "segfault",
+    "corrupt",
+    "unreachable",
+    "terminated",
+    "oom",
+    "unhandled",
+    "stacktrace",
+    "deadlock",
+    "SIGSEGV",
+    "SIGABRT",
+    "stack overflow",
+    "index out of bounds",
+    "nil pointer dereference",
+    "goroutine exit",
+    "nil pointer",
+    "runtime error",
+    "goexit",
+    "race condition",
+    "double free",
+]

tests/conftest.py

@@ -90,3 +90,11 @@ def close_open_nodes(attach_logs_on_fail):
                 crashed_containers.append(node.image)
             logger.error(f"Failed to stop container because of error {ex}")
     assert not crashed_containers, f"Containers {crashed_containers} crashed during the test!!!"
+
+
+@pytest.fixture(scope="function", autouse=True)
+def check_waku_log_errors():
+    yield
+    logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
+    for node in DS.waku_nodes:
+        node.check_waku_log_errors()