diff --git a/src/data_storage.py b/src/data_storage.py new file mode 100644 index 0000000..c6215e2 --- /dev/null +++ b/src/data_storage.py @@ -0,0 +1,3 @@ +# We use this class for global variables +class DS: + nomos_nodes = [] diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 17ff784..cd3f079 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -1,5 +1,6 @@ import os +from src.data_storage import DS from src.libs.custom_logger import get_custom_logger from tenacity import retry, stop_after_delay, wait_fixed @@ -7,6 +8,7 @@ from src.node.api_clients.rest import REST from src.node.docker_mananger import DockerManager from src.env_vars import DOCKER_LOG_DIR from src.node.node_vars import nomos_nodes +from src.test_data import LOG_ERROR_KEYWORDS logger = get_custom_logger(__name__) @@ -76,9 +78,10 @@ class NomosNode: name=self._container_name, ) - logger.debug(f"Container returned {self._container}") logger.debug(f"Started container from image {self._image_name}. " f"REST: {getattr(self, '_tcp_port', 'N/A')}") + DS.nomos_nodes.append(self) + @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) def stop(self): if self._container: @@ -132,3 +135,13 @@ class NomosNode: def info(self): return self._api.info() + + def check_nomos_log_errors(self, whitelist=None): + keywords = LOG_ERROR_KEYWORDS + + # If a whitelist is provided, remove those keywords from the keywords list + if whitelist: + keywords = [keyword for keyword in keywords if keyword not in whitelist] + + matches = self._docker_manager.search_log_for_keywords(self._log_path, keywords, False) + assert not matches, f"Found errors {matches}" diff --git a/src/test_data.py b/src/test_data.py new file mode 100644 index 0000000..e4d7665 --- /dev/null +++ b/src/test_data.py @@ -0,0 +1,26 @@ +from time import time +from datetime import datetime, timedelta + +LOG_ERROR_KEYWORDS = [ + "crash", + "fatal", + "panic", + "abort", + "segfault", + "corrupt", + "terminated", + "unhandled", + "stacktrace", + "deadlock", + "SIGSEGV", + "SIGABRT", + "stack overflow", + "index out of bounds", + "nil pointer dereference", + "goroutine exit", + "nil pointer", + "runtime error", + "goexit", + "race condition", + "double free", +] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..66d83d4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +import inspect +import glob +from src.libs.custom_logger import get_custom_logger +import os +import pytest +from datetime import datetime +from time import time +from uuid import uuid4 +from src.libs.common import attach_allure_file +import src.env_vars as env_vars +from src.data_storage import DS + +logger = get_custom_logger(__name__) + + +# See https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_makereport(item): + outcome = yield + rep = outcome.get_result() + if rep.when == "call": + setattr(item, "rep_call", rep) + return rep + return None + + +@pytest.fixture(scope="session", autouse=True) +def set_allure_env_variables(): + yield + if os.path.isdir("allure-results") and not os.path.isfile(os.path.join("allure-results", "environment.properties")): + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + with open(os.path.join("allure-results", "environment.properties"), "w") as outfile: + for attribute_name in dir(env_vars): + if attribute_name.isupper(): + attribute_value = getattr(env_vars, attribute_name) + outfile.write(f"{attribute_name}={attribute_value}\n") + + +@pytest.fixture(scope="function", autouse=True) +def test_id(request): + # setting up an unique test id to be used where needed + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + request.cls.test_id = f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}__{str(uuid4())}" + + +@pytest.fixture(scope="function", autouse=True) +def test_setup(request, test_id): + logger.debug(f"Running test: {request.node.name} with id: {request.cls.test_id}") + yield + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + for file in glob.glob(os.path.join(env_vars.DOCKER_LOG_DIR, "*")): + if os.path.getmtime(file) < time() - 3600: + logger.debug(f"Deleting old log file: {file}") + try: + os.remove(file) + except: + logger.error("Could not delete file") + + +@pytest.fixture(scope="function", autouse=True) +def attach_logs_on_fail(request): + yield + if env_vars.RUNNING_IN_CI and hasattr(request.node, "rep_call") and request.node.rep_call.failed: + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + logger.debug("Test failed, attempting to attach logs to the allure reports") + for file in glob.glob(os.path.join(env_vars.DOCKER_LOG_DIR, "*" + request.cls.test_id + "*")): + attach_allure_file(file) + + +@pytest.fixture(scope="function", autouse=True) +def close_open_nodes(attach_logs_on_fail): + DS.nomos_nodes = [] + yield + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + crashed_containers = [] + for node in DS.nomos_nodes: + try: + node.stop() + except Exception as ex: + if "No such container" in str(ex): + crashed_containers.append(node.image) + logger.error(f"Failed to stop container because of error {ex}") + assert not crashed_containers, f"Containers {crashed_containers} crashed during the test!!!" + + +@pytest.fixture(scope="function", autouse=True) +def check_nomos_log_errors(): + yield + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + for node in DS.nomos_nodes: + node.check_nomos_log_errors() diff --git a/tests/e2e/test_2node_alive.py b/tests/e2e/test_2node_alive.py index c8e2d01..9553332 100644 --- a/tests/e2e/test_2node_alive.py +++ b/tests/e2e/test_2node_alive.py @@ -22,7 +22,3 @@ class Test2NodeClAlive: except Exception as ex: logger.error(f"REST service did not become ready in time: {ex}") raise - - self.node1.stop() - self.node2.stop() - self.node3.stop()