From 08080a569ade43edc2e5abab158bd14008a70fe5 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 28 May 2025 15:16:53 +0800 Subject: [PATCH 01/14] test: unauthorized_node_cannot_receive_dispersed_data --- .../test_data_confidentiality.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/data_confidentiality/test_data_confidentiality.py diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py new file mode 100644 index 0000000..6efc54f --- /dev/null +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -0,0 +1,48 @@ +import json + +import pytest + +from src.client.nomos_cli import NomosCli +from src.env_vars import CONSENSUS_SLOT_TIME, NOMOS +from src.libs.common import delay, to_app_id, to_index +from src.libs.custom_logger import get_custom_logger +from src.node.nomos_node import NomosNode +from src.steps.common import ensure_nodes_ready +from src.steps.da import StepsDataAvailability +from src.test_data import DATA_TO_DISPERSE + +logger = get_custom_logger(__name__) + + +class TestDataConfidentiality(StepsDataAvailability): + main_nodes = [] + + @pytest.mark.usefixtures("setup_2_node_cluster") + def test_unauthorized_node_cannot_receive_dispersed_data(self): + self.disperse_data(DATA_TO_DISPERSE[1], to_app_id(1), to_index(0)) + delay(CONSENSUS_SLOT_TIME) + rcv_data = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) + rcv_data_json = json.dumps(rcv_data) + + decoded_data = NomosCli(command="reconstruct").run(input_values=[rcv_data_json], decode_only=True) + + assert DATA_TO_DISPERSE[1] == decoded_data, "Retrieved data are not same with original data" + self.node2.stop() + # Start new node with the same IP address + self.nodeX = NomosNode(NOMOS, "nomos_node_0") + self.nodeX.start() + + try: + ensure_nodes_ready(self.nodeX) + except Exception as ex: + logger.error(f"REST service did not become ready in time: {ex}") + raise + + delay(CONSENSUS_SLOT_TIME) + + self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0)) + delay(CONSENSUS_SLOT_TIME) + try: + rcv_data = self.get_data_range(self.nodeX, to_app_id(2), to_index(0), to_index(5)) + except AssertionError as ae: + assert "Get data range response is empty" in str(ae), "Get data range response should be empty" From 5b2ee8188339db12678d06d4b1ca2568cac18c20 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 29 May 2025 10:23:06 +0800 Subject: [PATCH 02/14] fix: add log prefix for nomos node --- src/client/nomos_cli.py | 2 +- src/client/proxy_client.py | 2 +- src/node/nomos_node.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/nomos_cli.py b/src/client/nomos_cli.py index cf5bf94..fa15bf0 100644 --- a/src/client/nomos_cli.py +++ b/src/client/nomos_cli.py @@ -30,7 +30,7 @@ class NomosCli: self._volumes = nomos_cli[command]["volumes"] self._entrypoint = nomos_cli[command]["entrypoint"] - container_name = "nomos-cli-" + generate_log_prefix() + container_name = "nomos-cli_" + generate_log_prefix() self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}__{self._image_name.replace('/', '_')}.log") self._docker_manager = DockerManager(self._image_name) self._container_name = container_name diff --git a/src/client/proxy_client.py b/src/client/proxy_client.py index 8794819..f1999b2 100644 --- a/src/client/proxy_client.py +++ b/src/client/proxy_client.py @@ -25,7 +25,7 @@ class ProxyClient: self._volumes = http_proxy[command]["volumes"] self._entrypoint = http_proxy[command]["entrypoint"] - container_name = "proxy-client-" + generate_log_prefix() + container_name = "proxy-client_" + generate_log_prefix() self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}__{self._image_name.replace('/', '_')}.log") self._docker_manager = DockerManager(self._image_name) self._container_name = container_name diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 3a5519d..4080b76 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -1,6 +1,7 @@ import os from src.data_storage import DS +from src.libs.common import generate_log_prefix from src.libs.custom_logger import get_custom_logger from tenacity import retry, stop_after_delay, wait_fixed @@ -31,7 +32,8 @@ class NomosNode: self._entrypoint = nomos_nodes[node_type]["entrypoint"] self._node_type = node_type - self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}__{self._image_name.replace('/', '_')}.log") + log_prefix = generate_log_prefix() + self._log_path = os.path.join(DOCKER_LOG_DIR, f"{container_name}_{log_prefix}__{self._image_name.replace('/', '_')}.log") self._docker_manager = DockerManager(self._image_name) self._container_name = container_name self._container = None From ce3babf65f1be2fd569f16c6f9f1e3bd16228382 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 29 May 2025 16:49:33 +0800 Subject: [PATCH 03/14] test: customized node --- cluster_config/scripts/run_customized_node.sh | 13 ++++++++++ src/env_vars.py | 1 + src/node/node_vars.py | 6 +++++ src/node/nomos_node.py | 3 +++ .../test_data_confidentiality.py | 26 ++++++++++--------- 5 files changed, 37 insertions(+), 12 deletions(-) create mode 100755 cluster_config/scripts/run_customized_node.sh diff --git a/cluster_config/scripts/run_customized_node.sh b/cluster_config/scripts/run_customized_node.sh new file mode 100755 index 0000000..8440bff --- /dev/null +++ b/cluster_config/scripts/run_customized_node.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +export CFG_FILE_PATH="/etc/nomos/config.yaml" \ + CFG_SERVER_ADDR="http://cfgsync:4400" \ + CFG_HOST_IP=$(hostname -i) \ + CFG_HOST_IDENTIFIER="validator-$(hostname -i)" \ + LOG_LEVEL="INFO" \ + RISC0_DEV_MODE=true + +/usr/bin/cfgsync-client && \ + exec /usr/bin/nomos-node /etc/nomos/config.yaml diff --git a/src/env_vars.py b/src/env_vars.py index 0bcc5c8..8a2358e 100644 --- a/src/env_vars.py +++ b/src/env_vars.py @@ -21,6 +21,7 @@ NOMOS_IMAGE = get_env_var("NOMOS_IMAGE", DEFAULT_NOMOS_IMAGE) DEFAULT_PROXY_IMAGE = "bitnami/configurable-http-proxy:latest" HTTP_PROXY_IMAGE = get_env_var("HTTP_PROXY_IMAGE", DEFAULT_PROXY_IMAGE) +NOMOS_CUSTOM = "nomos_custom" NOMOS = "nomos" NOMOS_EXECUTOR = "nomos_executor" CFGSYNC = "cfgsync" diff --git a/src/node/node_vars.py b/src/node/node_vars.py index a381084..6b1f71f 100644 --- a/src/node/node_vars.py +++ b/src/node/node_vars.py @@ -1,6 +1,12 @@ from src.env_vars import NOMOS_IMAGE nomos_nodes = { + "nomos_custom": { + "image": NOMOS_IMAGE, + "volumes": ["cluster_config:/etc/nomos", "./kzgrs/kzgrs_test_params:/kzgrs_test_params:z"], + "ports": ["3000/udp", "18080/tcp"], + "entrypoint": "/etc/nomos/scripts/run_customized_node.sh", + }, "nomos": { "image": NOMOS_IMAGE, "volumes": ["cluster_config:/etc/nomos", "./kzgrs/kzgrs_test_params:/kzgrs_test_params:z"], diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 4080b76..d61fdfc 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -128,6 +128,9 @@ class NomosNode: def name(self): return self._container_name + def get_archive(self, path): + return self._container.get_archive(path) + def api_port(self): return self._tcp_port diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 6efc54f..9389ff9 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -3,7 +3,7 @@ import json import pytest from src.client.nomos_cli import NomosCli -from src.env_vars import CONSENSUS_SLOT_TIME, NOMOS +from src.env_vars import CONSENSUS_SLOT_TIME, NOMOS, NOMOS_CUSTOM from src.libs.common import delay, to_app_id, to_index from src.libs.custom_logger import get_custom_logger from src.node.nomos_node import NomosNode @@ -27,22 +27,24 @@ class TestDataConfidentiality(StepsDataAvailability): decoded_data = NomosCli(command="reconstruct").run(input_values=[rcv_data_json], decode_only=True) assert DATA_TO_DISPERSE[1] == decoded_data, "Retrieved data are not same with original data" + + # Copy the config file from first node + cfg_file = open("./cluster_config/config.yaml", "wb") + stream, _stat = self.node2.get_archive("/config.yaml") + for chunk in stream: + cfg_file.write(chunk) + cfg_file.close() + self.node2.stop() - # Start new node with the same IP address - self.nodeX = NomosNode(NOMOS, "nomos_node_0") + + # Start new node with the same hostname and configuration + self.nodeX = NomosNode(NOMOS_CUSTOM, "nomos_node_0") self.nodeX.start() try: - ensure_nodes_ready(self.nodeX) + self.nodeX.ensure_ready() except Exception as ex: logger.error(f"REST service did not become ready in time: {ex}") raise - delay(CONSENSUS_SLOT_TIME) - - self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0)) - delay(CONSENSUS_SLOT_TIME) - try: - rcv_data = self.get_data_range(self.nodeX, to_app_id(2), to_index(0), to_index(5)) - except AssertionError as ae: - assert "Get data range response is empty" in str(ae), "Get data range response should be empty" + delay(600) From 410eb43bdba2a1814978adff4a0b96dd270a5b2e Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 29 May 2025 17:12:57 +0800 Subject: [PATCH 04/14] test: read config file from container as tar buffer --- cluster_config/scripts/run_customized_node.sh | 3 +-- .../test_data_confidentiality.py | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cluster_config/scripts/run_customized_node.sh b/cluster_config/scripts/run_customized_node.sh index 8440bff..734cb20 100755 --- a/cluster_config/scripts/run_customized_node.sh +++ b/cluster_config/scripts/run_customized_node.sh @@ -9,5 +9,4 @@ export CFG_FILE_PATH="/etc/nomos/config.yaml" \ LOG_LEVEL="INFO" \ RISC0_DEV_MODE=true -/usr/bin/cfgsync-client && \ - exec /usr/bin/nomos-node /etc/nomos/config.yaml +exec /usr/bin/nomos-node /etc/nomos/config.yaml diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 9389ff9..1562d05 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -1,4 +1,6 @@ +import io import json +import tarfile import pytest @@ -29,11 +31,18 @@ class TestDataConfidentiality(StepsDataAvailability): assert DATA_TO_DISPERSE[1] == decoded_data, "Retrieved data are not same with original data" # Copy the config file from first node - cfg_file = open("./cluster_config/config.yaml", "wb") stream, _stat = self.node2.get_archive("/config.yaml") - for chunk in stream: - cfg_file.write(chunk) - cfg_file.close() + + # Join stream into bytes and load into a memory buffer + tar_bytes = io.BytesIO(b"".join(stream)) + + # Extract and write only the actual text file + with tarfile.open(fileobj=tar_bytes) as tar: + member = tar.getmembers()[0] + file_obj = tar.extractfile(member) + if file_obj: + with open("./cluster_config/config.yaml", "wb") as f: + f.write(file_obj.read()) self.node2.stop() @@ -48,3 +57,12 @@ class TestDataConfidentiality(StepsDataAvailability): raise delay(600) + + # delay(CONSENSUS_SLOT_TIME) + # + # self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0)) + # delay(CONSENSUS_SLOT_TIME) + # try: + # rcv_data = self.get_data_range(self.nodeX, to_app_id(2), to_index(0), to_index(5)) + # except AssertionError as ae: + # assert "Get data range response is empty" in str(ae), "Get data range response should be empty" From f015334c4f584e9a8b85d24095cf2603565a6e47 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 29 May 2025 17:20:07 +0800 Subject: [PATCH 05/14] test: new node cannot receive dispersed data --- .../test_data_confidentiality.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 1562d05..1febc6c 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -46,7 +46,7 @@ class TestDataConfidentiality(StepsDataAvailability): self.node2.stop() - # Start new node with the same hostname and configuration + # Start new node with the same hostname and configuration as first node self.nodeX = NomosNode(NOMOS_CUSTOM, "nomos_node_0") self.nodeX.start() @@ -56,13 +56,10 @@ class TestDataConfidentiality(StepsDataAvailability): logger.error(f"REST service did not become ready in time: {ex}") raise - delay(600) - - # delay(CONSENSUS_SLOT_TIME) - # - # self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0)) - # delay(CONSENSUS_SLOT_TIME) - # try: - # rcv_data = self.get_data_range(self.nodeX, to_app_id(2), to_index(0), to_index(5)) - # except AssertionError as ae: - # assert "Get data range response is empty" in str(ae), "Get data range response should be empty" + # Confirm new node haven't received any dispersed data + self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0)) + delay(CONSENSUS_SLOT_TIME) + try: + _rcv_data = self.get_data_range(self.nodeX, to_app_id(2), to_index(0), to_index(5)) + except AssertionError as ae: + assert "Get data range response is empty" in str(ae), "Get data range response should be empty" From 7e7177fb8feb8a44eff6462d24c6d1a1f30edcd7 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 2 Jun 2025 12:36:25 +0800 Subject: [PATCH 06/14] fix: remove unnecessary imports --- tests/data_confidentiality/test_data_confidentiality.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 1febc6c..500ef71 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -5,11 +5,10 @@ import tarfile import pytest from src.client.nomos_cli import NomosCli -from src.env_vars import CONSENSUS_SLOT_TIME, NOMOS, NOMOS_CUSTOM +from src.env_vars import CONSENSUS_SLOT_TIME, NOMOS_CUSTOM from src.libs.common import delay, to_app_id, to_index from src.libs.custom_logger import get_custom_logger from src.node.nomos_node import NomosNode -from src.steps.common import ensure_nodes_ready from src.steps.da import StepsDataAvailability from src.test_data import DATA_TO_DISPERSE From 51b85276552399b58ed130e1fa6d15c265c9c0ba Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 2 Jun 2025 14:05:01 +0800 Subject: [PATCH 07/14] test: modify key value for nomos_node_0 --- requirements.txt | 2 +- .../test_data_confidentiality.py | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1621ad8..bb64b26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ setuptools==70.0.0 tenacity==8.2.3 typeguard==4.1.5 typing-inspect==0.9.0 -typing_extensions==4.9.0 +typing_extensions>=4.10 urllib3==2.2.2 virtualenv==20.25.0 Jinja2~=3.1.5 diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 500ef71..316b1ab 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -3,6 +3,7 @@ import json import tarfile import pytest +from ruamel.yaml import YAML from src.client.nomos_cli import NomosCli from src.env_vars import CONSENSUS_SLOT_TIME, NOMOS_CUSTOM @@ -15,6 +16,32 @@ from src.test_data import DATA_TO_DISPERSE logger = get_custom_logger(__name__) +def modify_key_value(file_path, yaml_key_path): + yaml = YAML() + yaml.preserve_quotes = True + + with open(file_path, "r") as f: + data = yaml.load(f) + + keys = yaml_key_path.split(".") + ref = data + for key in keys[:-1]: + if key not in ref: + raise KeyError(f"Key '{key}' not found in path '{'.'.join(keys)}'") + ref = ref[key] + + final_key = keys[-1] + if final_key not in ref: + raise KeyError(f"Key '{final_key}' not found in path '{'.'.join(keys)}'") + + old_value = ref[final_key] + # Swap last two characters + ref[final_key] = old_value[:-2] + old_value[-1] + old_value[-2] + + with open(file_path, "w") as f: + yaml.dump(data, f) + + class TestDataConfidentiality(StepsDataAvailability): main_nodes = [] @@ -45,6 +72,11 @@ class TestDataConfidentiality(StepsDataAvailability): self.node2.stop() + # Change the private key of the nomos_node_0 -> change its PeerId + modify_key_value("./cluster_config/config.yaml", "network.backend.node_key") + modify_key_value("./cluster_config/config.yaml", "blend.backend.node_key") + modify_key_value("./cluster_config/config.yaml", "da_network.backend.node_key") + # Start new node with the same hostname and configuration as first node self.nodeX = NomosNode(NOMOS_CUSTOM, "nomos_node_0") self.nodeX.start() From 78615798c0ac3bc29999283e1da4819e474c4613 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 2 Jun 2025 14:17:45 +0800 Subject: [PATCH 08/14] fix: comment with better explanation --- tests/data_confidentiality/test_data_confidentiality.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 316b1ab..fe57153 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -72,10 +72,9 @@ class TestDataConfidentiality(StepsDataAvailability): self.node2.stop() - # Change the private key of the nomos_node_0 -> change its PeerId - modify_key_value("./cluster_config/config.yaml", "network.backend.node_key") - modify_key_value("./cluster_config/config.yaml", "blend.backend.node_key") - modify_key_value("./cluster_config/config.yaml", "da_network.backend.node_key") + # Change the private key -> PeerId of the nomos_node_0. This would create a stranger to existing membership list. + for yaml_key_path in ["network.backend.node_key", "blend.backend.node_key", "da_network.backend.node_key"]: + modify_key_value("./cluster_config/config.yaml", yaml_key_path) # Start new node with the same hostname and configuration as first node self.nodeX = NomosNode(NOMOS_CUSTOM, "nomos_node_0") @@ -87,7 +86,7 @@ class TestDataConfidentiality(StepsDataAvailability): logger.error(f"REST service did not become ready in time: {ex}") raise - # Confirm new node haven't received any dispersed data + # Confirm new node haven't received any dispersed data as it is not on membership list. self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0)) delay(CONSENSUS_SLOT_TIME) try: From 0e388734ea96636134ef0f664fa26739d4539ce5 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 2 Jun 2025 14:34:44 +0800 Subject: [PATCH 09/14] fix: add ruamel.yaml to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index bb64b26..3834521 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ python-dotenv==1.0.1 pytest-dependency==0.6.0 PyYAML==6.0.1 requests==2.31.0 +ruamel.yaml==0.17 setuptools==70.0.0 tenacity==8.2.3 typeguard==4.1.5 From 8996b08ba056f861e4e6a155bcc26500577e9ecd Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 2 Jun 2025 14:41:33 +0800 Subject: [PATCH 10/14] fix: adjust ruamel.yaml version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3834521..1b260ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ python-dotenv==1.0.1 pytest-dependency==0.6.0 PyYAML==6.0.1 requests==2.31.0 -ruamel.yaml==0.17 +ruamel.yaml==0.17.21 setuptools==70.0.0 tenacity==8.2.3 typeguard==4.1.5 From 0c8a153c558b467bc32d34436a864abb0cdbd19b Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 3 Jun 2025 12:50:55 +0800 Subject: [PATCH 11/14] fix: add cluster variant selector --- src/steps/common.py | 4 ++++ tests/data_confidentiality/test_data_confidentiality.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/steps/common.py b/src/steps/common.py index d7ba494..2af620e 100644 --- a/src/steps/common.py +++ b/src/steps/common.py @@ -118,3 +118,7 @@ class StepsCommon: default_target = [f"http://{self.main_nodes[1 + i % 2].name()}:18080"] proxy_client.run(input_values=default_target) self.client_nodes.append(proxy_client) + + @pytest.fixture(params=["setup_2_node_cluster", "setup_4_node_cluster"]) + def setup_cluster_variant(self, request): + return request.getfixturevalue(request.param) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index fe57153..f3453ca 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -45,7 +45,7 @@ def modify_key_value(file_path, yaml_key_path): class TestDataConfidentiality(StepsDataAvailability): main_nodes = [] - @pytest.mark.usefixtures("setup_2_node_cluster") + @pytest.mark.usefixtures("setup_cluster_variant") def test_unauthorized_node_cannot_receive_dispersed_data(self): self.disperse_data(DATA_TO_DISPERSE[1], to_app_id(1), to_index(0)) delay(CONSENSUS_SLOT_TIME) From c90974118e6ed0865863978e4a72c9e0e2532da4 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 3 Jun 2025 13:23:27 +0800 Subject: [PATCH 12/14] fix: make modify_key_value more efficient --- .../test_data_confidentiality.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index f3453ca..05d5726 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -16,27 +16,28 @@ from src.test_data import DATA_TO_DISPERSE logger = get_custom_logger(__name__) -def modify_key_value(file_path, yaml_key_path): +def modify_key_value(file_path, yaml_key_paths): yaml = YAML() yaml.preserve_quotes = True with open(file_path, "r") as f: data = yaml.load(f) - keys = yaml_key_path.split(".") - ref = data - for key in keys[:-1]: - if key not in ref: - raise KeyError(f"Key '{key}' not found in path '{'.'.join(keys)}'") - ref = ref[key] + for yaml_key_path in yaml_key_paths: + keys = yaml_key_path.split(".") + ref = data + for key in keys[:-1]: + if key not in ref: + raise KeyError(f"Key '{key}' not found in path '{'.'.join(keys)}'") + ref = ref[key] - final_key = keys[-1] - if final_key not in ref: - raise KeyError(f"Key '{final_key}' not found in path '{'.'.join(keys)}'") + final_key = keys[-1] + if final_key not in ref: + raise KeyError(f"Key '{final_key}' not found in path '{'.'.join(keys)}'") - old_value = ref[final_key] - # Swap last two characters - ref[final_key] = old_value[:-2] + old_value[-1] + old_value[-2] + old_value = ref[final_key] + # Swap last two characters + ref[final_key] = old_value[:-2] + old_value[-1] + old_value[-2] with open(file_path, "w") as f: yaml.dump(data, f) @@ -73,8 +74,8 @@ class TestDataConfidentiality(StepsDataAvailability): self.node2.stop() # Change the private key -> PeerId of the nomos_node_0. This would create a stranger to existing membership list. - for yaml_key_path in ["network.backend.node_key", "blend.backend.node_key", "da_network.backend.node_key"]: - modify_key_value("./cluster_config/config.yaml", yaml_key_path) + yaml_key_paths = ["network.backend.node_key", "blend.backend.node_key", "da_network.backend.node_key"] + modify_key_value("./cluster_config/config.yaml", yaml_key_paths) # Start new node with the same hostname and configuration as first node self.nodeX = NomosNode(NOMOS_CUSTOM, "nomos_node_0") From fce8af0debc06d2a81d1a8ad0809754a74a769cc Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 4 Jun 2025 09:30:00 +0800 Subject: [PATCH 13/14] fix: add reusable extract_config for node --- src/node/nomos_node.py | 17 +++++++++++++++++ .../test_data_confidentiality.py | 15 +-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index d61fdfc..26fa2a2 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -1,4 +1,6 @@ +import io import os +import tarfile from src.data_storage import DS from src.libs.common import generate_log_prefix @@ -157,6 +159,21 @@ class NomosNode: else: logger.debug("No keyword matches found in the logs.") + def extract_config(self, target_file): + # Copy the config file from first node + stream, _stat = self.get_archive("/config.yaml") + + # Join stream into bytes and load into a memory buffer + tar_bytes = io.BytesIO(b"".join(stream)) + + # Extract and write only the actual config file + with tarfile.open(fileobj=tar_bytes) as tar: + member = tar.getmembers()[0] + file_obj = tar.extractfile(member) + if file_obj: + with open(f"{target_file}", "wb") as f: + f.write(file_obj.read()) + def send_dispersal_request(self, data): return self._api.da_disperse_data(data) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index 05d5726..e7149e1 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -57,20 +57,7 @@ class TestDataConfidentiality(StepsDataAvailability): assert DATA_TO_DISPERSE[1] == decoded_data, "Retrieved data are not same with original data" - # Copy the config file from first node - stream, _stat = self.node2.get_archive("/config.yaml") - - # Join stream into bytes and load into a memory buffer - tar_bytes = io.BytesIO(b"".join(stream)) - - # Extract and write only the actual text file - with tarfile.open(fileobj=tar_bytes) as tar: - member = tar.getmembers()[0] - file_obj = tar.extractfile(member) - if file_obj: - with open("./cluster_config/config.yaml", "wb") as f: - f.write(file_obj.read()) - + self.node2.extract_config("./cluster_config/config.yaml") self.node2.stop() # Change the private key -> PeerId of the nomos_node_0. This would create a stranger to existing membership list. From 26383148770b1db47831e370666450503500d988 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 4 Jun 2025 09:39:01 +0800 Subject: [PATCH 14/14] fix: refactor ensure_ready --- src/node/nomos_node.py | 6 +++++- src/steps/common.py | 14 ++------------ .../test_data_confidentiality.py | 7 +------ 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 26fa2a2..a65f0e4 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -116,7 +116,11 @@ class NomosNode: logger.info("REST service is ready !!") if self.is_nomos(): - check_ready() + try: + check_ready() + except Exception as ex: + logger.error(f"REST service did not become ready in time: {ex}") + raise def is_nomos(self): return "nomos" in self._container_name diff --git a/src/steps/common.py b/src/steps/common.py index 2af620e..89671eb 100644 --- a/src/steps/common.py +++ b/src/steps/common.py @@ -65,12 +65,7 @@ class StepsCommon: self.node3 = NomosNode(NOMOS_EXECUTOR, "nomos_node_1") self.main_nodes.extend([self.node1, self.node2, self.node3]) start_nodes(self.main_nodes) - - try: - ensure_nodes_ready(self.main_nodes[1:]) - except Exception as ex: - logger.error(f"REST service did not become ready in time: {ex}") - raise + ensure_nodes_ready(self.main_nodes[1:]) delay(CONSENSUS_SLOT_TIME) @@ -90,12 +85,7 @@ class StepsCommon: self.node5 = NomosNode(NOMOS_EXECUTOR, "nomos_node_3") self.main_nodes.extend([self.node1, self.node2, self.node3, self.node4, self.node5]) start_nodes(self.main_nodes) - - try: - ensure_nodes_ready(self.main_nodes[1:]) - except Exception as ex: - logger.error(f"REST service did not become ready in time: {ex}") - raise + ensure_nodes_ready(self.main_nodes[1:]) delay(CONSENSUS_SLOT_TIME) diff --git a/tests/data_confidentiality/test_data_confidentiality.py b/tests/data_confidentiality/test_data_confidentiality.py index e7149e1..9a2cc29 100644 --- a/tests/data_confidentiality/test_data_confidentiality.py +++ b/tests/data_confidentiality/test_data_confidentiality.py @@ -67,12 +67,7 @@ class TestDataConfidentiality(StepsDataAvailability): # Start new node with the same hostname and configuration as first node self.nodeX = NomosNode(NOMOS_CUSTOM, "nomos_node_0") self.nodeX.start() - - try: - self.nodeX.ensure_ready() - except Exception as ex: - logger.error(f"REST service did not become ready in time: {ex}") - raise + self.nodeX.ensure_ready() # Confirm new node haven't received any dispersed data as it is not on membership list. self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(2), to_index(0))