From f8061cf5846bdc5063b03552b30d4e7e1c85ea6f Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 7 Apr 2025 10:18:33 +0800 Subject: [PATCH 01/19] fix: update cfgsync template --- cluster_config/cfgsync-template.yaml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/cluster_config/cfgsync-template.yaml b/cluster_config/cfgsync-template.yaml index 7097f80..134eb09 100644 --- a/cluster_config/cfgsync-template.yaml +++ b/cluster_config/cfgsync-template.yaml @@ -11,26 +11,22 @@ subnetwork_size: {{ subnet_size }} dispersal_factor: {{ dispersal_factor }} num_samples: 1 num_subnets: {{ subnet_size }} -old_blobs_check_interval_secs: 5 -blobs_validity_duration_secs: 60 +old_blobs_check_interval_secs: "5.0" +blobs_validity_duration_secs: "60.0" global_params_path: "/kzgrs_test_params" min_dispersal_peers: 1 min_replication_peers: 1 -monitor_failure_time_window_secs: 5 -balancer_interval_secs: 5 +monitor_failure_time_window: "5.0" +balancer_interval: "5.0" # Dispersal mempool publish strategy mempool_publish_strategy: !SampleSubnetworks sample_threshold: {{ subnet_size }} - timeout: - secs: 2 - nanos: 0 - cooldown: - secs: 0 - nanos: 100000000 + timeout: "2.0" + cooldown: "0.0001" replication_settings: seen_message_cache_size: 204800 - seen_message_ttl_secs: "900.0" + seen_message_ttl: "900.0" # Tracing tracing_settings: From 19d4b04a5594c94868b9cde2f942380cb318f43a Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 7 Apr 2025 03:01:59 +0000 Subject: [PATCH 02/19] fix: remove secs from cfgsync template --- cluster_config/cfgsync-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster_config/cfgsync-template.yaml b/cluster_config/cfgsync-template.yaml index 134eb09..5f5ec63 100644 --- a/cluster_config/cfgsync-template.yaml +++ b/cluster_config/cfgsync-template.yaml @@ -11,8 +11,8 @@ subnetwork_size: {{ subnet_size }} dispersal_factor: {{ dispersal_factor }} num_samples: 1 num_subnets: {{ subnet_size }} -old_blobs_check_interval_secs: "5.0" -blobs_validity_duration_secs: "60.0" +old_blobs_check_interval: "5.0" +blobs_validity_duration: "60.0" global_params_path: "/kzgrs_test_params" min_dispersal_peers: 1 min_replication_peers: 1 From 6af92fa4493a6aa3c6db4cbd20d792bb3011a798 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 8 Apr 2025 14:05:33 +0800 Subject: [PATCH 03/19] fix: add get_shares_commitments to da --- src/libs/common.py | 6 ++++++ src/node/nomos_node.py | 3 +++ src/steps/da.py | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/src/libs/common.py b/src/libs/common.py index b089576..678f9f5 100644 --- a/src/libs/common.py +++ b/src/libs/common.py @@ -43,6 +43,12 @@ def to_app_id(n: int) -> list: return list(n.to_bytes(32, byteorder="big")) +def to_blob_id(n: int) -> list: + if n < 0: + raise ValueError("Input must be an unsigned integer (non-negative)") + return list(n.to_bytes(32, byteorder="big")) + + def random_divide_k(n, k): if n < k: raise ValueError(f"n={n} must be at least k={k} to split into {k} parts") diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index ce894b1..106ae2c 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -157,3 +157,6 @@ class NomosNode: def send_get_data_range_request(self, data): return self._api.da_get_range(data) + + def send_get_commitments_request(self, data): + return self._api.da_get_commitments(data) diff --git a/src/steps/da.py b/src/steps/da.py index 95509b6..fbf457a 100644 --- a/src/steps/da.py +++ b/src/steps/da.py @@ -28,6 +28,11 @@ def prepare_get_range_request(app_id, start_index, end_index): return query_data +def prepare_get_shares_commitments_request(blob_id): + query_data = {"blob_id": blob_id} + return query_data + + def response_contains_data(response): if response is None: return False @@ -101,3 +106,23 @@ class StepsDataAvailability(StepsCommon): return response return get_range() + + @allure.step + def get_shares_commitments(self, node, blob_id, **kwargs): + + timeout_duration = kwargs.get("timeout_duration", 65) + interval = kwargs.get("interval", 0.1) + + query = prepare_get_shares_commitments_request(blob_id) + + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(interval), reraise=True) + def get_commitments(): + try: + response = node.send_get_commitments_request(query) + except Exception as ex: + logger.error(f"Exception while retrieving commitments: {ex}") + raise + + return response + + return get_commitments() From 1fe6354ac60f8fd651b2af2f7cc479e0e1dbf76a Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 8 Apr 2025 14:17:18 +0800 Subject: [PATCH 04/19] fix: add get_storage_block to storage --- src/node/nomos_node.py | 3 +++ src/steps/storage.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/steps/storage.py diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 106ae2c..3e7df22 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -160,3 +160,6 @@ class NomosNode: def send_get_commitments_request(self, data): return self._api.da_get_commitments(data) + + def send_get_storage_block_request(self, data): + return self._api.storage_block(data) diff --git a/src/steps/storage.py b/src/steps/storage.py new file mode 100644 index 0000000..495ea0c --- /dev/null +++ b/src/steps/storage.py @@ -0,0 +1,34 @@ +import allure +from tenacity import retry, stop_after_delay, wait_fixed + +from src.libs.custom_logger import get_custom_logger +from src.steps.common import StepsCommon + +logger = get_custom_logger(__name__) + + +def prepare_get_storage_block_request(header_id): + query_data = {"header_id": header_id} + return query_data + + +class StepsStorage(StepsCommon): + @allure.step + def get_storage_block(self, node, header_id, **kwargs): + + timeout_duration = kwargs.get("timeout_duration", 65) + interval = kwargs.get("interval", 0.1) + + query = prepare_get_storage_block_request(header_id) + + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(interval), reraise=True) + def get_block(): + try: + response = node.send_get_storage_block_request(query) + except Exception as ex: + logger.error(f"Exception while retrieving commitments: {ex}") + raise + + return response + + return get_block() From a584d5e218e8f84e049efa90131a970c817f3ad6 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 8 Apr 2025 14:18:55 +0800 Subject: [PATCH 05/19] fix: log message --- src/steps/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/steps/storage.py b/src/steps/storage.py index 495ea0c..879595e 100644 --- a/src/steps/storage.py +++ b/src/steps/storage.py @@ -26,7 +26,7 @@ class StepsStorage(StepsCommon): try: response = node.send_get_storage_block_request(query) except Exception as ex: - logger.error(f"Exception while retrieving commitments: {ex}") + logger.error(f"Exception while retrieving storage block: {ex}") raise return response From c90328a5c3b2f361beaaf2e9f726b3ec8cb90405 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 9 Apr 2025 12:59:26 +0800 Subject: [PATCH 06/19] fix: add to_header_id conversion --- src/libs/common.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/libs/common.py b/src/libs/common.py index 678f9f5..e42db8a 100644 --- a/src/libs/common.py +++ b/src/libs/common.py @@ -32,21 +32,28 @@ def generate_log_prefix(): def to_index(n: int) -> list: - if n < 0: - raise ValueError("Input must be an unsigned integer (non-negative)") - return list(n.to_bytes(8, byteorder="big")) + return to_byte_list(n, 8) def to_app_id(n: int) -> list: - if n < 0: - raise ValueError("Input must be an unsigned integer (non-negative)") - return list(n.to_bytes(32, byteorder="big")) + return to_byte_list(n, 32) def to_blob_id(n: int) -> list: + return to_byte_list(n, 32) + + +def to_header_id(n: int) -> list: + return to_byte_list(n, 32) + + +def to_byte_list(n: int, l: int) -> list: if n < 0: raise ValueError("Input must be an unsigned integer (non-negative)") - return list(n.to_bytes(32, byteorder="big")) + if l < 1: + raise ValueError("Length must be an unsigned integer greater than 0") + + return list(n.to_bytes(l, byteorder="big")) def random_divide_k(n, k): From 14fccadc5471848ef6c4ebcb2259d9c6ea654d4f Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 9 Apr 2025 13:23:42 +0800 Subject: [PATCH 07/19] fix: add get_cryptarchia_headers to consensus --- src/steps/consensus.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/steps/consensus.py diff --git a/src/steps/consensus.py b/src/steps/consensus.py new file mode 100644 index 0000000..ab89364 --- /dev/null +++ b/src/steps/consensus.py @@ -0,0 +1,27 @@ +import allure +from tenacity import retry, stop_after_delay, wait_fixed + +from src.libs.custom_logger import get_custom_logger +from src.steps.common import StepsCommon + +logger = get_custom_logger(__name__) + + +class StepsConsensus(StepsCommon): + @allure.step + def get_cryptarchia_headers(self, node, from_header_id=None, to_header_id=None, **kwargs): + + timeout_duration = kwargs.get("timeout_duration", 65) + interval = kwargs.get("interval", 0.1) + + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(interval), reraise=True) + def get_headers(): + try: + response = node.send_get_cryptarchia_headers_request(from_header_id, to_header_id) + except Exception as ex: + logger.error(f"Exception while retrieving cryptarchia headers: {ex}") + raise + + return response + + return get_headers() From 8127ca86b5f0682d67165143b5562fb6a390ffa8 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 9 Apr 2025 14:48:06 +0800 Subject: [PATCH 08/19] fix: cryptarchia headers query - log body size only for size > 0 --- src/api_clients/base_client.py | 3 ++- src/api_clients/rest.py | 5 +++-- src/libs/common.py | 7 +++++-- src/node/nomos_node.py | 3 +++ src/steps/consensus.py | 18 +++++++++++++++++- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/api_clients/base_client.py b/src/api_clients/base_client.py index ddb68ee..2d6e257 100644 --- a/src/api_clients/base_client.py +++ b/src/api_clients/base_client.py @@ -40,4 +40,5 @@ class BaseClient: def print_request_size(self, data): body_size = len(data) if data else 0 body_kb = body_size / 1024 - logger.debug(f"Request body size: {body_kb:.2f}kB") + if body_size > 0: + logger.debug(f"Request body size: {body_kb:.2f}kB") diff --git a/src/api_clients/rest.py b/src/api_clients/rest.py index 280ebb9..506dd0c 100644 --- a/src/api_clients/rest.py +++ b/src/api_clients/rest.py @@ -33,8 +33,9 @@ class REST(BaseClient): response = self.rest_call("get", "cryptarchia/info") return response.json() - def cryptarchia_headers(self, from_header_id, to_header_id): - response = self.rest_call("get", f"cryptarchia/headers?from={quote(from_header_id, safe='')}" f"&to={quote(to_header_id, safe='')}") + def cryptarchia_headers(self, query): + path = f"cryptarchia/headers{'?' + query if query else ''}" + response = self.rest_call("get", path) return response.json() def da_add_share(self, data): diff --git a/src/libs/common.py b/src/libs/common.py index e42db8a..d0fcdaf 100644 --- a/src/libs/common.py +++ b/src/libs/common.py @@ -43,8 +43,11 @@ def to_blob_id(n: int) -> list: return to_byte_list(n, 32) -def to_header_id(n: int) -> list: - return to_byte_list(n, 32) +def to_header_id(n: int): + if n < 0: + raise ValueError("Input must be an unsigned integer (non-negative)") + + return n.to_bytes(32, byteorder="big").hex() def to_byte_list(n: int, l: int) -> list: diff --git a/src/node/nomos_node.py b/src/node/nomos_node.py index 3e7df22..d3f72ea 100644 --- a/src/node/nomos_node.py +++ b/src/node/nomos_node.py @@ -163,3 +163,6 @@ class NomosNode: def send_get_storage_block_request(self, data): return self._api.storage_block(data) + + def send_get_cryptarchia_headers_request(self, data): + return self._api.cryptarchia_headers(data) diff --git a/src/steps/consensus.py b/src/steps/consensus.py index ab89364..349666e 100644 --- a/src/steps/consensus.py +++ b/src/steps/consensus.py @@ -1,3 +1,5 @@ +from urllib.parse import quote + import allure from tenacity import retry, stop_after_delay, wait_fixed @@ -7,6 +9,18 @@ from src.steps.common import StepsCommon logger = get_custom_logger(__name__) +def prepare_get_cryptarchia_headers_request(from_header_id, to_header_id): + query_parts = [] + + if from_header_id is not None: + query_parts.append(f"from={quote(from_header_id, safe='')}") + + if to_header_id is not None: + query_parts.append(f"to={quote(to_header_id, safe='')}") + + return "&".join(query_parts) + + class StepsConsensus(StepsCommon): @allure.step def get_cryptarchia_headers(self, node, from_header_id=None, to_header_id=None, **kwargs): @@ -14,10 +28,12 @@ class StepsConsensus(StepsCommon): timeout_duration = kwargs.get("timeout_duration", 65) interval = kwargs.get("interval", 0.1) + query = prepare_get_cryptarchia_headers_request(from_header_id, to_header_id) + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(interval), reraise=True) def get_headers(): try: - response = node.send_get_cryptarchia_headers_request(from_header_id, to_header_id) + response = node.send_get_cryptarchia_headers_request(query) except Exception as ex: logger.error(f"Exception while retrieving cryptarchia headers: {ex}") raise From c847fcc222caf51acf85ca3b8ae5f790ae02c330 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 10 Apr 2025 15:13:29 +0800 Subject: [PATCH 09/19] fix: get_storage_block --- src/api_clients/rest.py | 2 +- src/steps/storage.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api_clients/rest.py b/src/api_clients/rest.py index 506dd0c..c231def 100644 --- a/src/api_clients/rest.py +++ b/src/api_clients/rest.py @@ -75,7 +75,7 @@ class REST(BaseClient): return response.json() def storage_block(self, query): - response = self.rest_call("get", "storage/block", json.dumps(query)) + response = self.rest_call("post", f"storage/block", json.dumps(query)) return response.json() def mempool_add_tx(self, data): diff --git a/src/steps/storage.py b/src/steps/storage.py index 879595e..bf81580 100644 --- a/src/steps/storage.py +++ b/src/steps/storage.py @@ -1,3 +1,5 @@ +from urllib.parse import quote + import allure from tenacity import retry, stop_after_delay, wait_fixed @@ -8,7 +10,7 @@ logger = get_custom_logger(__name__) def prepare_get_storage_block_request(header_id): - query_data = {"header_id": header_id} + query_data = f"{header_id}" return query_data From 9d83b1a2c89a90f0a898df754147e838a138fa8c Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 10 Apr 2025 15:14:10 +0800 Subject: [PATCH 10/19] test: get blob ids --- tests/protocol_compatibility/__init__.py | 0 .../test_api_compatibility.py | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/protocol_compatibility/__init__.py create mode 100644 tests/protocol_compatibility/test_api_compatibility.py diff --git a/tests/protocol_compatibility/__init__.py b/tests/protocol_compatibility/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py new file mode 100644 index 0000000..2517ced --- /dev/null +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -0,0 +1,35 @@ +import json +import pytest + +from src.client.nomos_cli import NomosCli +from src.libs.common import to_app_id, to_index, delay +from src.libs.custom_logger import get_custom_logger +from src.steps.consensus import StepsConsensus +from src.steps.da import StepsDataAvailability +from src.steps.storage import StepsStorage +from src.test_data import DATA_TO_DISPERSE + +logger = get_custom_logger(__name__) + + +class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): + main_nodes = [] + + @pytest.mark.usefixtures("setup_2_node_cluster") + def test_da_consensus_compatibility(self): + self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) + delay(5) + rcv_data = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) + logger.debug(f"shares: {rcv_data}") + headers = self.get_cryptarchia_headers(self.node2) + logger.debug(f"headers: {headers}") + # get storage blocks for headerID + blob_ids = [] + for header in headers: + block = self.get_storage_block(self.node2, header) + if block is not None and "bl_blobs" in block: + blobs = block["bl_blobs"] + for blob in blobs: + blob_ids.append(blob["id"]) + + logger.debug(f"blob ids: {blob_ids}") From e7c2141dd82fa2f36e23176b815699dfef1d132a Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 11 Apr 2025 11:05:12 +0800 Subject: [PATCH 11/19] test: consensus compatibility --- src/libs/custom_logger.py | 2 +- .../test_api_compatibility.py | 37 ++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/libs/custom_logger.py b/src/libs/custom_logger.py index 989548c..ec2f8e5 100644 --- a/src/libs/custom_logger.py +++ b/src/libs/custom_logger.py @@ -1,6 +1,6 @@ import logging -max_log_line_length = 5000 +max_log_line_length = 10000 def log_length_filter(max_length): diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index 2517ced..676617f 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -19,8 +19,23 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): def test_da_consensus_compatibility(self): self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) delay(5) - rcv_data = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) - logger.debug(f"shares: {rcv_data}") + shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) + logger.debug(f"shares: {shares}") + aggregated_column_commitments = [] + rows_commitments = [] + for index_share in shares: + if len(index_share[1]) != 0: + for share in index_share[1]: + a_c_c = share["aggregated_column_commitment"] + if a_c_c not in aggregated_column_commitments: + aggregated_column_commitments.append(a_c_c) + logger.debug(f"a_c_c: {a_c_c}") + + r_c = share["rows_commitments"] + if r_c not in rows_commitments: + rows_commitments.append(r_c) + logger.debug(f"r_c: {rows_commitments}") + headers = self.get_cryptarchia_headers(self.node2) logger.debug(f"headers: {headers}") # get storage blocks for headerID @@ -33,3 +48,21 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): blob_ids.append(blob["id"]) logger.debug(f"blob ids: {blob_ids}") + commitments = [] + for blob_id in blob_ids: + commitment = self.get_shares_commitments(self.node2, blob_id) + commitments.append(commitment) + logger.debug(f"commitments: {commitments}") + + rcv_aggregated_column_commitments = [] + rcv_rows_commitments = [] + for commitment in commitments: + rcv_aggregated_column_commitments.append(commitment["aggregated_column_commitment"]) + for rcv_rows_commitment in commitment["rows_commitments"]: + rcv_rows_commitments.append(rcv_rows_commitment) + + # Check commitments from shares match commitments ceceived + for a_c_c in aggregated_column_commitments: + assert a_c_c in rcv_aggregated_column_commitments + for r_c in rcv_rows_commitments: + assert r_c in rcv_rows_commitments From 8337506961c42d7aa16ce7ce583154b6efb2913f Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 11 Apr 2025 12:19:14 +0800 Subject: [PATCH 12/19] fix: rows_commitments collection --- .../test_api_compatibility.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index 676617f..818e7ae 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -29,16 +29,15 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): a_c_c = share["aggregated_column_commitment"] if a_c_c not in aggregated_column_commitments: aggregated_column_commitments.append(a_c_c) - logger.debug(f"a_c_c: {a_c_c}") r_c = share["rows_commitments"] - if r_c not in rows_commitments: - rows_commitments.append(r_c) - logger.debug(f"r_c: {rows_commitments}") + for commitment in r_c: + if commitment not in rows_commitments: + rows_commitments.append(commitment) headers = self.get_cryptarchia_headers(self.node2) - logger.debug(f"headers: {headers}") - # get storage blocks for headerID + + # Get storage blocks for headerIDs blob_ids = [] for header in headers: block = self.get_storage_block(self.node2, header) @@ -47,12 +46,11 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): for blob in blobs: blob_ids.append(blob["id"]) - logger.debug(f"blob ids: {blob_ids}") + # Get commitments for blob ids commitments = [] for blob_id in blob_ids: commitment = self.get_shares_commitments(self.node2, blob_id) commitments.append(commitment) - logger.debug(f"commitments: {commitments}") rcv_aggregated_column_commitments = [] rcv_rows_commitments = [] @@ -61,8 +59,9 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): for rcv_rows_commitment in commitment["rows_commitments"]: rcv_rows_commitments.append(rcv_rows_commitment) - # Check commitments from shares match commitments ceceived + # Check commitments from shares match commitments received based on consensus data for a_c_c in aggregated_column_commitments: assert a_c_c in rcv_aggregated_column_commitments - for r_c in rcv_rows_commitments: + + for r_c in rows_commitments: assert r_c in rcv_rows_commitments From 17371cf3c5749bef9f80c54bafc4b7c7c229a4de Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 11 Apr 2025 16:20:15 +0800 Subject: [PATCH 13/19] fix: optimize test_da_consensus_compatibility --- .../test_api_compatibility.py | 67 +++++++++++-------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index 818e7ae..88bdc46 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -1,7 +1,5 @@ -import json import pytest -from src.client.nomos_cli import NomosCli from src.libs.common import to_app_id, to_index, delay from src.libs.custom_logger import get_custom_logger from src.steps.consensus import StepsConsensus @@ -12,6 +10,37 @@ from src.test_data import DATA_TO_DISPERSE logger = get_custom_logger(__name__) +# Extract commitments from indexed shares +def extract_commitments(index_shares): + aggregated_column_commitments = [] + rows_commitments = [] + + for index, shares in index_shares: + for share in shares: + a_c_c = share["aggregated_column_commitment"] + if a_c_c not in aggregated_column_commitments: + aggregated_column_commitments.append(a_c_c) + + r_c = share["rows_commitments"] + for commitment in r_c: + if commitment not in rows_commitments: + rows_commitments.append(commitment) + + return aggregated_column_commitments, rows_commitments + + +# Parse commitments received by get_shares_commitments +def parse_commitments(commitments): + aggregated_column_commitments = [] + rows_commitments = [] + for commitment in commitments: + aggregated_column_commitments.append(commitment["aggregated_column_commitment"]) + for rows_commitment in commitment["rows_commitments"]: + rows_commitments.append(rows_commitment) + + return aggregated_column_commitments, rows_commitments + + class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): main_nodes = [] @@ -19,25 +48,13 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): def test_da_consensus_compatibility(self): self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) delay(5) - shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) - logger.debug(f"shares: {shares}") - aggregated_column_commitments = [] - rows_commitments = [] - for index_share in shares: - if len(index_share[1]) != 0: - for share in index_share[1]: - a_c_c = share["aggregated_column_commitment"] - if a_c_c not in aggregated_column_commitments: - aggregated_column_commitments.append(a_c_c) - - r_c = share["rows_commitments"] - for commitment in r_c: - if commitment not in rows_commitments: - rows_commitments.append(commitment) + index_shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) + column_commitments, rows_commitments = extract_commitments(index_shares) + # Get headers of received blocks headers = self.get_cryptarchia_headers(self.node2) - # Get storage blocks for headerIDs + # Get storage blocks for headers blob_ids = [] for header in headers: block = self.get_storage_block(self.node2, header) @@ -52,16 +69,8 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): commitment = self.get_shares_commitments(self.node2, blob_id) commitments.append(commitment) - rcv_aggregated_column_commitments = [] - rcv_rows_commitments = [] - for commitment in commitments: - rcv_aggregated_column_commitments.append(commitment["aggregated_column_commitment"]) - for rcv_rows_commitment in commitment["rows_commitments"]: - rcv_rows_commitments.append(rcv_rows_commitment) + rcv_column_commitments, rcv_rows_commitments = parse_commitments(commitments) # Check commitments from shares match commitments received based on consensus data - for a_c_c in aggregated_column_commitments: - assert a_c_c in rcv_aggregated_column_commitments - - for r_c in rows_commitments: - assert r_c in rcv_rows_commitments + assert all(c in rcv_column_commitments for c in column_commitments), "Not all aggregated column commitments are present" + assert all(r in rcv_rows_commitments for r in rows_commitments), "Not all rows commitments are present" From 21d5881d0ddb0d471adf2e502e0dadb4464b27df Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 11 Apr 2025 16:46:28 +0800 Subject: [PATCH 14/19] test: consensus compatibility across nodes --- .../test_api_compatibility.py | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index 88bdc46..112a0f4 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -51,10 +51,10 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): index_shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) column_commitments, rows_commitments = extract_commitments(index_shares) - # Get headers of received blocks + # Get consensus headers headers = self.get_cryptarchia_headers(self.node2) - # Get storage blocks for headers + # Get storage blocks for received headers and extract blob ids blob_ids = [] for header in headers: block = self.get_storage_block(self.node2, header) @@ -74,3 +74,34 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): # Check commitments from shares match commitments received based on consensus data assert all(c in rcv_column_commitments for c in column_commitments), "Not all aggregated column commitments are present" assert all(r in rcv_rows_commitments for r in rows_commitments), "Not all rows commitments are present" + + @pytest.mark.usefixtures("setup_4_node_cluster") + def test_da_consensus_compatibility_across_nodes(self): + self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) + delay(5) + index_shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) + column_commitments, rows_commitments = extract_commitments(index_shares) + + # Get consensus headers + headers = self.get_cryptarchia_headers(self.node3) + + # Get storage blocks for received headers and extract blob ids + blob_ids = [] + for header in headers: + block = self.get_storage_block(self.node3, header) + if block is not None and "bl_blobs" in block: + blobs = block["bl_blobs"] + for blob in blobs: + blob_ids.append(blob["id"]) + + # Get commitments for blob ids + commitments = [] + for blob_id in blob_ids: + commitment = self.get_shares_commitments(self.node3, blob_id) + commitments.append(commitment) + + rcv_column_commitments, rcv_rows_commitments = parse_commitments(commitments) + + # Check commitments from shares match commitments received based on consensus data + assert all(c in rcv_column_commitments for c in column_commitments), "Not all aggregated column commitments are present" + assert all(r in rcv_rows_commitments for r in rows_commitments), "Not all rows commitments are present" From 2dba68f5b188069fe45b023e8b38d9143131739f Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 11 Apr 2025 16:57:10 +0800 Subject: [PATCH 15/19] fix: optimize tests --- .../test_api_compatibility.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index 112a0f4..d8c4f1f 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -55,19 +55,16 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): headers = self.get_cryptarchia_headers(self.node2) # Get storage blocks for received headers and extract blob ids - blob_ids = [] - for header in headers: - block = self.get_storage_block(self.node2, header) - if block is not None and "bl_blobs" in block: - blobs = block["bl_blobs"] - for blob in blobs: - blob_ids.append(blob["id"]) + blob_ids = [ + blob["id"] + for header in headers + for block in [self.get_storage_block(self.node2, header)] + if block is not None and "bl_blobs" in block + for blob in block["bl_blobs"] + ] # Get commitments for blob ids - commitments = [] - for blob_id in blob_ids: - commitment = self.get_shares_commitments(self.node2, blob_id) - commitments.append(commitment) + commitments = [self.get_shares_commitments(self.node2, blob_id) for blob_id in blob_ids] rcv_column_commitments, rcv_rows_commitments = parse_commitments(commitments) @@ -86,19 +83,16 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): headers = self.get_cryptarchia_headers(self.node3) # Get storage blocks for received headers and extract blob ids - blob_ids = [] - for header in headers: - block = self.get_storage_block(self.node3, header) - if block is not None and "bl_blobs" in block: - blobs = block["bl_blobs"] - for blob in blobs: - blob_ids.append(blob["id"]) + blob_ids = [ + blob["id"] + for header in headers + for block in [self.get_storage_block(self.node3, header)] + if block is not None and "bl_blobs" in block + for blob in block["bl_blobs"] + ] # Get commitments for blob ids - commitments = [] - for blob_id in blob_ids: - commitment = self.get_shares_commitments(self.node3, blob_id) - commitments.append(commitment) + commitments = [self.get_shares_commitments(self.node3, blob_id) for blob_id in blob_ids] rcv_column_commitments, rcv_rows_commitments = parse_commitments(commitments) From ce6292413c316add017db09bd6e5458749486a04 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 11 Apr 2025 17:02:09 +0800 Subject: [PATCH 16/19] fix: rename test --- tests/protocol_compatibility/test_api_compatibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index d8c4f1f..58bf0d7 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -73,7 +73,7 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): assert all(r in rcv_rows_commitments for r in rows_commitments), "Not all rows commitments are present" @pytest.mark.usefixtures("setup_4_node_cluster") - def test_da_consensus_compatibility_across_nodes(self): + def test_da_across_nodes_consensus_compatibility(self): self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) delay(5) index_shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) From 9dc3950092357a5c02d090244d5c9d5eadb2936b Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 14 Apr 2025 12:44:26 +0800 Subject: [PATCH 17/19] fix: change subnet_size to 2 on 4 node cl --- tests/protocol_compatibility/test_api_compatibility.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index 58bf0d7..f8ef7f3 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -73,6 +73,7 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): assert all(r in rcv_rows_commitments for r in rows_commitments), "Not all rows commitments are present" @pytest.mark.usefixtures("setup_4_node_cluster") + @pytest.mark.parametrize("setup_4_node_cluster", [{"subnet_size": 2}], indirect=True) def test_da_across_nodes_consensus_compatibility(self): self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) delay(5) From 53f6441f37c9b13374a0f68ef1b8132e18a31ac4 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 14 Apr 2025 17:57:49 +0800 Subject: [PATCH 18/19] fix: change 4 node cl default parameters --- cluster_config/cfgsync-template.yaml | 2 +- src/steps/common.py | 14 +++++++++----- .../test_api_compatibility.py | 5 ++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cluster_config/cfgsync-template.yaml b/cluster_config/cfgsync-template.yaml index 5f5ec63..df3bf29 100644 --- a/cluster_config/cfgsync-template.yaml +++ b/cluster_config/cfgsync-template.yaml @@ -14,7 +14,7 @@ num_subnets: {{ subnet_size }} old_blobs_check_interval: "5.0" blobs_validity_duration: "60.0" global_params_path: "/kzgrs_test_params" -min_dispersal_peers: 1 +min_dispersal_peers: {{ min_dispersal_peers }} min_replication_peers: 1 monitor_failure_time_window: "5.0" balancer_interval: "5.0" diff --git a/src/steps/common.py b/src/steps/common.py index 71e4a35..d110e12 100644 --- a/src/steps/common.py +++ b/src/steps/common.py @@ -14,7 +14,7 @@ from jinja2 import Template logger = get_custom_logger(__name__) -def prepare_cluster_config(node_count, subnetwork_size=2, dispersal_factor=2): +def prepare_cluster_config(node_count, subnetwork_size=2, dispersal_factor=2, min_dispersal_peers=1): cwd = os.getcwd() config_dir = "cluster_config" @@ -22,7 +22,9 @@ def prepare_cluster_config(node_count, subnetwork_size=2, dispersal_factor=2): template_content = file.read() template = Template(template_content) - rendered = template.render(num_hosts=node_count, subnet_size=subnetwork_size, dispersal_factor=dispersal_factor) + rendered = template.render( + num_hosts=node_count, subnet_size=subnetwork_size, dispersal_factor=dispersal_factor, min_dispersal_peers=min_dispersal_peers + ) with open(f"{cwd}/{config_dir}/cfgsync.yaml", "w") as outfile: outfile.write(rendered) @@ -55,7 +57,8 @@ class StepsCommon: subnet_size = get_param_or_default(request, "subnet_size", 2) dispersal_factor = get_param_or_default(request, "dispersal_factor", 2) - prepare_cluster_config(2, subnet_size, dispersal_factor) + min_dispersal_peers = get_param_or_default(request, "min_dispersal_peers", 1) + prepare_cluster_config(2, subnet_size, dispersal_factor, min_dispersal_peers) self.node1 = NomosNode(CFGSYNC, "cfgsync") self.node2 = NomosNode(NOMOS, "nomos_node_0") @@ -76,8 +79,9 @@ class StepsCommon: logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") subnet_size = get_param_or_default(request, "subnet_size", 4) - dispersal_factor = get_param_or_default(request, "dispersal_factor", 1) - prepare_cluster_config(4, subnet_size, dispersal_factor) + dispersal_factor = get_param_or_default(request, "dispersal_factor", 2) + min_dispersal_peers = get_param_or_default(request, "min_dispersal_peers", 4) + prepare_cluster_config(4, subnet_size, dispersal_factor, min_dispersal_peers) self.node1 = NomosNode(CFGSYNC, "cfgsync") self.node2 = NomosNode(NOMOS, "nomos_node_0") diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index f8ef7f3..aa20093 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -73,10 +73,9 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): assert all(r in rcv_rows_commitments for r in rows_commitments), "Not all rows commitments are present" @pytest.mark.usefixtures("setup_4_node_cluster") - @pytest.mark.parametrize("setup_4_node_cluster", [{"subnet_size": 2}], indirect=True) - def test_da_across_nodes_consensus_compatibility(self): + def test_da_cross_nodes_consensus_compatibility(self): self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) - delay(5) + delay(10) index_shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) column_commitments, rows_commitments = extract_commitments(index_shares) From 71db7418c93f938fa2a7d281f08544dfabf7c748 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 14 Apr 2025 18:11:30 +0800 Subject: [PATCH 19/19] fix: set wait time back to 5 seconds --- tests/protocol_compatibility/test_api_compatibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/protocol_compatibility/test_api_compatibility.py b/tests/protocol_compatibility/test_api_compatibility.py index aa20093..bc05181 100644 --- a/tests/protocol_compatibility/test_api_compatibility.py +++ b/tests/protocol_compatibility/test_api_compatibility.py @@ -75,7 +75,7 @@ class TestApiCompatibility(StepsDataAvailability, StepsConsensus, StepsStorage): @pytest.mark.usefixtures("setup_4_node_cluster") def test_da_cross_nodes_consensus_compatibility(self): self.disperse_data(DATA_TO_DISPERSE[2], to_app_id(1), to_index(0)) - delay(10) + delay(5) index_shares = self.get_data_range(self.node2, to_app_id(1), to_index(0), to_index(5)) column_commitments, rows_commitments = extract_commitments(index_shares)