diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af7f256fb..7ff06833d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: - 'tools/**' - 'tests/all_tests_v2.nim' - 'tests/**' + - 'tests-e2e/**' docker: - 'docker/**' @@ -205,12 +206,13 @@ jobs: secrets: inherit - # Send API E2E tests run the wrapper suite against the liblogosdelivery.so - # built by the `build` job (downloaded as the `liblogosdelivery` artifact). - # Gated on `build`, so it is skipped on docker-only PRs where no lib is built. + # Send API E2E tests run the in-repo tests-e2e/ wrapper suite against the + # liblogosdelivery.so built by the `build` job (downloaded as the + # `liblogosdelivery` artifact). Gated on `build`, so it is skipped on + # docker-only PRs where no lib is built. send-api-e2e-tests: needs: build - uses: logos-messaging/logos-delivery-interop-tests/.github/workflows/send_api_e2e_PR.yml@SMOKE_TEST_2026.06.25 + uses: ./.github/workflows/e2e-api-tests.yml secrets: inherit lint: diff --git a/.github/workflows/e2e-api-tests.yml b/.github/workflows/e2e-api-tests.yml new file mode 100644 index 000000000..00c5a4cfe --- /dev/null +++ b/.github/workflows/e2e-api-tests.yml @@ -0,0 +1,34 @@ +name: e2e-api-tests + +on: + workflow_call: + +jobs: + api-e2e: + runs-on: ubuntu-22.04 + timeout-minutes: 45 + name: api-e2e + steps: + - name: Checkout code + uses: actions/checkout@v4 + + # Consumes the shared library built by ci.yml's `build` job (same run). + - name: Download liblogosdelivery.so + uses: actions/download-artifact@v4 + with: + name: liblogosdelivery + path: tests-e2e/vendor/logos-delivery-python-bindings/lib + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: tests-e2e/requirements.txt + + - name: Install Python deps + run: pip install -r tests-e2e/requirements.txt + + - name: Run API/e2e tests (non-docker subset) + working-directory: tests-e2e + run: pytest tests/wrappers_tests -m "not docker_required" --reruns 2 diff --git a/tests-e2e/README.md b/tests-e2e/README.md new file mode 100644 index 000000000..c52ea2a1d --- /dev/null +++ b/tests-e2e/README.md @@ -0,0 +1,32 @@ +# tests-e2e — logos-delivery API/e2e tests (Python) + +End-to-end tests for `liblogosdelivery`, driven through the Python bindings (C-FFI). +Migrated from [`logos-delivery-interop-tests`](https://github.com/logos-messaging/logos-delivery-interop-tests) (the wrapper / send-API suite). + +## Layout + +- `src/` — test framework (node wrappers, steps, helpers) +- `tests/wrappers_tests/` — the API/e2e scenario tests (`test_s02…s31`) +- `vendor/logos-delivery-python-bindings/waku/wrapper.py` — CFFI binding (`NodeWrapper`); it `dlopen`s `../lib/liblogosdelivery.so` + +## Run locally + +```bash +# 1. Build the shared library and place it where the binding looks for it +./tests-e2e/scripts/prepare_lib.sh + +# 2. Python env + deps +python -m venv .venv && source .venv/bin/activate +pip install -r tests-e2e/requirements.txt + +# 3. Run (from tests-e2e/) +cd tests-e2e +pytest tests/wrappers_tests -m "not docker_required" # 18 pure-binding tests +pytest tests/wrappers_tests -m docker_required # 3 tests that also need a Docker nwaku peer (S19/S20/S31) +``` + +## CI + +`.github/workflows/e2e-api-tests.yml` (called from `ci.yml`, `needs: build`) downloads the +`liblogosdelivery` artifact produced by the `build` job and runs the non-docker subset on every PR — +so a protocol change and its e2e test land in the same PR. diff --git a/tests-e2e/log/.gitkeep b/tests-e2e/log/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/pytest.ini b/tests-e2e/pytest.ini new file mode 100644 index 000000000..2406bfb06 --- /dev/null +++ b/tests-e2e/pytest.ini @@ -0,0 +1,15 @@ +[pytest] +addopts = --instafail --tb=short --color=auto +log_level = DEBUG +log_cli = True +norecursedirs = + vendor + nimbledeps + *.egg-info +log_file = log/test.log +log_cli_format = %(asctime)s.%(msecs)03d %(levelname)s [%(name)s] %(message)s +log_file_format = %(asctime)s.%(msecs)03d %(levelname)s [%(name)s] %(message)s +timeout = 300 +markers = + smoke: marks tests as smoke test (deselect with '-m "not smoke"') + docker_required: test requires Docker nodes (WakuNode) diff --git a/tests-e2e/requirements.txt b/tests-e2e/requirements.txt new file mode 100644 index 000000000..53c1e36af --- /dev/null +++ b/tests-e2e/requirements.txt @@ -0,0 +1,44 @@ +allure-pytest==2.13.2 +allure-python-commons==2.13.2 +attrs==23.1.0 +black==24.3.0 +certifi==2024.7.4 +cfgv==3.4.0 +charset-normalizer==3.3.2 +click==8.1.7 +distlib==0.3.8 +docker==7.0.0 +execnet==2.0.2 +filelock==3.13.1 +identify==2.5.33 +idna==3.7 +iniconfig==2.0.0 +marshmallow==3.20.1 +marshmallow-dataclass==8.6.0 +mypy-extensions==1.0.0 +nodeenv==1.8.0 +packaging==23.2 +pathspec==0.12.1 +platformdirs==4.1.0 +pluggy==1.3.0 +pre-commit==3.6.2 +pyright==1.1.352 +pytest==8.0.2 +pytest-instafail==0.5.0 +pytest-rerunfailures==13.0 +pytest-timeout==2.2.0 +pytest-xdist==3.5.0 +python-dotenv==1.0.1 +pytest-dependency==0.6.0 +PyYAML==6.0.1 +requests==2.31.0 +setuptools==70.0.0 +tenacity==8.2.3 +typeguard==4.1.5 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +urllib3==2.2.2 +virtualenv==20.25.0 +pytest-shard==0.1.2 +result==0.17.0 +cffi \ No newline at end of file diff --git a/tests-e2e/scripts/prepare_lib.sh b/tests-e2e/scripts/prepare_lib.sh new file mode 100755 index 000000000..62a6f2cd2 --- /dev/null +++ b/tests-e2e/scripts/prepare_lib.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Build liblogosdelivery from this repo and place it where the Python binding expects it. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LIBDIR="$ROOT/tests-e2e/vendor/logos-delivery-python-bindings/lib" + +cd "$ROOT" +make update +make V=1 liblogosdelivery + +mkdir -p "$LIBDIR" +if [ -f build/liblogosdelivery.so ]; then + cp build/liblogosdelivery.so "$LIBDIR/liblogosdelivery.so" +elif [ -f build/liblogosdelivery.dylib ]; then + cp build/liblogosdelivery.dylib "$LIBDIR/liblogosdelivery.dylib" + ln -sf liblogosdelivery.dylib "$LIBDIR/liblogosdelivery.so" +else + echo "ERROR: built library not found in build/" >&2 + exit 1 +fi +echo "Placed liblogosdelivery in $LIBDIR" diff --git a/tests-e2e/src/__init__.py b/tests-e2e/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/src/data_storage.py b/tests-e2e/src/data_storage.py new file mode 100644 index 000000000..1b69a4631 --- /dev/null +++ b/tests-e2e/src/data_storage.py @@ -0,0 +1,3 @@ +# We use this class for global variables +class DS: + waku_nodes = [] diff --git a/tests-e2e/src/env_vars.py b/tests-e2e/src/env_vars.py new file mode 100644 index 000000000..32e96a7e1 --- /dev/null +++ b/tests-e2e/src/env_vars.py @@ -0,0 +1,36 @@ +import os +from dotenv import load_dotenv + +load_dotenv() # This will load environment variables from a .env file if it exists + + +def get_env_var(var_name, default=None): + env_var = os.getenv(var_name, default) + if env_var in [None, ""]: + print(f"{var_name} is not set; using default value: {default}") + env_var = default + print(f"{var_name}: {env_var}") + return env_var + + +# Configuration constants. Need to be upercase to appear in reports +DEFAULT_NWAKU = "wakuorg/nwaku:latest" +STRESS_ENABLED = False +USE_WRAPPERS = True +NODE_1 = get_env_var("NODE_1", DEFAULT_NWAKU) +NODE_2 = get_env_var("NODE_2", DEFAULT_NWAKU) +ADDITIONAL_NODES = get_env_var("ADDITIONAL_NODES", f"{DEFAULT_NWAKU},{DEFAULT_NWAKU},{DEFAULT_NWAKU}") +# more nodes need to follow the NODE_X pattern +DOCKER_LOG_DIR = get_env_var("DOCKER_LOG_DIR", "./log/docker") +NETWORK_NAME = get_env_var("NETWORK_NAME", "waku") +SUBNET = get_env_var("SUBNET", "172.18.0.0/16") +IP_RANGE = get_env_var("IP_RANGE", "172.18.0.0/24") +GATEWAY = get_env_var("GATEWAY", "172.18.0.1") +RUNNING_IN_CI = get_env_var("CI") +API_REQUEST_TIMEOUT = get_env_var("API_REQUEST_TIMEOUT", 20) +RLN_CREDENTIALS = get_env_var("RLN_CREDENTIALS") +PG_USER = get_env_var("POSTGRES_USER", "postgres") +PG_PASS = get_env_var("POSTGRES_PASSWORD", "test123") + +# example for .env file +# RLN_CREDENTIALS = {"rln-relay-cred-password": "password", "rln-relay-eth-client-address": "https://rpc.sepolia.linea.build", "rln-relay-eth-contract-address": "0xB9cd878C90E49F797B4431fBF4fb333108CB90e6", "rln-relay-eth-private-key-1": "", "rln-relay-eth-private-key-2": "", "rln-relay-eth-private-key-3": "", "rln-relay-eth-private-key-4": "", "rln-relay-eth-private-key-5": ""} diff --git a/tests-e2e/src/libs/__init__.py b/tests-e2e/src/libs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/src/libs/common.py b/tests-e2e/src/libs/common.py new file mode 100644 index 000000000..cd0dfc19c --- /dev/null +++ b/tests-e2e/src/libs/common.py @@ -0,0 +1,52 @@ +import uuid +from datetime import datetime +from time import sleep +from src.libs.custom_logger import get_custom_logger +import os +import base64 +import allure + +logger = get_custom_logger(__name__) + + +def bytes_to_hex(byte_array): + return "".join(format(byte, "02x") for byte in byte_array) + + +def to_base64(input_data): + if isinstance(input_data, str): + input_bytes = input_data.encode() + elif isinstance(input_data, int): + input_bytes = str(input_data).encode() + elif isinstance(input_data, bytes): + input_bytes = input_data + else: + input_bytes = str(input_data).encode() + base64_encoded = base64.b64encode(input_bytes) + return base64_encoded.decode() + + +def to_hex(input_data): + if isinstance(input_data, str): + input_bytes = input_data.encode() + elif isinstance(input_data, int): + input_bytes = str(input_data).encode() + elif isinstance(input_data, bytes): + input_bytes = input_data + else: + input_bytes = str(input_data).encode() + return "0x" + input_bytes.hex() + + +def attach_allure_file(file): + logger.debug(f"Attaching file {file}") + allure.attach.file(file, name=os.path.basename(file), attachment_type=allure.attachment_type.TEXT) + + +def delay(num_seconds): + logger.debug(f"Sleeping for {num_seconds} seconds") + sleep(num_seconds) + + +def gen_step_id(): + return f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}__{str(uuid.uuid4())}" diff --git a/tests-e2e/src/libs/custom_logger.py b/tests-e2e/src/libs/custom_logger.py new file mode 100644 index 000000000..ecdd7ecc6 --- /dev/null +++ b/tests-e2e/src/libs/custom_logger.py @@ -0,0 +1,31 @@ +import logging + +max_log_line_length = 5000 + + +def log_length_filter(max_length): + class logLengthFilter(logging.Filter): + def filter(self, record): + if len(record.getMessage()) > max_length: + logging.getLogger(record.name).log( + record.levelno, f"Log line was discarded because it's longer than max_log_line_length={max_log_line_length}" + ) + return False + return True + + return logLengthFilter() + + +def get_custom_logger(name): + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("docker").setLevel(logging.WARNING) + logger = logging.getLogger(name) + logger.addFilter(log_length_filter(max_log_line_length)) + + # Define a formatter with millisecond precision + ch = logging.StreamHandler() + formatter = logging.Formatter("%(asctime)s.%(msecs)03d %(levelname)s [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + ch.setFormatter(formatter) + logger.addHandler(ch) + + return logger diff --git a/tests-e2e/src/node/__init__.py b/tests-e2e/src/node/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/src/node/api_clients/__init__.py b/tests-e2e/src/node/api_clients/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/src/node/api_clients/base_client.py b/tests-e2e/src/node/api_clients/base_client.py new file mode 100644 index 000000000..b731e0f3f --- /dev/null +++ b/tests-e2e/src/node/api_clients/base_client.py @@ -0,0 +1,37 @@ +import json +import requests +from src.env_vars import API_REQUEST_TIMEOUT +from src.libs.custom_logger import get_custom_logger + +logger = get_custom_logger(__name__) + + +class BaseClient: + def make_request(self, method, url, headers=None, data=None): + self.log_request_as_curl(method, url, headers, data) + response = requests.request(method.upper(), url, headers=headers, data=data, timeout=API_REQUEST_TIMEOUT) + try: + response.raise_for_status() + except requests.HTTPError as http_err: + logger.error(f"HTTP error occurred: {http_err}. Response content: {response.content}") + raise Exception(f"Error: {http_err} with response: {response.content}") + except Exception as err: + logger.error(f"An error occurred: {err}. Response content: {response.content}") + raise Exception(f"Error: {err} with response: {response.content}") + else: + logger.info(f"Response status code: {response.status_code}. Response content: {response.content}") + return response + + def log_request_as_curl(self, method, url, headers, data): + if data: + try: + data_dict = json.loads(data) + if "timestamp" in data_dict: + data_dict["timestamp"] = "TIMESTAMP_PLACEHOLDER" + data = json.dumps(data_dict) + data = data.replace('"TIMESTAMP_PLACEHOLDER"', "'$(date +%s%N)'") + except json.JSONDecodeError: + logger.error("Invalid JSON data provided") + headers_str_for_log = " ".join([f'-H "{key}: {value}"' for key, value in headers.items()]) if headers else "" + curl_cmd = f"curl -v -X {method.upper()} \"{url}\" {headers_str_for_log} -d '{data}'" + logger.info(curl_cmd) diff --git a/tests-e2e/src/node/api_clients/rest.py b/tests-e2e/src/node/api_clients/rest.py new file mode 100644 index 000000000..c0bd2f7f6 --- /dev/null +++ b/tests-e2e/src/node/api_clients/rest.py @@ -0,0 +1,190 @@ +from src.libs.custom_logger import get_custom_logger +import json +from urllib.parse import quote +from src.node.api_clients.base_client import BaseClient + +logger = get_custom_logger(__name__) + + +class REST(BaseClient): + def __init__(self, rest_port): + self._rest_port = rest_port + + def rest_call(self, method, endpoint, payload=None): + url = f"http://127.0.0.1:{self._rest_port}/{endpoint}" + headers = {"Content-Type": "application/json"} + return self.make_request(method, url, headers=headers, data=payload) + + def rest_call_text(self, method, endpoint, payload=None): + url = f"http://127.0.0.1:{self._rest_port}/{endpoint}" + headers = {"accept": "text/plain"} + return self.make_request(method, url, headers=headers, data=payload) + + def info(self): + info_response = self.rest_call("get", "debug/v1/info") + return info_response.json() + + def health(self): + health_response = self.rest_call("get", "health") + return health_response.content + + def get_peers(self): + get_peers_response = self.rest_call("get", "admin/v1/peers") + return get_peers_response.json() + + def add_peers(self, peers): + return self.rest_call("post", "admin/v1/peers", json.dumps(peers)) + + def set_relay_subscriptions(self, pubsub_topics): + return self.rest_call("post", "relay/v1/subscriptions", json.dumps(pubsub_topics)) + + def set_relay_auto_subscriptions(self, content_topics): + return self.rest_call("post", "relay/v1/auto/subscriptions", json.dumps(content_topics)) + + def delete_relay_subscriptions(self, pubsub_topics): + return self.rest_call("delete", "relay/v1/subscriptions", json.dumps(pubsub_topics)) + + def delete_relay_auto_subscriptions(self, content_topics): + return self.rest_call("delete", "relay/v1/auto/subscriptions", json.dumps(content_topics)) + + def send_relay_message(self, message, pubsub_topic): + return self.rest_call("post", f"relay/v1/messages/{quote(pubsub_topic, safe='')}", json.dumps(message)) + + def send_relay_auto_message(self, message): + return self.rest_call("post", "relay/v1/auto/messages", json.dumps(message)) + + def send_light_push_message(self, payload): + return self.rest_call("post", "lightpush/v1/message", json.dumps(payload)) + + def get_relay_messages(self, pubsub_topic): + get_messages_response = self.rest_call("get", f"relay/v1/messages/{quote(pubsub_topic, safe='')}") + return get_messages_response.json() + + def get_relay_auto_messages(self, content_topic): + get_messages_response = self.rest_call("get", f"relay/v1/auto/messages/{quote(content_topic, safe='')}") + return get_messages_response.json() + + def set_filter_subscriptions(self, subscription): + set_subscriptions_response = self.rest_call("post", "filter/v2/subscriptions", json.dumps(subscription)) + return set_subscriptions_response.json() + + def update_filter_subscriptions(self, subscription): + update_subscriptions_response = self.rest_call("put", "filter/v2/subscriptions", json.dumps(subscription)) + return update_subscriptions_response.json() + + def delete_filter_subscriptions(self, subscription): + delete_subscriptions_response = self.rest_call("delete", "filter/v2/subscriptions", json.dumps(subscription)) + return delete_subscriptions_response.json() + + def delete_all_filter_subscriptions(self, request_id): + delete_all_subscriptions_response = self.rest_call("delete", "filter/v2/subscriptions/all", json.dumps(request_id)) + return delete_all_subscriptions_response.json() + + def ping_filter_subscriptions(self, request_id): + ping_subscriptions_response = self.rest_call("get", f"filter/v2/subscriptions/{quote(request_id, safe='')}") + return ping_subscriptions_response.json() + + def get_filter_messages(self, content_topic, pubsub_topic=None): + if pubsub_topic is not None: + endpoint = f"filter/v2/messages/{quote(pubsub_topic, safe='')}/{quote(content_topic, safe='')}" + else: + endpoint = f"filter/v2/messages/{quote(content_topic, safe='')}" + get_messages_response = self.rest_call("get", endpoint) + return get_messages_response.json() + + def get_store_messages( + self, + peer_addr, + include_data, + pubsub_topic, + content_topics, + start_time, + end_time, + hashes, + cursor, + page_size, + ascending, + store_v, + encode_pubsubtopic=True, + **kwargs, + ): + base_url = f"store/{store_v}/messages" + params = [] + + if peer_addr is not None: + params.append(f"peerAddr={quote(peer_addr, safe='')}") + if include_data is not None: + params.append(f"includeData={include_data}") + if pubsub_topic is not None: + if encode_pubsubtopic: + params.append(f"pubsubTopic={quote(pubsub_topic, safe='')}") + else: + params.append(f"pubsubTopic={pubsub_topic}") + if content_topics is not None: + params.append(f"contentTopics={quote(content_topics, safe='')}") + if start_time is not None: + params.append(f"startTime={start_time}") + if end_time is not None: + params.append(f"endTime={end_time}") + if hashes is not None: + params.append(f"hashes={quote(hashes, safe='')}") + if cursor is not None: + params.append(f"cursor={quote(cursor, safe='')}") + if page_size is not None: + params.append(f"pageSize={page_size}") + if ascending is not None: + params.append(f"ascending={ascending}") + + # Append any additional keyword arguments to the parameters list + for key, value in kwargs.items(): + if value is not None: + params.append(f"{key}={quote(str(value), safe='')}") + + if params: + base_url += "?" + "&".join(params) + + get_messages_response = self.rest_call("get", base_url) + return get_messages_response.json() + + def set_log_level(self, log_level): + return self.rest_call("post", f"admin/v1/log-level/{quote(log_level)}") + + def get_service_peers(self): + return self.rest_call("get", "admin/v1/peers/service").json() + + def get_connected_peers(self): + return self.rest_call("get", "admin/v1/peers/connected").json() + + def get_connected_peers_on_shard(self, shard_id): + return self.rest_call("get", f"admin/v1/peers/connected/on/{quote(str(shard_id))}").json() + + def get_relay_peers(self): + return self.rest_call("get", "admin/v1/peers/relay").json() + + def get_relay_peers_on_shard(self, shard_id): + return self.rest_call("get", f"admin/v1/peers/relay/on/{quote(str(shard_id))}").json() + + def get_mesh_peers(self): + return self.rest_call("get", "admin/v1/peers/mesh").json() + + def get_mesh_peers_on_shard(self, shard_id): + return self.rest_call("get", f"admin/v1/peers/mesh/on/{quote(str(shard_id))}").json() + + def get_peer_stats(self): + return self.rest_call("get", "admin/v1/peers/stats").json() + + def get_filter_subscriptions(self): + return self.rest_call("get", "admin/v1/filter/subscriptions").json() + + def get_info(self): + return self.rest_call("get", "info").json() + + def get_version(self): + return self.rest_call("get", "version").text.strip() + + def get_debug_version(self): + return self.rest_call("get", "debug/v1/version").text.strip() + + def get_peer(self, peer_id: str): + resp = self.rest_call("get", f"admin/v1/peer/{peer_id}") + return resp.json() diff --git a/tests-e2e/src/node/docker_mananger.py b/tests-e2e/src/node/docker_mananger.py new file mode 100644 index 000000000..44329d098 --- /dev/null +++ b/tests-e2e/src/node/docker_mananger.py @@ -0,0 +1,145 @@ +import os +import re +import time +from src.libs.custom_logger import get_custom_logger +import random +import threading +import docker +from src.env_vars import NETWORK_NAME, SUBNET, IP_RANGE, GATEWAY +from docker.types import IPAMConfig, IPAMPool +from docker.errors import NotFound, APIError + +logger = get_custom_logger(__name__) + + +class DockerManager: + def __init__(self, image): + self._image = image + self._client = docker.from_env() + logger.debug(f"Docker client initialized with image {self._image}") + + def create_network(self, network_name=NETWORK_NAME): + logger.debug(f"Attempting to create or retrieve network {network_name}") + networks = self._client.networks.list(names=[network_name]) + if networks: + logger.debug(f"Network {network_name} already exists") + return networks[0] + + network = self._client.networks.create( + network_name, + driver="bridge", + ipam=IPAMConfig(driver="default", pool_configs=[IPAMPool(subnet=SUBNET, iprange=IP_RANGE, gateway=GATEWAY)]), + ) + logger.debug(f"Network {network_name} created") + return network + + def start_container(self, image_name, ports, args, log_path, container_ip, volumes, remove_container=True): + cli_args = [] + for key, value in args.items(): + if isinstance(value, list): # Check if value is a list + cli_args.extend([f"--{key}={item}" for item in value]) # Add a command for each item in the list + elif value is None: + cli_args.append(f"{key}") # Add simple command as it is passed in the key + else: + cli_args.append(f"--{key}={value}") # Add a single command + + port_bindings = {f"{port}/tcp": ("", port) for port in ports} + port_bindings_for_log = " ".join(f"-p {port}:{port}" for port in ports) + cli_args_str_for_log = " ".join(cli_args) + logger.debug(f"docker run -i -t {port_bindings_for_log} {image_name} {cli_args_str_for_log}") + container = self._client.containers.run( + image_name, command=cli_args, ports=port_bindings, detach=True, remove=remove_container, auto_remove=remove_container, volumes=volumes + ) + + network = self._client.networks.get(NETWORK_NAME) + logger.debug(f"docker network connect --ip {container_ip} {NETWORK_NAME} {container.id}") + network.connect(container, ipv4_address=container_ip) + + logger.debug(f"Container started with ID {container.short_id}. Setting up logs at {log_path}") + log_thread = threading.Thread(target=self._log_container_output, args=(container, log_path)) + log_thread.daemon = True + log_thread.start() + + return container + + def _log_container_output(self, container, log_path): + os.makedirs(os.path.dirname(log_path), exist_ok=True) + retry_count = 0 + start_time = time.time() + try: + with open(log_path, "wb+") as log_file: + while True: + if container.status in ["exited", "dead"]: + logger.info(f"Container {container.short_id} has stopped. Exiting log stream.") + return + try: + for chunk in container.logs(stream=True): + if chunk: + log_file.write(chunk) + log_file.flush() + start_time = time.time() + retry_count = 0 + else: + if time.time() - start_time > 5: + logger.warning(f"Log stream timeout for container {container.short_id}") + return + except (APIError, IOError) as e: + retry_count += 1 + if retry_count >= 5: + logger.error(f"Max retries reached for container {container.short_id}. Exiting log stream.") + return + time.sleep(0.2) + except Exception as e: + return + except Exception as e: + logger.error(f"Failed to set up logging for container {container.short_id}: {e}") + + def generate_ports(self, base_port=None, count=5): + if base_port is None: + base_port = random.randint(1024, 65535 - count) + ports = [str(base_port + i) for i in range(count)] + logger.debug(f"Generated ports {ports}") + return ports + + @staticmethod + def generate_random_ext_ip(): + base_ip_fragments = ["172", "18"] + ext_ip = ".".join(base_ip_fragments + [str(random.randint(0, 255)) for _ in range(2)]) + logger.debug(f"Generated random external IP {ext_ip}") + return ext_ip + + def is_container_running(self, container): + try: + refreshed_container = self._client.containers.get(container.id) + return refreshed_container.status == "running" + except NotFound: + logger.error(f"Container with ID {container.id} not found") + return False + + @property + def image(self): + return self._image + + def search_log_for_keywords(self, log_path, keywords, use_regex=False): + matches = {keyword: [] for keyword in keywords} + + # Open the log file and search line by line + with open(log_path, "r") as log_file: + for line in log_file: + for keyword in keywords: + if use_regex: + if re.search(keyword, line, re.IGNORECASE): + matches[keyword].append(line.strip()) + else: + if keyword.lower() in line.lower(): + matches[keyword].append(line.strip()) + + # Check if there were any matches + if any(matches[keyword] for keyword in keywords): + for keyword, lines in matches.items(): + if lines: + logger.debug(f"Found matches for keyword '{keyword}': {lines}") + return matches + else: + logger.debug("No errors found in the waku logs.") + return None diff --git a/tests-e2e/src/node/store_response.py b/tests-e2e/src/node/store_response.py new file mode 100644 index 000000000..d6274ce78 --- /dev/null +++ b/tests-e2e/src/node/store_response.py @@ -0,0 +1,103 @@ +class StoreResponse: + def __init__(self, store_response, node): + self.response = store_response + self.node = node + + @property + def request_id(self): + try: + if self.node.is_nwaku(): + return self.response.get("requestId") + else: + return self.response.get("request_id") + except: + return None + + @property + def status_code(self): + try: + if self.node.is_nwaku(): + return self.response.get("statusCode") + else: + return self.response.get("status_code") + except: + return None + + @property + def status_desc(self): + try: + if self.node.is_nwaku(): + return self.response.get("statusDesc") + else: + return self.response.get("status_desc") + except: + return None + + @property + def messages(self): + try: + return self.response.get("messages") + except: + return None + + @property + def pagination_cursor(self): + try: + if self.node.is_nwaku(): + return self.response.get("paginationCursor") + else: + return self.response.get("pagination_cursor") + except: + return None + + def message_hash(self, index): + if self.messages is not None: + if self.node.is_nwaku(): + return self.messages[index]["messageHash"] + else: + return self.messages[index]["message_hash"] + else: + return None + + def message_content(self, index): + try: + if self.messages is not None: + payload = self.messages[index]["message"]["contentTopic"] + return payload + else: + return None + except IndexError: + return None + + def message_payload(self, index): + try: + if self.messages is not None: + payload = self.messages[index]["message"]["payload"] + return payload + else: + return None + except IndexError: + return None + + def message_at(self, index): + try: + if self.messages is not None: + message = self.messages[index]["message"] + return message + else: + return None + except IndexError: + return None + + def message_pubsub_topic(self, index): + if self.messages is not None: + if self.node.is_nwaku(): + return self.messages[index]["pubsubTopic"] + else: + return self.messages[index]["pubsub_topic"] + else: + return None + + @property + def resp_json(self): + return self.response diff --git a/tests-e2e/src/node/waku_message.py b/tests-e2e/src/node/waku_message.py new file mode 100644 index 000000000..ea6703ccf --- /dev/null +++ b/tests-e2e/src/node/waku_message.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass, field +from marshmallow_dataclass import class_schema +from typing import Optional, Union +import math +import allure + + +@dataclass +class MessageRpcResponse: + payload: str + contentTopic: str + version: Optional[int] + timestamp: Optional[int] + ephemeral: Optional[bool] + meta: Optional[str] + proof: Optional[str] = field(default_factory=dict) + rateLimitProof: Optional[str] = field(default_factory=dict) + rate_limit_proof: Optional[dict] = field(default_factory=dict) + + +class WakuMessage: + def __init__(self, message_response, schema=MessageRpcResponse): + self.schema = schema + self.received_messages = message_response + self.message_rpc_response_schema = class_schema(self.schema)() + + @allure.step + def assert_received_message(self, sent_message, index=0): + message = self.message_rpc_response_schema.load(self.received_messages[index]) + + def assert_fail_message(field_name): + return f"Incorrect field: {field_name}. Published: {sent_message[field_name]} Received: {getattr(message, field_name)}" + + assert message.payload == sent_message["payload"], assert_fail_message("payload") + assert message.contentTopic == sent_message["contentTopic"], assert_fail_message("contentTopic") + if sent_message.get("timestamp") is not None: + if isinstance(sent_message["timestamp"], float): + assert math.isclose(float(message.timestamp), sent_message["timestamp"], rel_tol=1e-9), assert_fail_message("timestamp") + else: + assert str(message.timestamp) == str(sent_message["timestamp"]), assert_fail_message("timestamp") + if "version" in sent_message: + assert str(message.version) == str(sent_message["version"]), assert_fail_message("version") + if "meta" in sent_message: + assert str(message.meta) == str(sent_message["meta"]), assert_fail_message("meta") + if "ephemeral" in sent_message: + assert str(message.ephemeral) == str(sent_message["ephemeral"]), assert_fail_message("ephemeral") diff --git a/tests-e2e/src/node/waku_node.py b/tests-e2e/src/node/waku_node.py new file mode 100644 index 000000000..8e73fe4cd --- /dev/null +++ b/tests-e2e/src/node/waku_node.py @@ -0,0 +1,683 @@ +import errno +import json +import os +import random +import re +import shutil +import string +import subprocess +import pytest +import requests +from src.libs.common import delay +from src.libs.custom_logger import get_custom_logger +from tenacity import retry, stop_after_delay, wait_fixed, sleep +from docker.errors import NotFound as DockerNotFound +from src.node.api_clients.rest import REST +from src.node.docker_mananger import DockerManager +from src.env_vars import DOCKER_LOG_DIR +from src.data_storage import DS +from src.test_data import DEFAULT_CLUSTER_ID, LOG_ERROR_KEYWORDS, VALID_PUBSUB_TOPICS + +logger = get_custom_logger(__name__) + + +def select_private_key(prv_keys, key_id): + for key in prv_keys: + if key.endswith(key_id): + return key + + raise ValueError("No matching key was found") + + +def sanitize_docker_flags(input_flags): + output_flags = {} + for key, value in input_flags.items(): + key = key.replace("_", "-") + output_flags[key] = value + + return output_flags + + +@retry(stop=stop_after_delay(180), wait=wait_fixed(0.5), reraise=True) +def rln_credential_store_ready(creds_file_path, single_check=False, require_credentials=False): + if os.path.exists(creds_file_path): + subprocess.run(["sudo", "-n", "chmod", "a+r", creds_file_path], check=False) + if require_credentials: + try: + with open(creds_file_path, "r", encoding="utf-8") as creds_file: + keystore_data = json.load(creds_file) + except (OSError, json.JSONDecodeError) as ex: + if single_check: + return False + raise ValueError(f"Failed to parse RLN keystore at {creds_file_path}: {ex}") + + credentials = keystore_data.get("credentials", {}) if isinstance(keystore_data, dict) else {} + if not credentials: + if single_check: + return False + raise ValueError(f"RLN keystore exists but has no credentials yet: {creds_file_path}") + + return True + elif not single_check: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), creds_file_path) + + return False + + +def peer_info2multiaddr(peer, is_nwaku=True): + if is_nwaku: + return peer["multiaddr"] + else: + return peer["multiaddrs"][0] + + +def peer_info2id(peer, is_nwaku=True): + return peer_info2multiaddr(peer, is_nwaku).split("/")[-1] + + +def multiaddr2id(multiaddr): + return multiaddr.split("/")[-1] + + +def resolve_sharding_flags(kwargs): + if "pubsub_topic" in kwargs and kwargs["pubsub_topic"]: + pubsub_topic = kwargs["pubsub_topic"] + if not "cluster_id" in kwargs: + try: + if isinstance(pubsub_topic, list): + pubsub_topic = pubsub_topic[0] + cluster_id = pubsub_topic.split("/")[4] + logger.debug(f"Cluster id was resolved to: {cluster_id}") + kwargs["cluster_id"] = cluster_id + except Exception as ex: + raise Exception("Could not resolve cluster_id from pubsub_topic") + return kwargs + + +class WakuNode: + def __init__(self, docker_image, docker_log_prefix=""): + self._image_name = docker_image + self._log_path = os.path.join(DOCKER_LOG_DIR, f"{docker_log_prefix}__{self._image_name.replace('/', '_')}.log") + self._docker_manager = DockerManager(self._image_name) + self._container = None + self.rln_membership_index = None + self.start_args = {} + self._rln_creds_set = False + logger.debug(f"WakuNode instance initialized with log path {self._log_path}") + + @retry(stop=stop_after_delay(60), wait=wait_fixed(0.1), reraise=True) + def start(self, wait_for_node_sec=20, **kwargs): + logger.debug("Starting Node...") + default_args, remove_container = self._prepare_start_context(**kwargs) + self._start_docker(default_args, remove_container, wait_for_node_sec) + + def _prepare_start_context(self, **kwargs): + self._docker_manager.create_network() + self._ext_ip = self._docker_manager.generate_random_ext_ip() + self._ports = self._docker_manager.generate_ports() + self._rest_port = self._ports[0] + self._tcp_port = self._ports[1] + self._websocket_port = self._ports[2] + self._discv5_port = self._ports[3] + self._metrics_port = self._ports[4] + self._api = REST(self._rest_port) + self._volumes = [] + + default_args = { + "listen-address": "0.0.0.0", + "rest": "true", + "rest-admin": "true", + "websocket-support": "true", + "log-level": "TRACE", + "rest-relay-cache-capacity": "100", + "websocket-port": self._websocket_port, + "rest-port": self._rest_port, + "tcp-port": self._tcp_port, + "discv5-udp-port": self._discv5_port, + "rest-address": "0.0.0.0", + "nat": f"extip:{self._ext_ip}", + "peer-exchange": "true", + "discv5-discovery": "true", + "cluster-id": DEFAULT_CLUSTER_ID, + "rln-creds-id": None, + "rln-creds-source": None, + "nodekey": self.generate_random_nodekey(), + # "max-connections": "50", + # "filter-subscription-timeout": "600", + } + + store_sync = kwargs.pop("store_sync", "false").lower() == "true" + store_sync_range = kwargs.pop("store_sync_range", None) + store_sync_interval = kwargs.pop("store_sync_interval", None) + + if store_sync: + default_args["store-sync"] = "true" + default_args.setdefault("store", "true") + + if store_sync_range: + default_args["store-sync-range"] = store_sync_range + + if store_sync_interval: + default_args["store-sync-interval"] = store_sync_interval + + if self.is_nwaku(): + nwaku_args = { + "shard": "0", + "metrics-server": "true", + "metrics-server-address": "0.0.0.0", + "metrics-server-port": self._metrics_port, + "metrics-logging": "true", + } + default_args.update(nwaku_args) + else: + raise NotImplementedError("Not implemented for this node type") + + if "remove_container" in kwargs: + remove_container = kwargs["remove_container"] + del kwargs["remove_container"] + else: + remove_container = True + + kwargs = self.parse_peer_persistence_config(kwargs) + kwargs = resolve_sharding_flags(kwargs) + + default_args.update(sanitize_docker_flags(kwargs)) + + if self.is_nwaku() and "pubsub-topic" in default_args: + logger.debug("Removing pubsub-topic from nwaku args") + del default_args["pubsub-topic"] + + rln_args, rln_creds_set, keystore_path = self.parse_rln_credentials(default_args, False) + self._rln_creds_set = rln_creds_set + + default_args.pop("rln-creds-id", None) + default_args.pop("rln-creds-source", None) + default_args.pop("rln-keystore-prefix", None) + + if rln_creds_set: + rln_credential_store_ready(keystore_path, require_credentials=True) + default_args.update(rln_args) + else: + logger.info(f"RLN credentials not set or credential store not available, starting without RLN") + + self.start_args = dict(default_args) + return default_args, remove_container + + def _start_docker(self, default_args, remove_container, wait_for_node_sec): + logger.debug(f"Using volumes {self._volumes}") + + self._container = self._docker_manager.start_container( + self._docker_manager.image, + ports=self._ports, + args=default_args, + log_path=self._log_path, + container_ip=self._ext_ip, + volumes=self._volumes, + remove_container=remove_container, + ) + logger.debug(f"Started container from image {self._image_name}. REST: {self._rest_port}") + DS.waku_nodes.append(self) + delay(1) + try: + self.ensure_ready(timeout_duration=wait_for_node_sec, rln_required=self._rln_creds_set) + except Exception as ex: + logger.error(f"REST service did not become ready in time: {ex}") + raise + + @retry(stop=stop_after_delay(250), wait=wait_fixed(0.1), reraise=True) + def register_rln(self, **kwargs): + logger.debug("Registering RLN credentials...") + self._docker_manager.create_network() + self._ext_ip = self._docker_manager.generate_random_ext_ip() + self._ports = self._docker_manager.generate_ports() + self._rest_port = self._ports[0] + self._api = REST(self._rest_port) + self._volumes = [] + + default_args = {"rln-creds-id": None, "rln-creds-source": None, "rln-relay-user-message-limit-registration": 100} + + default_args.update(sanitize_docker_flags(kwargs)) + + rln_args, rln_creds_set, keystore_path = self.parse_rln_credentials(default_args, True) + + if rln_creds_set: + self._container = self._docker_manager.start_container( + self._docker_manager.image, self._ports, rln_args, self._log_path, self._ext_ip, self._volumes + ) + + logger.debug(f"Executed container from image {self._image_name}. REST: {self._rest_port} to register RLN") + + logger.debug(f"Waiting for keystore {keystore_path}") + try: + rln_credential_store_ready(keystore_path, require_credentials=True) + self.rln_membership_index = str(self.get_rln_membership_index_from_log()) + logger.debug(f"Detected RLN membership index from registration logs: {self.rln_membership_index}") + self.stop() + except Exception as ex: + logger.error(f"File {keystore_path} with RLN credentials did not become available in time {ex}") + raise + else: + logger.warn("RLN credentials not set, no action performed") + + return self.rln_membership_index + + @retry(stop=stop_after_delay(10), wait=wait_fixed(0.2), reraise=True) + def get_rln_membership_index_from_log(self): + if not os.path.exists(self._log_path): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self._log_path) + + with open(self._log_path, "r", encoding="utf-8", errors="ignore") as log_file: + log_data = log_file.read() + + matches = re.findall(r"membershipIndex=(\d+)", log_data) + if not matches: + raise ValueError("Could not infer RLN membership index from registration logs") + + return int(matches[-1]) + + @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) + def stop(self): + if self._container: + logger.debug(f"Stopping container with id {self._container.short_id}") + try: + self._container.stop() + except DockerNotFound: + logger.debug(f"Container {self._container.short_id} already exited and removed, treating as stopped.") + self._container = None + return + try: + self._container.remove() + except: + pass + self._container = None + logger.debug("Container stopped.") + + @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True) + def kill(self): + if self._container: + logger.debug(f"Killing container with id {self._container.short_id}") + self._container.kill() + try: + self._container.remove() + except: + pass + self._container = None + logger.debug("Container killed.") + + def restart(self): + if self._container: + logger.debug(f"Restarting container with id {self._container.short_id}") + self._container.restart() + + def pause(self): + if self._container: + logger.debug(f"Pausing container with id {self._container.short_id}") + self._container.pause() + + def unpause(self): + if self._container: + logger.debug(f"Unpause container with id {self._container.short_id}") + self._container.unpause() + + def ensure_ready(self, timeout_duration=10, rln_required=False): + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(0.1), reraise=True) + def check_healthy(node=self): + self.health_response = node.health() + + try: + self.health_response = json.loads(self.health_response) + except Exception as ex: + raise AttributeError(f"Unknown health response format {ex}") + + if self.health_response.get("nodeHealth") != "READY": + raise AssertionError("Waiting for the node health status: READY") + + for p in self.health_response.get("protocolsHealth"): + if rln_required and "Rln Relay" in p: + if p["Rln Relay"] != "READY": + raise AssertionError("Waiting for the Rln relay status: READY") + # TODO: Remove once Rln Relay reflects true RLN status + sleep(20) + + logger.info("Node protocols are initialized !!") + + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(0.1), reraise=True) + def check_ready(node=self): + node.info_response = node.info() + logger.info("REST service is ready !!") + + if self.is_nwaku(): + check_healthy() + check_ready() + + def get_id(self): + try: + return self.info_response["listenAddresses"][0].split("/")[-1] + except Exception as ex: + raise AttributeError(f"Could not find ID in the info call because of error: {str(ex)}") + + def get_enr_uri(self): + try: + return self.info_response["enrUri"] + except Exception as ex: + raise AttributeError(f"Could not find enrUri in the info call because of error: {str(ex)}") + + def get_multiaddr_with_id(self): + addresses = self.info_response.get("listenAddresses", []) + ws_address = next((addr for addr in addresses if "/ws" not in addr), None) + if ws_address: + identifier = ws_address.split("/p2p/")[-1] + new_address = f"{self.get_tcp_address()}/p2p/{identifier}" + return new_address + else: + raise AttributeError("No '/ws' address found") + + def get_tcp_address(self): + return f"/ip4/{self._ext_ip}/tcp/{self._tcp_port}" + + def subscribe_content_topic(self, content_topic: str): + return self._api.set_relay_auto_subscriptions([content_topic]) + + def unsubscribe_content_topic(self, content_topic: str): + return self._api.delete_relay_auto_subscriptions([content_topic]) + + def send_message(self, message: dict): + return self._api.send_relay_auto_message(message) + + def info(self): + return self._api.info() + + def health(self): + return self._api.health() + + def get_peers(self): + return self._api.get_peers() + + def add_peers(self, peers): + return self._api.add_peers(peers) + + def set_relay_subscriptions(self, pubsub_topics): + return self._api.set_relay_subscriptions(pubsub_topics) + + def set_relay_auto_subscriptions(self, content_topics): + return self._api.set_relay_auto_subscriptions(content_topics) + + def delete_relay_subscriptions(self, pubsub_topics): + return self._api.delete_relay_subscriptions(pubsub_topics) + + def delete_relay_auto_subscriptions(self, content_topics): + return self._api.delete_relay_auto_subscriptions(content_topics) + + def send_relay_message(self, message, pubsub_topic): + return self._api.send_relay_message(message, pubsub_topic) + + def send_relay_auto_message(self, message): + return self._api.send_relay_auto_message(message) + + def send_light_push_message(self, payload): + return self._api.send_light_push_message(payload) + + def get_relay_messages(self, pubsub_topic): + return self._api.get_relay_messages(pubsub_topic) + + def get_relay_auto_messages(self, content_topic): + return self._api.get_relay_auto_messages(content_topic) + + def set_filter_subscriptions(self, subscription): + return self._api.set_filter_subscriptions(subscription) + + def update_filter_subscriptions(self, subscription): + return self._api.update_filter_subscriptions(subscription) + + def delete_filter_subscriptions(self, subscription): + return self._api.delete_filter_subscriptions(subscription) + + def delete_all_filter_subscriptions(self, request_id): + return self._api.delete_all_filter_subscriptions(request_id) + + def ping_filter_subscriptions(self, request_id): + return self._api.ping_filter_subscriptions(request_id) + + def get_filter_messages(self, content_topic, pubsub_topic=None): + return self._api.get_filter_messages(content_topic, pubsub_topic) + + def get_store_messages( + self, + peer_addr=None, + include_data=None, + pubsub_topic=None, + content_topics=None, + start_time=None, + end_time=None, + hashes=None, + cursor=None, + page_size=None, + ascending=None, + store_v="v3", + **kwargs, + ): + return self._api.get_store_messages( + peer_addr=peer_addr, + include_data=include_data, + pubsub_topic=pubsub_topic, + content_topics=content_topics, + start_time=start_time, + end_time=end_time, + hashes=hashes, + cursor=cursor, + page_size=page_size, + ascending=ascending, + store_v=store_v, + **kwargs, + ) + + def get_metrics(self): + if self.is_nwaku(): + metrics = requests.get(f"http://localhost:{self._metrics_port}/metrics") + metrics.raise_for_status() + return metrics.content.decode("utf-8") + else: + pytest.skip(f"This method doesn't exist for node {self.type()}") + + @property + def image(self): + return self._image_name + + def type(self): + if self.is_nwaku(): + return "nwaku" + else: + raise ValueError("Unknown node type!!!") + + def is_nwaku(self): + return "nwaku" in self.image + + def prepare_rln_storage_paths(self, cwd, keystore_prefix, selected_id, reset_existing=False): + keystore_dir = os.path.join(cwd, f"keystore_{keystore_prefix}_{selected_id}") + rln_tree_dir = os.path.join(cwd, f"rln_tree_{keystore_prefix}_{selected_id}") + + if reset_existing: + for path, path_name in [(keystore_dir, "keystore"), (rln_tree_dir, "rln tree")]: + if os.path.exists(path): + logger.warning(f"Resetting existing RLN {path_name} directory before registration: {path}") + shutil.rmtree(path, ignore_errors=True) + + os.makedirs(keystore_dir, exist_ok=True) + os.makedirs(rln_tree_dir, exist_ok=True) + + return keystore_dir, rln_tree_dir + + def parse_rln_credentials(self, default_args, is_registration): + rln_args = {} + keystore_path = None + + rln_creds_source = default_args["rln-creds-source"] + selected_id = default_args["rln-creds-id"] + + if rln_creds_source is None or selected_id is None: + logger.debug(f"RLN credentials were not set") + return rln_args, False, keystore_path + + imported_creds = json.loads(rln_creds_source) + rln_chain_id = imported_creds.get("rln-relay-chain-id") + if rln_chain_id is None: + eth_client_address = imported_creds.get("rln-relay-eth-client-address", "") + if "linea" in eth_client_address: + rln_chain_id = "59141" + elif "sepolia" in eth_client_address: + rln_chain_id = "11155111" + + if len(imported_creds) < 4 or any(value is None for value in imported_creds.values()): + logger.warn(f"One or more of required RLN credentials were not set properly") + return rln_args, False, keystore_path + + eth_private_key = select_private_key(imported_creds, selected_id) + + cwd = os.getcwd() + keystore_prefix = default_args.get("rln-keystore-prefix") + + if not keystore_prefix: + logger.warn("rln-keystore-prefix is missing, cannot mount RLN state and keystore") + return rln_args, False, keystore_path + + keystore_dir, rln_tree_dir = self.prepare_rln_storage_paths(cwd, keystore_prefix, selected_id, reset_existing=is_registration) + + if self.is_nwaku(): + if is_registration: + rln_args.update( + { + "generateRlnKeystore": None, + "--execute": None, + "rln-relay-user-message-limit": default_args["rln-relay-user-message-limit-registration"], + } + ) + else: + rln_args.update( + { + "rln-relay": "true", + } + ) + + if not default_args.get("rln-relay-dynamic") and not is_registration: + rln_args.update( + { + "rln-relay-cred-path": "/keystore/keystore.json", + "rln-relay-cred-password": imported_creds["rln-relay-cred-password"], + "rln-relay-eth-client-address": imported_creds["rln-relay-eth-client-address"], + "rln-relay-eth-contract-address": imported_creds["rln-relay-eth-contract-address"], + } + ) + else: + rln_args.update( + { + "rln-relay-cred-path": "/keystore/keystore.json", + "rln-relay-cred-password": imported_creds["rln-relay-cred-password"], + "rln-relay-eth-client-address": imported_creds["rln-relay-eth-client-address"], + "rln-relay-eth-contract-address": imported_creds["rln-relay-eth-contract-address"], + "rln-relay-eth-private-key": imported_creds[eth_private_key], + } + ) + + if rln_chain_id is not None: + rln_args["rln-relay-chain-id"] = str(rln_chain_id) + + keystore_path = os.path.join(keystore_dir, "keystore.json") + + self._volumes.extend( + [ + f"{rln_tree_dir}:/etc/rln_tree", + f"{keystore_dir}:/keystore", + ] + ) + + else: + raise NotImplementedError("Not implemented for type other than Nim Waku ") + + return rln_args, True, keystore_path + + def parse_peer_persistence_config(self, kwargs): + if kwargs.get("peer_persistence") == "true": + cwd = os.getcwd() + # Please note, as of now, peerdb is stored directly at / which is not shareable between containers. + # Volume related code is usable after https://github.com/waku-org/nwaku/issues/2792 would be resolved. + self._volumes.extend( + [ + cwd + "/peerdb" + ":/shared", + ] + ) + + shutil.rmtree(cwd + "/peerdb") + + return kwargs + + @property + def container(self): + return self._container + + def generate_random_nodekey(self): + # Define the set of hexadecimal characters + hex_chars = string.hexdigits.lower() + # Generate a random 64-character string from the hex characters + random_key = "".join(random.choice(hex_chars) for _ in range(64)) + return random_key + + def search_waku_log_for_string(self, search_pattern, use_regex=False): + return self._docker_manager.search_log_for_keywords(self._log_path, [search_pattern], use_regex) + + def check_waku_log_errors(self, whitelist=None): + keywords = LOG_ERROR_KEYWORDS + + # If a whitelist is provided, remove those keywords from the keywords list + if whitelist: + keywords = [keyword for keyword in keywords if keyword not in whitelist] + + matches = self._docker_manager.search_log_for_keywords(self._log_path, keywords, False) + assert not matches, f"Found errors {matches}" + + def set_log_level(self, log_level): + return self._api.set_log_level(log_level) + + def get_service_peers(self): + return self._api.get_service_peers() + + def get_connected_peers(self): + return self._api.get_connected_peers() + + def get_connected_peers_on_shard(self, shard_id): + return self._api.get_connected_peers_on_shard(shard_id) + + def get_relay_peers(self): + return self._api.get_relay_peers() + + def get_relay_peers_on_shard(self, shard_id): + return self._api.get_relay_peers_on_shard(shard_id) + + def get_mesh_peers(self): + return self._api.get_mesh_peers() + + def get_mesh_peers_on_shard(self, shard_id): + return self._api.get_mesh_peers_on_shard(shard_id) + + def get_peer_stats(self): + return self._api.get_peer_stats() + + def get_filter_subscriptions(self): + return self._api.get_filter_subscriptions() + + def get_info(self): + return self._api.get_info() + + def get_version(self): + return self._api.get_version() + + def get_debug_version(self): + return self._api.get_debug_version() + + def get_peer_info(self, peer_id: str): + return self._api.get_peer(peer_id) + + @property + def container_id(self) -> str: + if not self._container: + raise RuntimeError("Node container not started yet") + return self._container.id diff --git a/tests-e2e/src/node/wrapper_helpers.py b/tests-e2e/src/node/wrapper_helpers.py new file mode 100644 index 000000000..7c9dd1a2d --- /dev/null +++ b/tests-e2e/src/node/wrapper_helpers.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import json +import threading +import time +from typing import Optional +from src.libs.common import to_base64 + +DEFAULT_CONTENT_TOPIC = "/test/1/default/proto" +DEFAULT_PAYLOAD = to_base64("test payload") +EVENT_PROPAGATED = "message_propagated" +EVENT_SENT = "message_sent" +EVENT_ERROR = "message_error" + +# MaxTimeInCache from send_service.nim. +MAX_TIME_IN_CACHE_S = 60.0 +# Extra slack to cover the background retry loop tick after the window expires. +CACHE_EXPIRY_SLACK_S = 10.0 +ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S = MAX_TIME_IN_CACHE_S + CACHE_EXPIRY_SLACK_S +RETRY_WINDOW_EXPIRED_MSG = "Unable to send within retry time window" + + +class EventCollector: + """Thread-safe collector for async node events. + + Pass `collector.event_callback` as the `event_cb` argument to + WrapperManager.create_and_start(). Every event fired by the library + is decoded from JSON and appended to `self.events`. + """ + + def __init__(self): + self._lock = threading.Lock() + self.events: list[dict] = [] + + def event_callback(self, ret: int, raw: bytes) -> None: + try: + payload = json.loads(raw.decode("utf-8")) + except Exception: + payload = {"_raw": raw.decode("utf-8", errors="replace"), "_ret": ret} + + with self._lock: + self.events.append(payload) + + def get_events_for_request(self, request_id: str) -> list[dict]: + with self._lock: + return [e for e in self.events if e.get("requestId") == request_id] + + def snapshot(self) -> list[dict]: + """Return a thread-safe copy of all collected events. + + Use this whenever you need to iterate over every event (rather than + events for a single request_id). Iterating `self.events` directly is + unsafe because `event_callback` appends from the wrapper's event + thread. + """ + with self._lock: + return list(self.events) + + +def is_propagated_event(event: dict) -> bool: + return event.get("eventType") == EVENT_PROPAGATED + + +def is_sent_event(event: dict) -> bool: + return event.get("eventType") == EVENT_SENT + + +def is_error_event(event: dict) -> bool: + return event.get("eventType") == EVENT_ERROR + + +def wait_for_event( + collector: EventCollector, + request_id: str, + predicate, + timeout_s: float, + poll_interval_s: float = 0.5, +) -> Optional[dict]: + """Poll until an event matching `predicate` arrives for `request_id`, + or until `timeout_s` elapses. Returns the matching event or None. + """ + deadline = time.monotonic() + timeout_s + + while True: + for event in collector.get_events_for_request(request_id): + if predicate(event): + return event + if time.monotonic() >= deadline: + return None + time.sleep(poll_interval_s) + + +def wait_for_propagated(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]: + return wait_for_event(collector, request_id, is_propagated_event, timeout_s) + + +def wait_for_sent(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]: + return wait_for_event(collector, request_id, is_sent_event, timeout_s) + + +def wait_for_error(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]: + return wait_for_event(collector, request_id, is_error_event, timeout_s) + + +def assert_no_error(collector: EventCollector, request_id: str, context: str = "") -> None: + """Assert that no message_error event is currently buffered for `request_id`.""" + event = wait_for_error(collector, request_id, timeout_s=0) + suffix = f" ({context})" if context else "" + assert event is None, f"Unexpected message_error event{suffix}: {event}" + + +def assert_no_sent(collector: EventCollector, request_id: str, context: str = "") -> None: + """Assert that no message_sent event is currently buffered for `request_id`.""" + event = wait_for_sent(collector, request_id, timeout_s=0) + suffix = f" ({context})" if context else "" + assert event is None, f"Unexpected message_sent event{suffix}: {event}" + + +def assert_no_propagated(collector: EventCollector, request_id: str, context: str = "") -> None: + """Assert that no message_propagated event is currently buffered for `request_id`.""" + event = wait_for_propagated(collector, request_id, timeout_s=0) + suffix = f" ({context})" if context else "" + assert event is None, f"Unexpected message_propagated event{suffix}: {event}" + + +def wait_for_connected( + collector: EventCollector, + timeout_s: float = 10.0, + poll_interval_s: float = 0.3, +) -> Optional[dict]: + """Wait until a connection_status_change event with PartiallyConnected or Connected arrives.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + for event in collector.snapshot(): + if event.get("eventType") == "connection_status_change" and event.get("connectionStatus") in ("PartiallyConnected", "Connected"): + return event + time.sleep(poll_interval_s) + return None + + +TERMINAL_EVENT_TYPES = {EVENT_PROPAGATED, EVENT_SENT, EVENT_ERROR} + + +def assert_event_invariants(collector: EventCollector, request_id: str) -> None: + """Check per-request event invariants (issue #163): + - All events carry the correct requestId. + - No duplicate terminal events (Propagated, Sent, Error). + - Sent never appears before Propagated. + """ + events = collector.get_events_for_request(request_id) + assert events, f"No events found for request {request_id}" + + counts: dict[str, int] = {} + first_index: dict[str, int] = {} + for i, event in enumerate(events): + assert event.get("requestId") == request_id, ( + f"Event at index {i} has wrong requestId: " f"expected {request_id!r}, got {event.get('requestId')!r}" + ) + event_type = event.get("eventType", "") + if event_type in TERMINAL_EVENT_TYPES: + counts[event_type] = counts.get(event_type, 0) + 1 + if event_type not in first_index: + first_index[event_type] = i + + for event_type, count in counts.items(): + assert count == 1, f"Duplicate {event_type} events for request {request_id}: " f"got {count}, expected 1. Events: {events}" + + if EVENT_SENT in first_index and EVENT_PROPAGATED in first_index: + assert first_index[EVENT_PROPAGATED] < first_index[EVENT_SENT], ( + f"message_sent (index {first_index[EVENT_SENT]}) arrived before " + f"message_propagated (index {first_index[EVENT_PROPAGATED]}) " + f"for request {request_id}. Events: {events}" + ) + + +def get_node_multiaddr(node) -> str: + """Return the TCP multiaddr (with peer-id) from a WrapperManager node. + + Asserts that the wrapper returned exactly one address. If the wrapper ever + starts returning multiple addresses (newline/comma-separated or a JSON + list), this fails loudly instead of silently passing a malformed string + downstream to staticnodes / add_peers. + """ + result = node.get_node_info_raw("MyMultiaddresses") + if result.is_err(): + raise RuntimeError(f"get_node_info_raw failed: {result.err()}") + + addr = result.ok_value.strip() + if not addr or not addr.startswith("/"): + raise RuntimeError(f"Unexpected multiaddr format: {addr!r}") + + if "\n" in addr or "," in addr or addr.startswith("["): + raise AssertionError(f"Expected a single multiaddr from MyMultiaddresses, got multiple: {addr!r}") + + return addr + + +def create_message_bindings(**overrides) -> dict: + envelope = { + "contentTopic": DEFAULT_CONTENT_TOPIC, + "payload": DEFAULT_PAYLOAD, + "ephemeral": False, + } + envelope.update(overrides) + return envelope + + +def assert_no_unknown_request_ids(collector: EventCollector, issued_request_ids) -> None: + """Cross-association guard: every event carrying a requestId must belong + to one of the request ids we issued. Catches events that get attached to + the wrong request id under concurrency. + """ + issued = set(issued_request_ids) + for event in collector.snapshot(): + event_request_id = event.get("requestId") + if event_request_id is None: + continue + assert event_request_id in issued, f"Event carries an unknown requestId={event_request_id!r}, " f"not in issued set {issued}. Event: {event}" diff --git a/tests-e2e/src/node/wrappers_manager.py b/tests-e2e/src/node/wrappers_manager.py new file mode 100644 index 000000000..a0d20e0d6 --- /dev/null +++ b/tests-e2e/src/node/wrappers_manager.py @@ -0,0 +1,98 @@ +import sys +from pathlib import Path +from result import Result, Ok, Err + +_BINDINGS_PATH = Path(__file__).resolve().parents[2] / "vendor" / "logos-delivery-python-bindings" / "waku" +if str(_BINDINGS_PATH) not in sys.path: + sys.path.insert(0, str(_BINDINGS_PATH)) + +from wrapper import NodeWrapper as _NodeWrapper # type: ignore[import] + +"""" +thin manager/wrapper layer around NodeWrapper from the bindings. +It simplifies create, start, and interaction with a Waku node while returning consistent Result objects (Ok / Err). +""" + + +class WrapperManager: + def __init__(self, node: _NodeWrapper): + self._node = node + + @classmethod + def create( + cls, + config: dict, + event_cb=None, + *, + timeout_s: float = 20.0, + ) -> Result["WrapperManager", str]: + result = _NodeWrapper.create_node(config, event_cb, timeout_s=timeout_s) + if result.is_err(): + return Err(result.err()) + return Ok(cls(result.ok_value)) + + @classmethod + def create_and_start( + cls, + config: dict, + event_cb=None, + *, + timeout_s: float = 20.0, + ) -> Result["WrapperManager", str]: + result = _NodeWrapper.create_and_start(config, event_cb, timeout_s=timeout_s) + if result.is_err(): + return Err(result.err()) + return Ok(cls(result.ok_value)) + + def __enter__(self) -> "WrapperManager": + return self + + def __exit__(self, *_) -> None: + self.stop_and_destroy() + + def start_node(self, *, timeout_s: float = 20.0) -> Result[int, str]: + return self._node.start_node(timeout_s=timeout_s) + + def stop_node(self, *, timeout_s: float = 20.0) -> Result[int, str]: + return self._node.stop_node(timeout_s=timeout_s) + + def destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]: + return self._node.destroy(timeout_s=timeout_s) + + def stop_and_destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]: + return self._node.stop_and_destroy(timeout_s=timeout_s) + + def subscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]: + return self._node.subscribe_content_topic(content_topic, timeout_s=timeout_s) + + def unsubscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]: + return self._node.unsubscribe_content_topic(content_topic, timeout_s=timeout_s) + + def send_message(self, message: dict, *, timeout_s: float = 20.0) -> Result[str, str]: + return self._node.send_message(message, timeout_s=timeout_s) + + def get_available_node_info_ids(self, *, timeout_s: float = 20.0) -> Result[list[str], str]: + return self._node.get_available_node_info_ids(timeout_s=timeout_s) + + def get_node_info(self, node_info_id: str, *, timeout_s: float = 20.0) -> Result[dict, str]: + return self._node.get_node_info(node_info_id, timeout_s=timeout_s) + + def get_node_info_raw(self, node_info_id: str, *, timeout_s: float = 20.0) -> Result[str, str]: + """Like get_node_info but returns the raw string without JSON parsing.""" + from wrapper import lib, ffi, _new_cb_state, _wait_cb_raw # type: ignore[import] + + state = _new_cb_state() + cb = self._node._make_waiting_cb(state) + rc = lib.logosdelivery_get_node_info(self._node.ctx, cb, ffi.NULL, node_info_id.encode("utf-8")) + if rc != 0: + return Err(f"get_node_info_raw: immediate call failed (ret={rc})") + wait_result = _wait_cb_raw(state, "get_node_info_raw", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + cb_ret, cb_msg = wait_result.ok_value + if cb_ret != 0: + return Err(f"get_node_info_raw: callback failed (ret={cb_ret})") + return Ok(cb_msg.decode("utf-8") if cb_msg else "") + + def get_available_configs(self, *, timeout_s: float = 20.0) -> Result[dict, str]: + return self._node.get_available_configs(timeout_s=timeout_s) diff --git a/tests-e2e/src/postgres_setup.py b/tests-e2e/src/postgres_setup.py new file mode 100644 index 000000000..97d47c954 --- /dev/null +++ b/tests-e2e/src/postgres_setup.py @@ -0,0 +1,44 @@ +import docker +import os +from src.env_vars import NETWORK_NAME, PG_PASS, PG_USER +from src.libs.custom_logger import get_custom_logger + +logger = get_custom_logger(__name__) + + +def start_postgres(): + pg_env = {"POSTGRES_USER": PG_USER, "POSTGRES_PASSWORD": PG_PASS} + + base_path = os.path.abspath(".") + volumes = {os.path.join(base_path, "postgresql"): {"bind": "/var/lib/postgresql/data", "mode": "Z"}} + + client = docker.from_env() + + postgres_container = client.containers.run( + "postgres:15.4-alpine3.18", + name="postgres", + environment=pg_env, + volumes=volumes, + command="postgres", + ports={"5432/tcp": ("127.0.0.1", 5432)}, + restart_policy={"Name": "on-failure", "MaximumRetryCount": 5}, + healthcheck={ + "Test": ["CMD-SHELL", "pg_isready -U postgres -d postgres"], + "Interval": 30000000000, # 30 seconds in nanoseconds + "Timeout": 60000000000, # 60 seconds in nanoseconds + "Retries": 5, + "StartPeriod": 80000000000, # 80 seconds in nanoseconds + }, + detach=True, + network_mode=NETWORK_NAME, + ) + + logger.debug("Postgres container started") + + return postgres_container + + +def stop_postgres(postgres_container): + postgres_container.stop() + postgres_container.remove() + logger.debug("Postgres container stopped and removed.") diff --git a/tests-e2e/src/steps/__init__.py b/tests-e2e/src/steps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/src/steps/common.py b/tests-e2e/src/steps/common.py new file mode 100644 index 000000000..ccdfcf533 --- /dev/null +++ b/tests-e2e/src/steps/common.py @@ -0,0 +1,80 @@ +import base64 +import hashlib +import inspect +from time import time +from time import time_ns +import allure +import pytest +from datetime import timedelta, datetime +from tenacity import retry, stop_after_delay, wait_fixed +from src.libs.common import delay, to_base64 +from src.libs.custom_logger import get_custom_logger + +logger = get_custom_logger(__name__) + + +class StepsCommon: + @pytest.fixture(scope="function", autouse=True) + def common_setup(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + if not hasattr(self, "test_payload"): + self.test_payload = "Default Payload" + if not hasattr(self, "test_content_topic"): + self.test_content_topic = "/test/1/default/proto" + + @allure.step + @retry(stop=stop_after_delay(20), wait=wait_fixed(0.5), reraise=True) + def add_node_peer(self, node, multiaddr_list, shards=[0, 1, 2, 3, 4, 5, 6, 7, 8]): + if node.is_nwaku(): + for multiaddr in multiaddr_list: + node.add_peers([multiaddr]) + + @allure.step + @retry(stop=stop_after_delay(70), wait=wait_fixed(1), reraise=True) + def wait_for_autoconnection(self, node_list, hard_wait=None): + for node in node_list: + get_peers = node.get_peers() + assert len(get_peers) >= 1 + if hard_wait: + delay(hard_wait) + + @allure.step + def create_message(self, **kwargs): + ts_ns = time_ns() + ts_ns = int(f"{ts_ns:019d}") + message = {"payload": to_base64(self.test_payload), "contentTopic": self.test_content_topic, "timestamp": ts_ns} + message.update(kwargs) + return message + + @allure.step + def compute_message_hash(self, pubsub_topic, msg, hash_type="hex"): + ctx = hashlib.sha256() + ctx.update(pubsub_topic.encode("utf-8")) + ctx.update(base64.b64decode(msg["payload"])) + ctx.update(msg["contentTopic"].encode("utf-8")) + if "meta" in msg: + ctx.update(base64.b64decode(msg["meta"])) + ctx.update(int(msg["timestamp"]).to_bytes(8, byteorder="big")) + hash_bytes = ctx.digest() + if hash_type == "hex": + return "0x" + hash_bytes.hex() + else: + return base64.b64encode(hash_bytes).decode("utf-8") + + def get_time_list_pass(self): + ts_pass = [ + {"description": "3 sec Past", "value": int((datetime.now() - timedelta(seconds=3)).timestamp() * 1e9)}, + {"description": "1 sec Past", "value": int((datetime.now() - timedelta(seconds=1)).timestamp() * 1e9)}, + {"description": "0.1 sec Past", "value": int((datetime.now() - timedelta(seconds=0.1)).timestamp() * 1e9)}, + {"description": "0.1 sec Future", "value": int((datetime.now() + timedelta(seconds=0.1)).timestamp() * 1e9)}, + {"description": "2 sec Future", "value": int((datetime.now() + timedelta(seconds=2)).timestamp() * 1e9)}, + {"description": "10 sec Future", "value": int((datetime.now() + timedelta(seconds=10)).timestamp() * 1e9)}, + ] + return ts_pass + + def get_time_list_fail(self): + ts_fail = [ + {"description": "20 sec Past", "value": int((datetime.now() - timedelta(seconds=20)).timestamp() * 1e9)}, + {"description": "40 sec Future", "value": int((datetime.now() + timedelta(seconds=40)).timestamp() * 1e9)}, + ] + return ts_fail diff --git a/tests-e2e/src/steps/filter.py b/tests-e2e/src/steps/filter.py new file mode 100644 index 000000000..8a5d90c41 --- /dev/null +++ b/tests-e2e/src/steps/filter.py @@ -0,0 +1,197 @@ +import inspect +from uuid import uuid4 +from src.libs.custom_logger import get_custom_logger +from time import time +import pytest +import allure +from src.libs.common import to_base64, delay +from src.node.waku_message import WakuMessage +from src.env_vars import NODE_1, NODE_2, ADDITIONAL_NODES +from src.node.waku_node import WakuNode +from tenacity import retry, stop_after_delay, wait_fixed +from src.steps.common import StepsCommon +from src.test_data import VALID_PUBSUB_TOPICS + +logger = get_custom_logger(__name__) + + +class StepsFilter(StepsCommon): + test_pubsub_topic = VALID_PUBSUB_TOPICS[1] + second_pubsub_topic = VALID_PUBSUB_TOPICS[2] + another_cluster_pubsub_topic = "/waku/2/rs/2/2" + test_content_topic = "/test/1/waku-filter/proto" + second_content_topic = "/test/2/waku-filter/proto" + test_payload = "Filter works!!" + + @pytest.fixture(scope="function", autouse=True) + def filter_setup(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.main_nodes = [] + self.optional_nodes = [] + + @pytest.fixture(scope="function") + def setup_main_relay_node(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.relay_node_start(NODE_1) + + @pytest.fixture(scope="function") + def setup_main_filter_node(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}") + self.node2.start(relay="false", discv5_bootstrap_node=self.enr_uri, filternode=self.multiaddr_with_id) + self.add_node_peer(self.node2, [self.multiaddr_with_id]) + self.main_nodes.append(self.node2) + + @pytest.fixture(scope="function") + def subscribe_main_nodes(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.wait_for_subscriptions_on_main_nodes([self.test_content_topic]) + + @pytest.fixture(scope="function") + @retry(stop=stop_after_delay(20), wait=wait_fixed(1), reraise=True) + def filter_warm_up(self): + try: + self.ping_filter_subscriptions("1") + except Exception as ex: + if "peer has no subscriptions" in str(ex): + logger.info("WARM UP successful!!") + else: + raise TimeoutError(f"WARM UP FAILED WITH: {ex}") + + def relay_node_start(self, node): + self.node1 = WakuNode(node, f"node1_{self.test_id}") + start_args = {"relay": "true", "filter": "true"} + self.node1.start(**start_args) + self.enr_uri = self.node1.get_enr_uri() + self.multiaddr_with_id = self.node1.get_multiaddr_with_id() + return self.node1 + + def setup_optional_filter_nodes(self, node_list=ADDITIONAL_NODES): + if node_list: + nodes = [node.strip() for node in node_list.split(",") if node] + else: + pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test") + for index, node in enumerate(nodes): + node = WakuNode(node, f"node{index + 3}_{self.test_id}") + node.start(relay="false", discv5_bootstrap_node=self.enr_uri, filternode=self.multiaddr_with_id) + self.add_node_peer(node, [self.multiaddr_with_id]) + self.optional_nodes.append(node) + + @allure.step + def check_published_message_reaches_filter_peer( + self, message=None, pubsub_topic=None, message_propagation_delay=0.1, sender=None, peer_list=None + ): + if message is None: + message = self.create_message() + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if not sender: + sender = self.node1 + if not peer_list: + peer_list = self.main_nodes + self.optional_nodes + + sender.send_relay_message(message, pubsub_topic) + delay(message_propagation_delay) + for index, peer in enumerate(peer_list): + logger.debug(f"Checking that peer NODE_{index + 2}:{peer.image} can find the published message") + get_messages_response = self.get_filter_messages(message["contentTopic"], pubsub_topic=pubsub_topic, node=peer) + assert get_messages_response, f"Peer NODE_{index + 2}:{peer.image} couldn't find any messages" + assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" + waku_message = WakuMessage(get_messages_response) + waku_message.assert_received_message(message) + + @allure.step + def check_publish_without_filter_subscription(self, message=None, pubsub_topic=None, peer_list=None): + try: + self.check_published_message_reaches_filter_peer(message=message, pubsub_topic=pubsub_topic, peer_list=peer_list) + raise AssertionError("Publish with no subscription worked!!!") + except Exception as ex: + assert "Bad Request" in str(ex) or "Not Found" in str(ex) or "couldn't find any messages" in str(ex) + + @allure.step + def wait_for_subscriptions_on_main_nodes(self, content_topic_list, pubsub_topic=None): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + self.node1.set_relay_subscriptions([pubsub_topic]) + request_id = str(uuid4()) + filter_sub_response = self.create_filter_subscription_with_retry( + {"requestId": request_id, "contentFilters": content_topic_list, "pubsubTopic": pubsub_topic} + ) + assert filter_sub_response["requestId"] == request_id + assert filter_sub_response["statusDesc"] in ["OK"] + + @allure.step + def subscribe_optional_filter_nodes(self, content_topic_list, pubsub_topic=None): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + for node in self.optional_nodes: + request_id = str(uuid4()) + self.create_filter_subscription_with_retry( + {"requestId": request_id, "contentFilters": content_topic_list, "pubsubTopic": pubsub_topic}, node=node + ) + + @retry(stop=stop_after_delay(60), wait=wait_fixed(1), reraise=True) + @allure.step + def create_filter_subscription_with_retry(self, subscription, node=None): + return self.create_filter_subscription(subscription, node) + + @allure.step + def create_filter_subscription(self, subscription, node=None): + if node is None: + node = self.node2 + return node.set_filter_subscriptions(subscription) + + @allure.step + def update_filter_subscription(self, subscription, node=None): + if node is None: + node = self.node2 + return node.update_filter_subscriptions(subscription) + + @allure.step + def delete_filter_subscription(self, subscription, status=None, node=None): + if node is None: + node = self.node2 + delete_sub_response = node.delete_filter_subscriptions(subscription) + assert delete_sub_response["requestId"] == subscription["requestId"] + if status is None: + assert delete_sub_response["statusDesc"] in ["OK"] + else: + assert status in delete_sub_response["statusDesc"] + + @allure.step + def delete_all_filter_subscriptions(self, request_id, node=None): + if node is None: + node = self.node2 + delete_sub_response = node.delete_all_filter_subscriptions(request_id) + assert delete_sub_response["requestId"] == request_id["requestId"] + assert delete_sub_response["statusDesc"] in ["OK"] + + @allure.step + def ping_filter_subscriptions(self, request_id, node=None): + if node is None: + node = self.node2 + ping_sub_response = node.ping_filter_subscriptions(request_id) + assert ping_sub_response["requestId"] == request_id + assert ping_sub_response["statusDesc"] in ["OK"] + + def ping_without_filter_subscription(self, node=None): + try: + self.ping_filter_subscriptions(str(uuid4()), node=node) + raise AssertionError("Ping without any subscription worked") + except Exception as ex: + assert "peer has no subscription" in str(ex) or "ping request failed" in str(ex) + + @allure.step + def add_new_relay_subscription(self, pubsub_topics, node=None): + if node is None: + node = self.node1 + self.node1.set_relay_subscriptions(pubsub_topics) + + @allure.step + def get_filter_messages(self, content_topic, pubsub_topic=None, node=None): + if node is None: + node = self.node2 + if node.is_nwaku(): + return node.get_filter_messages(content_topic) + else: + raise NotImplementedError("Not implemented for this node type") diff --git a/tests-e2e/src/steps/light_push.py b/tests-e2e/src/steps/light_push.py new file mode 100644 index 000000000..fd7da42d0 --- /dev/null +++ b/tests-e2e/src/steps/light_push.py @@ -0,0 +1,148 @@ +import inspect +from src.libs.custom_logger import get_custom_logger +from time import time +import pytest +import allure +from src.libs.common import to_base64, delay +from src.node.waku_message import WakuMessage +from src.env_vars import ( + ADDITIONAL_NODES, + NODE_1, + NODE_2, +) +from src.node.waku_node import WakuNode +from src.steps.common import StepsCommon +from src.test_data import VALID_PUBSUB_TOPICS +from tenacity import retry, stop_after_delay, wait_fixed + +logger = get_custom_logger(__name__) + + +class StepsLightPush(StepsCommon): + test_content_topic = "/myapp/1/latest/proto" + test_pubsub_topic = VALID_PUBSUB_TOPICS[0] + test_payload = "Light push works!!" + + @pytest.fixture(scope="function", autouse=True) + def light_push_setup(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.main_receiving_nodes = [] + self.optional_nodes = [] + self.multiaddr_list = [] + + @allure.step + def start_receiving_node(self, image, node_index, **kwargs): + node = WakuNode(image, f"receiving_node{node_index}_{self.test_id}") + node.start(**kwargs) + if kwargs["relay"] == "true": + self.main_receiving_nodes.extend([node]) + self.add_node_peer(node, self.multiaddr_list) + self.multiaddr_list.extend([node.get_multiaddr_with_id()]) + return node + + @allure.step + def setup_lightpush_node(self, image, node_index, **kwargs): + node = WakuNode(image, f"lightpush_node{node_index}_{self.test_id}") + node.start(discv5_bootstrap_node=self.enr_uri, lightpushnode=self.multiaddr_list[0], **kwargs) + if kwargs["relay"] == "true": + self.main_receiving_nodes.extend([node]) + self.add_node_peer(node, self.multiaddr_list) + return node + + @allure.step + def setup_first_receiving_node(self, lightpush="true", relay="true", **kwargs): + self.receiving_node1 = self.start_receiving_node(NODE_1, node_index=1, lightpush=lightpush, relay=relay, **kwargs) + self.enr_uri = self.receiving_node1.get_enr_uri() + + @allure.step + def setup_second_receiving_node(self, lightpush, relay, **kwargs): + self.receiving_node2 = self.start_receiving_node(NODE_1, node_index=2, lightpush=lightpush, relay=relay, **kwargs) + + @allure.step + def setup_additional_receiving_nodes(self, node_list=ADDITIONAL_NODES, **kwargs): + if node_list: + nodes = [node.strip() for node in node_list.split(",") if node] + else: + pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test") + for index, node in enumerate(nodes): + self.start_receiving_node(node, node_index=index + 2, lightpush="true", relay="true", pubsub_topic=self.test_pubsub_topic, **kwargs) + + @allure.step + def setup_first_lightpush_node(self, lightpush="true", relay="true", **kwargs): + self.light_push_node1 = self.setup_lightpush_node(NODE_2, node_index=1, lightpush=lightpush, relay=relay, **kwargs) + + @allure.step + def setup_second_lightpush_node(self, lightpush="true", relay="true", **kwargs): + self.light_push_node2 = self.setup_lightpush_node(NODE_2, node_index=2, lightpush=lightpush, relay=relay, **kwargs) + + @allure.step + def setup_additional_lightpush_nodes(self, node_list=ADDITIONAL_NODES, **kwargs): + if node_list: + nodes = [node.strip() for node in node_list.split(",") if node] + else: + pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test") + self.additional_lightpush_nodes = [] + for index, node in enumerate(nodes): + node = self.setup_lightpush_node(node, node_index=index + 2, lightpush="true", relay="true", **kwargs) + self.additional_lightpush_nodes.append(node) + + @allure.step + def subscribe_to_pubsub_topics_via_relay(self, node=None, pubsub_topics=None): + if pubsub_topics is None: + pubsub_topics = [self.test_pubsub_topic] + if not node: + node = self.main_receiving_nodes + if isinstance(node, list): + for node in node: + node.set_relay_subscriptions(pubsub_topics) + else: + node.set_relay_subscriptions(pubsub_topics) + + @allure.step + def subscribe_to_pubsub_topics_via_filter(self, node, pubsub_topic=None, content_topic=None): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if content_topic is None: + content_topic = [self.test_content_topic] + subscription = {"requestId": "1", "contentFilters": content_topic, "pubsubTopic": pubsub_topic} + node.set_filter_subscriptions(subscription) + + @allure.step + def check_light_pushed_message_reaches_receiving_peer( + self, pubsub_topic=None, message=None, message_propagation_delay=0.1, sender=None, peer_list=None + ): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if not sender: + sender = self.light_push_node1 + if not peer_list: + peer_list = self.main_receiving_nodes + self.optional_nodes + payload = self.create_payload(pubsub_topic, message) + logger.debug("Lightpushing message") + sender.send_light_push_message(payload) + delay(message_propagation_delay) + for index, peer in enumerate(peer_list): + logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the lightpushed message") + get_messages_response = peer.get_relay_messages(pubsub_topic) + assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" + assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" + waku_message = WakuMessage(get_messages_response) + waku_message.assert_received_message(payload["message"]) + + @allure.step + def create_payload(self, pubsub_topic=None, message=None, **kwargs): + if message is None: + message = self.create_message() + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + payload = {"pubsubTopic": pubsub_topic, "message": message} + payload.update(kwargs) + return payload + + @allure.step + @retry(stop=stop_after_delay(120), wait=wait_fixed(1), reraise=True) + def subscribe_and_light_push_with_retry(self): + self.subscribe_to_pubsub_topics_via_relay() + self.light_push_node1.send_light_push_message(self.create_payload()) + get_messages_response = self.main_receiving_nodes[0].get_relay_messages(self.test_pubsub_topic) + assert len(get_messages_response) >= 1, f"Expected al least 1 message but got {len(get_messages_response)}" diff --git a/tests-e2e/src/steps/metrics.py b/tests-e2e/src/steps/metrics.py new file mode 100644 index 000000000..f53686999 --- /dev/null +++ b/tests-e2e/src/steps/metrics.py @@ -0,0 +1,68 @@ +import re +from src.libs.custom_logger import get_custom_logger +import allure +from tenacity import retry, stop_after_delay, wait_fixed + +from src.test_data import METRICS_WITH_INITIAL_VALUE_ZERO + + +logger = get_custom_logger(__name__) + + +class StepsMetrics: + @allure.step + def check_metric(self, node, metric_name, expected_value, exact=False): + logger.debug(f"Checking metric: {metric_name} has {expected_value}") + response = node.get_metrics() + lines = response.split("\n") + actual_value = None + for line in lines: + if line.startswith(metric_name): + parts = line.split(" ") + if len(parts) >= 2: + actual_value = float(parts[1]) + break + if actual_value is None: + raise AttributeError(f"Metric '{metric_name}' not found") + logger.debug(f"Found metric: {metric_name} with value {actual_value}") + if exact: + assert actual_value == expected_value, f"Expected value for '{metric_name}' is {expected_value}, but got {actual_value}" + else: + assert actual_value >= expected_value, f"Expected value for '{metric_name}' is >= {expected_value}, but got {actual_value}" + + @allure.step + def wait_for_metric(self, node, metric_name, expected_value, timeout_duration=90): + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(1), reraise=True) + def check_metric_with_retry(): + self.check_metric(node, metric_name, expected_value) + + check_metric_with_retry() + + def validate_initial_metrics(self, node): + metrics_data = node.get_metrics() + + # Regular expression to match metric lines, accounting for optional labels + metric_pattern = re.compile(r"^(?P[a-zA-Z0-9_:]+(?:{[^}]+})?)\s+(?P[0-9]+\.?[0-9]*)$", re.MULTILINE) + + # Dictionary to store the metrics and their values + metrics_dict = {} + for match in metric_pattern.finditer(metrics_data): + metric_name = match.group("metric_name") + value = float(match.group("value")) + metrics_dict[metric_name] = value + + errors = [] + # Assert that specific metrics have a value of 0.0 + for metric in METRICS_WITH_INITIAL_VALUE_ZERO: + if metric not in metrics_dict: + errors.append(f"Metric {metric} is missing from the metrics data") + elif metrics_dict[metric] != 0.0: + errors.append(f"Expected {metric} to be 0.0, but got {metrics_dict[metric]}") + + # Assert that all other metrics have a value greater than 0.0 + for metric, value in metrics_dict.items(): + if metric not in METRICS_WITH_INITIAL_VALUE_ZERO and value <= 0.0: + errors.append(f"Expected {metric} to have a positive value, but got {value}") + + assert not errors, f"Metrics validation failed:\n" + "\n".join(errors) + logger.debug(f"All metrics are present and have valid values.") diff --git a/tests-e2e/src/steps/network_conditions.py b/tests-e2e/src/steps/network_conditions.py new file mode 100644 index 000000000..b96d9a0cc --- /dev/null +++ b/tests-e2e/src/steps/network_conditions.py @@ -0,0 +1,201 @@ +import subprocess +from src.env_vars import NETWORK_NAME +from src.libs.custom_logger import get_custom_logger + +logger = get_custom_logger(__name__) + + +class TrafficController: + def _pid(self, node) -> int: + if not node.container: + raise RuntimeError("Node container not started yet") + + node.container.reload() + pid = node.container.attrs.get("State", {}).get("Pid") + if not pid or pid == 0: + raise RuntimeError("Container PID not available (container not running?)") + return int(pid) + + def _exec(self, node, tc_args: list[str], iface: str = "eth0"): + pid = self._pid(node) + + cmd = ["sudo", "-n", "nsenter", "-t", str(pid), "-n", "tc"] + tc_args + logger.info(f"TC exec: {cmd}") + + res = subprocess.run(cmd, capture_output=True, text=True) + if res.returncode != 0: + raise RuntimeError(f"TC failed: {' '.join(cmd)}\n" f"stdout: {res.stdout}\n" f"stderr: {res.stderr}") + + return res.stdout + + def log_tc_stats(self, node, iface: str = "eth0"): + """ + Log tc statistics for an interface (best-effort). + Useful to confirm netem loss/delay counters (sent/dropped/etc.). + """ + try: + out = self._exec(node, ["-s", "qdisc", "show", "dev", iface], iface=iface) + out = (out or "").strip() + if out: + logger.debug(f"tc -s qdisc show dev {iface}:\n{out}") + else: + logger.debug(f"tc -s qdisc show dev {iface}: (no output)") + except Exception as e: + logger.debug(f"Failed to read tc stats for {iface}: {e}") + + def clear(self, node, iface: str = "eth0"): + try: + self._exec(node, ["qdisc", "del", "dev", iface, "root"], iface=iface) + except RuntimeError as e: + msg = str(e) + if "Cannot delete qdisc with handle of zero" in msg or "No such file or directory" in msg: + return + raise + + def add_latency(self, node, ms: int, iface: str = "eth0"): + self.clear(node, iface=iface) + self._exec(node, ["qdisc", "add", "dev", iface, "root", "netem", "delay", f"{ms}ms"], iface=iface) + + def add_packet_loss(self, node, percent: float, iface: str = "eth0"): + self.clear(node, iface=iface) + + self._exec( + node, + ["qdisc", "add", "dev", iface, "root", "netem", "loss", f"{percent}%"], + iface=iface, + ) + try: + stats = self._exec(node, ["-s", "qdisc", "show", "dev", iface], iface=iface) + if stats is not None: + if isinstance(stats, (bytes, bytearray)): + stats = stats.decode(errors="replace") + logger.debug(f"tc -s qdisc show dev {iface}:\n{stats}") + else: + logger.debug(f"Executed: tc -s qdisc show dev {iface} (no output returned by _exec)") + except Exception as e: + logger.debug(f"Failed to read tc stats for {iface}: {e}") + + def add_bandwidth(self, node, rate: str, iface: str = "eth0"): + self.clear(node, iface=iface) + self._exec( + node, + ["qdisc", "add", "dev", iface, "root", "tbf", "rate", rate, "burst", "32kbit", "limit", "12500"], + iface=iface, + ) + + def add_packet_loss_correlated( + self, + node, + percent: float, + correlation: float, + iface: str = "eth0", + ): + self.clear(node, iface=iface) + self._exec( + node, + [ + "qdisc", + "add", + "dev", + iface, + "root", + "netem", + "loss", + f"{percent}%", + f"{correlation}%", + ], + iface=iface, + ) + + def add_packet_reordering( + self, + node, + percent: int = 25, + correlation: int = 50, + delay_ms: int = 10, + iface: str = "eth0", + ): + self.clear(node, iface=iface) + + self._exec( + node, + [ + "qdisc", + "add", + "dev", + iface, + "root", + "netem", + "delay", + f"{delay_ms}ms", + "reorder", + f"{percent}%", + f"{correlation}%", + ], + iface=iface, + ) + + def _p2p_iface(self, node) -> str: + """ + Return the name of the container interface attached to the waku + network (where libp2p traffic flows). + + DockerManager attaches each node to two networks: the default bridge + (where host-published ports land, typically `eth0`) and the waku + network (where inter-container libp2p/gossipsub traffic flows, typically + `eth1`). tc on the default bridge only affects REST control plane; for + a packet loss test targeting libp2p we need the waku interface. + + This helper resolves the correct interface by looking up the node's + waku-network IP via Docker and matching it against `ip -o -4 addr` + output from inside the container. + """ + if not node.container: + raise RuntimeError("Node container not started yet") + node.container.reload() + networks = node.container.attrs.get("NetworkSettings", {}).get("Networks", {}) + waku_net = networks.get(NETWORK_NAME) + if not waku_net or not waku_net.get("IPAddress"): + raise RuntimeError(f"Container is not attached to the '{NETWORK_NAME}' docker network") + waku_ip = waku_net["IPAddress"] + + exit_code, output = node.container.exec_run(["ip", "-o", "-4", "addr"]) + if exit_code != 0: + raise RuntimeError(f"ip addr failed inside container: {output}") + for line in output.decode().splitlines(): + if f" {waku_ip}/" in line: + tokens = line.split() + if len(tokens) >= 2: + return tokens[1] + raise RuntimeError(f"No interface inside container holds waku IP {waku_ip}") + + def clear_p2p(self, node): + """ + Remove any tc rule previously installed on the node's waku (libp2p) + interface. Paired with add_packet_loss_p2p_only / + add_packet_loss_correlated_p2p_only. + """ + self.clear(node, iface=self._p2p_iface(node)) + + def add_packet_loss_p2p_only(self, node, percent: float): + """ + Apply uncorrelated packet loss to the waku (libp2p) network interface + of a node. REST API traffic rides a separate docker interface and is + not affected, so the test harness's control plane stays reliable. + """ + iface = self._p2p_iface(node) + self.clear(node, iface=iface) + self._exec(node, f"qdisc add dev {iface} root netem loss {percent}%".split(), iface=iface) + + def add_packet_loss_correlated_p2p_only(self, node, percent: float, correlation: float): + """ + Correlated packet loss on the waku (libp2p) network interface. See + add_packet_loss_p2p_only for why REST stays unaffected. + """ + iface = self._p2p_iface(node) + self.clear(node, iface=iface) + self._exec( + node, + f"qdisc add dev {iface} root netem loss {percent}% {correlation}%".split(), + iface=iface, + ) diff --git a/tests-e2e/src/steps/peer_exchange.py b/tests-e2e/src/steps/peer_exchange.py new file mode 100644 index 000000000..6504ba3ef --- /dev/null +++ b/tests-e2e/src/steps/peer_exchange.py @@ -0,0 +1,30 @@ +from uuid import uuid4 + +from src.env_vars import NODE_2 +from src.node.waku_node import WakuNode + +import allure + +from src.steps.relay import StepsRelay + + +class StepsPeerExchange(StepsRelay): + responder_multiaddr = "" + + @allure.step + def setup_third_node_as_peer_exchange_requester(self, **kwargs): + self.node3 = WakuNode(NODE_2, f"node3_{self.test_id}") + self.node3.start( + relay="false", + peer_exchange_node=self.responder_multiaddr, + **kwargs, + ) + self.add_node_peer(self.node3, [self.multiaddr_with_id]) + self.main_nodes.extend([self.node3]) + + @allure.step + def setup_fourth_node_as_filter(self, **kwargs): + self.node4 = WakuNode(NODE_2, f"node4_{self.test_id}") + self.node4.start(relay="false", **kwargs) + self.add_node_peer(self.node4, [self.multiaddr_with_id]) + self.main_nodes.extend([self.node4]) diff --git a/tests-e2e/src/steps/relay.py b/tests-e2e/src/steps/relay.py new file mode 100644 index 000000000..840461bbe --- /dev/null +++ b/tests-e2e/src/steps/relay.py @@ -0,0 +1,187 @@ +import inspect +from src.libs.custom_logger import get_custom_logger +from time import time +import pytest +import allure +from src.libs.common import to_base64, delay +from src.node.waku_message import WakuMessage +from src.env_vars import ( + NODE_1, + NODE_2, + ADDITIONAL_NODES, +) +from src.node.waku_node import WakuNode +from tenacity import retry, stop_after_delay, wait_fixed +from src.steps.common import StepsCommon +from src.test_data import VALID_PUBSUB_TOPICS + +logger = get_custom_logger(__name__) + + +class StepsRelay(StepsCommon): + test_pubsub_topic = VALID_PUBSUB_TOPICS[1] + test_content_topic = "/test/1/waku-relay/proto" + test_payload = "Relay works!!" + + @pytest.fixture(scope="function", autouse=True) + def relay_setup(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.main_nodes = [] + self.optional_nodes = [] + + @pytest.fixture(scope="function") + def setup_main_relay_nodes(self, request): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.node1 = WakuNode(NODE_1, f"node1_{request.cls.test_id}") + self.node1.start(relay="true") + self.enr_uri = self.node1.get_enr_uri() + self.multiaddr_with_id = self.node1.get_multiaddr_with_id() + self.node2 = WakuNode(NODE_2, f"node2_{request.cls.test_id}") + self.node2.start(relay="true", discv5_bootstrap_node=self.enr_uri) + self.add_node_peer(self.node2, [self.multiaddr_with_id]) + self.main_nodes.extend([self.node1, self.node2]) + + @pytest.fixture(scope="function") + def setup_optional_relay_nodes(self, request): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + if ADDITIONAL_NODES: + nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")] + else: + pytest.skip("ADDITIONAL_NODES is empty, cannot run test") + for index, node in enumerate(nodes): + node = WakuNode(node, f"node{index + 3}_{request.cls.test_id}") + node.start(relay="true", discv5_bootstrap_node=self.enr_uri) + self.add_node_peer(node, [self.multiaddr_with_id]) + self.optional_nodes.append(node) + + @pytest.fixture(scope="function") + def subscribe_main_relay_nodes(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic]) + + @pytest.fixture(scope="function") + def subscribe_optional_relay_nodes(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.ensure_relay_subscriptions_on_nodes(self.optional_nodes, [self.test_pubsub_topic]) + + @pytest.fixture(scope="function") + def relay_warm_up(self): + try: + self.wait_for_published_message_to_reach_relay_peer() + logger.info("WARM UP successful!!") + except Exception as ex: + raise TimeoutError(f"WARM UP FAILED WITH: {ex}") + + # Refactor candidate + @allure.step + def setup_first_relay_node(self, **kwargs): + self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}") + self.node1.start(relay="true", **kwargs) + self.enr_uri = self.node1.get_enr_uri() + self.multiaddr_with_id = self.node1.get_multiaddr_with_id() + self.main_nodes.extend([self.node1]) + + # Refactor candidate + @allure.step + def setup_second_relay_node(self, **kwargs): + self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}") + self.node2.start( + relay="true", + discv5_bootstrap_node=self.enr_uri, + **kwargs, + ) + self.add_node_peer(self.node2, [self.multiaddr_with_id]) + self.main_nodes.extend([self.node2]) + + @allure.step + def setup_third_relay_node(self, **kwargs): + self.node3 = WakuNode(NODE_1, f"node3_{self.test_id}") + self.node3.start( + relay="true", + discv5_bootstrap_node=self.enr_uri, + **kwargs, + ) + self.add_node_peer(self.node3, [self.multiaddr_with_id]) + self.optional_nodes.extend([self.node3]) + + # this method should be used only for the tests that use the relay_warm_up fixture + # otherwise use wait_for_published_message_to_reach_relay_peer + @allure.step + def check_published_message_reaches_relay_peer(self, message=None, pubsub_topic=None, message_propagation_delay=0.1, sender=None, peer_list=None): + if message is None: + message = self.create_message() + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if not sender: + sender = self.node1 + if not peer_list: + peer_list = self.main_nodes + self.optional_nodes + + sender.send_relay_message(message, pubsub_topic) + delay(message_propagation_delay) + for index, peer in enumerate(peer_list): + logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the published message") + get_messages_response = peer.get_relay_messages(pubsub_topic) + assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" + assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" + waku_message = WakuMessage(get_messages_response) + waku_message.assert_received_message(message) + + @allure.step + def check_publish_without_relay_subscription(self, pubsub_topic): + try: + self.node1.send_relay_message(self.create_message(), pubsub_topic) + raise AssertionError("Publish with no subscription worked!!!") + except Exception as ex: + assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex) + + # we need much bigger timeout in CI because we run tests in parallel there and the machine itself is slower + @allure.step + def wait_for_published_message_to_reach_relay_peer( + self, timeout_duration=120, time_between_retries=1, pubsub_topic=None, sender=None, peer_list=None + ): + @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(time_between_retries), reraise=True) + def publish_and_check_relay_peer(): + message = {"payload": to_base64(self.test_payload), "contentTopic": self.test_content_topic, "timestamp": int(time() * 1e9)} + self.check_published_message_reaches_relay_peer(message, pubsub_topic=pubsub_topic, sender=sender, peer_list=peer_list) + + publish_and_check_relay_peer() + + @allure.step + def ensure_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list): + for node in node_list: + node.set_relay_subscriptions(pubsub_topic_list) + + @allure.step + def delete_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list): + for node in node_list: + node.delete_relay_subscriptions(pubsub_topic_list) + + @allure.step + @retry(stop=stop_after_delay(120), wait=wait_fixed(1), reraise=True) + def subscribe_and_publish_with_retry(self, node_list, pubsub_topic_list): + self.ensure_relay_subscriptions_on_nodes(node_list, pubsub_topic_list) + self.check_published_message_reaches_relay_peer() + + @allure.step + def setup_main_nodes(self, **kwargs): + self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}") + self.node1.start(relay="true", **kwargs) + self.enr_uri = self.node1.get_enr_uri() + self.multiaddr_with_id = self.node1.get_multiaddr_with_id() + self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}") + self.node2.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs) + self.add_node_peer(self.node2, [self.multiaddr_with_id]) + self.main_nodes.extend([self.node1, self.node2]) + + @allure.step + def setup_optional_nodes(self, **kwargs): + if ADDITIONAL_NODES: + nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")] + else: + pytest.skip("ADDITIONAL_NODES is empty, cannot run test") + for index, node in enumerate(nodes): + node = WakuNode(node, f"node{index + 3}_{self.test_id}") + node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs) + self.add_node_peer(node, [self.multiaddr_with_id]) + self.optional_nodes.append(node) diff --git a/tests-e2e/src/steps/rln.py b/tests-e2e/src/steps/rln.py new file mode 100644 index 000000000..1bf051df2 --- /dev/null +++ b/tests-e2e/src/steps/rln.py @@ -0,0 +1,203 @@ +import os +import inspect +import random +import string + +import pytest +import allure + +from src.steps.common import StepsCommon +from src.test_data import PUBSUB_TOPICS_RLN +from src.env_vars import DEFAULT_NWAKU, RLN_CREDENTIALS, NODE_1, NODE_2, ADDITIONAL_NODES +from src.libs.common import gen_step_id, delay +from src.libs.custom_logger import get_custom_logger +from src.node.waku_node import WakuNode, rln_credential_store_ready + +logger = get_custom_logger(__name__) + + +class StepsRLN(StepsCommon): + test_pubsub_topic = PUBSUB_TOPICS_RLN[0] + test_content_topic = "/test/1/waku-rln-relay/proto" + test_payload = "RLN relay works!!" + + main_nodes = [] + optional_nodes = [] + multiaddr_list = [] + lightpush_nodes = [] + keystore_prefixes = [] + rln_membership_indexes = [] + + @allure.step + def generate_keystore_prefixes(self, count=2): + new_prefixes = [] + for _ in range(count): + new_prefixes.append("".join(random.choices(string.ascii_lowercase, k=4))) + + return new_prefixes + + @allure.step + def register_rln_relay_nodes(self, count, orig_prefixes): + if count > 0: + self.keystore_prefixes = self.generate_keystore_prefixes(count) + self.rln_membership_indexes = [] + for i, prefix in enumerate(self.keystore_prefixes): + membership_index = self.register_rln_single_node(prefix=prefix, rln_creds_source=RLN_CREDENTIALS, rln_creds_id=f"{i+1}") + self.rln_membership_indexes.append(membership_index) + else: + self.keystore_prefixes = orig_prefixes.get("keystore_prefixes", []) + self.rln_membership_indexes = orig_prefixes.get("rln_membership_indexes", []) + + return { + "keystore_prefixes": self.keystore_prefixes, + "rln_membership_indexes": self.rln_membership_indexes, + } + + @allure.step + def setup_main_rln_relay_nodes(self, **kwargs): + self.setup_first_rln_relay_node(**kwargs) + self.setup_second_rln_relay_node(**kwargs) + + @allure.step + def setup_first_rln_relay_node(self, **kwargs): + self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}") + self.node1.start( + relay="true", + rln_creds_source=RLN_CREDENTIALS, + rln_creds_id="1", + rln_relay_membership_index=self.resolve_rln_membership_index(0, **kwargs), + rln_keystore_prefix=self.keystore_prefixes[0], + **kwargs, + ) + self.enr_uri = self.node1.get_enr_uri() + self.multiaddr_with_id = self.node1.get_multiaddr_with_id() + self.main_nodes.extend([self.node1]) + + self.multiaddr_list.extend([self.node1.get_multiaddr_with_id()]) + + @allure.step + def setup_second_rln_relay_node(self, **kwargs): + self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}") + self.node2.start( + relay="true", + discv5_bootstrap_node=self.enr_uri, + rln_creds_source=RLN_CREDENTIALS, + rln_creds_id="2", + rln_relay_membership_index=self.resolve_rln_membership_index(1, **kwargs), + rln_keystore_prefix=self.keystore_prefixes[1], + **kwargs, + ) + self.add_node_peer(self.node2, [self.multiaddr_with_id]) + self.main_nodes.extend([self.node2]) + + @allure.step + def setup_optional_rln_relay_nodes(self, **kwargs): + if ADDITIONAL_NODES: + nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")] + else: + pytest.skip("ADDITIONAL_NODES is empty, cannot run test") + if len(nodes) > 3: + logger.debug("More than 3 nodes are not supported for RLN tests, using first 3") + nodes = nodes[:3] + for index, node in enumerate(nodes): + node = WakuNode(node, f"node{index + 3}_{self.test_id}") + node.start( + relay="true", + discv5_bootstrap_node=self.enr_uri, + rln_creds_source=RLN_CREDENTIALS, + rln_creds_id=f"{index + 3}", + rln_relay_membership_index=self.resolve_rln_membership_index(index + 2, **kwargs), + rln_keystore_prefix=self.keystore_prefixes[index + 2], + **kwargs, + ) + self.add_node_peer(node, [self.multiaddr_with_id]) + self.optional_nodes.append(node) + + @allure.step + def setup_second_rln_lightpush_node(self, relay="true", **kwargs): + self.light_push_node2 = WakuNode(NODE_2, f"lightpush_node2_{self.test_id}") + self.light_push_node2.start( + relay=relay, + discv5_bootstrap_node=self.enr_uri, + lightpush="true", + lightpushnode=self.multiaddr_list[0], + rln_creds_source=RLN_CREDENTIALS, + rln_creds_id="2", + rln_relay_membership_index=self.resolve_rln_membership_index(1, **kwargs), + rln_keystore_prefix=self.keystore_prefixes[1], + **kwargs, + ) + if relay == "true": + self.main_nodes.extend([self.light_push_node2]) + self.lightpush_nodes.extend([self.light_push_node2]) + self.add_node_peer(self.light_push_node2, self.multiaddr_list) + + @allure.step + def register_rln_single_node(self, prefix="", **kwargs): + logger.debug("Registering RLN credentials for single node") + self.node = WakuNode(DEFAULT_NWAKU, f"node_{gen_step_id()}") + return self.node.register_rln(rln_keystore_prefix=prefix, rln_creds_source=kwargs["rln_creds_source"], rln_creds_id=kwargs["rln_creds_id"]) + + @allure.step + def resolve_rln_membership_index(self, index, **kwargs): + explicit_index = kwargs.get("rln_relay_membership_index") + if explicit_index is not None: + return str(explicit_index) + + if len(self.rln_membership_indexes) > index and self.rln_membership_indexes[index] is not None: + inferred_index = str(self.rln_membership_indexes[index]) + logger.debug(f"Using inferred RLN membership index for position {index}: {inferred_index}") + return inferred_index + + raise ValueError( + f"RLN membership index for position {index} is not available. " + "Register credentials and persist rln_membership_indexes together with keystore_prefixes before node startup." + ) + + @allure.step + def check_rln_registration(self, prefix, key_id): + cwd = os.getcwd() + creds_file_path = f"{cwd}/keystore_{prefix}_{key_id}/keystore.json" + try: + rln_credential_store_ready(creds_file_path) + except Exception as ex: + logger.error(f"Credentials at {creds_file_path} not available: {ex}") + raise + + @allure.step + def publish_message(self, message=None, pubsub_topic=None, sender=None, use_lightpush=False): + if message is None: + message = self.create_message() + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if not sender: + sender = self.node1 + + if use_lightpush: + payload = self.create_payload(pubsub_topic, message) + sender.send_light_push_message(payload) + else: + sender.send_relay_message(message, pubsub_topic) + + @allure.step + def ensure_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list): + for node in node_list: + node.set_relay_subscriptions(pubsub_topic_list) + + @allure.step + def subscribe_main_relay_nodes(self): + self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic]) + + @allure.step + def subscribe_optional_relay_nodes(self): + self.ensure_relay_subscriptions_on_nodes(self.optional_nodes, [self.test_pubsub_topic]) + + @allure.step + def create_payload(self, pubsub_topic=None, message=None, **kwargs): + if message is None: + message = self.create_message() + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + payload = {"pubsubTopic": pubsub_topic, "message": message} + payload.update(kwargs) + return payload diff --git a/tests-e2e/src/steps/sharding.py b/tests-e2e/src/steps/sharding.py new file mode 100644 index 000000000..7c8aabc25 --- /dev/null +++ b/tests-e2e/src/steps/sharding.py @@ -0,0 +1,212 @@ +import inspect +from uuid import uuid4 +from src.libs.custom_logger import get_custom_logger +from time import time +import pytest +import allure +from src.libs.common import to_base64, delay +from src.node.waku_message import WakuMessage +from src.env_vars import ( + DEFAULT_NWAKU, + NODE_2, + ADDITIONAL_NODES, +) +from src.node.waku_node import WakuNode +from src.steps.common import StepsCommon +from src.steps.relay import StepsRelay + +logger = get_custom_logger(__name__) + + +class StepsSharding(StepsRelay): + test_content_topic = "/myapp/1/latest/proto" + test_pubsub_topic = "/waku/2/rs/199/0" + test_payload = "Sharding works!!" + auto_cluster = 199 + num_shards_in_network = 8 + + @pytest.fixture(scope="function", autouse=True) + def sharding_setup(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.main_nodes = [] + self.optional_nodes = [] + self.main_filter_nodes = [] + self.optional_filter_nodes = [] + + @allure.step + def setup_first_relay_node_with_filter(self, **kwargs): + self.setup_first_relay_node(filter="true", **kwargs) + + @allure.step + def setup_second_node_as_filter(self, **kwargs): + self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}") + self.node2.start(relay="false", discv5_bootstrap_node=self.enr_uri, filternode=self.multiaddr_with_id, **kwargs) + self.add_node_peer(self.node2, [self.multiaddr_with_id]) + self.main_filter_nodes.extend([self.node2]) + + @allure.step + def setup_main_relay_nodes(self, **kwargs): + self.setup_first_relay_node_with_filter(**kwargs) + self.setup_second_relay_node(**kwargs) + + @allure.step + def setup_optional_relay_nodes(self, **kwargs): + if ADDITIONAL_NODES: + nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")] + else: + pytest.skip("ADDITIONAL_NODES is empty, cannot run test") + for index, node in enumerate(nodes): + node = WakuNode(node, f"node{index + 3}_{self.test_id}") + node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs) + self.add_node_peer(node, [self.multiaddr_with_id]) + self.optional_nodes.append(node) + + @allure.step + def setup_nwaku_relay_nodes(self, num_nodes, **kwargs): + for index in range(num_nodes): + node = WakuNode(DEFAULT_NWAKU, f"node{index + 3}_{self.test_id}") + node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs) + self.add_node_peer(node, [self.multiaddr_with_id]) + self.optional_nodes.append(node) + + @allure.step + def subscribe_relay_node(self, node, content_topics, pubsub_topics): + if content_topics: + node.set_relay_auto_subscriptions(content_topics) + elif pubsub_topics: + node.set_relay_subscriptions(pubsub_topics) + else: + raise AttributeError("content_topics or pubsub_topics need to be passed") + + @allure.step + def subscribe_first_relay_node(self, content_topics=None, pubsub_topics=None): + self.subscribe_relay_node(self.node1, content_topics, pubsub_topics) + + @allure.step + def subscribe_second_relay_node(self, content_topics=None, pubsub_topics=None): + self.subscribe_relay_node(self.node2, content_topics, pubsub_topics) + + @allure.step + def subscribe_main_relay_nodes(self, content_topics=None, pubsub_topics=None): + for node in self.main_nodes: + self.subscribe_relay_node(node, content_topics, pubsub_topics) + + @allure.step + def subscribe_optional_relay_nodes(self, content_topics=None, pubsub_topics=None): + for node in self.optional_nodes: + self.subscribe_relay_node(node, content_topics, pubsub_topics) + + @allure.step + def unsubscribe_relay_node(self, node, content_topics, pubsub_topics): + if content_topics: + node.delete_relay_auto_subscriptions(content_topics) + elif pubsub_topics: + node.delete_relay_subscriptions(pubsub_topics) + else: + raise AttributeError("content_topics or pubsub_topics need to be passed") + + @allure.step + def unsubscribe_first_relay_node(self, content_topics=None, pubsub_topics=None): + self.unsubscribe_relay_node(self.node1, content_topics, pubsub_topics) + + @allure.step + def unsubscribe_second_relay_node(self, content_topics=None, pubsub_topics=None): + self.unsubscribe_relay_node(self.node2, content_topics, pubsub_topics) + + @allure.step + def unsubscribe_main_relay_nodes(self, content_topics=None, pubsub_topics=None): + for node in self.main_nodes: + self.unsubscribe_relay_node(node, content_topics, pubsub_topics) + + @allure.step + def unsubscribe_optional_relay_nodes(self, content_topics=None, pubsub_topics=None): + for node in self.optional_nodes: + self.unsubscribe_relay_node(node, content_topics, pubsub_topics) + + @allure.step + def subscribe_filter_node(self, node, content_topics=None, pubsub_topic=None): + subscription = {"requestId": str(uuid4()), "contentFilters": content_topics, "pubsubTopic": pubsub_topic} + node.set_filter_subscriptions(subscription) + + @allure.step + def relay_message(self, node, message, pubsub_topic=None): + if pubsub_topic: + node.send_relay_message(message, pubsub_topic) + else: + node.send_relay_auto_message(message) + + @allure.step + def retrieve_relay_message(self, node, content_topic=None, pubsub_topic=None): + if content_topic: + return node.get_relay_auto_messages(content_topic) + elif pubsub_topic: + return node.get_relay_messages(pubsub_topic) + else: + raise AttributeError("content_topic or pubsub_topic needs to be passed") + + @allure.step + def check_published_message_reaches_relay_peer(self, content_topic=None, pubsub_topic=None, sender=None, peer_list=None): + message = self.create_message(contentTopic=content_topic) if content_topic else self.create_message() + if not sender: + sender = self.node1 + if not peer_list: + peer_list = self.main_nodes + self.optional_nodes + + self.relay_message(sender, message, pubsub_topic) + delay(0.1) + for index, peer in enumerate(peer_list): + logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the published message") + get_messages_response = self.retrieve_relay_message(peer, content_topic, pubsub_topic) + assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages" + assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" + waku_message = WakuMessage(get_messages_response) + waku_message.assert_received_message(message) + + @allure.step + def get_filter_messages(self, content_topic, pubsub_topic=None, node=None): + if node is None: + node = self.node2 + if node.is_nwaku(): + return node.get_filter_messages(content_topic) + else: + raise NotImplementedError("Not implemented for this node type") + + @allure.step + def check_published_message_reaches_filter_peer(self, content_topic=None, pubsub_topic=None, sender=None, peer_list=None): + message = self.create_message(contentTopic=content_topic) if content_topic else self.create_message() + if not sender: + sender = self.node1 + if not peer_list: + peer_list = self.main_filter_nodes + self.optional_filter_nodes + + self.relay_message(sender, message, pubsub_topic) + delay(0.1) + for index, peer in enumerate(peer_list): + logger.debug(f"Checking that peer NODE_{index + 2}:{peer.image} can find the published message") + get_messages_response = self.get_filter_messages(message["contentTopic"], pubsub_topic=pubsub_topic, node=peer) + assert get_messages_response, f"Peer NODE_{index + 2}:{peer.image} couldn't find any messages" + assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}" + waku_message = WakuMessage(get_messages_response) + waku_message.assert_received_message(message) + + @allure.step + def check_published_message_doesnt_reach_relay_peer(self, pubsub_topic=None, content_topic=None): + try: + self.check_published_message_reaches_relay_peer(pubsub_topic=pubsub_topic, content_topic=content_topic) + raise AssertionError("Retrieving messages on not subscribed content topic worked!!!") + except Exception as ex: + error_message = str(ex) + expected_errors = [ + "Not Found", + "NoPeersToPublish", + "Failed to publish: publish failed in relay: NoPeersToPublish", + ] + assert any(expected in error_message for expected in expected_errors), error_message + + @allure.step + def check_publish_fails_on_not_subscribed_pubsub_topic(self, pubsub_topic): + try: + self.check_published_message_reaches_relay_peer(pubsub_topic=pubsub_topic) + raise AssertionError("Publishing messages on unsubscribed shard worked!!!") + except Exception as ex: + assert "Failed to publish: Node not subscribed to topic" in str(ex), "Expected Bad Request because the node is not subscribed" diff --git a/tests-e2e/src/steps/store.py b/tests-e2e/src/steps/store.py new file mode 100644 index 000000000..311932ae5 --- /dev/null +++ b/tests-e2e/src/steps/store.py @@ -0,0 +1,382 @@ +import inspect + +import requests + +from src.libs.custom_logger import get_custom_logger +import pytest +import allure +from src.libs.common import delay +from src.node.store_response import StoreResponse +from src.node.waku_message import WakuMessage +from src.env_vars import ( + ADDITIONAL_NODES, + NODE_1, + NODE_2, +) +from src.node.waku_node import WakuNode +from src.steps.common import StepsCommon +from src.test_data import VALID_PUBSUB_TOPICS +from tenacity import retry, stop_after_delay, wait_fixed + +logger = get_custom_logger(__name__) + + +class StepsStore(StepsCommon): + test_content_topic = "/myapp/1/latest/proto" + test_pubsub_topic = VALID_PUBSUB_TOPICS[0] + test_payload = "Store works!!" + + @pytest.fixture(scope="function", autouse=True) + def store_setup(self): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.main_publishing_nodes = [] + self.store_nodes = [] + self.optional_nodes = [] + self.multiaddr_list = [] + + @pytest.fixture(scope="function", autouse=False) + def node_setup(self, store_setup): + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + self.setup_first_publishing_node(store="true", relay="true") + self.setup_first_store_node(store="true", relay="true") + self.subscribe_to_pubsub_topics_via_relay(node=self.main_publishing_nodes) + + @allure.step + def start_publishing_node(self, image, node_index, **kwargs): + node = WakuNode(image, f"publishing_node{node_index}_{self.test_id}") + node.start(**kwargs) + if kwargs["relay"] == "true": + self.main_publishing_nodes.extend([node]) + if kwargs["store"] == "true": + self.store_nodes.extend([node]) + self.add_node_peer(node, self.multiaddr_list) + self.multiaddr_list.extend([node.get_multiaddr_with_id()]) + return node + + @allure.step + def setup_store_node(self, image, node_index, **kwargs): + node = WakuNode(image, f"store_node{node_index}_{self.test_id}") + node.start(discv5_bootstrap_node=self.enr_uri, storenode=self.multiaddr_list[0], **kwargs) + if kwargs["relay"] == "true": + self.main_publishing_nodes.extend([node]) + self.store_nodes.extend([node]) + self.add_node_peer(node, self.multiaddr_list) + return node + + @allure.step + def setup_first_publishing_node(self, store="true", relay="true", **kwargs): + self.publishing_node1 = self.start_publishing_node(NODE_1, node_index=1, store=store, relay=relay, **kwargs) + self.enr_uri = self.publishing_node1.get_enr_uri() + + @allure.step + def setup_second_publishing_node(self, store, relay, **kwargs): + self.publishing_node2 = self.start_publishing_node(NODE_1, node_index=2, store=store, relay=relay, **kwargs) + + @allure.step + def setup_additional_publishing_nodes(self, node_list=ADDITIONAL_NODES, **kwargs): + if node_list: + nodes = [node.strip() for node in node_list.split(",") if node] + else: + pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test") + for index, node in enumerate(nodes): + self.start_publishing_node(node, node_index=index + 2, store="true", relay="true", **kwargs) + + @allure.step + def setup_first_store_node(self, store="true", relay="true", **kwargs): + self.store_node1 = self.setup_store_node(NODE_2, node_index=1, store=store, relay=relay, **kwargs) + + @allure.step + def setup_second_store_node(self, store="true", relay="false", **kwargs): + self.store_node2 = self.setup_store_node(NODE_2, node_index=2, store=store, relay=relay, **kwargs) + + @allure.step + def setup_additional_store_nodes(self, node_list=ADDITIONAL_NODES, **kwargs): + if node_list: + nodes = [node.strip() for node in node_list.split(",") if node] + else: + pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test") + self.additional_store_nodes = [] + for index, node in enumerate(nodes): + node = self.setup_store_node(node, node_index=index + 2, store="true", relay="false", **kwargs) + self.additional_store_nodes.append(node) + + @allure.step + def subscribe_to_pubsub_topics_via_relay(self, node=None, pubsub_topics=None): + if pubsub_topics is None: + pubsub_topics = [self.test_pubsub_topic] + if not node: + node = self.main_publishing_nodes + if isinstance(node, list): + for node in node: + node.set_relay_subscriptions(pubsub_topics) + else: + node.set_relay_subscriptions(pubsub_topics) + + @allure.step + def subscribe_to_pubsub_topics_via_filter(self, node, pubsub_topic=None, content_topic=None): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if content_topic is None: + content_topic = [self.test_content_topic] + subscription = {"requestId": "1", "contentFilters": content_topic, "pubsubTopic": pubsub_topic} + node.set_filter_subscriptions(subscription) + + @allure.step + def publish_message(self, via="relay", pubsub_topic=None, message=None, message_propagation_delay=0.2, sender=None): + self.message = self.create_message() if message is None else message + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if not sender: + sender = self.publishing_node1 + if via == "relay": + logger.debug("Relaying message") + sender.send_relay_message(self.message, pubsub_topic) + elif via == "lightpush": + payload = self.create_payload(pubsub_topic, self.message) + sender.send_light_push_message(payload) + delay(message_propagation_delay) + return self.message + + @retry(stop=stop_after_delay(30), wait=wait_fixed(1), reraise=True) + @allure.step + def get_messages_from_store_with_retry(self, node): + return self.get_messages_from_store(node, page_size=5) + + @allure.step + def get_messages_from_store( + self, + node=None, + peer_addr=None, + include_data=None, + pubsub_topic=None, + content_topics=None, + start_time=None, + end_time=None, + hashes=None, + cursor=None, + page_size=None, + ascending="true", + store_v="v3", + **kwargs, + ): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + store_response = node.get_store_messages( + peer_addr=peer_addr, + include_data=include_data, + pubsub_topic=pubsub_topic, + content_topics=content_topics, + start_time=start_time, + end_time=end_time, + hashes=hashes, + cursor=cursor, + page_size=page_size, + ascending=ascending, + store_v=store_v, + **kwargs, + ) + store_response = StoreResponse(store_response, node) + assert store_response.request_id is not None, "Request id is missing" + assert store_response.status_code, "Status code is missing" + assert store_response.status_desc, "Status desc is missing" + return store_response + + @allure.step + def check_published_message_is_stored( + self, + store_node=None, + peer_addr=None, + include_data=None, + pubsub_topic=None, + content_topics=None, + start_time=None, + end_time=None, + hashes=None, + cursor=None, + page_size=None, + ascending=None, + store_v="v3", + messages_to_check=None, + **kwargs, + ): + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if messages_to_check is None: + messages_to_check = [self.message] + if store_node is None: + store_node = self.store_nodes + elif not isinstance(store_node, list): + store_node = [store_node] + for node in store_node: + logger.debug(f"Checking that peer {node.image} can find the stored messages") + self.store_response = self.get_messages_from_store( + node=node, + peer_addr=peer_addr, + include_data=include_data, + pubsub_topic=pubsub_topic, + content_topics=content_topics, + start_time=start_time, + end_time=end_time, + hashes=hashes, + cursor=cursor, + page_size=page_size, + ascending=ascending, + store_v=store_v, + **kwargs, + ) + + logger.debug(f"messages length is {len(self.store_response.messages)}") + assert self.store_response.messages, f"Peer {node.image} couldn't find any messages. Actual response: {self.store_response.resp_json}" + assert len(self.store_response.messages) >= len( + messages_to_check + ), f"Expected at least {len(messages_to_check)} messages but got {len(self.store_response.messages)}" + + # Determine the range of indices for message comparison + if len(messages_to_check) == 1: + indices = [-1] # Use the last message in the store response for a single message + else: + indices = range(len(messages_to_check)) # Use corresponding indices for multiple messages + + # Iterate through messages_to_check and their respective indices + for idx, message_to_check in zip(indices, messages_to_check): + if store_v == "v1": + waku_message = WakuMessage([self.store_response.messages[idx]]) + waku_message.assert_received_message(message_to_check) + else: + hash_type = "hex" if node.is_nwaku() else "base64" + expected_hash = self.compute_message_hash(pubsub_topic, message_to_check, hash_type=hash_type) + actual_hash = self.store_response.message_hash(idx) + assert ( + expected_hash == actual_hash + ), f"Message hash at index {idx} returned by store doesn't match the computed message hash {expected_hash}. Actual hash: {actual_hash}" + + @allure.step + def check_sent_message_is_stored( + self, + expected_hashes, + store_node=None, + peer_addr=None, + include_data=None, + pubsub_topic=None, + content_topics=None, + start_time=None, + end_time=None, + hashes=None, + cursor=None, + page_size=None, + ascending=None, + store_v="v3", + **kwargs, + ): + """Verify that messages with the given hashes are present in the store.""" + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + if isinstance(expected_hashes, str): + expected_hashes = [expected_hashes] + if store_node is None: + store_node = self.store_nodes + elif not isinstance(store_node, list): + store_node = [store_node] + + for node in store_node: + logger.debug(f"Checking that peer {node.image} can find the stored messages by hash") + self.store_response = self.get_messages_from_store( + node=node, + peer_addr=peer_addr, + include_data=include_data, + pubsub_topic=pubsub_topic, + content_topics=content_topics, + start_time=start_time, + end_time=end_time, + hashes=hashes, + cursor=cursor, + page_size=page_size, + ascending=ascending, + store_v=store_v, + **kwargs, + ) + + logger.debug(f"messages length is {len(self.store_response.messages)}") + assert self.store_response.messages, f"Peer {node.image} couldn't find any messages. " f"Actual response: {self.store_response.resp_json}" + assert len(self.store_response.messages) >= len(expected_hashes), ( + f"Expected at least {len(expected_hashes)} messages " f"but got {len(self.store_response.messages)}" + ) + + actual_hashes = [self.store_response.message_hash(i) for i in range(len(self.store_response.messages))] + for expected_hash in expected_hashes: + assert expected_hash in actual_hashes, f"Expected hash {expected_hash} not found in store. " f"Actual hashes: {actual_hashes}" + + @allure.step + def check_store_returns_empty_response(self, pubsub_topic=None): + if not pubsub_topic: + pubsub_topic = self.test_pubsub_topic + try: + self.check_published_message_is_stored(pubsubTopic=pubsub_topic, page_size=5, ascending="true") + except Exception as ex: + assert "couldn't find any messages" in str(ex) + + @allure.step + def create_payload(self, pubsub_topic=None, message=None, **kwargs): + if message is None: + message = self.create_message() + if pubsub_topic is None: + pubsub_topic = self.test_pubsub_topic + payload = {"pubsubTopic": pubsub_topic, "message": message} + payload.update(kwargs) + return payload + + @allure.step + def get_store_messages_with_errors( + self, + node=None, + peer_addr=None, + include_data=None, + pubsub_topic=None, + content_topics=None, + start_time=None, + end_time=None, + hashes=None, + cursor=None, + page_size=None, + ascending="true", + store_v="v3", + **kwargs, + ): + """ + This method calls the original get_store_messages and returns the actual + error response from the service, if present. + """ + try: + # Call the original get_store_messages method + store_response = node.get_store_messages( + peer_addr=peer_addr, + include_data=include_data, + pubsub_topic=pubsub_topic, + content_topics=content_topics, + start_time=start_time, + end_time=end_time, + hashes=hashes, + cursor=cursor, + page_size=page_size, + ascending=ascending, + store_v=store_v, + **kwargs, + ) + + # Check if the response has a status code >= 400, indicating an error + if store_response.status_code >= 400: + # Return the status code and the plain text error message directly + return {"status_code": store_response.status_code, "error_message": store_response.text} # Handling plain text response + + # Otherwise, return the successful response as JSON + response_json = store_response.json() + response_json["status_code"] = store_response.status_code + return response_json + + except requests.exceptions.HTTPError as http_err: + # Handle HTTP errors separately + return {"status_code": http_err.response.status_code, "error_message": http_err.response.text} + + except Exception as e: + # Handle unexpected errors and return as 500 + return {"status_code": 500, "error_message": str(e)} diff --git a/tests-e2e/src/test_data.py b/tests-e2e/src/test_data.py new file mode 100644 index 000000000..659dd39ae --- /dev/null +++ b/tests-e2e/src/test_data.py @@ -0,0 +1,356 @@ +from time import time +from datetime import datetime, timedelta + +NOW = datetime.now() + +SAMPLE_INPUTS = [ + {"description": "A simple string", "value": "Hello World!"}, + {"description": "An integer", "value": "1234567890"}, + {"description": "A dictionary", "value": '{"key": "value"}'}, + {"description": "Chinese characters", "value": "这是一些中文"}, + {"description": "Emojis", "value": "🚀🌟✨"}, + {"description": "Lorem ipsum text", "value": "Lorem ipsum dolor sit amet"}, + {"description": "HTML content", "value": "Hello"}, + {"description": "Cyrillic characters", "value": "\u041f\u0440\u0438\u0432\u0435\u0442"}, + {"description": "Base64 encoded string", "value": "Base64==dGVzdA=="}, + {"description": "Binary data", "value": "d29ya2luZyB3aXRoIGJpbmFyeSBkYXRh: \x50\x51"}, + {"description": "Special characters with whitespace", "value": "\t\nSpecial\tCharacters\n"}, + {"description": "Boolean false as a string", "value": "False"}, + {"description": "A float number", "value": "3.1415926535"}, + {"description": "A list", "value": "[1, 2, 3, 4, 5]"}, + {"description": "Hexadecimal number as a string", "value": "0xDEADBEEF"}, + {"description": "Email format", "value": "user@example.com"}, + {"description": "URL format", "value": "http://example.com"}, + {"description": "Date and time in ISO format", "value": "2023-11-01T12:00:00Z"}, + {"description": "String with escaped quotes", "value": '"Escaped" \\"quotes\\"'}, + {"description": "A regular expression", "value": "Regular expression: ^[a-z0-9_-]{3,16}$"}, + {"description": "A very long string", "value": "x" * 1000}, + {"description": "A JSON string", "value": '{"name": "John", "age": 30, "city": "New York"}'}, + {"description": "A Unix path", "value": "/usr/local/bin"}, + {"description": "A Windows path", "value": "C:\\Windows\\System32"}, + {"description": "An SQL query", "value": "SELECT * FROM users WHERE id = 1;"}, + {"description": "JavaScript code snippet", "value": "function test() { console.log('Hello World'); }"}, + {"description": "A CSS snippet", "value": "body { background-color: #fff; }"}, + {"description": "A Python one-liner", "value": "print('Hello World')"}, + {"description": "An IP address", "value": "192.168.1.1"}, + {"description": "A domain name", "value": "www.example.com"}, + {"description": "A user agent string", "value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}, + {"description": "A credit card number", "value": "1234-5678-9012-3456"}, + {"description": "A phone number", "value": "+1234567890"}, + {"description": "A UUID", "value": "123e4567-e89b-12d3-a456-426614174000"}, + {"description": "A hashtag", "value": "#helloWorld"}, + {"description": "A Twitter handle", "value": "@username"}, + {"description": "A password", "value": "P@ssw0rd!"}, + {"description": "A date in common format", "value": "01/11/2023"}, + {"description": "A time string", "value": "12:00:00"}, + {"description": "A mathematical equation", "value": "E = mc^2"}, +] + +INVALID_PAYLOADS = [ + {"description": "Empty string", "value": ""}, + {"description": "Unecoded text", "value": "Hello World!"}, + {"description": "A dictionary", "value": {"key": "YWFh"}}, + {"description": "An integer", "value": 1234567890}, + {"description": "A list", "value": ["YWFh"]}, + {"description": "A bool", "value": True}, +] + +INVALID_CONTENT_TOPICS = [ + {"description": "Empty string", "value": ""}, + {"description": "A dictionary", "value": {"key": "YWFh"}}, + {"description": "An integer", "value": 1234567890}, + {"description": "A list", "value": ["YWFh"]}, + {"description": "A bool", "value": True}, +] + +CONTENT_TOPICS_DIFFERENT_SHARDS = [ + "/myapp/1/latest/proto", # resolves to shard 0 + "/waku/2/content/test.js", # resolves to shard 1 + "/app/22/sometopic/someencoding", # resolves to shard 2 + "/toychat/2/huilong/proto", # resolves to shard 3 + "/statusim/1/community/cbor", # resolves to shard 4 + "/app/27/sometopic/someencoding", # resolves to shard 5 + "/app/29/sometopic/someencoding", # resolves to shard 6 + "/app/20/sometopic/someencoding", # resolves to shard 7 +] + +CONTENT_TOPICS_SHARD_0 = [ + "/newsService/1.0/weekly/protobuf", + "/newsService/1.0/alerts/xml", + "/newsService/1.0/updates/json", + "/newsService/2.0/alerts/json", + "/newsService/2.0/summaries/xml", + "/newsService/2.0/highlights/yaml", + "/newsService/3.0/weekly/json", + "/newsService/3.0/summaries/xml", +] + +CONTENT_TOPICS_SHARD_7 = [ + "/newsService/2.0/alerts/yaml", + "/newsService/2.0/highlights/xml", + "/newsService/3.0/daily/protobuf", + "/newsService/3.0/alerts/xml", + "/newsService/3.0/updates/protobuf", + "/newsService/3.0/reviews/xml", + "/newsService/4.0/alerts/yaml", + "/newsService/4.0/updates/yaml", +] + +DEFAULT_CLUSTER_ID = "198" +VALID_PUBSUB_TOPICS = [ + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/9", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/25", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1000", +] + +PUBSUB_TOPICS_STORE = [ + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/2", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/3", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/4", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/5", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/6", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/7", + f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/8", +] + +INVALID_PUBSUB_TOPICS = ["/test/2/rs/0/1", "/waku/3/rs/0/1", "/waku/2/test/0/1", "/waku/2/rs/3/b", "/waku/2/rs/0"] + +PUBSUB_TOPICS_DIFFERENT_CLUSTERS = [ + "/waku/2/rs/2/0", + "/waku/2/rs/2/1", + "/waku/2/rs/5/0", + "/waku/2/rs/5/1", + "/waku/2/rs/5/999", + "/waku/2/rs/8/0", + "/waku/2/rs/999/999", +] + +PUBSUB_TOPICS_SAME_CLUSTER = [ + "/waku/2/rs/199/0", + "/waku/2/rs/199/1", + "/waku/2/rs/199/2", + "/waku/2/rs/199/3", + "/waku/2/rs/199/4", + "/waku/2/rs/199/5", + "/waku/2/rs/199/6", + "/waku/2/rs/199/7", +] + +PUBSUB_TOPICS_WRONG_FORMAT = [ + {"description": "A dictionary", "value": {"key": "YWFh"}}, + {"description": "An integer", "value": 1234567890}, + {"description": "A list", "value": ["YWFh"]}, + {"description": "A bool", "value": True}, +] + +SAMPLE_TIMESTAMPS = [ + {"description": "Now", "value": int(time() * 1e9), "valid_for": ["nwaku"]}, + { + "description": "Far future", + "value": int((NOW + timedelta(days=365 * 10)).timestamp() * 1e9), + "valid_for": ["nwaku"], + }, # 10 years from now + {"description": "Recent past", "value": int((NOW - timedelta(hours=1)).timestamp() * 1e9), "valid_for": ["nwaku"]}, # 1 hour ago + {"description": "Near future", "value": int((NOW + timedelta(hours=1)).timestamp() * 1e9), "valid_for": ["nwaku"]}, # 1 hour ahead + {"description": "Positive number", "value": 1, "valid_for": ["nwaku"]}, + {"description": "Negative number", "value": -1, "valid_for": ["nwaku"]}, + {"description": "DST change", "value": int(datetime(2020, 3, 8, 2, 0, 0).timestamp() * 1e9), "valid_for": ["nwaku"]}, # DST starts + {"description": "Timestamp as string number", "value": str(int(time() * 1e9)), "valid_for": []}, + {"description": "Invalid large number", "value": 2**63, "valid_for": []}, + {"description": "Float number", "value": float(time() * 1e9), "valid_for": []}, + {"description": "Array instead of timestamp", "value": [int(time() * 1e9)], "valid_for": []}, + {"description": "Object instead of timestamp", "value": {"time": int(time() * 1e9)}, "valid_for": []}, + {"description": "ISO 8601 timestamp", "value": "2023-12-26T10:58:51", "valid_for": []}, + {"description": "Missing", "value": None, "valid_for": []}, +] + +PUBSUB_TOPICS_RLN = [f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0"] + +LOG_ERROR_KEYWORDS = [ + "crash", + "fatal", + "panic", + "abort", + "segfault", + "corrupt", + "terminated", + "unhandled", + "stacktrace", + "deadlock", + "SIGSEGV", + "SIGABRT", + "stack overflow", + "index out of bounds", + "nil pointer dereference", + "goroutine exit", + "nil pointer", + "runtime error", + "goexit", + "race condition", + "double free", +] + +METRICS_WITH_INITIAL_VALUE_ZERO = [ + "libp2p_peers", + "libp2p_failed_upgrades_incoming_total", + "libp2p_failed_upgrades_outgoing_total", + "libp2p_total_dial_attempts_total", + "libp2p_successful_dials_total", + "libp2p_failed_dials_total", + "waku_rln_messages_total_total", + "waku_rln_spam_messages_total_total", + "waku_rln_proof_verification_total_total", + "waku_rln_number_registered_memberships", + "waku_rln_proof_verification_duration_seconds", + "waku_rln_proof_generation_duration_seconds", + "waku_rln_instance_creation_duration_seconds", + "waku_rln_membership_insertion_duration_seconds", + "waku_rln_membership_credentials_import_duration_seconds", + "waku_rln_remaining_proofs_per_epoch", + "waku_rln_total_generated_proofs", + "libp2p_pubsub_sig_verify_success_total", + "libp2p_pubsub_sig_verify_failure_total", + "libp2p_pubsub_disconnects_over_non_priority_queue_limit_total", + "libp2p_pubsub_peers", + "libp2p_pubsub_unsubscriptions_total", + "libp2p_pubsub_validation_success_total", + "libp2p_pubsub_validation_failure_total", + "libp2p_pubsub_validation_ignore_total", + "libp2p_pubsub_broadcast_iwant_total", + "libp2p_pubsub_received_iwant_total", + "libp2p_gossipsub_cache_window_size", + 'libp2p_gossipsub_peers_per_topic_mesh{topic="other"}', + 'libp2p_gossipsub_peers_per_topic_fanout{topic="other"}', + 'libp2p_gossipsub_peers_per_topic_gossipsub{topic="other"}', + "libp2p_gossipsub_low_peers_topics", + "libp2p_gossipsub_healthy_peers_topics", + "libp2p_gossipsub_failed_publish_total", + "libp2p_gossipsub_invalid_topic_subscription_total", + "libp2p_gossipsub_duplicate_during_validation_total", + "libp2p_gossipsub_idontwant_saved_messages_total", + "libp2p_gossipsub_duplicate_total", + "libp2p_gossipsub_received_total", + "libp2p_rendezvous_register_total", + "libp2p_rendezvous_discover_total", + "libp2p_rendezvous_registered", + "libp2p_rendezvous_namespaces", + "waku_peer_store_size", + "waku_total_unique_peers", + "presto_server_missing_requests_count", + "presto_server_invalid_requests_count", + 'waku_archive_messages{type="stored"}', + "waku_archive_insert_duration_seconds_sum", + "waku_archive_insert_duration_seconds_count", + 'waku_archive_insert_duration_seconds_bucket{le="0.005"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.01"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.025"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.05"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.075"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.1"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.25"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.5"}', + 'waku_archive_insert_duration_seconds_bucket{le="0.75"}', + 'waku_archive_insert_duration_seconds_bucket{le="1.0"}', + 'waku_archive_insert_duration_seconds_bucket{le="2.5"}', + 'waku_archive_insert_duration_seconds_bucket{le="5.0"}', + 'waku_archive_insert_duration_seconds_bucket{le="7.5"}', + 'waku_archive_insert_duration_seconds_bucket{le="10.0"}', + 'waku_archive_insert_duration_seconds_bucket{le="+Inf"}', + "waku_archive_query_duration_seconds_sum", + "waku_archive_query_duration_seconds_count", + 'waku_archive_query_duration_seconds_bucket{le="0.005"}', + 'waku_archive_query_duration_seconds_bucket{le="0.01"}', + 'waku_archive_query_duration_seconds_bucket{le="0.025"}', + 'waku_archive_query_duration_seconds_bucket{le="0.05"}', + 'waku_archive_query_duration_seconds_bucket{le="0.075"}', + 'waku_archive_query_duration_seconds_bucket{le="0.1"}', + 'waku_archive_query_duration_seconds_bucket{le="0.25"}', + 'waku_archive_query_duration_seconds_bucket{le="0.5"}', + 'waku_archive_query_duration_seconds_bucket{le="0.75"}', + 'waku_archive_query_duration_seconds_bucket{le="1.0"}', + 'waku_archive_query_duration_seconds_bucket{le="2.5"}', + 'waku_archive_query_duration_seconds_bucket{le="5.0"}', + 'waku_archive_query_duration_seconds_bucket{le="7.5"}', + 'waku_archive_query_duration_seconds_bucket{le="10.0"}', + 'waku_archive_query_duration_seconds_bucket{le="+Inf"}', + "waku_filter_subscriptions", + "waku_filter_handle_message_duration_seconds_sum", + "waku_filter_handle_message_duration_seconds_count", + 'waku_filter_handle_message_duration_seconds_bucket{le="0.005"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.01"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.025"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.05"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.075"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.1"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.25"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.5"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="0.75"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="1.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="2.5"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="5.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="7.5"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="10.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="+Inf"}', + "discovery_session_lru_cache_hits_total", + "discovery_session_lru_cache_misses_total", + "discovery_session_decrypt_failures_total", + "discovery_unsolicited_messages_total", + "discovery_enr_auto_update_total", + "waku_px_peers_received_total", + "waku_px_peers_received_unknown", + "waku_px_peers_sent_total", + "waku_px_peers", + "waku_histogram_message_size_sum", + "waku_histogram_message_size_count", + 'waku_histogram_message_size_bucket{le="0.0"}', + 'waku_histogram_message_size_bucket{le="1.0"}', + 'waku_histogram_message_size_bucket{le="3.0"}', + 'waku_histogram_message_size_bucket{le="5.0"}', + 'waku_histogram_message_size_bucket{le="15.0"}', + 'waku_histogram_message_size_bucket{le="50.0"}', + 'waku_histogram_message_size_bucket{le="75.0"}', + 'waku_histogram_message_size_bucket{le="100.0"}', + 'waku_histogram_message_size_bucket{le="125.0"}', + 'waku_histogram_message_size_bucket{le="150.0"}', + 'waku_histogram_message_size_bucket{le="500.0"}', + 'waku_histogram_message_size_bucket{le="700.0"}', + 'waku_histogram_message_size_bucket{le="1000.0"}', + 'waku_histogram_message_size_bucket{le="+Inf"}', + "waku_lightpush_peers", + "waku_filter_peers", + "waku_store_peers", + "waku_px_peers", + "waku_dnsdisc_discovered", + "rendezvousPeerFoundTotal_total", + "reconciliation_roundtrips_sum", + "reconciliation_roundtrips_count", + "reconciliation_differences_sum", + "reconciliation_differences_count", + 'reconciliation_roundtrips_bucket{le="1.0"}', + 'reconciliation_roundtrips_bucket{le="2.0"}', + 'reconciliation_roundtrips_bucket{le="3.0"}', + 'reconciliation_roundtrips_bucket{le="5.0"}', + 'reconciliation_roundtrips_bucket{le="8.0"}', + 'reconciliation_roundtrips_bucket{le="13.0"}', + 'reconciliation_roundtrips_bucket{le="+Inf"}', + 'reconciliation_differences_bucket{le="0.0"}', + 'reconciliation_differences_bucket{le="10.0"}', + 'reconciliation_differences_bucket{le="50.0"}', + 'reconciliation_differences_bucket{le="100.0"}', + 'reconciliation_differences_bucket{le="500.0"}', + 'reconciliation_differences_bucket{le="1000.0"}', + 'reconciliation_differences_bucket{le="5000.0"}', + 'reconciliation_differences_bucket{le="+Inf"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="15.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="20.0"}', + 'waku_filter_handle_message_duration_seconds_bucket{le="30.0"}', + "total_messages_cached", + "waku_store_queries_total", + "mix_pool_size", + "libp2p_gossipsub_imreceiving_saved_messages_total", + "postgres_payload_size_bytes", +] diff --git a/tests-e2e/tests/__init__.py b/tests-e2e/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/tests/conftest.py b/tests-e2e/tests/conftest.py new file mode 100644 index 000000000..5c42669dd --- /dev/null +++ b/tests-e2e/tests/conftest.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +import inspect +import glob +from src.libs.custom_logger import get_custom_logger +import os +import pytest +from datetime import datetime +from time import time +from uuid import uuid4 +from src.libs.common import attach_allure_file +import src.env_vars as env_vars +from src.data_storage import DS +from src.postgres_setup import start_postgres, stop_postgres + +logger = get_custom_logger(__name__) + + +# See https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_makereport(item): + outcome = yield + rep = outcome.get_result() + if rep.when == "call": + setattr(item, "rep_call", rep) + return rep + return None + + +@pytest.fixture(scope="session", autouse=True) +def set_allure_env_variables(): + yield + if os.path.isdir("allure-results") and not os.path.isfile(os.path.join("allure-results", "environment.properties")): + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + with open(os.path.join("allure-results", "environment.properties"), "w") as outfile: + for attribute_name in dir(env_vars): + if attribute_name.isupper(): + attribute_value = getattr(env_vars, attribute_name) + outfile.write(f"{attribute_name}={attribute_value}\n") + + +@pytest.fixture(scope="function", autouse=False) +def start_postgres_container(): + pg_container = start_postgres() + yield + stop_postgres(pg_container) + + +@pytest.fixture(scope="function", autouse=True) +def test_id(request): + # setting up an unique test id to be used where needed + logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}") + request.cls.test_id = f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}__{str(uuid4())}" + + +@pytest.fixture(scope="function", autouse=True) +def test_setup(request, test_id): + logger.debug(f"Running test: {request.node.name} with id: {request.cls.test_id}") + yield + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + for file in glob.glob(os.path.join(env_vars.DOCKER_LOG_DIR, "*")): + if os.path.getmtime(file) < time() - 3600: + logger.debug(f"Deleting old log file: {file}") + try: + os.remove(file) + except: + logger.error("Could not delete file") + + +@pytest.fixture(scope="function", autouse=True) +def attach_logs_on_fail(request): + yield + if env_vars.RUNNING_IN_CI and hasattr(request.node, "rep_call") and request.node.rep_call.failed: + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + logger.debug("Test failed, attempting to attach logs to the allure reports") + for file in glob.glob(os.path.join(env_vars.DOCKER_LOG_DIR, "*" + request.cls.test_id + "*")): + attach_allure_file(file) + + +@pytest.fixture(scope="function", autouse=True) +def close_open_nodes(attach_logs_on_fail): + DS.waku_nodes = [] + yield + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + crashed_containers = [] + for node in DS.waku_nodes: + try: + node.stop() + except Exception as ex: + if "No such container" in str(ex): + crashed_containers.append(node.image) + logger.error(f"Failed to stop container because of error {ex}") + assert not crashed_containers, f"Containers {crashed_containers} crashed during the test!!!" + + +@pytest.fixture(scope="function", autouse=True) +def check_waku_log_errors(): + yield + logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}") + for node in DS.waku_nodes: + node.check_waku_log_errors() diff --git a/tests-e2e/tests/wrappers_tests/conftest.py b/tests-e2e/tests/wrappers_tests/conftest.py new file mode 100644 index 000000000..3d2e19e38 --- /dev/null +++ b/tests-e2e/tests/wrappers_tests/conftest.py @@ -0,0 +1,35 @@ +import socket +import pytest +from src.test_data import DEFAULT_CLUSTER_ID + + +def free_port(): + """Return a currently-unbound TCP/UDP port from the OS.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def build_node_config(**overrides): + config = { + "logLevel": "DEBUG", + "listenAddress": "0.0.0.0", + "tcpPort": free_port(), + "discv5UdpPort": free_port(), + "restPort": free_port(), + "restAddress": "0.0.0.0", + "clusterId": DEFAULT_CLUSTER_ID, + "relay": True, + "store": True, + "filter": False, + "lightpush": False, + "peerExchange": False, + "discv5Discovery": False, + } + config.update(overrides) + return config + + +@pytest.fixture +def node_config(): + return build_node_config() diff --git a/tests-e2e/tests/wrappers_tests/test_basic_life_cycle.py b/tests-e2e/tests/wrappers_tests/test_basic_life_cycle.py new file mode 100644 index 000000000..eb2026438 --- /dev/null +++ b/tests-e2e/tests/wrappers_tests/test_basic_life_cycle.py @@ -0,0 +1,26 @@ +import pytest +from src.node.wrappers_manager import WrapperManager + + +@pytest.mark.smoke +class TestLogosDeliveryLifecycle: + def _create_start_node(self, node_config): + result = WrapperManager.create_and_start(config=node_config) + assert result.is_ok(), f"Failed to create and start node: {result.err()}" + return result.ok_value + + def test_create_start_and_stop_node(self, node_config): + node = self._create_start_node(node_config) + + stop_result = node.stop_and_destroy() + assert stop_result.is_ok(), f"Failed to stop and destroy node: {stop_result.err()}" + + def test_stop_node_without_destroy(self, node_config): + node = self._create_start_node(node_config) + + try: + stop_result = node.stop_node() + assert stop_result.is_ok(), f"Failed to stop node: {stop_result.err()}" + finally: + destroy_result = node.destroy() + assert destroy_result.is_ok(), f"Failed to destroy node: {destroy_result.err()}" diff --git a/tests-e2e/tests/wrappers_tests/test_send_e2e_part1.py b/tests-e2e/tests/wrappers_tests/test_send_e2e_part1.py new file mode 100644 index 000000000..c998c73df --- /dev/null +++ b/tests-e2e/tests/wrappers_tests/test_send_e2e_part1.py @@ -0,0 +1,1018 @@ +from concurrent.futures import ThreadPoolExecutor + +import pytest +from src.env_vars import NODE_2 +from src.steps.common import StepsCommon +from src.libs.common import delay, to_base64 +from src.libs.custom_logger import get_custom_logger +from src.node.waku_node import WakuNode +from src.node.wrappers_manager import WrapperManager +from src.node.wrapper_helpers import ( + EventCollector, + assert_event_invariants, + create_message_bindings, + get_node_multiaddr, + wait_for_connected, + wait_for_propagated, + wait_for_sent, + wait_for_error, +) +from src.steps.store import StepsStore +from tests.wrappers_tests.conftest import free_port + +logger = get_custom_logger(__name__) + +## max time to wait after sending the message +PROPAGATED_TIMEOUT_S = 30.0 +SENT_TIMEOUT_S = 10.0 +NO_SENT_OBSERVATION_S = 5.0 +SENT_AFTER_STORE_TIMEOUT_S = 60.0 +NO_STORE_OBSERVATION_S = 60.0 +RECOVERY_TIMEOUT_S = 45.0 + +# S20 stabilization delays for gossipsub mesh formation. +MESH_STABILIZATION_S = 10 +STORE_JOIN_STABILIZATION_S = 10 + +# MaxTimeInCache from send_service.nim. +MAX_TIME_IN_CACHE_S = 60.0 +# Extra slack to cover the background retry loop tick after the window expires. +CACHE_EXPIRY_SLACK_S = 10.0 +ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S = MAX_TIME_IN_CACHE_S + CACHE_EXPIRY_SLACK_S +RETRY_WINDOW_EXPIRED_MSG = "Unable to send within retry time window" + +# S30: concurrent sends on the same content topic during initial auto-subscribe. +S30_CONCURRENT_SENDS = 5 +S30_CONTENT_TOPIC = "/test/1/s30-concurrent/proto" + +# S31: concurrent sends across mixed topics during peer churn. +S31_BURST_SIZE = 8 +S31_CONTENT_TOPICS = [ + "/test/1/s31-topic-a/proto", + "/test/1/s31-topic-b/proto", + "/test/1/s31-topic-c/proto", + "/test/1/s31-topic-d/proto", + "/test/1/s31-topic-e/proto", + "/test/1/s31-topic-f/proto", + "/test/1/s31-topic-g/proto", + "/test/1/s31-topic-h/proto", +] + + +class TestSendBeforeRelay(StepsStore): + def test_s17_send_before_relay_peers_joins(self, node_config): + """ + S17: sender starts isolated, calls send() + - send() returns Ok(RequestId) immediately + - Propagated event eventually arrives + """ + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + message = create_message_bindings() + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId) even with no peers, got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + # Step 2: start a relay peer with store enabled. + relay_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender_node)], + "portsShift": 1, + "store": True, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + # Match the gating part2's tests use: wait until the sender + # actually reports Connected/PartiallyConnected before asserting + # on propagation. Without this, the wait_for_propagated poll can + # miss the event because the sender's mesh hasn't formed yet. + assert wait_for_connected(sender_collector) is not None, ( + f"Sender did not reach Connected/PartiallyConnected after " f"relay peer joined. Collected events: {sender_collector.events}" + ) + + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent received within {PROPAGATED_TIMEOUT_S}s " + f"after relay peer joined. Collected events: {sender_collector.events}" + ) + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=SENT_TIMEOUT_S, + ) + assert sent_event is not None, ( + f"No MessageSentEvent received within {SENT_TIMEOUT_S}s " + f"from a store-enabled relay peer. Collected events: {sender_collector.events}" + ) + + assert_event_invariants(sender_collector, request_id) + + @pytest.mark.docker_required + @pytest.mark.xfail(reason="fails to republish after store peer joins mesh see https://github.com/logos-messaging/logos-delivery/issues/3848") + def test_s19_store_peer_appears_after_propagation(self, node_config): + """ + S19: a store peer comes online later. + - send() returns Ok(RequestId) immediately + - Propagated --- relay peer + - Sent when store peer is reachable + """ + sender_collector = EventCollector() + + node_config.update({"relay": True, "store": False, "discv5Discovery": False, "numShardsInNetwork": 1, "reliabilityEnabled": True}) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + # relay peer + relay_config = { + **node_config, + "tcpPort": free_port(), + "discv5UdpPort": free_port(), + "restPort": free_port(), + "staticnodes": [get_node_multiaddr(sender_node)], + "store": False, + "reliabilityEnabled": True, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value as relay_peer: + # Wait until the sender actually reports a connection before + # sending. Without this, send() can race the static-peer + # dial on slower runners (same gate S17 uses). + assert wait_for_connected(sender_collector) is not None, ( + f"Sender did not reach Connected/PartiallyConnected after " f"relay peer joined. Collected events: {sender_collector.events}" + ) + message = create_message_bindings() + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId), got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + # Propagated should arrive via the relay peer. + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent received within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + + early_sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=NO_SENT_OBSERVATION_S, + ) + assert early_sent_event is None, f"MessageSentEvent arrived before any store peer was reachable. " f"Event: {early_sent_event}" + + # Store peer + store_node = WakuNode(NODE_2, f"store_node") + store_node.start(relay="true", store="true", discv5_discovery="false", cluster_id=node_config["clusterId"], shard=0) + store_node.set_relay_subscriptions([self.test_pubsub_topic]) + relay_multiaddr = get_node_multiaddr(relay_peer) + sender_multiaddr = get_node_multiaddr(sender_node) + store_node.add_peers([relay_multiaddr, sender_multiaddr]) + self.wait_for_autoconnection([store_node], hard_wait=40) + delay(3) + + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=SENT_AFTER_STORE_TIMEOUT_S, + ) + + assert sent_event is not None, ( + f"No MessageSentEvent received within {SENT_AFTER_STORE_TIMEOUT_S}s " + f"after store peer joined. Collected events: {sender_collector.events}" + ) + + self.check_published_message_is_stored( + store_node=store_node, + pubsub_topic=self.test_pubsub_topic, + messages_to_check=[message], + page_size=5, + ascending="true", + ) + + assert_event_invariants(sender_collector, request_id) + + @pytest.mark.docker_required + @pytest.mark.skip(reason="Forcing the miss store round not possible") + def test_s20_store_misses_initially_then_retry_succeeds(self, node_config): + """ + S20: relay propagation succeeds, the first store query misses + (the store peer is reachable but does not yet have the message), + a later retry republishes through the relay mesh, and the store + peer then archives it. + + Covers state flow: + SuccessfullyPropagated -> NextRoundRetry + -> SuccessfullyPropagated -> SuccessfullyValidated + + """ + sender_collector = EventCollector() + store_node = WakuNode(NODE_2, f"s20_store_node_{self.test_id}") + store_node.start( + relay="true", + store="true", + discv5_discovery="false", + cluster_id=node_config["clusterId"], + shard=0, + ) + store_multiaddr = store_node.get_multiaddr_with_id() + + node_config.update( + { + "relay": True, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "reliabilityEnabled": True, + "storenode": store_multiaddr, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + relay_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender_node)], + "portsShift": 1, + "store": False, + } + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value as relay_peer: + # Wait for the sender to see the relay peer before publishing. + assert wait_for_connected(sender_collector) is not None, ( + f"Sender did not reach Connected/PartiallyConnected. " f"Collected events: {sender_collector.events}" + ) + + # Let the gossipsub mesh form between sender and relay peer. + delay(MESH_STABILIZATION_S) + + message = create_message_bindings(ephemeral=False) + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId), got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + # Round 1: propagation succeeds via the relay peer. + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + + # The store peer is reachable for queries but never received + # the message via gossipsub, so the first store query must + # miss and Sent must NOT arrive yet. + early_sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=NO_SENT_OBSERVATION_S, + ) + assert early_sent_event is None, ( + f"MessageSentEvent arrived before the store could have the message. " + f"Initial store query should have missed. Event: {early_sent_event}" + ) + + # Now subscribe the store to the test topic and wire it into + # the relay mesh so the next retry round's republish reaches + # the store via gossipsub. + store_node.set_relay_subscriptions([self.test_pubsub_topic]) + store_node.add_peers([get_node_multiaddr(sender_node), get_node_multiaddr(relay_peer)]) + self.wait_for_autoconnection([store_node], hard_wait=10) + delay(STORE_JOIN_STABILIZATION_S) + + # Round 2: retry republishes, store archives, next query hits. + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=SENT_AFTER_STORE_TIMEOUT_S, + ) + assert sent_event is not None, ( + f"No MessageSentEvent within {SENT_AFTER_STORE_TIMEOUT_S}s " + f"after the store joined the relay mesh. The retry round " + f"should have republished and the store should have archived. " + f"Collected events: {sender_collector.events}" + ) + + self.check_published_message_is_stored( + store_node=store_node, + pubsub_topic=self.test_pubsub_topic, + messages_to_check=[message], + page_size=5, + ascending="true", + ) + + assert_event_invariants(sender_collector, request_id) + + def test_s21_error_when_retry_window_expires(self, node_config): + """ + S21: delivery retry window expires before any valid path recovers. + """ + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "lightpush": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + message = create_message_bindings() + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId) even with no peers, got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + # No peer + error_event = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S, + ) + assert error_event is not None, ( + f"No MessageErrorEvent received within {ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S}s " + f"(MaxTimeInCache={MAX_TIME_IN_CACHE_S}s + slack). " + f"Collected events: {sender_collector.events}" + ) + logger.info(f"S21 received error event: {error_event}") + + assert error_event.get("error") == RETRY_WINDOW_EXPIRED_MSG, ( + f"Unexpected error message in message_error event.\n" + f"Expected: {RETRY_WINDOW_EXPIRED_MSG!r}\n" + f"Got: {error_event.get('error')!r}\n" + f"Full event: {error_event}" + ) + + assert_event_invariants(sender_collector, request_id) + + def test_s22_non_ephemeral_message_with_reliability_disabled(self, node_config): + """ + S22: non-ephemeral message with reliabilityEnabled disabled. + - propagation path exists ,reliabilityEnabled = false. + - Expected: Ok(RequestId), Propagated event only, no Sent event. + Note: S17 already covers the positive path of this test with reliabilityEnabled=True. + """ + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "reliabilityEnabled": False, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + relay_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender_node)], + "portsShift": 1, + "store": True, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + # Wait for the sender to actually establish the mesh before + # publishing, matching part2's pattern. Otherwise the publish + # races with mesh formation and message_propagated may not fire. + assert wait_for_connected(sender_collector) is not None, ( + f"Sender did not reach Connected/PartiallyConnected. " f"Collected events: {sender_collector.events}" + ) + + message = create_message_bindings(ephemeral=False) + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId), got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent received within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=NO_SENT_OBSERVATION_S, + ) + assert sent_event is None, ( + f"Unexpected MessageSentEvent received when reliabilityEnabled is disabled.\n" + f"Sent event: {sent_event}\n" + f"Collected events: {sender_collector.events}" + ) + + assert_event_invariants(sender_collector, request_id) + + def test_s23_no_sent_event_when_relay_has_no_store(self, node_config): + """ + S23: non-ephemeral message, reliability enabled, no store peer ever reachable. + - Expected: Ok(RequestId), Propagated event only, no Sent and no terminal error. + """ + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "reliabilityEnabled": True, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + message = create_message_bindings(ephemeral=False) + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId) even with no peers, got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + relay_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender_node)], + "portsShift": 1, + "store": False, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent received within {PROPAGATED_TIMEOUT_S}s " + f"after relay peer joined. Collected events: {sender_collector.events}" + ) + + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=NO_STORE_OBSERVATION_S, + ) + assert sent_event is None, ( + f"Unexpected MessageSentEvent within {NO_STORE_OBSERVATION_S}s " + f"when relay peer has store=false.\n" + f"Sent event: {sent_event}\n" + f"Collected events: {sender_collector.events}" + ) + + # Regression guard: current behavior must NOT convert "no store + # reachable" into an immediate terminal error. If a future change + # starts emitting one, this assertion will catch it. + error_event = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=0, + ) + assert error_event is None, ( + f"Unexpected terminal error event when no store peer is reachable. " + f"S23 expects silent behavior (Propagated only).\n" + f"Error event: {error_event}\n" + f"Collected events: {sender_collector.events}" + ) + + assert_event_invariants(sender_collector, request_id) + + def test_s24_ephemeral_message_with_reachable_store(self, node_config): + """ + S24: ephemeral message, reliability enabled, reachable store peer. + - Setup: propagation path exists, relay peer has store=True (reachable), + - Expected: Ok(RequestId), Propagated event only, no Sent event. + """ + + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "reliabilityEnabled": True, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + relay_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender_node)], + "portsShift": 1, + "store": True, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + message = create_message_bindings(ephemeral=True) + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId), got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent received within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=NO_STORE_OBSERVATION_S, + ) + assert sent_event is None, ( + f"Unexpected MessageSentEvent for an ephemeral message. " + f"Ephemeral messages must never be store-validated.\n" + f"Sent event: {sent_event}\n" + f"Collected events: {sender_collector.events}" + ) + + assert_event_invariants(sender_collector, request_id) + + def test_s26_lightpush_peer_churn_alternate_remains(self, node_config): + """ + S26: multiple lightpush peers, the selected one disappears, + an alternate remains. + + Topology (3 peers + sender): + - peer1: relay + lightpush. The lightpush server initially selected + by the sender. Stopped mid-test to simulate churn. + - relay_peer: relay-only. Kept alive throughout the test as a + stable gossipsub mesh neighbour, so that after peer1 disappears + peer2 still has a relay path to propagate the message. + - peer2: relay + lightpush. The surviving lightpush server that + must take over once peer1 is gone. + - sender: edge node with peer1 and peer2 as static lightpush peers. + """ + sender_collector = EventCollector() + peer1_config = { + **node_config, + "relay": True, + "lightpush": True, + "store": False, + "filter": False, + "discv5Discovery": True, + "numShardsInNetwork": 1, + "portsShift": 1, + "discv5UdpPort": free_port(), + } + peer1_result = WrapperManager.create_and_start(config=peer1_config) + assert peer1_result.is_ok(), f"Failed to start lightpush peer1: {peer1_result.err()}" + peer1 = peer1_result.ok_value + + relay_config = { + **node_config, + "relay": True, + "lightpush": False, + "store": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "portsShift": 4, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value as relay_peer: + peer2_config = { + **peer1_config, + "staticnodes": [ + get_node_multiaddr(peer1), + get_node_multiaddr(relay_peer), + ], + "portsShift": 2, + "discv5UdpPort": free_port(), + } + + peer2_result = WrapperManager.create_and_start(config=peer2_config) + assert peer2_result.is_ok(), f"Failed to start lightpush peer2: {peer2_result.err()}" + + with peer2_result.ok_value as peer2: + sender_config = { + **node_config, + "mode": "Edge", + "relay": False, + "store": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "portsShift": 3, + "staticnodes": [ + get_node_multiaddr(peer1), + get_node_multiaddr(peer2), + ], + } + + sender_result = WrapperManager.create_and_start( + config=sender_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + delay(2) + stop_result = peer1.stop_and_destroy() + assert stop_result.is_ok(), f"Failed to stop peer1: {stop_result.err()}" + delay(2) + + message = create_message_bindings() + send_result = sender_node.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId) during peer churn, got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + # Expect Propagated via the surviving lightpush peer (peer2). + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent within {PROPAGATED_TIMEOUT_S}s " + f"after the selected lightpush peer disappeared. " + f"Collected events: {sender_collector.events}" + ) + + error_event = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=0, + ) + assert error_event is None, f"Unexpected message_error event during peer churn: {error_event}" + + assert_event_invariants(sender_collector, request_id) + + def test_s30_concurrent_sends_during_auto_subscribe(self, node_config): + """ + S30: concurrent sends on the same content topic during initial auto-subscribe. + - Sender starts unsubscribed to the target topic. + - Several send() calls are issued at nearly the same time. + - Each call must return Ok(RequestId) with a unique id. + - Each request id must get its own propagated event, + with no dropped or cross-associated events. + """ + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + # Relay peer so the sender has a propagation path. + relay_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender_node)], + "portsShift": 1, + } + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + # Build one message per send, with distinct payloads so we can + # detect any cross-association between request ids and events. + messages = [ + create_message_bindings( + contentTopic=S30_CONTENT_TOPIC, + payload=to_base64(f"s30-concurrent-{i}"), + ) + for i in range(S30_CONCURRENT_SENDS) + ] + + # Fire all sends concurrently. The sender is not yet subscribed + # to S30_CONTENT_TOPIC, so this exercises the auto-subscribe path + # under contention. + with ThreadPoolExecutor(max_workers=S30_CONCURRENT_SENDS) as pool: + send_results = list(pool.map(sender_node.send_message, messages)) + + # Every send must return Ok(RequestId). + request_ids = [] + for i, send_result in enumerate(send_results): + assert send_result.is_ok(), f"Concurrent send #{i} failed: {send_result.err()}" + request_id = send_result.ok_value + assert request_id, f"Concurrent send #{i} returned an empty RequestId" + request_ids.append(request_id) + + # Request ids must be unique across concurrent sends. + assert len(set(request_ids)) == len(request_ids), f"Duplicate RequestIds returned by concurrent sends: {request_ids}" + + # Each request id must get its own propagated event and no error. + for request_id in request_ids: + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent for request_id={request_id} " + f"within {PROPAGATED_TIMEOUT_S}s. " + f"Collected events: {sender_collector.events}" + ) + + error_event = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=0, + ) + assert error_event is None, f"Unexpected message_error for request_id={request_id}: {error_event}" + + # Cross-association guard: every event with a requestId must + # belong to exactly one of the request ids we issued. + issued = set(request_ids) + for event in sender_collector.snapshot(): + event_request_id = event.get("requestId") + if event_request_id is None: + continue + assert event_request_id in issued, ( + f"Event carries an unknown requestId={event_request_id!r}, " f"not in issued set {issued}. Event: {event}" + ) + + # Per-request invariants apply to every concurrent send + # (correct requestId, no duplicate terminal events, + # Sent never before Propagated). + for request_id in request_ids: + assert_event_invariants(sender_collector, request_id) + + @pytest.mark.docker_required + def test_s31_concurrent_sends_mixed_topics_during_churn(self, node_config): + """ + S31: concurrent sends across mixed content topics during peer churn. + """ + sender_collector = EventCollector() + + relay_peer = WakuNode(NODE_2, f"s31_relay_peer_{self.test_id}") + relay_peer.start(relay="true", discv5_discovery="false") + relay_peer.set_relay_subscriptions([self.test_pubsub_topic]) + + lightpush_peer = WakuNode(NODE_2, f"s31_lightpush_peer_{self.test_id}") + lightpush_peer.start(relay="true", lightpush="true", discv5_discovery="false") + lightpush_peer.set_relay_subscriptions([self.test_pubsub_topic]) + + store_peer = WakuNode(NODE_2, f"s31_store_peer_{self.test_id}") + store_peer.start(relay="true", store="true", discv5_discovery="false") + store_peer.set_relay_subscriptions([self.test_pubsub_topic]) + + churn_peers = [relay_peer, lightpush_peer, store_peer] + + # Mesh docker peers so a lightpushed message can fan out to the store peer. + peer_multiaddrs = [p.get_multiaddr_with_id() for p in churn_peers] + for peer in churn_peers: + others = [a for a in peer_multiaddrs if a != peer.get_multiaddr_with_id()] + peer.add_peers(others) + + node_config.update( + { + "mode": "Edge", + "relay": False, + "store": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "lightpushnode": lightpush_peer.get_multiaddr_with_id(), + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender_node: + sender_multiaddr = get_node_multiaddr(sender_node) + for peer in churn_peers: + peer.add_peers([sender_multiaddr]) + delay(3) # let docker peers connect to the sender + + all_request_ids: list[str] = [] + phase1_ids = self._s31_fire_burst(sender_node, phase_label="phase1") + all_request_ids.extend(phase1_ids) + + for peer in churn_peers: + peer.restart() + delay(1) # small window so the restart is actually in-flight + phase2_ids = self._s31_fire_burst(sender_node, phase_label="phase2") + all_request_ids.extend(phase2_ids) + + # Wait for all peers to be ready again and re-attach the sender. + for peer in churn_peers: + peer.ensure_ready(timeout_duration=20) + peer.add_peers([sender_multiaddr]) + + peer_multiaddrs = [p.get_multiaddr_with_id() for p in churn_peers] + for peer in churn_peers: + others = [a for a in peer_multiaddrs if a != peer.get_multiaddr_with_id()] + peer.add_peers(others) + delay(3) + + phase3_ids = self._s31_fire_burst(sender_node, phase_label="phase3") + all_request_ids.extend(phase3_ids) + + assert len(set(all_request_ids)) == len(all_request_ids), f"Duplicate RequestIds across bursts: {all_request_ids}" + + # Phase 1 ran before any churn, so the mesh was stable — standard timeout. + # Phase 3 ran right after restart + re-attach, so the mesh needed to + # re-stabilize — use the recovery timeout to avoid CI flakiness. + phase_timeouts = [ + (phase1_ids, PROPAGATED_TIMEOUT_S), + (phase3_ids, RECOVERY_TIMEOUT_S), + ] + for request_ids, timeout_s in phase_timeouts: + for request_id in request_ids: + propagated_event = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=timeout_s, + ) + assert propagated_event is not None, ( + f"No MessagePropagatedEvent for stable-phase " + f"request_id={request_id} within {timeout_s}s. " + f"Collected events: {sender_collector.events}" + ) + + error_event = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=0, + ) + assert error_event is None, f"Unexpected message_error event for stable-phase " f"request_id={request_id}: {error_event}" + + for request_id in phase2_ids: + error_event = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=0, + ) + assert error_event is None, f"Unexpected terminal message_error for phase-2 " f"request_id={request_id} after recovery: {error_event}" + + issued = set(all_request_ids) + for event in sender_collector.snapshot(): + event_request_id = event.get("requestId") + if event_request_id is None: + continue + assert event_request_id in issued, ( + f"Event carries an unknown requestId={event_request_id!r}, " f"not in issued set {issued}. Event: {event}" + ) + + # Use the hash the wrapper emitted on message_sent so the store + # lookup matches the exact bytes that were actually published. + phase3_hashes = [] + for request_id in phase3_ids: + sent_event = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=RECOVERY_TIMEOUT_S, + ) + assert sent_event is not None, ( + f"No message_sent event for phase-3 request_id={request_id} " + f"within {RECOVERY_TIMEOUT_S}s. Collected events: {sender_collector.events}" + ) + msg_hash = sent_event.get("messageHash") + assert msg_hash, f"message_sent event missing messageHash: {sent_event}" + phase3_hashes.append(msg_hash) + + # 3 phases × S31_BURST_SIZE messages, so the page must fit them all, + # otherwise phase-3 hashes (which sort last in ascending order) get cut off. + self.check_sent_message_is_stored( + expected_hashes=phase3_hashes, + store_node=store_peer, + pubsub_topic=self.test_pubsub_topic, + page_size=S31_BURST_SIZE * 3, + ascending="true", + ) + + # Per-request invariants apply across all phases, including the + # retry-path bursts (phase 2). If retries ever emit duplicate + # Propagated events or reorder Sent before Propagated, this catches it. + for request_id in all_request_ids: + assert_event_invariants(sender_collector, request_id) + + def _s31_fire_burst(self, sender_node, *, phase_label: str) -> list[str]: + """Fire S31_BURST_SIZE concurrent sends, one per topic in S31_CONTENT_TOPICS. + Returns the list of RequestIds. Asserts every send returned Ok.""" + messages = [ + self.create_message( + contentTopic=S31_CONTENT_TOPICS[i], + payload=to_base64(f"s31-{phase_label}-{i}"), + ) + for i in range(S31_BURST_SIZE) + ] + + with ThreadPoolExecutor(max_workers=S31_BURST_SIZE) as pool: + send_results = list(pool.map(sender_node.send_message, messages)) + + request_ids = [] + for i, send_result in enumerate(send_results): + assert send_result.is_ok(), f"{phase_label}: concurrent send #{i} failed: {send_result.err()}" + request_id = send_result.ok_value + assert request_id, f"{phase_label}: concurrent send #{i} returned an empty RequestId" + request_ids.append(request_id) + + return request_ids diff --git a/tests-e2e/tests/wrappers_tests/test_send_e2e_part2.py b/tests-e2e/tests/wrappers_tests/test_send_e2e_part2.py new file mode 100644 index 000000000..d7e3f088c --- /dev/null +++ b/tests-e2e/tests/wrappers_tests/test_send_e2e_part2.py @@ -0,0 +1,699 @@ +import base64 +import pytest +from src.steps.common import StepsCommon +from src.libs.common import delay, to_base64 +from src.libs.custom_logger import get_custom_logger +from src.node.wrappers_manager import WrapperManager +from src.node.wrapper_helpers import ( + EventCollector, + assert_event_invariants, + create_message_bindings, + get_node_multiaddr, + wait_for_connected, + wait_for_propagated, + wait_for_sent, + wait_for_error, +) +from tests.wrappers_tests.conftest import build_node_config + +logger = get_custom_logger(__name__) + +PROPAGATED_TIMEOUT_S = 30.0 +SENT_TIMEOUT_S = 10.0 +NO_SENT_OBSERVATION_S = 5.0 +SENT_AFTER_STORE_TIMEOUT_S = 60.0 +OVERSIZED_PAYLOAD_BYTES = 200 * 1024 +RECOVERY_TIMEOUT_S = 45.0 +SERVICE_DOWN_SETTLE_S = 3.0 + +# MaxTimeInCache from send_service.nim. +MAX_TIME_IN_CACHE_S = 60.0 +# Extra slack to cover the background retry loop tick after the window expires. +CACHE_EXPIRY_SLACK_S = 10.0 +ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S = MAX_TIME_IN_CACHE_S + CACHE_EXPIRY_SLACK_S +RETRY_WINDOW_EXPIRED_MSG = "Unable to send within retry time window" + + +class TestS02AutoSubscribeOnFirstSend(StepsCommon): + """ + S02 — Auto-subscribe on first send. + Sender never calls subscribe_content_topic() before send(). + The send API must auto-subscribe to the content topic used in the message. + Expected: send() returns Ok(RequestId), message_propagated arrives. + """ + + def test_s02_send_without_explicit_subscribe(self, node_config): + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "lightpush": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender: + peer_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender)], + "portsShift": 1, + } + + peer_result = WrapperManager.create_and_start(config=peer_config) + assert peer_result.is_ok(), f"Failed to start relay peer: {peer_result.err()}" + + with peer_result.ok_value: + assert wait_for_connected(sender_collector) is not None, "Sender did not reach Connected/PartiallyConnected state" + + message = create_message_bindings( + payload=to_base64("S02 auto-subscribe test payload"), + contentTopic="/test/1/s02-auto-subscribe/proto", + ) + + send_result = sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error event: {error}" + + +class TestS06CoreSenderRelayOnly(StepsCommon): + """ + S06 — Core sender with relay peers only, no store. + Sender has local relay enabled and is connected to one relay peer. + Expected: send() returns Ok(RequestId), message_propagated event arrives, + no message_sent (store disabled), no message_error. + """ + + def test_s06_relay_propagation_without_store(self, node_config): + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "lightpush": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "reliabilityEnabled": True, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender: + peer_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender)], + "portsShift": 1, + } + + peer_result = WrapperManager.create_and_start(config=peer_config) + assert peer_result.is_ok(), f"Failed to start relay peer: {peer_result.err()}" + + with peer_result.ok_value: + assert wait_for_connected(sender_collector) is not None, "Sender did not reach Connected/PartiallyConnected state" + + message = create_message_bindings( + payload=to_base64("S06 relay-only test payload"), + contentTopic="/test/1/s06-relay-only/proto", + ) + + send_result = sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error event: {error}" + + sent = wait_for_sent(sender_collector, request_id, timeout_s=0) + assert sent is None, f"Unexpected message_sent event (store is disabled): {sent}" + + assert_event_invariants(sender_collector, request_id) + + +class TestS07CoreSenderRelayAndStore(StepsCommon): + """ + S07 — Core sender with relay peers and store peer, reliability enabled. + Sender relays message to a store-capable peer; delivery service validates + the message reached the store via p2p reliability check. + Expected: Propagated, then Sent. + """ + + def test_s07_relay_propagation_with_store_validation(self, node_config): + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "lightpush": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + "reliabilityEnabled": True, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender: + peer_config = { + **node_config, + "staticnodes": [get_node_multiaddr(sender)], + "portsShift": 1, + "store": True, + } + + peer_result = WrapperManager.create_and_start(config=peer_config) + assert peer_result.is_ok(), f"Failed to start store peer: {peer_result.err()}" + + with peer_result.ok_value: + message = create_message_bindings( + payload=to_base64("S07 relay+store test payload"), + contentTopic="/test/1/s07-relay-store/proto", + ) + + send_result = sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + sent = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=SENT_TIMEOUT_S, + ) + assert sent is not None, ( + f"No message_sent event within {SENT_TIMEOUT_S}s after propagation. " f"Collected events: {sender_collector.events}" + ) + assert sent["requestId"] == request_id + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error event: {error}" + + assert_event_invariants(sender_collector, request_id) + + +class TestRelayToLightpushFallback(StepsCommon): + """S08/S09 — Relay-to-lightpush fallback. + + Sender has relay enabled but zero gossipsub relay peers. + A lightpush peer is reachable via lightpushnode (no staticnodes). + Relay fails with NO_PEERS_TO_RELAY, lightpush fallback succeeds + in the same processing pass. + + Topology: + [Service] relay=True, lightpush=True + [RelayPeer] relay=True, staticnodes=[service] (gives service gossipsub mesh) + [Sender] relay=True, lightpush=True, lightpushnode=service + (no staticnodes → zero gossipsub relay peers → fallback) + """ + + def test_s08_relay_fallback_to_lightpush(self, node_config): + """S08: no store peer → Propagated only.""" + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "lightpush": True, + "store": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + service_result = WrapperManager.create_and_start(config=node_config) + assert service_result.is_ok(), f"Failed to start service: {service_result.err()}" + + with service_result.ok_value as service: + service_addr = get_node_multiaddr(service) + + relay_config = { + **node_config, + "lightpush": False, + "staticnodes": [service_addr], + "portsShift": 1, + } + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + sender_config = { + **node_config, + "lightpushnode": service_addr, + "portsShift": 2, + } + sender_result = WrapperManager.create_and_start( + config=sender_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender: + message = create_message_bindings() + send_result = sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error event: {error}" + + sent = wait_for_sent(sender_collector, request_id, timeout_s=0) + assert sent is None, f"Unexpected message_sent event (no store peer): {sent}" + + assert_event_invariants(sender_collector, request_id) + + def test_s09_relay_fallback_to_lightpush_with_store_validation(self, node_config): + """S09: S08 + store peer + reliability → Propagated, then Sent.""" + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "lightpush": True, + "store": True, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + service_result = WrapperManager.create_and_start(config=node_config) + assert service_result.is_ok(), f"Failed to start service: {service_result.err()}" + + with service_result.ok_value as service: + service_addr = get_node_multiaddr(service) + + relay_config = { + **node_config, + "lightpush": False, + "store": False, + "staticnodes": [service_addr], + "portsShift": 1, + } + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + sender_config = {**node_config, "reliabilityEnabled": True, "storenode": service_addr, "portsShift": 2, "store": False} + sender_result = WrapperManager.create_and_start( + config=sender_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender: + message = create_message_bindings() + send_result = sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + sent = wait_for_sent( + collector=sender_collector, + request_id=request_id, + timeout_s=SENT_AFTER_STORE_TIMEOUT_S, + ) + assert sent is not None, ( + f"No message_sent event within {SENT_AFTER_STORE_TIMEOUT_S}s " + f"after propagation. Collected events: {sender_collector.events}" + ) + assert sent["requestId"] == request_id + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error event: {error}" + + assert_event_invariants(sender_collector, request_id) + + +class TestS10EdgeSenderLightpushOnly(StepsCommon): + """ + S10 — Edge sender with lightpush path only, no store peer. + Edge sender has no local relay; it publishes via a lightpush service node. + Expected: Propagated only (no Sent, no Error). + """ + + def test_s10_edge_lightpush_propagation(self, node_config): + sender_collector = EventCollector() + + common = { + "store": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + + service_config = build_node_config(relay=True, lightpush=True, **common) + + service_result = WrapperManager.create_and_start(config=service_config) + assert service_result.is_ok(), f"Failed to start service node: {service_result.err()}" + + with service_result.ok_value as service_node: + service_multiaddr = get_node_multiaddr(service_node) + + relay_config = build_node_config( + relay=True, + staticnodes=[service_multiaddr], + **common, + ) + + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + edge_config = build_node_config( + mode="Edge", + relay=False, + staticnodes=[service_multiaddr], + **common, + ) + + edge_result = WrapperManager.create_and_start( + config=edge_config, + event_cb=sender_collector.event_callback, + ) + assert edge_result.is_ok(), f"Failed to start edge sender: {edge_result.err()}" + + with edge_result.ok_value as edge_sender: + message = create_message_bindings( + payload=to_base64("S10 edge lightpush test payload"), + contentTopic="/test/1/s10-edge-lightpush/proto", + ) + + send_result = edge_sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + sent = wait_for_sent(sender_collector, request_id, timeout_s=NO_SENT_OBSERVATION_S) + assert sent is None, f"Unexpected message_sent event (no store peer): {sent}" + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error event: {error}" + + assert_event_invariants(sender_collector, request_id) + + +class TestS12IsolatedSenderNoPeers(StepsCommon): + """ + S12 — Isolated sender, no peers. + Sender has relay enabled but zero relay peers and zero lightpush peers. + Expected: send() returns Ok(RequestId), but eventually a message_error + event arrives (no route to propagate). + """ + + def test_s12_send_with_no_peers_produces_error(self, node_config): + sender_collector = EventCollector() + + node_config.update( + { + "relay": True, + "store": False, + "lightpush": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + ) + + sender_result = WrapperManager.create_and_start( + config=node_config, + event_cb=sender_collector.event_callback, + ) + assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}" + + with sender_result.ok_value as sender: + message = create_message_bindings( + payload=to_base64("S12 isolated sender payload"), + contentTopic="/test/1/s12-isolated/proto", + ) + + send_result = sender.send_message(message=message) + assert send_result.is_ok(), f"send() must return Ok(RequestId) even with no peers, got: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + error = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S, + ) + assert error is not None, ( + f"No message_error event within {ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S}s " + f"(MaxTimeInCache={MAX_TIME_IN_CACHE_S}s + slack) for isolated sender. " + f"Collected events: {sender_collector.events}" + ) + assert error["requestId"] == request_id + + propagated = wait_for_propagated(sender_collector, request_id, timeout_s=0) + assert propagated is None, f"Unexpected message_propagated event for isolated sender: {propagated}" + + +class TestS14LightpushNonRetryableError(StepsCommon): + """ + S14 — Lightpush non-retryable error via oversized message. + Edge sender publishes a message exceeding DefaultMaxWakuMessageSize (150KiB) + through a lightpush service node. The server validates message size and + returns INVALID_MESSAGE (420), a non-retryable error. + Expected: send() returns Ok(RequestId), then message_error event. + """ + + def test_s14_oversized_message_triggers_error(self): + sender_collector = EventCollector() + + common = { + "store": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + + service_config = build_node_config(relay=True, lightpush=True, **common) + service_result = WrapperManager.create_and_start(config=service_config) + assert service_result.is_ok(), f"Failed to start service: {service_result.err()}" + + with service_result.ok_value as service: + service_multiaddr = get_node_multiaddr(service) + + edge_config = build_node_config( + mode="Edge", + relay=False, + staticnodes=[service_multiaddr], + **common, + ) + edge_result = WrapperManager.create_and_start( + config=edge_config, + event_cb=sender_collector.event_callback, + ) + assert edge_result.is_ok(), f"Failed to start edge sender: {edge_result.err()}" + + with edge_result.ok_value as edge_sender: + oversized_payload = base64.b64encode(b"x" * OVERSIZED_PAYLOAD_BYTES).decode() + message = create_message_bindings( + payload=oversized_payload, + contentTopic="/test/1/s14-oversized/proto", + ) + + send_result = edge_sender.send_message(message=message) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + error = wait_for_error( + collector=sender_collector, + request_id=request_id, + timeout_s=PROPAGATED_TIMEOUT_S, + ) + assert error is not None, ( + f"No message_error event within {PROPAGATED_TIMEOUT_S}s " + f"after sending oversized message. " + f"Collected events: {sender_collector.events}" + ) + assert error["requestId"] == request_id + logger.info(f"S14 received error event: {error}") + + error_msg = error.get("error", "").lower() + assert "size exceeded" in error_msg, f"Error message doesn't indicate size violation: {error}" + + propagated = wait_for_propagated(sender_collector, request_id, timeout_s=0) + assert propagated is None, f"Unexpected message_propagated for an invalid message: {propagated}" + + assert_event_invariants(sender_collector, request_id) + + +class TestS15LightpushRetryableErrorRecovery(StepsCommon): + """ + S15 — Lightpush retryable error + recovery. + Edge sender publishes via a lightpush service node that has NO relay peers. + The service accepts the lightpush request but returns NO_PEERS_TO_RELAY — + a retryable error (explicitly listed in the S15 spec). The message enters + the retry loop. A relay peer then joins the service node, and the next + retry succeeds. + Expected: send() returns Ok(RequestId), then eventually Propagated. + """ + + def test_s15_lightpush_retryable_error_then_recovery(self): + sender_collector = EventCollector() + + common = { + "store": False, + "filter": False, + "discv5Discovery": False, + "numShardsInNetwork": 1, + } + + service_config = build_node_config(relay=True, lightpush=True, **common) + service_result = WrapperManager.create_and_start(config=service_config) + assert service_result.is_ok(), f"Failed to start service: {service_result.err()}" + + with service_result.ok_value as service: + service_multiaddr = get_node_multiaddr(service) + + edge_config = build_node_config( + mode="Edge", + relay=False, + staticnodes=[service_multiaddr], + **common, + ) + edge_result = WrapperManager.create_and_start( + config=edge_config, + event_cb=sender_collector.event_callback, + ) + assert edge_result.is_ok(), f"Failed to start edge sender: {edge_result.err()}" + + with edge_result.ok_value as edge_sender: + msg = create_message_bindings( + payload=to_base64("S15 retryable error recovery"), + contentTopic="/test/1/s15-recovery/proto", + ) + send_result = edge_sender.send_message(message=msg) + assert send_result.is_ok(), f"send() failed: {send_result.err()}" + request_id = send_result.ok_value + assert request_id, "send() returned an empty RequestId" + + delay(SERVICE_DOWN_SETTLE_S) + + early_propagated = wait_for_propagated(sender_collector, request_id, timeout_s=0) + assert early_propagated is None, ( + f"message_propagated arrived before relay peer joined — " f"retryable error path was not exercised: {early_propagated}" + ) + + relay_config = build_node_config( + relay=True, + staticnodes=[service_multiaddr], + **common, + ) + relay_result = WrapperManager.create_and_start(config=relay_config) + assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}" + + with relay_result.ok_value: + propagated = wait_for_propagated( + collector=sender_collector, + request_id=request_id, + timeout_s=RECOVERY_TIMEOUT_S, + ) + assert propagated is not None, ( + f"No message_propagated within {RECOVERY_TIMEOUT_S}s " + f"after relay peer joined. " + f"Collected events: {sender_collector.events}" + ) + assert propagated["requestId"] == request_id + + error = wait_for_error(sender_collector, request_id, timeout_s=0) + assert error is None, f"Unexpected message_error after recovery: {error}" + + assert_event_invariants(sender_collector, request_id) diff --git a/tests-e2e/vendor/logos-delivery-python-bindings/lib/.gitkeep b/tests-e2e/vendor/logos-delivery-python-bindings/lib/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests-e2e/vendor/logos-delivery-python-bindings/waku/wrapper.py b/tests-e2e/vendor/logos-delivery-python-bindings/waku/wrapper.py new file mode 100644 index 000000000..d23238833 --- /dev/null +++ b/tests-e2e/vendor/logos-delivery-python-bindings/waku/wrapper.py @@ -0,0 +1,392 @@ +import json +import threading + +from cffi import FFI +from pathlib import Path +from result import Result, Ok, Err + +ffi = FFI() + +ffi.cdef( + """ +typedef void (*FFICallBack)(int callerRet, const char *msg, size_t len, void *userData); + +void *logosdelivery_create_node( + const char *configJson, + FFICallBack callback, + void *userData +); + +int logosdelivery_start_node( + void *ctx, + FFICallBack callback, + void *userData +); + +int logosdelivery_stop_node( + void *ctx, + FFICallBack callback, + void *userData +); + +void logosdelivery_set_event_callback( + void *ctx, + FFICallBack callback, + void *userData +); + +int logosdelivery_destroy( + void *ctx, + FFICallBack callback, + void *userData +); + +int logosdelivery_subscribe( + void *ctx, + FFICallBack callback, + void *userData, + const char *contentTopic +); + +int logosdelivery_unsubscribe( + void *ctx, + FFICallBack callback, + void *userData, + const char *contentTopic +); + +int logosdelivery_send( + void *ctx, + FFICallBack callback, + void *userData, + const char *messageJson +); + +int logosdelivery_get_available_node_info_ids( + void *ctx, + FFICallBack callback, + void *userData +); + +int logosdelivery_get_node_info( + void *ctx, + FFICallBack callback, + void *userData, + const char *nodeInfoId +); + +int logosdelivery_get_available_configs( + void *ctx, + FFICallBack callback, + void *userData +); +""" +) + +_repo_root = Path(__file__).resolve().parents[1] +lib = ffi.dlopen(str(_repo_root / "lib" / "liblogosdelivery.so")) + +CallbackType = ffi.callback("void(int, const char*, size_t, void*)") + + +def _new_cb_state(): + return { + "done": threading.Event(), + "ret": None, + "msg": b"", + } + + +def _wait_cb_raw( + state, + op_name: str, + timeout_s: float = 20.0, +) -> Result[tuple[int, bytes], str]: + ok = state["done"].wait(timeout_s) + if not ok: + return Err(f"{op_name}: timeout after {timeout_s}s") + + if state["ret"] is None: + return Err(f"{op_name}: callback ret is None") + + return Ok((state["ret"], state["msg"])) + + +def _wait_cb_ok(state, op_name: str, timeout_s: float = 20.0) -> Result[int, str]: + wait_result = _wait_cb_raw(state, op_name, timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + cb_ret, cb_msg = wait_result.ok_value + if cb_ret != 0: + return Err(f"callback failed in _wait_cb_ok: {op_name} (ret={cb_ret}) msg={cb_msg!r}") + + return Ok(cb_ret) + + +class NodeWrapper: + def __init__(self, ctx, config_buffer, event_cb_handler): + self.ctx = ctx + self._config_buffer = config_buffer + self._event_cb_handler = event_cb_handler + + @staticmethod + def _make_waiting_cb(state): + def c_cb(ret, char_p, length, userData): + msg = ffi.buffer(char_p, length)[:] if char_p != ffi.NULL else b"" + + if not state["done"].is_set(): + state["ret"] = int(ret) + state["msg"] = msg + state["done"].set() + + return CallbackType(c_cb) + + @staticmethod + def _make_event_cb(py_callback): + def c_cb(ret, char_p, length, userData): + msg = ffi.buffer(char_p, length)[:] if char_p != ffi.NULL else b"" + py_callback(int(ret), msg) + + return CallbackType(c_cb) + + @classmethod + def create_node( + cls, + config: dict, + event_cb=None, + *, + timeout_s: float = 20.0, + ) -> Result["NodeWrapper", str]: + config_json = json.dumps(config, separators=(",", ":"), ensure_ascii=False) + config_buffer = ffi.new("char[]", config_json.encode("utf-8")) + + state = _new_cb_state() + cb = cls._make_waiting_cb(state) + + ctx = lib.logosdelivery_create_node( + config_buffer, + cb, + ffi.NULL, + ) + + if ctx == ffi.NULL: + return Err("create_node: ctx is NULL") + + wait_result = _wait_cb_ok(state, "create_node", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + event_cb_handler = None + if event_cb is not None: + event_cb_handler = cls._make_event_cb(event_cb) + lib.logosdelivery_set_event_callback( + ctx, + event_cb_handler, + ffi.NULL, + ) + + return Ok(cls(ctx, config_buffer, event_cb_handler)) + + @classmethod + def create_and_start( + cls, + config: dict, + event_cb=None, + *, + timeout_s: float = 20.0, + ) -> Result["NodeWrapper", str]: + node_result = cls.create_node( + config=config, + event_cb=event_cb, + timeout_s=timeout_s, + ) + if node_result.is_err(): + return Err(node_result.err()) + + node = node_result.ok_value + + start_result = node.start_node(timeout_s=timeout_s) + if start_result.is_err(): + return Err(start_result.err()) + + return Ok(node) + + def start_node(self, *, timeout_s: float = 20.0) -> Result[int, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_start_node(self.ctx, cb, ffi.NULL) + if rc != 0: + return Err(f"start_node: immediate call failed (ret={rc})") + + return _wait_cb_ok(state, "start_node", timeout_s) + + def stop_node(self, *, timeout_s: float = 20.0) -> Result[int, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_stop_node(self.ctx, cb, ffi.NULL) + if rc != 0: + return Err(f"stop_node: immediate call failed (ret={rc})") + + return _wait_cb_ok(state, "stop_node", timeout_s) + + def destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_destroy(self.ctx, cb, ffi.NULL) + if rc != 0: + return Err(f"destroy: immediate call failed (ret={rc})") + + wait_result = _wait_cb_ok(state, "destroy", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + self.ctx = ffi.NULL + return wait_result + + def stop_and_destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]: + stop_result = self.stop_node(timeout_s=timeout_s) + if stop_result.is_err(): + return Err(stop_result.err()) + + return self.destroy(timeout_s=timeout_s) + + def subscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_subscribe( + self.ctx, + cb, + ffi.NULL, + content_topic.encode("utf-8"), + ) + if rc != 0: + return Err(f"subscribe_content_topic: immediate call failed (ret={rc})") + + return _wait_cb_ok(state, f"subscribe({content_topic})", timeout_s) + + def unsubscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_unsubscribe( + self.ctx, + cb, + ffi.NULL, + content_topic.encode("utf-8"), + ) + if rc != 0: + return Err(f"unsubscribe_content_topic: immediate call failed (ret={rc})") + + return _wait_cb_ok(state, f"unsubscribe({content_topic})", timeout_s) + + def send_message(self, message: dict, *, timeout_s: float = 20.0) -> Result[str, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + message_json = json.dumps(message, separators=(",", ":"), ensure_ascii=False) + + rc = lib.logosdelivery_send( + self.ctx, + cb, + ffi.NULL, + message_json.encode("utf-8"), + ) + if rc != 0: + return Err(f"send_message: immediate call failed (ret={rc})") + + wait_result = _wait_cb_raw(state, "send_message", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + cb_ret, cb_msg = wait_result.ok_value + if cb_ret != 0: + return Err(f"send_message: callback failed (ret={cb_ret}) msg={cb_msg!r}") + + request_id = cb_msg.decode("utf-8") if cb_msg else "" + return Ok(request_id) + + def get_available_node_info_ids(self, *, timeout_s: float = 20.0) -> Result[list[str], str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_get_available_node_info_ids(self.ctx, cb, ffi.NULL) + if rc != 0: + return Err(f"get_available_node_info_ids: immediate call failed (ret={rc})") + + wait_result = _wait_cb_raw(state, "get_available_node_info_ids", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + cb_ret, cb_msg = wait_result.ok_value + if cb_ret != 0: + return Err(f"get_available_node_info_ids: callback failed (ret={cb_ret})") + if not cb_msg: + return Err("get_available_node_info_ids: empty response") + + try: + return Ok(json.loads(cb_msg.decode("utf-8").strip().lstrip("@"))) + except Exception as e: + return Err(f"get_available_node_info_ids: invalid response: {e}") + + def get_node_info(self, node_info_id: str, *, timeout_s: float = 20.0) -> Result[dict, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_get_node_info( + self.ctx, + cb, + ffi.NULL, + node_info_id.encode("utf-8"), + ) + if rc != 0: + return Err(f"get_node_info: immediate call failed (ret={rc})") + + wait_result = _wait_cb_raw(state, "get_node_info", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + cb_ret, cb_msg = wait_result.ok_value + if cb_ret != 0: + return Err(f"get_node_info: callback failed (ret={cb_ret}) msg={cb_msg!r}") + + if not cb_msg: + return Err("get_node_info: empty response") + + try: + result = json.loads(cb_msg.decode("utf-8")) + except Exception as e: + return Err(f"get_node_info: invalid json: {e}") + + return Ok(result) + + def get_available_configs(self, *, timeout_s: float = 20.0) -> Result[dict, str]: + state = _new_cb_state() + cb = self._make_waiting_cb(state) + + rc = lib.logosdelivery_get_available_configs(self.ctx, cb, ffi.NULL) + if rc != 0: + return Err(f"get_available_configs: immediate call failed (ret={rc})") + + wait_result = _wait_cb_raw(state, "get_available_configs", timeout_s) + if wait_result.is_err(): + return Err(wait_result.err()) + + cb_ret, cb_msg = wait_result.ok_value + if cb_ret != 0: + return Err(f"get_available_configs: callback failed (ret={cb_ret}) msg={cb_msg!r}") + + if not cb_msg: + return Err("get_available_configs: empty response") + + try: + result = json.loads(cb_msg.decode("utf-8")) + except Exception as e: + return Err(f"get_available_configs: invalid json: {e}") + + return Ok(result) +