Merge 40cbf87cff4fe439eba8893f5f9c5c039e2c8c54 into 9827be59905bb86abae59705202b5b16816bf805

.github/workflows/ci.yml

@@ -46,6 +46,7 @@ jobs:
           - 'tools/**'
           - 'tests/all_tests_v2.nim'
           - 'tests/**'
+          - 'tests-e2e/**'
           docker:
           - 'docker/**'
 
@@ -205,12 +206,13 @@ jobs:
 
     secrets: inherit
 
-  # Send API E2E tests run the wrapper suite against the liblogosdelivery.so
-  # built by the `build` job (downloaded as the `liblogosdelivery` artifact).
-  # Gated on `build`, so it is skipped on docker-only PRs where no lib is built.
+  # Send API E2E tests run the in-repo tests-e2e/ wrapper suite against the
+  # liblogosdelivery.so built by the `build` job (downloaded as the
+  # `liblogosdelivery` artifact). Gated on `build`, so it is skipped on
+  # docker-only PRs where no lib is built.
   send-api-e2e-tests:
     needs: build
-    uses: logos-messaging/logos-delivery-interop-tests/.github/workflows/send_api_e2e_PR.yml@SMOKE_TEST_2026.06.25
+    uses: ./.github/workflows/e2e-api-tests.yml
     secrets: inherit
 
   lint:

.github/workflows/e2e-api-tests.yml

@@ -0,0 +1,34 @@
+name: e2e-api-tests
+
+on:
+  workflow_call:
+
+jobs:
+  api-e2e:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    name: api-e2e
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Consumes the shared library built by ci.yml's `build` job (same run).
+      - name: Download liblogosdelivery.so
+        uses: actions/download-artifact@v4
+        with:
+          name: liblogosdelivery
+          path: tests-e2e/vendor/logos-delivery-python-bindings/lib
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+          cache-dependency-path: tests-e2e/requirements.txt
+
+      - name: Install Python deps
+        run: pip install -r tests-e2e/requirements.txt
+
+      - name: Run API/e2e tests (non-docker subset)
+        working-directory: tests-e2e
+        run: pytest tests/wrappers_tests -m "not docker_required" --reruns 2

tests-e2e/README.md

@@ -0,0 +1,32 @@
+# tests-e2e — logos-delivery API/e2e tests (Python)
+
+End-to-end tests for `liblogosdelivery`, driven through the Python bindings (C-FFI).
+Migrated from [`logos-delivery-interop-tests`](https://github.com/logos-messaging/logos-delivery-interop-tests) (the wrapper / send-API suite).
+
+## Layout
+
+- `src/` — test framework (node wrappers, steps, helpers)
+- `tests/wrappers_tests/` — the API/e2e scenario tests (`test_s02…s31`)
+- `vendor/logos-delivery-python-bindings/waku/wrapper.py` — CFFI binding (`NodeWrapper`); it `dlopen`s `../lib/liblogosdelivery.so`
+
+## Run locally
+
+```bash
+# 1. Build the shared library and place it where the binding looks for it
+./tests-e2e/scripts/prepare_lib.sh
+
+# 2. Python env + deps
+python -m venv .venv && source .venv/bin/activate
+pip install -r tests-e2e/requirements.txt
+
+# 3. Run (from tests-e2e/)
+cd tests-e2e
+pytest tests/wrappers_tests -m "not docker_required"   # 18 pure-binding tests
+pytest tests/wrappers_tests -m docker_required          # 3 tests that also need a Docker nwaku peer (S19/S20/S31)
+```
+
+## CI
+
+`.github/workflows/e2e-api-tests.yml` (called from `ci.yml`, `needs: build`) downloads the
+`liblogosdelivery` artifact produced by the `build` job and runs the non-docker subset on every PR —
+so a protocol change and its e2e test land in the same PR.

tests-e2e/pytest.ini

@@ -0,0 +1,15 @@
+[pytest]
+addopts = --instafail --tb=short --color=auto
+log_level = DEBUG
+log_cli = True
+norecursedirs =
+    vendor
+    nimbledeps
+    *.egg-info
+log_file = log/test.log
+log_cli_format = %(asctime)s.%(msecs)03d %(levelname)s [%(name)s] %(message)s
+log_file_format = %(asctime)s.%(msecs)03d %(levelname)s [%(name)s] %(message)s
+timeout = 300
+markers =
+    smoke: marks tests as smoke test (deselect with '-m "not smoke"')
+    docker_required: test requires Docker nodes (WakuNode)

tests-e2e/requirements.txt

@@ -0,0 +1,44 @@
+allure-pytest==2.13.2
+allure-python-commons==2.13.2
+attrs==23.1.0
+black==24.3.0
+certifi==2024.7.4
+cfgv==3.4.0
+charset-normalizer==3.3.2
+click==8.1.7
+distlib==0.3.8
+docker==7.0.0
+execnet==2.0.2
+filelock==3.13.1
+identify==2.5.33
+idna==3.7
+iniconfig==2.0.0
+marshmallow==3.20.1
+marshmallow-dataclass==8.6.0
+mypy-extensions==1.0.0
+nodeenv==1.8.0
+packaging==23.2
+pathspec==0.12.1
+platformdirs==4.1.0
+pluggy==1.3.0
+pre-commit==3.6.2
+pyright==1.1.352
+pytest==8.0.2
+pytest-instafail==0.5.0
+pytest-rerunfailures==13.0
+pytest-timeout==2.2.0
+pytest-xdist==3.5.0
+python-dotenv==1.0.1
+pytest-dependency==0.6.0
+PyYAML==6.0.1
+requests==2.31.0
+setuptools==70.0.0
+tenacity==8.2.3
+typeguard==4.1.5
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+urllib3==2.2.2
+virtualenv==20.25.0
+pytest-shard==0.1.2
+result==0.17.0
+cffi

tests-e2e/scripts/prepare_lib.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Build liblogosdelivery from this repo and place it where the Python binding expects it.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LIBDIR="$ROOT/tests-e2e/vendor/logos-delivery-python-bindings/lib"
+
+cd "$ROOT"
+make update
+make V=1 liblogosdelivery
+
+mkdir -p "$LIBDIR"
+if [ -f build/liblogosdelivery.so ]; then
+  cp build/liblogosdelivery.so "$LIBDIR/liblogosdelivery.so"
+elif [ -f build/liblogosdelivery.dylib ]; then
+  cp build/liblogosdelivery.dylib "$LIBDIR/liblogosdelivery.dylib"
+  ln -sf liblogosdelivery.dylib "$LIBDIR/liblogosdelivery.so"
+else
+  echo "ERROR: built library not found in build/" >&2
+  exit 1
+fi
+echo "Placed liblogosdelivery in $LIBDIR"

tests-e2e/src/data_storage.py

@@ -0,0 +1,3 @@
+# We use this class for global variables
+class DS:
+    waku_nodes = []

tests-e2e/src/env_vars.py

@@ -0,0 +1,36 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()  # This will load environment variables from a .env file if it exists
+
+
+def get_env_var(var_name, default=None):
+    env_var = os.getenv(var_name, default)
+    if env_var in [None, ""]:
+        print(f"{var_name} is not set; using default value: {default}")
+        env_var = default
+    print(f"{var_name}: {env_var}")
+    return env_var
+
+
+# Configuration constants. Need to be upercase to appear in reports
+DEFAULT_NWAKU = "wakuorg/nwaku:latest"
+STRESS_ENABLED = False
+USE_WRAPPERS = True
+NODE_1 = get_env_var("NODE_1", DEFAULT_NWAKU)
+NODE_2 = get_env_var("NODE_2", DEFAULT_NWAKU)
+ADDITIONAL_NODES = get_env_var("ADDITIONAL_NODES", f"{DEFAULT_NWAKU},{DEFAULT_NWAKU},{DEFAULT_NWAKU}")
+# more nodes need to follow the NODE_X pattern
+DOCKER_LOG_DIR = get_env_var("DOCKER_LOG_DIR", "./log/docker")
+NETWORK_NAME = get_env_var("NETWORK_NAME", "waku")
+SUBNET = get_env_var("SUBNET", "172.18.0.0/16")
+IP_RANGE = get_env_var("IP_RANGE", "172.18.0.0/24")
+GATEWAY = get_env_var("GATEWAY", "172.18.0.1")
+RUNNING_IN_CI = get_env_var("CI")
+API_REQUEST_TIMEOUT = get_env_var("API_REQUEST_TIMEOUT", 20)
+RLN_CREDENTIALS = get_env_var("RLN_CREDENTIALS")
+PG_USER = get_env_var("POSTGRES_USER", "postgres")
+PG_PASS = get_env_var("POSTGRES_PASSWORD", "test123")
+
+# example for .env file
+# RLN_CREDENTIALS = {"rln-relay-cred-password": "password", "rln-relay-eth-client-address": "https://rpc.sepolia.linea.build",  "rln-relay-eth-contract-address": "0xB9cd878C90E49F797B4431fBF4fb333108CB90e6",  "rln-relay-eth-private-key-1": "",  "rln-relay-eth-private-key-2": "", "rln-relay-eth-private-key-3": "", "rln-relay-eth-private-key-4": "", "rln-relay-eth-private-key-5": ""}

tests-e2e/src/libs/common.py

@@ -0,0 +1,52 @@
+import uuid
+from datetime import datetime
+from time import sleep
+from src.libs.custom_logger import get_custom_logger
+import os
+import base64
+import allure
+
+logger = get_custom_logger(__name__)
+
+
+def bytes_to_hex(byte_array):
+    return "".join(format(byte, "02x") for byte in byte_array)
+
+
+def to_base64(input_data):
+    if isinstance(input_data, str):
+        input_bytes = input_data.encode()
+    elif isinstance(input_data, int):
+        input_bytes = str(input_data).encode()
+    elif isinstance(input_data, bytes):
+        input_bytes = input_data
+    else:
+        input_bytes = str(input_data).encode()
+    base64_encoded = base64.b64encode(input_bytes)
+    return base64_encoded.decode()
+
+
+def to_hex(input_data):
+    if isinstance(input_data, str):
+        input_bytes = input_data.encode()
+    elif isinstance(input_data, int):
+        input_bytes = str(input_data).encode()
+    elif isinstance(input_data, bytes):
+        input_bytes = input_data
+    else:
+        input_bytes = str(input_data).encode()
+    return "0x" + input_bytes.hex()
+
+
+def attach_allure_file(file):
+    logger.debug(f"Attaching file {file}")
+    allure.attach.file(file, name=os.path.basename(file), attachment_type=allure.attachment_type.TEXT)
+
+
+def delay(num_seconds):
+    logger.debug(f"Sleeping for {num_seconds} seconds")
+    sleep(num_seconds)
+
+
+def gen_step_id():
+    return f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}__{str(uuid.uuid4())}"

tests-e2e/src/libs/custom_logger.py

@@ -0,0 +1,31 @@
+import logging
+
+max_log_line_length = 5000
+
+
+def log_length_filter(max_length):
+    class logLengthFilter(logging.Filter):
+        def filter(self, record):
+            if len(record.getMessage()) > max_length:
+                logging.getLogger(record.name).log(
+                    record.levelno, f"Log line was discarded because it's longer than max_log_line_length={max_log_line_length}"
+                )
+                return False
+            return True
+
+    return logLengthFilter()
+
+
+def get_custom_logger(name):
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("docker").setLevel(logging.WARNING)
+    logger = logging.getLogger(name)
+    logger.addFilter(log_length_filter(max_log_line_length))
+
+    # Define a formatter with millisecond precision
+    ch = logging.StreamHandler()
+    formatter = logging.Formatter("%(asctime)s.%(msecs)03d %(levelname)s [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    return logger

tests-e2e/src/node/api_clients/base_client.py

@@ -0,0 +1,37 @@
+import json
+import requests
+from src.env_vars import API_REQUEST_TIMEOUT
+from src.libs.custom_logger import get_custom_logger
+
+logger = get_custom_logger(__name__)
+
+
+class BaseClient:
+    def make_request(self, method, url, headers=None, data=None):
+        self.log_request_as_curl(method, url, headers, data)
+        response = requests.request(method.upper(), url, headers=headers, data=data, timeout=API_REQUEST_TIMEOUT)
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as http_err:
+            logger.error(f"HTTP error occurred: {http_err}. Response content: {response.content}")
+            raise Exception(f"Error: {http_err} with response: {response.content}")
+        except Exception as err:
+            logger.error(f"An error occurred: {err}. Response content: {response.content}")
+            raise Exception(f"Error: {err} with response: {response.content}")
+        else:
+            logger.info(f"Response status code: {response.status_code}. Response content: {response.content}")
+        return response
+
+    def log_request_as_curl(self, method, url, headers, data):
+        if data:
+            try:
+                data_dict = json.loads(data)
+                if "timestamp" in data_dict:
+                    data_dict["timestamp"] = "TIMESTAMP_PLACEHOLDER"
+                data = json.dumps(data_dict)
+                data = data.replace('"TIMESTAMP_PLACEHOLDER"', "'$(date +%s%N)'")
+            except json.JSONDecodeError:
+                logger.error("Invalid JSON data provided")
+        headers_str_for_log = " ".join([f'-H "{key}: {value}"' for key, value in headers.items()]) if headers else ""
+        curl_cmd = f"curl -v -X {method.upper()} \"{url}\" {headers_str_for_log} -d '{data}'"
+        logger.info(curl_cmd)

tests-e2e/src/node/api_clients/rest.py

@@ -0,0 +1,190 @@
+from src.libs.custom_logger import get_custom_logger
+import json
+from urllib.parse import quote
+from src.node.api_clients.base_client import BaseClient
+
+logger = get_custom_logger(__name__)
+
+
+class REST(BaseClient):
+    def __init__(self, rest_port):
+        self._rest_port = rest_port
+
+    def rest_call(self, method, endpoint, payload=None):
+        url = f"http://127.0.0.1:{self._rest_port}/{endpoint}"
+        headers = {"Content-Type": "application/json"}
+        return self.make_request(method, url, headers=headers, data=payload)
+
+    def rest_call_text(self, method, endpoint, payload=None):
+        url = f"http://127.0.0.1:{self._rest_port}/{endpoint}"
+        headers = {"accept": "text/plain"}
+        return self.make_request(method, url, headers=headers, data=payload)
+
+    def info(self):
+        info_response = self.rest_call("get", "debug/v1/info")
+        return info_response.json()
+
+    def health(self):
+        health_response = self.rest_call("get", "health")
+        return health_response.content
+
+    def get_peers(self):
+        get_peers_response = self.rest_call("get", "admin/v1/peers")
+        return get_peers_response.json()
+
+    def add_peers(self, peers):
+        return self.rest_call("post", "admin/v1/peers", json.dumps(peers))
+
+    def set_relay_subscriptions(self, pubsub_topics):
+        return self.rest_call("post", "relay/v1/subscriptions", json.dumps(pubsub_topics))
+
+    def set_relay_auto_subscriptions(self, content_topics):
+        return self.rest_call("post", "relay/v1/auto/subscriptions", json.dumps(content_topics))
+
+    def delete_relay_subscriptions(self, pubsub_topics):
+        return self.rest_call("delete", "relay/v1/subscriptions", json.dumps(pubsub_topics))
+
+    def delete_relay_auto_subscriptions(self, content_topics):
+        return self.rest_call("delete", "relay/v1/auto/subscriptions", json.dumps(content_topics))
+
+    def send_relay_message(self, message, pubsub_topic):
+        return self.rest_call("post", f"relay/v1/messages/{quote(pubsub_topic, safe='')}", json.dumps(message))
+
+    def send_relay_auto_message(self, message):
+        return self.rest_call("post", "relay/v1/auto/messages", json.dumps(message))
+
+    def send_light_push_message(self, payload):
+        return self.rest_call("post", "lightpush/v1/message", json.dumps(payload))
+
+    def get_relay_messages(self, pubsub_topic):
+        get_messages_response = self.rest_call("get", f"relay/v1/messages/{quote(pubsub_topic, safe='')}")
+        return get_messages_response.json()
+
+    def get_relay_auto_messages(self, content_topic):
+        get_messages_response = self.rest_call("get", f"relay/v1/auto/messages/{quote(content_topic, safe='')}")
+        return get_messages_response.json()
+
+    def set_filter_subscriptions(self, subscription):
+        set_subscriptions_response = self.rest_call("post", "filter/v2/subscriptions", json.dumps(subscription))
+        return set_subscriptions_response.json()
+
+    def update_filter_subscriptions(self, subscription):
+        update_subscriptions_response = self.rest_call("put", "filter/v2/subscriptions", json.dumps(subscription))
+        return update_subscriptions_response.json()
+
+    def delete_filter_subscriptions(self, subscription):
+        delete_subscriptions_response = self.rest_call("delete", "filter/v2/subscriptions", json.dumps(subscription))
+        return delete_subscriptions_response.json()
+
+    def delete_all_filter_subscriptions(self, request_id):
+        delete_all_subscriptions_response = self.rest_call("delete", "filter/v2/subscriptions/all", json.dumps(request_id))
+        return delete_all_subscriptions_response.json()
+
+    def ping_filter_subscriptions(self, request_id):
+        ping_subscriptions_response = self.rest_call("get", f"filter/v2/subscriptions/{quote(request_id, safe='')}")
+        return ping_subscriptions_response.json()
+
+    def get_filter_messages(self, content_topic, pubsub_topic=None):
+        if pubsub_topic is not None:
+            endpoint = f"filter/v2/messages/{quote(pubsub_topic, safe='')}/{quote(content_topic, safe='')}"
+        else:
+            endpoint = f"filter/v2/messages/{quote(content_topic, safe='')}"
+        get_messages_response = self.rest_call("get", endpoint)
+        return get_messages_response.json()
+
+    def get_store_messages(
+        self,
+        peer_addr,
+        include_data,
+        pubsub_topic,
+        content_topics,
+        start_time,
+        end_time,
+        hashes,
+        cursor,
+        page_size,
+        ascending,
+        store_v,
+        encode_pubsubtopic=True,
+        **kwargs,
+    ):
+        base_url = f"store/{store_v}/messages"
+        params = []
+
+        if peer_addr is not None:
+            params.append(f"peerAddr={quote(peer_addr, safe='')}")
+        if include_data is not None:
+            params.append(f"includeData={include_data}")
+        if pubsub_topic is not None:
+            if encode_pubsubtopic:
+                params.append(f"pubsubTopic={quote(pubsub_topic, safe='')}")
+            else:
+                params.append(f"pubsubTopic={pubsub_topic}")
+        if content_topics is not None:
+            params.append(f"contentTopics={quote(content_topics, safe='')}")
+        if start_time is not None:
+            params.append(f"startTime={start_time}")
+        if end_time is not None:
+            params.append(f"endTime={end_time}")
+        if hashes is not None:
+            params.append(f"hashes={quote(hashes, safe='')}")
+        if cursor is not None:
+            params.append(f"cursor={quote(cursor, safe='')}")
+        if page_size is not None:
+            params.append(f"pageSize={page_size}")
+        if ascending is not None:
+            params.append(f"ascending={ascending}")
+
+        # Append any additional keyword arguments to the parameters list
+        for key, value in kwargs.items():
+            if value is not None:
+                params.append(f"{key}={quote(str(value), safe='')}")
+
+        if params:
+            base_url += "?" + "&".join(params)
+
+        get_messages_response = self.rest_call("get", base_url)
+        return get_messages_response.json()
+
+    def set_log_level(self, log_level):
+        return self.rest_call("post", f"admin/v1/log-level/{quote(log_level)}")
+
+    def get_service_peers(self):
+        return self.rest_call("get", "admin/v1/peers/service").json()
+
+    def get_connected_peers(self):
+        return self.rest_call("get", "admin/v1/peers/connected").json()
+
+    def get_connected_peers_on_shard(self, shard_id):
+        return self.rest_call("get", f"admin/v1/peers/connected/on/{quote(str(shard_id))}").json()
+
+    def get_relay_peers(self):
+        return self.rest_call("get", "admin/v1/peers/relay").json()
+
+    def get_relay_peers_on_shard(self, shard_id):
+        return self.rest_call("get", f"admin/v1/peers/relay/on/{quote(str(shard_id))}").json()
+
+    def get_mesh_peers(self):
+        return self.rest_call("get", "admin/v1/peers/mesh").json()
+
+    def get_mesh_peers_on_shard(self, shard_id):
+        return self.rest_call("get", f"admin/v1/peers/mesh/on/{quote(str(shard_id))}").json()
+
+    def get_peer_stats(self):
+        return self.rest_call("get", "admin/v1/peers/stats").json()
+
+    def get_filter_subscriptions(self):
+        return self.rest_call("get", "admin/v1/filter/subscriptions").json()
+
+    def get_info(self):
+        return self.rest_call("get", "info").json()
+
+    def get_version(self):
+        return self.rest_call("get", "version").text.strip()
+
+    def get_debug_version(self):
+        return self.rest_call("get", "debug/v1/version").text.strip()
+
+    def get_peer(self, peer_id: str):
+        resp = self.rest_call("get", f"admin/v1/peer/{peer_id}")
+        return resp.json()

tests-e2e/src/node/docker_mananger.py

@@ -0,0 +1,145 @@
+import os
+import re
+import time
+from src.libs.custom_logger import get_custom_logger
+import random
+import threading
+import docker
+from src.env_vars import NETWORK_NAME, SUBNET, IP_RANGE, GATEWAY
+from docker.types import IPAMConfig, IPAMPool
+from docker.errors import NotFound, APIError
+
+logger = get_custom_logger(__name__)
+
+
+class DockerManager:
+    def __init__(self, image):
+        self._image = image
+        self._client = docker.from_env()
+        logger.debug(f"Docker client initialized with image {self._image}")
+
+    def create_network(self, network_name=NETWORK_NAME):
+        logger.debug(f"Attempting to create or retrieve network {network_name}")
+        networks = self._client.networks.list(names=[network_name])
+        if networks:
+            logger.debug(f"Network {network_name} already exists")
+            return networks[0]
+
+        network = self._client.networks.create(
+            network_name,
+            driver="bridge",
+            ipam=IPAMConfig(driver="default", pool_configs=[IPAMPool(subnet=SUBNET, iprange=IP_RANGE, gateway=GATEWAY)]),
+        )
+        logger.debug(f"Network {network_name} created")
+        return network
+
+    def start_container(self, image_name, ports, args, log_path, container_ip, volumes, remove_container=True):
+        cli_args = []
+        for key, value in args.items():
+            if isinstance(value, list):  # Check if value is a list
+                cli_args.extend([f"--{key}={item}" for item in value])  # Add a command for each item in the list
+            elif value is None:
+                cli_args.append(f"{key}")  # Add simple command as it is passed in the key
+            else:
+                cli_args.append(f"--{key}={value}")  # Add a single command
+
+        port_bindings = {f"{port}/tcp": ("", port) for port in ports}
+        port_bindings_for_log = " ".join(f"-p {port}:{port}" for port in ports)
+        cli_args_str_for_log = " ".join(cli_args)
+        logger.debug(f"docker run -i -t {port_bindings_for_log} {image_name} {cli_args_str_for_log}")
+        container = self._client.containers.run(
+            image_name, command=cli_args, ports=port_bindings, detach=True, remove=remove_container, auto_remove=remove_container, volumes=volumes
+        )
+
+        network = self._client.networks.get(NETWORK_NAME)
+        logger.debug(f"docker network connect --ip {container_ip} {NETWORK_NAME} {container.id}")
+        network.connect(container, ipv4_address=container_ip)
+
+        logger.debug(f"Container started with ID {container.short_id}. Setting up logs at {log_path}")
+        log_thread = threading.Thread(target=self._log_container_output, args=(container, log_path))
+        log_thread.daemon = True
+        log_thread.start()
+
+        return container
+
+    def _log_container_output(self, container, log_path):
+        os.makedirs(os.path.dirname(log_path), exist_ok=True)
+        retry_count = 0
+        start_time = time.time()
+        try:
+            with open(log_path, "wb+") as log_file:
+                while True:
+                    if container.status in ["exited", "dead"]:
+                        logger.info(f"Container {container.short_id} has stopped. Exiting log stream.")
+                        return
+                    try:
+                        for chunk in container.logs(stream=True):
+                            if chunk:
+                                log_file.write(chunk)
+                                log_file.flush()
+                                start_time = time.time()
+                                retry_count = 0
+                            else:
+                                if time.time() - start_time > 5:
+                                    logger.warning(f"Log stream timeout for container {container.short_id}")
+                                    return
+                    except (APIError, IOError) as e:
+                        retry_count += 1
+                        if retry_count >= 5:
+                            logger.error(f"Max retries reached for container {container.short_id}. Exiting log stream.")
+                            return
+                        time.sleep(0.2)
+                    except Exception as e:
+                        return
+        except Exception as e:
+            logger.error(f"Failed to set up logging for container {container.short_id}: {e}")
+
+    def generate_ports(self, base_port=None, count=5):
+        if base_port is None:
+            base_port = random.randint(1024, 65535 - count)
+        ports = [str(base_port + i) for i in range(count)]
+        logger.debug(f"Generated ports {ports}")
+        return ports
+
+    @staticmethod
+    def generate_random_ext_ip():
+        base_ip_fragments = ["172", "18"]
+        ext_ip = ".".join(base_ip_fragments + [str(random.randint(0, 255)) for _ in range(2)])
+        logger.debug(f"Generated random external IP {ext_ip}")
+        return ext_ip
+
+    def is_container_running(self, container):
+        try:
+            refreshed_container = self._client.containers.get(container.id)
+            return refreshed_container.status == "running"
+        except NotFound:
+            logger.error(f"Container with ID {container.id} not found")
+            return False
+
+    @property
+    def image(self):
+        return self._image
+
+    def search_log_for_keywords(self, log_path, keywords, use_regex=False):
+        matches = {keyword: [] for keyword in keywords}
+
+        # Open the log file and search line by line
+        with open(log_path, "r") as log_file:
+            for line in log_file:
+                for keyword in keywords:
+                    if use_regex:
+                        if re.search(keyword, line, re.IGNORECASE):
+                            matches[keyword].append(line.strip())
+                    else:
+                        if keyword.lower() in line.lower():
+                            matches[keyword].append(line.strip())
+
+        # Check if there were any matches
+        if any(matches[keyword] for keyword in keywords):
+            for keyword, lines in matches.items():
+                if lines:
+                    logger.debug(f"Found matches for keyword '{keyword}': {lines}")
+            return matches
+        else:
+            logger.debug("No errors found in the waku logs.")
+            return None

tests-e2e/src/node/store_response.py

@@ -0,0 +1,103 @@
+class StoreResponse:
+    def __init__(self, store_response, node):
+        self.response = store_response
+        self.node = node
+
+    @property
+    def request_id(self):
+        try:
+            if self.node.is_nwaku():
+                return self.response.get("requestId")
+            else:
+                return self.response.get("request_id")
+        except:
+            return None
+
+    @property
+    def status_code(self):
+        try:
+            if self.node.is_nwaku():
+                return self.response.get("statusCode")
+            else:
+                return self.response.get("status_code")
+        except:
+            return None
+
+    @property
+    def status_desc(self):
+        try:
+            if self.node.is_nwaku():
+                return self.response.get("statusDesc")
+            else:
+                return self.response.get("status_desc")
+        except:
+            return None
+
+    @property
+    def messages(self):
+        try:
+            return self.response.get("messages")
+        except:
+            return None
+
+    @property
+    def pagination_cursor(self):
+        try:
+            if self.node.is_nwaku():
+                return self.response.get("paginationCursor")
+            else:
+                return self.response.get("pagination_cursor")
+        except:
+            return None
+
+    def message_hash(self, index):
+        if self.messages is not None:
+            if self.node.is_nwaku():
+                return self.messages[index]["messageHash"]
+            else:
+                return self.messages[index]["message_hash"]
+        else:
+            return None
+
+    def message_content(self, index):
+        try:
+            if self.messages is not None:
+                payload = self.messages[index]["message"]["contentTopic"]
+                return payload
+            else:
+                return None
+        except IndexError:
+            return None
+
+    def message_payload(self, index):
+        try:
+            if self.messages is not None:
+                payload = self.messages[index]["message"]["payload"]
+                return payload
+            else:
+                return None
+        except IndexError:
+            return None
+
+    def message_at(self, index):
+        try:
+            if self.messages is not None:
+                message = self.messages[index]["message"]
+                return message
+            else:
+                return None
+        except IndexError:
+            return None
+
+    def message_pubsub_topic(self, index):
+        if self.messages is not None:
+            if self.node.is_nwaku():
+                return self.messages[index]["pubsubTopic"]
+            else:
+                return self.messages[index]["pubsub_topic"]
+        else:
+            return None
+
+    @property
+    def resp_json(self):
+        return self.response

tests-e2e/src/node/waku_message.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass, field
+from marshmallow_dataclass import class_schema
+from typing import Optional, Union
+import math
+import allure
+
+
+@dataclass
+class MessageRpcResponse:
+    payload: str
+    contentTopic: str
+    version: Optional[int]
+    timestamp: Optional[int]
+    ephemeral: Optional[bool]
+    meta: Optional[str]
+    proof: Optional[str] = field(default_factory=dict)
+    rateLimitProof: Optional[str] = field(default_factory=dict)
+    rate_limit_proof: Optional[dict] = field(default_factory=dict)
+
+
+class WakuMessage:
+    def __init__(self, message_response, schema=MessageRpcResponse):
+        self.schema = schema
+        self.received_messages = message_response
+        self.message_rpc_response_schema = class_schema(self.schema)()
+
+    @allure.step
+    def assert_received_message(self, sent_message, index=0):
+        message = self.message_rpc_response_schema.load(self.received_messages[index])
+
+        def assert_fail_message(field_name):
+            return f"Incorrect field: {field_name}. Published: {sent_message[field_name]} Received: {getattr(message, field_name)}"
+
+        assert message.payload == sent_message["payload"], assert_fail_message("payload")
+        assert message.contentTopic == sent_message["contentTopic"], assert_fail_message("contentTopic")
+        if sent_message.get("timestamp") is not None:
+            if isinstance(sent_message["timestamp"], float):
+                assert math.isclose(float(message.timestamp), sent_message["timestamp"], rel_tol=1e-9), assert_fail_message("timestamp")
+            else:
+                assert str(message.timestamp) == str(sent_message["timestamp"]), assert_fail_message("timestamp")
+        if "version" in sent_message:
+            assert str(message.version) == str(sent_message["version"]), assert_fail_message("version")
+        if "meta" in sent_message:
+            assert str(message.meta) == str(sent_message["meta"]), assert_fail_message("meta")
+        if "ephemeral" in sent_message:
+            assert str(message.ephemeral) == str(sent_message["ephemeral"]), assert_fail_message("ephemeral")

tests-e2e/src/node/waku_node.py

@@ -0,0 +1,683 @@
+import errno
+import json
+import os
+import random
+import re
+import shutil
+import string
+import subprocess
+import pytest
+import requests
+from src.libs.common import delay
+from src.libs.custom_logger import get_custom_logger
+from tenacity import retry, stop_after_delay, wait_fixed, sleep
+from docker.errors import NotFound as DockerNotFound
+from src.node.api_clients.rest import REST
+from src.node.docker_mananger import DockerManager
+from src.env_vars import DOCKER_LOG_DIR
+from src.data_storage import DS
+from src.test_data import DEFAULT_CLUSTER_ID, LOG_ERROR_KEYWORDS, VALID_PUBSUB_TOPICS
+
+logger = get_custom_logger(__name__)
+
+
+def select_private_key(prv_keys, key_id):
+    for key in prv_keys:
+        if key.endswith(key_id):
+            return key
+
+    raise ValueError("No matching key was found")
+
+
+def sanitize_docker_flags(input_flags):
+    output_flags = {}
+    for key, value in input_flags.items():
+        key = key.replace("_", "-")
+        output_flags[key] = value
+
+    return output_flags
+
+
+@retry(stop=stop_after_delay(180), wait=wait_fixed(0.5), reraise=True)
+def rln_credential_store_ready(creds_file_path, single_check=False, require_credentials=False):
+    if os.path.exists(creds_file_path):
+        subprocess.run(["sudo", "-n", "chmod", "a+r", creds_file_path], check=False)
+        if require_credentials:
+            try:
+                with open(creds_file_path, "r", encoding="utf-8") as creds_file:
+                    keystore_data = json.load(creds_file)
+            except (OSError, json.JSONDecodeError) as ex:
+                if single_check:
+                    return False
+                raise ValueError(f"Failed to parse RLN keystore at {creds_file_path}: {ex}")
+
+            credentials = keystore_data.get("credentials", {}) if isinstance(keystore_data, dict) else {}
+            if not credentials:
+                if single_check:
+                    return False
+                raise ValueError(f"RLN keystore exists but has no credentials yet: {creds_file_path}")
+
+        return True
+    elif not single_check:
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), creds_file_path)
+
+    return False
+
+
+def peer_info2multiaddr(peer, is_nwaku=True):
+    if is_nwaku:
+        return peer["multiaddr"]
+    else:
+        return peer["multiaddrs"][0]
+
+
+def peer_info2id(peer, is_nwaku=True):
+    return peer_info2multiaddr(peer, is_nwaku).split("/")[-1]
+
+
+def multiaddr2id(multiaddr):
+    return multiaddr.split("/")[-1]
+
+
+def resolve_sharding_flags(kwargs):
+    if "pubsub_topic" in kwargs and kwargs["pubsub_topic"]:
+        pubsub_topic = kwargs["pubsub_topic"]
+        if not "cluster_id" in kwargs:
+            try:
+                if isinstance(pubsub_topic, list):
+                    pubsub_topic = pubsub_topic[0]
+                cluster_id = pubsub_topic.split("/")[4]
+                logger.debug(f"Cluster id was resolved to: {cluster_id}")
+                kwargs["cluster_id"] = cluster_id
+            except Exception as ex:
+                raise Exception("Could not resolve cluster_id from pubsub_topic")
+    return kwargs
+
+
+class WakuNode:
+    def __init__(self, docker_image, docker_log_prefix=""):
+        self._image_name = docker_image
+        self._log_path = os.path.join(DOCKER_LOG_DIR, f"{docker_log_prefix}__{self._image_name.replace('/', '_')}.log")
+        self._docker_manager = DockerManager(self._image_name)
+        self._container = None
+        self.rln_membership_index = None
+        self.start_args = {}
+        self._rln_creds_set = False
+        logger.debug(f"WakuNode instance initialized with log path {self._log_path}")
+
+    @retry(stop=stop_after_delay(60), wait=wait_fixed(0.1), reraise=True)
+    def start(self, wait_for_node_sec=20, **kwargs):
+        logger.debug("Starting Node...")
+        default_args, remove_container = self._prepare_start_context(**kwargs)
+        self._start_docker(default_args, remove_container, wait_for_node_sec)
+
+    def _prepare_start_context(self, **kwargs):
+        self._docker_manager.create_network()
+        self._ext_ip = self._docker_manager.generate_random_ext_ip()
+        self._ports = self._docker_manager.generate_ports()
+        self._rest_port = self._ports[0]
+        self._tcp_port = self._ports[1]
+        self._websocket_port = self._ports[2]
+        self._discv5_port = self._ports[3]
+        self._metrics_port = self._ports[4]
+        self._api = REST(self._rest_port)
+        self._volumes = []
+
+        default_args = {
+            "listen-address": "0.0.0.0",
+            "rest": "true",
+            "rest-admin": "true",
+            "websocket-support": "true",
+            "log-level": "TRACE",
+            "rest-relay-cache-capacity": "100",
+            "websocket-port": self._websocket_port,
+            "rest-port": self._rest_port,
+            "tcp-port": self._tcp_port,
+            "discv5-udp-port": self._discv5_port,
+            "rest-address": "0.0.0.0",
+            "nat": f"extip:{self._ext_ip}",
+            "peer-exchange": "true",
+            "discv5-discovery": "true",
+            "cluster-id": DEFAULT_CLUSTER_ID,
+            "rln-creds-id": None,
+            "rln-creds-source": None,
+            "nodekey": self.generate_random_nodekey(),
+            # "max-connections": "50",
+            # "filter-subscription-timeout": "600",
+        }
+
+        store_sync = kwargs.pop("store_sync", "false").lower() == "true"
+        store_sync_range = kwargs.pop("store_sync_range", None)
+        store_sync_interval = kwargs.pop("store_sync_interval", None)
+
+        if store_sync:
+            default_args["store-sync"] = "true"
+            default_args.setdefault("store", "true")
+
+        if store_sync_range:
+            default_args["store-sync-range"] = store_sync_range
+
+        if store_sync_interval:
+            default_args["store-sync-interval"] = store_sync_interval
+
+        if self.is_nwaku():
+            nwaku_args = {
+                "shard": "0",
+                "metrics-server": "true",
+                "metrics-server-address": "0.0.0.0",
+                "metrics-server-port": self._metrics_port,
+                "metrics-logging": "true",
+            }
+            default_args.update(nwaku_args)
+        else:
+            raise NotImplementedError("Not implemented for this node type")
+
+        if "remove_container" in kwargs:
+            remove_container = kwargs["remove_container"]
+            del kwargs["remove_container"]
+        else:
+            remove_container = True
+
+        kwargs = self.parse_peer_persistence_config(kwargs)
+        kwargs = resolve_sharding_flags(kwargs)
+
+        default_args.update(sanitize_docker_flags(kwargs))
+
+        if self.is_nwaku() and "pubsub-topic" in default_args:
+            logger.debug("Removing pubsub-topic from nwaku args")
+            del default_args["pubsub-topic"]
+
+        rln_args, rln_creds_set, keystore_path = self.parse_rln_credentials(default_args, False)
+        self._rln_creds_set = rln_creds_set
+
+        default_args.pop("rln-creds-id", None)
+        default_args.pop("rln-creds-source", None)
+        default_args.pop("rln-keystore-prefix", None)
+
+        if rln_creds_set:
+            rln_credential_store_ready(keystore_path, require_credentials=True)
+            default_args.update(rln_args)
+        else:
+            logger.info(f"RLN credentials not set or credential store not available, starting without RLN")
+
+        self.start_args = dict(default_args)
+        return default_args, remove_container
+
+    def _start_docker(self, default_args, remove_container, wait_for_node_sec):
+        logger.debug(f"Using volumes {self._volumes}")
+
+        self._container = self._docker_manager.start_container(
+            self._docker_manager.image,
+            ports=self._ports,
+            args=default_args,
+            log_path=self._log_path,
+            container_ip=self._ext_ip,
+            volumes=self._volumes,
+            remove_container=remove_container,
+        )
+        logger.debug(f"Started container from image {self._image_name}. REST: {self._rest_port}")
+        DS.waku_nodes.append(self)
+        delay(1)
+        try:
+            self.ensure_ready(timeout_duration=wait_for_node_sec, rln_required=self._rln_creds_set)
+        except Exception as ex:
+            logger.error(f"REST service did not become ready in time: {ex}")
+            raise
+
+    @retry(stop=stop_after_delay(250), wait=wait_fixed(0.1), reraise=True)
+    def register_rln(self, **kwargs):
+        logger.debug("Registering RLN credentials...")
+        self._docker_manager.create_network()
+        self._ext_ip = self._docker_manager.generate_random_ext_ip()
+        self._ports = self._docker_manager.generate_ports()
+        self._rest_port = self._ports[0]
+        self._api = REST(self._rest_port)
+        self._volumes = []
+
+        default_args = {"rln-creds-id": None, "rln-creds-source": None, "rln-relay-user-message-limit-registration": 100}
+
+        default_args.update(sanitize_docker_flags(kwargs))
+
+        rln_args, rln_creds_set, keystore_path = self.parse_rln_credentials(default_args, True)
+
+        if rln_creds_set:
+            self._container = self._docker_manager.start_container(
+                self._docker_manager.image, self._ports, rln_args, self._log_path, self._ext_ip, self._volumes
+            )
+
+            logger.debug(f"Executed container from image {self._image_name}. REST: {self._rest_port} to register RLN")
+
+            logger.debug(f"Waiting for keystore {keystore_path}")
+            try:
+                rln_credential_store_ready(keystore_path, require_credentials=True)
+                self.rln_membership_index = str(self.get_rln_membership_index_from_log())
+                logger.debug(f"Detected RLN membership index from registration logs: {self.rln_membership_index}")
+                self.stop()
+            except Exception as ex:
+                logger.error(f"File {keystore_path} with RLN credentials did not become available in time {ex}")
+                raise
+        else:
+            logger.warn("RLN credentials not set, no action performed")
+
+        return self.rln_membership_index
+
+    @retry(stop=stop_after_delay(10), wait=wait_fixed(0.2), reraise=True)
+    def get_rln_membership_index_from_log(self):
+        if not os.path.exists(self._log_path):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self._log_path)
+
+        with open(self._log_path, "r", encoding="utf-8", errors="ignore") as log_file:
+            log_data = log_file.read()
+
+        matches = re.findall(r"membershipIndex=(\d+)", log_data)
+        if not matches:
+            raise ValueError("Could not infer RLN membership index from registration logs")
+
+        return int(matches[-1])
+
+    @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True)
+    def stop(self):
+        if self._container:
+            logger.debug(f"Stopping container with id {self._container.short_id}")
+            try:
+                self._container.stop()
+            except DockerNotFound:
+                logger.debug(f"Container {self._container.short_id} already exited and removed, treating as stopped.")
+                self._container = None
+                return
+            try:
+                self._container.remove()
+            except:
+                pass
+            self._container = None
+            logger.debug("Container stopped.")
+
+    @retry(stop=stop_after_delay(5), wait=wait_fixed(0.1), reraise=True)
+    def kill(self):
+        if self._container:
+            logger.debug(f"Killing container with id {self._container.short_id}")
+            self._container.kill()
+            try:
+                self._container.remove()
+            except:
+                pass
+            self._container = None
+            logger.debug("Container killed.")
+
+    def restart(self):
+        if self._container:
+            logger.debug(f"Restarting container with id {self._container.short_id}")
+            self._container.restart()
+
+    def pause(self):
+        if self._container:
+            logger.debug(f"Pausing container with id {self._container.short_id}")
+            self._container.pause()
+
+    def unpause(self):
+        if self._container:
+            logger.debug(f"Unpause container with id {self._container.short_id}")
+            self._container.unpause()
+
+    def ensure_ready(self, timeout_duration=10, rln_required=False):
+        @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(0.1), reraise=True)
+        def check_healthy(node=self):
+            self.health_response = node.health()
+
+            try:
+                self.health_response = json.loads(self.health_response)
+            except Exception as ex:
+                raise AttributeError(f"Unknown health response format {ex}")
+
+            if self.health_response.get("nodeHealth") != "READY":
+                raise AssertionError("Waiting for the node health status: READY")
+
+            for p in self.health_response.get("protocolsHealth"):
+                if rln_required and "Rln Relay" in p:
+                    if p["Rln Relay"] != "READY":
+                        raise AssertionError("Waiting for the Rln relay status: READY")
+                    # TODO: Remove once Rln Relay reflects true RLN status
+                    sleep(20)
+
+            logger.info("Node protocols are initialized !!")
+
+        @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(0.1), reraise=True)
+        def check_ready(node=self):
+            node.info_response = node.info()
+            logger.info("REST service is ready !!")
+
+        if self.is_nwaku():
+            check_healthy()
+        check_ready()
+
+    def get_id(self):
+        try:
+            return self.info_response["listenAddresses"][0].split("/")[-1]
+        except Exception as ex:
+            raise AttributeError(f"Could not find ID in the info call because of error: {str(ex)}")
+
+    def get_enr_uri(self):
+        try:
+            return self.info_response["enrUri"]
+        except Exception as ex:
+            raise AttributeError(f"Could not find enrUri in the info call because of error: {str(ex)}")
+
+    def get_multiaddr_with_id(self):
+        addresses = self.info_response.get("listenAddresses", [])
+        ws_address = next((addr for addr in addresses if "/ws" not in addr), None)
+        if ws_address:
+            identifier = ws_address.split("/p2p/")[-1]
+            new_address = f"{self.get_tcp_address()}/p2p/{identifier}"
+            return new_address
+        else:
+            raise AttributeError("No '/ws' address found")
+
+    def get_tcp_address(self):
+        return f"/ip4/{self._ext_ip}/tcp/{self._tcp_port}"
+
+    def subscribe_content_topic(self, content_topic: str):
+        return self._api.set_relay_auto_subscriptions([content_topic])
+
+    def unsubscribe_content_topic(self, content_topic: str):
+        return self._api.delete_relay_auto_subscriptions([content_topic])
+
+    def send_message(self, message: dict):
+        return self._api.send_relay_auto_message(message)
+
+    def info(self):
+        return self._api.info()
+
+    def health(self):
+        return self._api.health()
+
+    def get_peers(self):
+        return self._api.get_peers()
+
+    def add_peers(self, peers):
+        return self._api.add_peers(peers)
+
+    def set_relay_subscriptions(self, pubsub_topics):
+        return self._api.set_relay_subscriptions(pubsub_topics)
+
+    def set_relay_auto_subscriptions(self, content_topics):
+        return self._api.set_relay_auto_subscriptions(content_topics)
+
+    def delete_relay_subscriptions(self, pubsub_topics):
+        return self._api.delete_relay_subscriptions(pubsub_topics)
+
+    def delete_relay_auto_subscriptions(self, content_topics):
+        return self._api.delete_relay_auto_subscriptions(content_topics)
+
+    def send_relay_message(self, message, pubsub_topic):
+        return self._api.send_relay_message(message, pubsub_topic)
+
+    def send_relay_auto_message(self, message):
+        return self._api.send_relay_auto_message(message)
+
+    def send_light_push_message(self, payload):
+        return self._api.send_light_push_message(payload)
+
+    def get_relay_messages(self, pubsub_topic):
+        return self._api.get_relay_messages(pubsub_topic)
+
+    def get_relay_auto_messages(self, content_topic):
+        return self._api.get_relay_auto_messages(content_topic)
+
+    def set_filter_subscriptions(self, subscription):
+        return self._api.set_filter_subscriptions(subscription)
+
+    def update_filter_subscriptions(self, subscription):
+        return self._api.update_filter_subscriptions(subscription)
+
+    def delete_filter_subscriptions(self, subscription):
+        return self._api.delete_filter_subscriptions(subscription)
+
+    def delete_all_filter_subscriptions(self, request_id):
+        return self._api.delete_all_filter_subscriptions(request_id)
+
+    def ping_filter_subscriptions(self, request_id):
+        return self._api.ping_filter_subscriptions(request_id)
+
+    def get_filter_messages(self, content_topic, pubsub_topic=None):
+        return self._api.get_filter_messages(content_topic, pubsub_topic)
+
+    def get_store_messages(
+        self,
+        peer_addr=None,
+        include_data=None,
+        pubsub_topic=None,
+        content_topics=None,
+        start_time=None,
+        end_time=None,
+        hashes=None,
+        cursor=None,
+        page_size=None,
+        ascending=None,
+        store_v="v3",
+        **kwargs,
+    ):
+        return self._api.get_store_messages(
+            peer_addr=peer_addr,
+            include_data=include_data,
+            pubsub_topic=pubsub_topic,
+            content_topics=content_topics,
+            start_time=start_time,
+            end_time=end_time,
+            hashes=hashes,
+            cursor=cursor,
+            page_size=page_size,
+            ascending=ascending,
+            store_v=store_v,
+            **kwargs,
+        )
+
+    def get_metrics(self):
+        if self.is_nwaku():
+            metrics = requests.get(f"http://localhost:{self._metrics_port}/metrics")
+            metrics.raise_for_status()
+            return metrics.content.decode("utf-8")
+        else:
+            pytest.skip(f"This method doesn't exist for node {self.type()}")
+
+    @property
+    def image(self):
+        return self._image_name
+
+    def type(self):
+        if self.is_nwaku():
+            return "nwaku"
+        else:
+            raise ValueError("Unknown node type!!!")
+
+    def is_nwaku(self):
+        return "nwaku" in self.image
+
+    def prepare_rln_storage_paths(self, cwd, keystore_prefix, selected_id, reset_existing=False):
+        keystore_dir = os.path.join(cwd, f"keystore_{keystore_prefix}_{selected_id}")
+        rln_tree_dir = os.path.join(cwd, f"rln_tree_{keystore_prefix}_{selected_id}")
+
+        if reset_existing:
+            for path, path_name in [(keystore_dir, "keystore"), (rln_tree_dir, "rln tree")]:
+                if os.path.exists(path):
+                    logger.warning(f"Resetting existing RLN {path_name} directory before registration: {path}")
+                    shutil.rmtree(path, ignore_errors=True)
+
+        os.makedirs(keystore_dir, exist_ok=True)
+        os.makedirs(rln_tree_dir, exist_ok=True)
+
+        return keystore_dir, rln_tree_dir
+
+    def parse_rln_credentials(self, default_args, is_registration):
+        rln_args = {}
+        keystore_path = None
+
+        rln_creds_source = default_args["rln-creds-source"]
+        selected_id = default_args["rln-creds-id"]
+
+        if rln_creds_source is None or selected_id is None:
+            logger.debug(f"RLN credentials were not set")
+            return rln_args, False, keystore_path
+
+        imported_creds = json.loads(rln_creds_source)
+        rln_chain_id = imported_creds.get("rln-relay-chain-id")
+        if rln_chain_id is None:
+            eth_client_address = imported_creds.get("rln-relay-eth-client-address", "")
+            if "linea" in eth_client_address:
+                rln_chain_id = "59141"
+            elif "sepolia" in eth_client_address:
+                rln_chain_id = "11155111"
+
+        if len(imported_creds) < 4 or any(value is None for value in imported_creds.values()):
+            logger.warn(f"One or more of required RLN credentials were not set properly")
+            return rln_args, False, keystore_path
+
+        eth_private_key = select_private_key(imported_creds, selected_id)
+
+        cwd = os.getcwd()
+        keystore_prefix = default_args.get("rln-keystore-prefix")
+
+        if not keystore_prefix:
+            logger.warn("rln-keystore-prefix is missing, cannot mount RLN state and keystore")
+            return rln_args, False, keystore_path
+
+        keystore_dir, rln_tree_dir = self.prepare_rln_storage_paths(cwd, keystore_prefix, selected_id, reset_existing=is_registration)
+
+        if self.is_nwaku():
+            if is_registration:
+                rln_args.update(
+                    {
+                        "generateRlnKeystore": None,
+                        "--execute": None,
+                        "rln-relay-user-message-limit": default_args["rln-relay-user-message-limit-registration"],
+                    }
+                )
+            else:
+                rln_args.update(
+                    {
+                        "rln-relay": "true",
+                    }
+                )
+
+            if not default_args.get("rln-relay-dynamic") and not is_registration:
+                rln_args.update(
+                    {
+                        "rln-relay-cred-path": "/keystore/keystore.json",
+                        "rln-relay-cred-password": imported_creds["rln-relay-cred-password"],
+                        "rln-relay-eth-client-address": imported_creds["rln-relay-eth-client-address"],
+                        "rln-relay-eth-contract-address": imported_creds["rln-relay-eth-contract-address"],
+                    }
+                )
+            else:
+                rln_args.update(
+                    {
+                        "rln-relay-cred-path": "/keystore/keystore.json",
+                        "rln-relay-cred-password": imported_creds["rln-relay-cred-password"],
+                        "rln-relay-eth-client-address": imported_creds["rln-relay-eth-client-address"],
+                        "rln-relay-eth-contract-address": imported_creds["rln-relay-eth-contract-address"],
+                        "rln-relay-eth-private-key": imported_creds[eth_private_key],
+                    }
+                )
+
+            if rln_chain_id is not None:
+                rln_args["rln-relay-chain-id"] = str(rln_chain_id)
+
+            keystore_path = os.path.join(keystore_dir, "keystore.json")
+
+            self._volumes.extend(
+                [
+                    f"{rln_tree_dir}:/etc/rln_tree",
+                    f"{keystore_dir}:/keystore",
+                ]
+            )
+
+        else:
+            raise NotImplementedError("Not implemented for type other than Nim Waku ")
+
+        return rln_args, True, keystore_path
+
+    def parse_peer_persistence_config(self, kwargs):
+        if kwargs.get("peer_persistence") == "true":
+            cwd = os.getcwd()
+            # Please note, as of now, peerdb is stored directly at / which is not shareable between containers.
+            # Volume related code is usable after https://github.com/waku-org/nwaku/issues/2792 would be resolved.
+            self._volumes.extend(
+                [
+                    cwd + "/peerdb" + ":/shared",
+                ]
+            )
+
+            shutil.rmtree(cwd + "/peerdb")
+
+        return kwargs
+
+    @property
+    def container(self):
+        return self._container
+
+    def generate_random_nodekey(self):
+        # Define the set of hexadecimal characters
+        hex_chars = string.hexdigits.lower()
+        # Generate a random 64-character string from the hex characters
+        random_key = "".join(random.choice(hex_chars) for _ in range(64))
+        return random_key
+
+    def search_waku_log_for_string(self, search_pattern, use_regex=False):
+        return self._docker_manager.search_log_for_keywords(self._log_path, [search_pattern], use_regex)
+
+    def check_waku_log_errors(self, whitelist=None):
+        keywords = LOG_ERROR_KEYWORDS
+
+        # If a whitelist is provided, remove those keywords from the keywords list
+        if whitelist:
+            keywords = [keyword for keyword in keywords if keyword not in whitelist]
+
+        matches = self._docker_manager.search_log_for_keywords(self._log_path, keywords, False)
+        assert not matches, f"Found errors {matches}"
+
+    def set_log_level(self, log_level):
+        return self._api.set_log_level(log_level)
+
+    def get_service_peers(self):
+        return self._api.get_service_peers()
+
+    def get_connected_peers(self):
+        return self._api.get_connected_peers()
+
+    def get_connected_peers_on_shard(self, shard_id):
+        return self._api.get_connected_peers_on_shard(shard_id)
+
+    def get_relay_peers(self):
+        return self._api.get_relay_peers()
+
+    def get_relay_peers_on_shard(self, shard_id):
+        return self._api.get_relay_peers_on_shard(shard_id)
+
+    def get_mesh_peers(self):
+        return self._api.get_mesh_peers()
+
+    def get_mesh_peers_on_shard(self, shard_id):
+        return self._api.get_mesh_peers_on_shard(shard_id)
+
+    def get_peer_stats(self):
+        return self._api.get_peer_stats()
+
+    def get_filter_subscriptions(self):
+        return self._api.get_filter_subscriptions()
+
+    def get_info(self):
+        return self._api.get_info()
+
+    def get_version(self):
+        return self._api.get_version()
+
+    def get_debug_version(self):
+        return self._api.get_debug_version()
+
+    def get_peer_info(self, peer_id: str):
+        return self._api.get_peer(peer_id)
+
+    @property
+    def container_id(self) -> str:
+        if not self._container:
+            raise RuntimeError("Node container not started yet")
+        return self._container.id

tests-e2e/src/node/wrapper_helpers.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import json
+import threading
+import time
+from typing import Optional
+from src.libs.common import to_base64
+
+DEFAULT_CONTENT_TOPIC = "/test/1/default/proto"
+DEFAULT_PAYLOAD = to_base64("test payload")
+EVENT_PROPAGATED = "message_propagated"
+EVENT_SENT = "message_sent"
+EVENT_ERROR = "message_error"
+
+# MaxTimeInCache from send_service.nim.
+MAX_TIME_IN_CACHE_S = 60.0
+# Extra slack to cover the background retry loop tick after the window expires.
+CACHE_EXPIRY_SLACK_S = 10.0
+ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S = MAX_TIME_IN_CACHE_S + CACHE_EXPIRY_SLACK_S
+RETRY_WINDOW_EXPIRED_MSG = "Unable to send within retry time window"
+
+
+class EventCollector:
+    """Thread-safe collector for async node events.
+
+    Pass `collector.event_callback` as the `event_cb` argument to
+    WrapperManager.create_and_start().  Every event fired by the library
+    is decoded from JSON and appended to `self.events`.
+    """
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self.events: list[dict] = []
+
+    def event_callback(self, ret: int, raw: bytes) -> None:
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except Exception:
+            payload = {"_raw": raw.decode("utf-8", errors="replace"), "_ret": ret}
+
+        with self._lock:
+            self.events.append(payload)
+
+    def get_events_for_request(self, request_id: str) -> list[dict]:
+        with self._lock:
+            return [e for e in self.events if e.get("requestId") == request_id]
+
+    def snapshot(self) -> list[dict]:
+        """Return a thread-safe copy of all collected events.
+
+        Use this whenever you need to iterate over every event (rather than
+        events for a single request_id). Iterating `self.events` directly is
+        unsafe because `event_callback` appends from the wrapper's event
+        thread.
+        """
+        with self._lock:
+            return list(self.events)
+
+
+def is_propagated_event(event: dict) -> bool:
+    return event.get("eventType") == EVENT_PROPAGATED
+
+
+def is_sent_event(event: dict) -> bool:
+    return event.get("eventType") == EVENT_SENT
+
+
+def is_error_event(event: dict) -> bool:
+    return event.get("eventType") == EVENT_ERROR
+
+
+def wait_for_event(
+    collector: EventCollector,
+    request_id: str,
+    predicate,
+    timeout_s: float,
+    poll_interval_s: float = 0.5,
+) -> Optional[dict]:
+    """Poll until an event matching `predicate` arrives for `request_id`,
+    or until `timeout_s` elapses.  Returns the matching event or None.
+    """
+    deadline = time.monotonic() + timeout_s
+
+    while True:
+        for event in collector.get_events_for_request(request_id):
+            if predicate(event):
+                return event
+        if time.monotonic() >= deadline:
+            return None
+        time.sleep(poll_interval_s)
+
+
+def wait_for_propagated(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]:
+    return wait_for_event(collector, request_id, is_propagated_event, timeout_s)
+
+
+def wait_for_sent(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]:
+    return wait_for_event(collector, request_id, is_sent_event, timeout_s)
+
+
+def wait_for_error(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]:
+    return wait_for_event(collector, request_id, is_error_event, timeout_s)
+
+
+def assert_no_error(collector: EventCollector, request_id: str, context: str = "") -> None:
+    """Assert that no message_error event is currently buffered for `request_id`."""
+    event = wait_for_error(collector, request_id, timeout_s=0)
+    suffix = f" ({context})" if context else ""
+    assert event is None, f"Unexpected message_error event{suffix}: {event}"
+
+
+def assert_no_sent(collector: EventCollector, request_id: str, context: str = "") -> None:
+    """Assert that no message_sent event is currently buffered for `request_id`."""
+    event = wait_for_sent(collector, request_id, timeout_s=0)
+    suffix = f" ({context})" if context else ""
+    assert event is None, f"Unexpected message_sent event{suffix}: {event}"
+
+
+def assert_no_propagated(collector: EventCollector, request_id: str, context: str = "") -> None:
+    """Assert that no message_propagated event is currently buffered for `request_id`."""
+    event = wait_for_propagated(collector, request_id, timeout_s=0)
+    suffix = f" ({context})" if context else ""
+    assert event is None, f"Unexpected message_propagated event{suffix}: {event}"
+
+
+def wait_for_connected(
+    collector: EventCollector,
+    timeout_s: float = 10.0,
+    poll_interval_s: float = 0.3,
+) -> Optional[dict]:
+    """Wait until a connection_status_change event with PartiallyConnected or Connected arrives."""
+    deadline = time.monotonic() + timeout_s
+    while time.monotonic() < deadline:
+        for event in collector.snapshot():
+            if event.get("eventType") == "connection_status_change" and event.get("connectionStatus") in ("PartiallyConnected", "Connected"):
+                return event
+        time.sleep(poll_interval_s)
+    return None
+
+
+TERMINAL_EVENT_TYPES = {EVENT_PROPAGATED, EVENT_SENT, EVENT_ERROR}
+
+
+def assert_event_invariants(collector: EventCollector, request_id: str) -> None:
+    """Check per-request event invariants (issue #163):
+    - All events carry the correct requestId.
+    - No duplicate terminal events (Propagated, Sent, Error).
+    - Sent never appears before Propagated.
+    """
+    events = collector.get_events_for_request(request_id)
+    assert events, f"No events found for request {request_id}"
+
+    counts: dict[str, int] = {}
+    first_index: dict[str, int] = {}
+    for i, event in enumerate(events):
+        assert event.get("requestId") == request_id, (
+            f"Event at index {i} has wrong requestId: " f"expected {request_id!r}, got {event.get('requestId')!r}"
+        )
+        event_type = event.get("eventType", "")
+        if event_type in TERMINAL_EVENT_TYPES:
+            counts[event_type] = counts.get(event_type, 0) + 1
+            if event_type not in first_index:
+                first_index[event_type] = i
+
+    for event_type, count in counts.items():
+        assert count == 1, f"Duplicate {event_type} events for request {request_id}: " f"got {count}, expected 1. Events: {events}"
+
+    if EVENT_SENT in first_index and EVENT_PROPAGATED in first_index:
+        assert first_index[EVENT_PROPAGATED] < first_index[EVENT_SENT], (
+            f"message_sent (index {first_index[EVENT_SENT]}) arrived before "
+            f"message_propagated (index {first_index[EVENT_PROPAGATED]}) "
+            f"for request {request_id}. Events: {events}"
+        )
+
+
+def get_node_multiaddr(node) -> str:
+    """Return the TCP multiaddr (with peer-id) from a WrapperManager node.
+
+    Asserts that the wrapper returned exactly one address. If the wrapper ever
+    starts returning multiple addresses (newline/comma-separated or a JSON
+    list), this fails loudly instead of silently passing a malformed string
+    downstream to staticnodes / add_peers.
+    """
+    result = node.get_node_info_raw("MyMultiaddresses")
+    if result.is_err():
+        raise RuntimeError(f"get_node_info_raw failed: {result.err()}")
+
+    addr = result.ok_value.strip()
+    if not addr or not addr.startswith("/"):
+        raise RuntimeError(f"Unexpected multiaddr format: {addr!r}")
+
+    if "\n" in addr or "," in addr or addr.startswith("["):
+        raise AssertionError(f"Expected a single multiaddr from MyMultiaddresses, got multiple: {addr!r}")
+
+    return addr
+
+
+def create_message_bindings(**overrides) -> dict:
+    envelope = {
+        "contentTopic": DEFAULT_CONTENT_TOPIC,
+        "payload": DEFAULT_PAYLOAD,
+        "ephemeral": False,
+    }
+    envelope.update(overrides)
+    return envelope
+
+
+def assert_no_unknown_request_ids(collector: EventCollector, issued_request_ids) -> None:
+    """Cross-association guard: every event carrying a requestId must belong
+    to one of the request ids we issued. Catches events that get attached to
+    the wrong request id under concurrency.
+    """
+    issued = set(issued_request_ids)
+    for event in collector.snapshot():
+        event_request_id = event.get("requestId")
+        if event_request_id is None:
+            continue
+        assert event_request_id in issued, f"Event carries an unknown requestId={event_request_id!r}, " f"not in issued set {issued}. Event: {event}"

tests-e2e/src/node/wrappers_manager.py

@@ -0,0 +1,98 @@
+import sys
+from pathlib import Path
+from result import Result, Ok, Err
+
+_BINDINGS_PATH = Path(__file__).resolve().parents[2] / "vendor" / "logos-delivery-python-bindings" / "waku"
+if str(_BINDINGS_PATH) not in sys.path:
+    sys.path.insert(0, str(_BINDINGS_PATH))
+
+from wrapper import NodeWrapper as _NodeWrapper  # type: ignore[import]
+
+""""
+thin manager/wrapper layer around NodeWrapper from the bindings.
+It simplifies  create, start, and interaction with a Waku node while returning consistent Result objects (Ok / Err).
+"""
+
+
+class WrapperManager:
+    def __init__(self, node: _NodeWrapper):
+        self._node = node
+
+    @classmethod
+    def create(
+        cls,
+        config: dict,
+        event_cb=None,
+        *,
+        timeout_s: float = 20.0,
+    ) -> Result["WrapperManager", str]:
+        result = _NodeWrapper.create_node(config, event_cb, timeout_s=timeout_s)
+        if result.is_err():
+            return Err(result.err())
+        return Ok(cls(result.ok_value))
+
+    @classmethod
+    def create_and_start(
+        cls,
+        config: dict,
+        event_cb=None,
+        *,
+        timeout_s: float = 20.0,
+    ) -> Result["WrapperManager", str]:
+        result = _NodeWrapper.create_and_start(config, event_cb, timeout_s=timeout_s)
+        if result.is_err():
+            return Err(result.err())
+        return Ok(cls(result.ok_value))
+
+    def __enter__(self) -> "WrapperManager":
+        return self
+
+    def __exit__(self, *_) -> None:
+        self.stop_and_destroy()
+
+    def start_node(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        return self._node.start_node(timeout_s=timeout_s)
+
+    def stop_node(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        return self._node.stop_node(timeout_s=timeout_s)
+
+    def destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        return self._node.destroy(timeout_s=timeout_s)
+
+    def stop_and_destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        return self._node.stop_and_destroy(timeout_s=timeout_s)
+
+    def subscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]:
+        return self._node.subscribe_content_topic(content_topic, timeout_s=timeout_s)
+
+    def unsubscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]:
+        return self._node.unsubscribe_content_topic(content_topic, timeout_s=timeout_s)
+
+    def send_message(self, message: dict, *, timeout_s: float = 20.0) -> Result[str, str]:
+        return self._node.send_message(message, timeout_s=timeout_s)
+
+    def get_available_node_info_ids(self, *, timeout_s: float = 20.0) -> Result[list[str], str]:
+        return self._node.get_available_node_info_ids(timeout_s=timeout_s)
+
+    def get_node_info(self, node_info_id: str, *, timeout_s: float = 20.0) -> Result[dict, str]:
+        return self._node.get_node_info(node_info_id, timeout_s=timeout_s)
+
+    def get_node_info_raw(self, node_info_id: str, *, timeout_s: float = 20.0) -> Result[str, str]:
+        """Like get_node_info but returns the raw string without JSON parsing."""
+        from wrapper import lib, ffi, _new_cb_state, _wait_cb_raw  # type: ignore[import]
+
+        state = _new_cb_state()
+        cb = self._node._make_waiting_cb(state)
+        rc = lib.logosdelivery_get_node_info(self._node.ctx, cb, ffi.NULL, node_info_id.encode("utf-8"))
+        if rc != 0:
+            return Err(f"get_node_info_raw: immediate call failed (ret={rc})")
+        wait_result = _wait_cb_raw(state, "get_node_info_raw", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+        cb_ret, cb_msg = wait_result.ok_value
+        if cb_ret != 0:
+            return Err(f"get_node_info_raw: callback failed (ret={cb_ret})")
+        return Ok(cb_msg.decode("utf-8") if cb_msg else "")
+
+    def get_available_configs(self, *, timeout_s: float = 20.0) -> Result[dict, str]:
+        return self._node.get_available_configs(timeout_s=timeout_s)

tests-e2e/src/postgres_setup.py

@@ -0,0 +1,44 @@
+import docker
+import os
+from src.env_vars import NETWORK_NAME, PG_PASS, PG_USER
+from src.libs.custom_logger import get_custom_logger
+
+logger = get_custom_logger(__name__)
+
+
+def start_postgres():
+    pg_env = {"POSTGRES_USER": PG_USER, "POSTGRES_PASSWORD": PG_PASS}
+
+    base_path = os.path.abspath(".")
+    volumes = {os.path.join(base_path, "postgresql"): {"bind": "/var/lib/postgresql/data", "mode": "Z"}}
+
+    client = docker.from_env()
+
+    postgres_container = client.containers.run(
+        "postgres:15.4-alpine3.18",
+        name="postgres",
+        environment=pg_env,
+        volumes=volumes,
+        command="postgres",
+        ports={"5432/tcp": ("127.0.0.1", 5432)},
+        restart_policy={"Name": "on-failure", "MaximumRetryCount": 5},
+        healthcheck={
+            "Test": ["CMD-SHELL", "pg_isready -U postgres -d postgres"],
+            "Interval": 30000000000,  # 30 seconds in nanoseconds
+            "Timeout": 60000000000,  # 60 seconds in nanoseconds
+            "Retries": 5,
+            "StartPeriod": 80000000000,  # 80 seconds in nanoseconds
+        },
+        detach=True,
+        network_mode=NETWORK_NAME,
+    )
+
+    logger.debug("Postgres container started")
+
+    return postgres_container
+
+
+def stop_postgres(postgres_container):
+    postgres_container.stop()
+    postgres_container.remove()
+    logger.debug("Postgres container stopped and removed.")

tests-e2e/src/steps/common.py

@@ -0,0 +1,80 @@
+import base64
+import hashlib
+import inspect
+from time import time
+from time import time_ns
+import allure
+import pytest
+from datetime import timedelta, datetime
+from tenacity import retry, stop_after_delay, wait_fixed
+from src.libs.common import delay, to_base64
+from src.libs.custom_logger import get_custom_logger
+
+logger = get_custom_logger(__name__)
+
+
+class StepsCommon:
+    @pytest.fixture(scope="function", autouse=True)
+    def common_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        if not hasattr(self, "test_payload"):
+            self.test_payload = "Default Payload"
+        if not hasattr(self, "test_content_topic"):
+            self.test_content_topic = "/test/1/default/proto"
+
+    @allure.step
+    @retry(stop=stop_after_delay(20), wait=wait_fixed(0.5), reraise=True)
+    def add_node_peer(self, node, multiaddr_list, shards=[0, 1, 2, 3, 4, 5, 6, 7, 8]):
+        if node.is_nwaku():
+            for multiaddr in multiaddr_list:
+                node.add_peers([multiaddr])
+
+    @allure.step
+    @retry(stop=stop_after_delay(70), wait=wait_fixed(1), reraise=True)
+    def wait_for_autoconnection(self, node_list, hard_wait=None):
+        for node in node_list:
+            get_peers = node.get_peers()
+            assert len(get_peers) >= 1
+        if hard_wait:
+            delay(hard_wait)
+
+    @allure.step
+    def create_message(self, **kwargs):
+        ts_ns = time_ns()
+        ts_ns = int(f"{ts_ns:019d}")
+        message = {"payload": to_base64(self.test_payload), "contentTopic": self.test_content_topic, "timestamp": ts_ns}
+        message.update(kwargs)
+        return message
+
+    @allure.step
+    def compute_message_hash(self, pubsub_topic, msg, hash_type="hex"):
+        ctx = hashlib.sha256()
+        ctx.update(pubsub_topic.encode("utf-8"))
+        ctx.update(base64.b64decode(msg["payload"]))
+        ctx.update(msg["contentTopic"].encode("utf-8"))
+        if "meta" in msg:
+            ctx.update(base64.b64decode(msg["meta"]))
+        ctx.update(int(msg["timestamp"]).to_bytes(8, byteorder="big"))
+        hash_bytes = ctx.digest()
+        if hash_type == "hex":
+            return "0x" + hash_bytes.hex()
+        else:
+            return base64.b64encode(hash_bytes).decode("utf-8")
+
+    def get_time_list_pass(self):
+        ts_pass = [
+            {"description": "3 sec Past", "value": int((datetime.now() - timedelta(seconds=3)).timestamp() * 1e9)},
+            {"description": "1 sec Past", "value": int((datetime.now() - timedelta(seconds=1)).timestamp() * 1e9)},
+            {"description": "0.1 sec Past", "value": int((datetime.now() - timedelta(seconds=0.1)).timestamp() * 1e9)},
+            {"description": "0.1 sec Future", "value": int((datetime.now() + timedelta(seconds=0.1)).timestamp() * 1e9)},
+            {"description": "2 sec Future", "value": int((datetime.now() + timedelta(seconds=2)).timestamp() * 1e9)},
+            {"description": "10 sec Future", "value": int((datetime.now() + timedelta(seconds=10)).timestamp() * 1e9)},
+        ]
+        return ts_pass
+
+    def get_time_list_fail(self):
+        ts_fail = [
+            {"description": "20 sec Past", "value": int((datetime.now() - timedelta(seconds=20)).timestamp() * 1e9)},
+            {"description": "40 sec Future", "value": int((datetime.now() + timedelta(seconds=40)).timestamp() * 1e9)},
+        ]
+        return ts_fail

tests-e2e/src/steps/filter.py

@@ -0,0 +1,197 @@
+import inspect
+from uuid import uuid4
+from src.libs.custom_logger import get_custom_logger
+from time import time
+import pytest
+import allure
+from src.libs.common import to_base64, delay
+from src.node.waku_message import WakuMessage
+from src.env_vars import NODE_1, NODE_2, ADDITIONAL_NODES
+from src.node.waku_node import WakuNode
+from tenacity import retry, stop_after_delay, wait_fixed
+from src.steps.common import StepsCommon
+from src.test_data import VALID_PUBSUB_TOPICS
+
+logger = get_custom_logger(__name__)
+
+
+class StepsFilter(StepsCommon):
+    test_pubsub_topic = VALID_PUBSUB_TOPICS[1]
+    second_pubsub_topic = VALID_PUBSUB_TOPICS[2]
+    another_cluster_pubsub_topic = "/waku/2/rs/2/2"
+    test_content_topic = "/test/1/waku-filter/proto"
+    second_content_topic = "/test/2/waku-filter/proto"
+    test_payload = "Filter works!!"
+
+    @pytest.fixture(scope="function", autouse=True)
+    def filter_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.main_nodes = []
+        self.optional_nodes = []
+
+    @pytest.fixture(scope="function")
+    def setup_main_relay_node(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.relay_node_start(NODE_1)
+
+    @pytest.fixture(scope="function")
+    def setup_main_filter_node(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(relay="false", discv5_bootstrap_node=self.enr_uri, filternode=self.multiaddr_with_id)
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.append(self.node2)
+
+    @pytest.fixture(scope="function")
+    def subscribe_main_nodes(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.wait_for_subscriptions_on_main_nodes([self.test_content_topic])
+
+    @pytest.fixture(scope="function")
+    @retry(stop=stop_after_delay(20), wait=wait_fixed(1), reraise=True)
+    def filter_warm_up(self):
+        try:
+            self.ping_filter_subscriptions("1")
+        except Exception as ex:
+            if "peer has no subscriptions" in str(ex):
+                logger.info("WARM UP successful!!")
+            else:
+                raise TimeoutError(f"WARM UP FAILED WITH: {ex}")
+
+    def relay_node_start(self, node):
+        self.node1 = WakuNode(node, f"node1_{self.test_id}")
+        start_args = {"relay": "true", "filter": "true"}
+        self.node1.start(**start_args)
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        return self.node1
+
+    def setup_optional_filter_nodes(self, node_list=ADDITIONAL_NODES):
+        if node_list:
+            nodes = [node.strip() for node in node_list.split(",") if node]
+        else:
+            pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test")
+        for index, node in enumerate(nodes):
+            node = WakuNode(node, f"node{index + 3}_{self.test_id}")
+            node.start(relay="false", discv5_bootstrap_node=self.enr_uri, filternode=self.multiaddr_with_id)
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)
+
+    @allure.step
+    def check_published_message_reaches_filter_peer(
+        self, message=None, pubsub_topic=None, message_propagation_delay=0.1, sender=None, peer_list=None
+    ):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if not sender:
+            sender = self.node1
+        if not peer_list:
+            peer_list = self.main_nodes + self.optional_nodes
+
+        sender.send_relay_message(message, pubsub_topic)
+        delay(message_propagation_delay)
+        for index, peer in enumerate(peer_list):
+            logger.debug(f"Checking that peer NODE_{index + 2}:{peer.image} can find the published message")
+            get_messages_response = self.get_filter_messages(message["contentTopic"], pubsub_topic=pubsub_topic, node=peer)
+            assert get_messages_response, f"Peer NODE_{index + 2}:{peer.image} couldn't find any messages"
+            assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}"
+            waku_message = WakuMessage(get_messages_response)
+            waku_message.assert_received_message(message)
+
+    @allure.step
+    def check_publish_without_filter_subscription(self, message=None, pubsub_topic=None, peer_list=None):
+        try:
+            self.check_published_message_reaches_filter_peer(message=message, pubsub_topic=pubsub_topic, peer_list=peer_list)
+            raise AssertionError("Publish with no subscription worked!!!")
+        except Exception as ex:
+            assert "Bad Request" in str(ex) or "Not Found" in str(ex) or "couldn't find any messages" in str(ex)
+
+    @allure.step
+    def wait_for_subscriptions_on_main_nodes(self, content_topic_list, pubsub_topic=None):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        self.node1.set_relay_subscriptions([pubsub_topic])
+        request_id = str(uuid4())
+        filter_sub_response = self.create_filter_subscription_with_retry(
+            {"requestId": request_id, "contentFilters": content_topic_list, "pubsubTopic": pubsub_topic}
+        )
+        assert filter_sub_response["requestId"] == request_id
+        assert filter_sub_response["statusDesc"] in ["OK"]
+
+    @allure.step
+    def subscribe_optional_filter_nodes(self, content_topic_list, pubsub_topic=None):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        for node in self.optional_nodes:
+            request_id = str(uuid4())
+            self.create_filter_subscription_with_retry(
+                {"requestId": request_id, "contentFilters": content_topic_list, "pubsubTopic": pubsub_topic}, node=node
+            )
+
+    @retry(stop=stop_after_delay(60), wait=wait_fixed(1), reraise=True)
+    @allure.step
+    def create_filter_subscription_with_retry(self, subscription, node=None):
+        return self.create_filter_subscription(subscription, node)
+
+    @allure.step
+    def create_filter_subscription(self, subscription, node=None):
+        if node is None:
+            node = self.node2
+        return node.set_filter_subscriptions(subscription)
+
+    @allure.step
+    def update_filter_subscription(self, subscription, node=None):
+        if node is None:
+            node = self.node2
+        return node.update_filter_subscriptions(subscription)
+
+    @allure.step
+    def delete_filter_subscription(self, subscription, status=None, node=None):
+        if node is None:
+            node = self.node2
+        delete_sub_response = node.delete_filter_subscriptions(subscription)
+        assert delete_sub_response["requestId"] == subscription["requestId"]
+        if status is None:
+            assert delete_sub_response["statusDesc"] in ["OK"]
+        else:
+            assert status in delete_sub_response["statusDesc"]
+
+    @allure.step
+    def delete_all_filter_subscriptions(self, request_id, node=None):
+        if node is None:
+            node = self.node2
+        delete_sub_response = node.delete_all_filter_subscriptions(request_id)
+        assert delete_sub_response["requestId"] == request_id["requestId"]
+        assert delete_sub_response["statusDesc"] in ["OK"]
+
+    @allure.step
+    def ping_filter_subscriptions(self, request_id, node=None):
+        if node is None:
+            node = self.node2
+        ping_sub_response = node.ping_filter_subscriptions(request_id)
+        assert ping_sub_response["requestId"] == request_id
+        assert ping_sub_response["statusDesc"] in ["OK"]
+
+    def ping_without_filter_subscription(self, node=None):
+        try:
+            self.ping_filter_subscriptions(str(uuid4()), node=node)
+            raise AssertionError("Ping without any subscription worked")
+        except Exception as ex:
+            assert "peer has no subscription" in str(ex) or "ping request failed" in str(ex)
+
+    @allure.step
+    def add_new_relay_subscription(self, pubsub_topics, node=None):
+        if node is None:
+            node = self.node1
+        self.node1.set_relay_subscriptions(pubsub_topics)
+
+    @allure.step
+    def get_filter_messages(self, content_topic, pubsub_topic=None, node=None):
+        if node is None:
+            node = self.node2
+        if node.is_nwaku():
+            return node.get_filter_messages(content_topic)
+        else:
+            raise NotImplementedError("Not implemented for this node type")

tests-e2e/src/steps/light_push.py

@@ -0,0 +1,148 @@
+import inspect
+from src.libs.custom_logger import get_custom_logger
+from time import time
+import pytest
+import allure
+from src.libs.common import to_base64, delay
+from src.node.waku_message import WakuMessage
+from src.env_vars import (
+    ADDITIONAL_NODES,
+    NODE_1,
+    NODE_2,
+)
+from src.node.waku_node import WakuNode
+from src.steps.common import StepsCommon
+from src.test_data import VALID_PUBSUB_TOPICS
+from tenacity import retry, stop_after_delay, wait_fixed
+
+logger = get_custom_logger(__name__)
+
+
+class StepsLightPush(StepsCommon):
+    test_content_topic = "/myapp/1/latest/proto"
+    test_pubsub_topic = VALID_PUBSUB_TOPICS[0]
+    test_payload = "Light push works!!"
+
+    @pytest.fixture(scope="function", autouse=True)
+    def light_push_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.main_receiving_nodes = []
+        self.optional_nodes = []
+        self.multiaddr_list = []
+
+    @allure.step
+    def start_receiving_node(self, image, node_index, **kwargs):
+        node = WakuNode(image, f"receiving_node{node_index}_{self.test_id}")
+        node.start(**kwargs)
+        if kwargs["relay"] == "true":
+            self.main_receiving_nodes.extend([node])
+        self.add_node_peer(node, self.multiaddr_list)
+        self.multiaddr_list.extend([node.get_multiaddr_with_id()])
+        return node
+
+    @allure.step
+    def setup_lightpush_node(self, image, node_index, **kwargs):
+        node = WakuNode(image, f"lightpush_node{node_index}_{self.test_id}")
+        node.start(discv5_bootstrap_node=self.enr_uri, lightpushnode=self.multiaddr_list[0], **kwargs)
+        if kwargs["relay"] == "true":
+            self.main_receiving_nodes.extend([node])
+        self.add_node_peer(node, self.multiaddr_list)
+        return node
+
+    @allure.step
+    def setup_first_receiving_node(self, lightpush="true", relay="true", **kwargs):
+        self.receiving_node1 = self.start_receiving_node(NODE_1, node_index=1, lightpush=lightpush, relay=relay, **kwargs)
+        self.enr_uri = self.receiving_node1.get_enr_uri()
+
+    @allure.step
+    def setup_second_receiving_node(self, lightpush, relay, **kwargs):
+        self.receiving_node2 = self.start_receiving_node(NODE_1, node_index=2, lightpush=lightpush, relay=relay, **kwargs)
+
+    @allure.step
+    def setup_additional_receiving_nodes(self, node_list=ADDITIONAL_NODES, **kwargs):
+        if node_list:
+            nodes = [node.strip() for node in node_list.split(",") if node]
+        else:
+            pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test")
+        for index, node in enumerate(nodes):
+            self.start_receiving_node(node, node_index=index + 2, lightpush="true", relay="true", pubsub_topic=self.test_pubsub_topic, **kwargs)
+
+    @allure.step
+    def setup_first_lightpush_node(self, lightpush="true", relay="true", **kwargs):
+        self.light_push_node1 = self.setup_lightpush_node(NODE_2, node_index=1, lightpush=lightpush, relay=relay, **kwargs)
+
+    @allure.step
+    def setup_second_lightpush_node(self, lightpush="true", relay="true", **kwargs):
+        self.light_push_node2 = self.setup_lightpush_node(NODE_2, node_index=2, lightpush=lightpush, relay=relay, **kwargs)
+
+    @allure.step
+    def setup_additional_lightpush_nodes(self, node_list=ADDITIONAL_NODES, **kwargs):
+        if node_list:
+            nodes = [node.strip() for node in node_list.split(",") if node]
+        else:
+            pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test")
+        self.additional_lightpush_nodes = []
+        for index, node in enumerate(nodes):
+            node = self.setup_lightpush_node(node, node_index=index + 2, lightpush="true", relay="true", **kwargs)
+            self.additional_lightpush_nodes.append(node)
+
+    @allure.step
+    def subscribe_to_pubsub_topics_via_relay(self, node=None, pubsub_topics=None):
+        if pubsub_topics is None:
+            pubsub_topics = [self.test_pubsub_topic]
+        if not node:
+            node = self.main_receiving_nodes
+        if isinstance(node, list):
+            for node in node:
+                node.set_relay_subscriptions(pubsub_topics)
+        else:
+            node.set_relay_subscriptions(pubsub_topics)
+
+    @allure.step
+    def subscribe_to_pubsub_topics_via_filter(self, node, pubsub_topic=None, content_topic=None):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if content_topic is None:
+            content_topic = [self.test_content_topic]
+        subscription = {"requestId": "1", "contentFilters": content_topic, "pubsubTopic": pubsub_topic}
+        node.set_filter_subscriptions(subscription)
+
+    @allure.step
+    def check_light_pushed_message_reaches_receiving_peer(
+        self, pubsub_topic=None, message=None, message_propagation_delay=0.1, sender=None, peer_list=None
+    ):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if not sender:
+            sender = self.light_push_node1
+        if not peer_list:
+            peer_list = self.main_receiving_nodes + self.optional_nodes
+        payload = self.create_payload(pubsub_topic, message)
+        logger.debug("Lightpushing message")
+        sender.send_light_push_message(payload)
+        delay(message_propagation_delay)
+        for index, peer in enumerate(peer_list):
+            logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the lightpushed message")
+            get_messages_response = peer.get_relay_messages(pubsub_topic)
+            assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages"
+            assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}"
+            waku_message = WakuMessage(get_messages_response)
+            waku_message.assert_received_message(payload["message"])
+
+    @allure.step
+    def create_payload(self, pubsub_topic=None, message=None, **kwargs):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        payload = {"pubsubTopic": pubsub_topic, "message": message}
+        payload.update(kwargs)
+        return payload
+
+    @allure.step
+    @retry(stop=stop_after_delay(120), wait=wait_fixed(1), reraise=True)
+    def subscribe_and_light_push_with_retry(self):
+        self.subscribe_to_pubsub_topics_via_relay()
+        self.light_push_node1.send_light_push_message(self.create_payload())
+        get_messages_response = self.main_receiving_nodes[0].get_relay_messages(self.test_pubsub_topic)
+        assert len(get_messages_response) >= 1, f"Expected al least 1 message but got {len(get_messages_response)}"

tests-e2e/src/steps/metrics.py

@@ -0,0 +1,68 @@
+import re
+from src.libs.custom_logger import get_custom_logger
+import allure
+from tenacity import retry, stop_after_delay, wait_fixed
+
+from src.test_data import METRICS_WITH_INITIAL_VALUE_ZERO
+
+
+logger = get_custom_logger(__name__)
+
+
+class StepsMetrics:
+    @allure.step
+    def check_metric(self, node, metric_name, expected_value, exact=False):
+        logger.debug(f"Checking metric: {metric_name} has {expected_value}")
+        response = node.get_metrics()
+        lines = response.split("\n")
+        actual_value = None
+        for line in lines:
+            if line.startswith(metric_name):
+                parts = line.split(" ")
+                if len(parts) >= 2:
+                    actual_value = float(parts[1])
+                    break
+        if actual_value is None:
+            raise AttributeError(f"Metric '{metric_name}' not found")
+        logger.debug(f"Found metric: {metric_name} with value {actual_value}")
+        if exact:
+            assert actual_value == expected_value, f"Expected value for '{metric_name}' is {expected_value}, but got {actual_value}"
+        else:
+            assert actual_value >= expected_value, f"Expected value for '{metric_name}' is >= {expected_value}, but got {actual_value}"
+
+    @allure.step
+    def wait_for_metric(self, node, metric_name, expected_value, timeout_duration=90):
+        @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(1), reraise=True)
+        def check_metric_with_retry():
+            self.check_metric(node, metric_name, expected_value)
+
+        check_metric_with_retry()
+
+    def validate_initial_metrics(self, node):
+        metrics_data = node.get_metrics()
+
+        # Regular expression to match metric lines, accounting for optional labels
+        metric_pattern = re.compile(r"^(?P<metric_name>[a-zA-Z0-9_:]+(?:{[^}]+})?)\s+(?P<value>[0-9]+\.?[0-9]*)$", re.MULTILINE)
+
+        # Dictionary to store the metrics and their values
+        metrics_dict = {}
+        for match in metric_pattern.finditer(metrics_data):
+            metric_name = match.group("metric_name")
+            value = float(match.group("value"))
+            metrics_dict[metric_name] = value
+
+        errors = []
+        # Assert that specific metrics have a value of 0.0
+        for metric in METRICS_WITH_INITIAL_VALUE_ZERO:
+            if metric not in metrics_dict:
+                errors.append(f"Metric {metric} is missing from the metrics data")
+            elif metrics_dict[metric] != 0.0:
+                errors.append(f"Expected {metric} to be 0.0, but got {metrics_dict[metric]}")
+
+        # Assert that all other metrics have a value greater than 0.0
+        for metric, value in metrics_dict.items():
+            if metric not in METRICS_WITH_INITIAL_VALUE_ZERO and value <= 0.0:
+                errors.append(f"Expected {metric} to have a positive value, but got {value}")
+
+        assert not errors, f"Metrics validation failed:\n" + "\n".join(errors)
+        logger.debug(f"All metrics are present and have valid values.")

tests-e2e/src/steps/network_conditions.py

@@ -0,0 +1,201 @@
+import subprocess
+from src.env_vars import NETWORK_NAME
+from src.libs.custom_logger import get_custom_logger
+
+logger = get_custom_logger(__name__)
+
+
+class TrafficController:
+    def _pid(self, node) -> int:
+        if not node.container:
+            raise RuntimeError("Node container not started yet")
+
+        node.container.reload()
+        pid = node.container.attrs.get("State", {}).get("Pid")
+        if not pid or pid == 0:
+            raise RuntimeError("Container PID not available (container not running?)")
+        return int(pid)
+
+    def _exec(self, node, tc_args: list[str], iface: str = "eth0"):
+        pid = self._pid(node)
+
+        cmd = ["sudo", "-n", "nsenter", "-t", str(pid), "-n", "tc"] + tc_args
+        logger.info(f"TC exec: {cmd}")
+
+        res = subprocess.run(cmd, capture_output=True, text=True)
+        if res.returncode != 0:
+            raise RuntimeError(f"TC failed: {' '.join(cmd)}\n" f"stdout: {res.stdout}\n" f"stderr: {res.stderr}")
+
+        return res.stdout
+
+    def log_tc_stats(self, node, iface: str = "eth0"):
+        """
+        Log tc statistics for an interface (best-effort).
+        Useful to confirm netem loss/delay counters (sent/dropped/etc.).
+        """
+        try:
+            out = self._exec(node, ["-s", "qdisc", "show", "dev", iface], iface=iface)
+            out = (out or "").strip()
+            if out:
+                logger.debug(f"tc -s qdisc show dev {iface}:\n{out}")
+            else:
+                logger.debug(f"tc -s qdisc show dev {iface}: (no output)")
+        except Exception as e:
+            logger.debug(f"Failed to read tc stats for {iface}: {e}")
+
+    def clear(self, node, iface: str = "eth0"):
+        try:
+            self._exec(node, ["qdisc", "del", "dev", iface, "root"], iface=iface)
+        except RuntimeError as e:
+            msg = str(e)
+            if "Cannot delete qdisc with handle of zero" in msg or "No such file or directory" in msg:
+                return
+            raise
+
+    def add_latency(self, node, ms: int, iface: str = "eth0"):
+        self.clear(node, iface=iface)
+        self._exec(node, ["qdisc", "add", "dev", iface, "root", "netem", "delay", f"{ms}ms"], iface=iface)
+
+    def add_packet_loss(self, node, percent: float, iface: str = "eth0"):
+        self.clear(node, iface=iface)
+
+        self._exec(
+            node,
+            ["qdisc", "add", "dev", iface, "root", "netem", "loss", f"{percent}%"],
+            iface=iface,
+        )
+        try:
+            stats = self._exec(node, ["-s", "qdisc", "show", "dev", iface], iface=iface)
+            if stats is not None:
+                if isinstance(stats, (bytes, bytearray)):
+                    stats = stats.decode(errors="replace")
+                logger.debug(f"tc -s qdisc show dev {iface}:\n{stats}")
+            else:
+                logger.debug(f"Executed: tc -s qdisc show dev {iface} (no output returned by _exec)")
+        except Exception as e:
+            logger.debug(f"Failed to read tc stats for {iface}: {e}")
+
+    def add_bandwidth(self, node, rate: str, iface: str = "eth0"):
+        self.clear(node, iface=iface)
+        self._exec(
+            node,
+            ["qdisc", "add", "dev", iface, "root", "tbf", "rate", rate, "burst", "32kbit", "limit", "12500"],
+            iface=iface,
+        )
+
+    def add_packet_loss_correlated(
+        self,
+        node,
+        percent: float,
+        correlation: float,
+        iface: str = "eth0",
+    ):
+        self.clear(node, iface=iface)
+        self._exec(
+            node,
+            [
+                "qdisc",
+                "add",
+                "dev",
+                iface,
+                "root",
+                "netem",
+                "loss",
+                f"{percent}%",
+                f"{correlation}%",
+            ],
+            iface=iface,
+        )
+
+    def add_packet_reordering(
+        self,
+        node,
+        percent: int = 25,
+        correlation: int = 50,
+        delay_ms: int = 10,
+        iface: str = "eth0",
+    ):
+        self.clear(node, iface=iface)
+
+        self._exec(
+            node,
+            [
+                "qdisc",
+                "add",
+                "dev",
+                iface,
+                "root",
+                "netem",
+                "delay",
+                f"{delay_ms}ms",
+                "reorder",
+                f"{percent}%",
+                f"{correlation}%",
+            ],
+            iface=iface,
+        )
+
+    def _p2p_iface(self, node) -> str:
+        """
+        Return the name of the container interface attached to the waku
+        network (where libp2p traffic flows).
+
+        DockerManager attaches each node to two networks: the default bridge
+        (where host-published ports land, typically `eth0`) and the waku
+        network (where inter-container libp2p/gossipsub traffic flows, typically
+        `eth1`). tc on the default bridge only affects REST control plane; for
+        a packet loss test targeting libp2p we need the waku interface.
+
+        This helper resolves the correct interface by looking up the node's
+        waku-network IP via Docker and matching it against `ip -o -4 addr`
+        output from inside the container.
+        """
+        if not node.container:
+            raise RuntimeError("Node container not started yet")
+        node.container.reload()
+        networks = node.container.attrs.get("NetworkSettings", {}).get("Networks", {})
+        waku_net = networks.get(NETWORK_NAME)
+        if not waku_net or not waku_net.get("IPAddress"):
+            raise RuntimeError(f"Container is not attached to the '{NETWORK_NAME}' docker network")
+        waku_ip = waku_net["IPAddress"]
+
+        exit_code, output = node.container.exec_run(["ip", "-o", "-4", "addr"])
+        if exit_code != 0:
+            raise RuntimeError(f"ip addr failed inside container: {output}")
+        for line in output.decode().splitlines():
+            if f" {waku_ip}/" in line:
+                tokens = line.split()
+                if len(tokens) >= 2:
+                    return tokens[1]
+        raise RuntimeError(f"No interface inside container holds waku IP {waku_ip}")
+
+    def clear_p2p(self, node):
+        """
+        Remove any tc rule previously installed on the node's waku (libp2p)
+        interface. Paired with add_packet_loss_p2p_only /
+        add_packet_loss_correlated_p2p_only.
+        """
+        self.clear(node, iface=self._p2p_iface(node))
+
+    def add_packet_loss_p2p_only(self, node, percent: float):
+        """
+        Apply uncorrelated packet loss to the waku (libp2p) network interface
+        of a node. REST API traffic rides a separate docker interface and is
+        not affected, so the test harness's control plane stays reliable.
+        """
+        iface = self._p2p_iface(node)
+        self.clear(node, iface=iface)
+        self._exec(node, f"qdisc add dev {iface} root netem loss {percent}%".split(), iface=iface)
+
+    def add_packet_loss_correlated_p2p_only(self, node, percent: float, correlation: float):
+        """
+        Correlated packet loss on the waku (libp2p) network interface. See
+        add_packet_loss_p2p_only for why REST stays unaffected.
+        """
+        iface = self._p2p_iface(node)
+        self.clear(node, iface=iface)
+        self._exec(
+            node,
+            f"qdisc add dev {iface} root netem loss {percent}% {correlation}%".split(),
+            iface=iface,
+        )

tests-e2e/src/steps/peer_exchange.py

@@ -0,0 +1,30 @@
+from uuid import uuid4
+
+from src.env_vars import NODE_2
+from src.node.waku_node import WakuNode
+
+import allure
+
+from src.steps.relay import StepsRelay
+
+
+class StepsPeerExchange(StepsRelay):
+    responder_multiaddr = ""
+
+    @allure.step
+    def setup_third_node_as_peer_exchange_requester(self, **kwargs):
+        self.node3 = WakuNode(NODE_2, f"node3_{self.test_id}")
+        self.node3.start(
+            relay="false",
+            peer_exchange_node=self.responder_multiaddr,
+            **kwargs,
+        )
+        self.add_node_peer(self.node3, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node3])
+
+    @allure.step
+    def setup_fourth_node_as_filter(self, **kwargs):
+        self.node4 = WakuNode(NODE_2, f"node4_{self.test_id}")
+        self.node4.start(relay="false", **kwargs)
+        self.add_node_peer(self.node4, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node4])

tests-e2e/src/steps/relay.py

@@ -0,0 +1,187 @@
+import inspect
+from src.libs.custom_logger import get_custom_logger
+from time import time
+import pytest
+import allure
+from src.libs.common import to_base64, delay
+from src.node.waku_message import WakuMessage
+from src.env_vars import (
+    NODE_1,
+    NODE_2,
+    ADDITIONAL_NODES,
+)
+from src.node.waku_node import WakuNode
+from tenacity import retry, stop_after_delay, wait_fixed
+from src.steps.common import StepsCommon
+from src.test_data import VALID_PUBSUB_TOPICS
+
+logger = get_custom_logger(__name__)
+
+
+class StepsRelay(StepsCommon):
+    test_pubsub_topic = VALID_PUBSUB_TOPICS[1]
+    test_content_topic = "/test/1/waku-relay/proto"
+    test_payload = "Relay works!!"
+
+    @pytest.fixture(scope="function", autouse=True)
+    def relay_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.main_nodes = []
+        self.optional_nodes = []
+
+    @pytest.fixture(scope="function")
+    def setup_main_relay_nodes(self, request):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.node1 = WakuNode(NODE_1, f"node1_{request.cls.test_id}")
+        self.node1.start(relay="true")
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        self.node2 = WakuNode(NODE_2, f"node2_{request.cls.test_id}")
+        self.node2.start(relay="true", discv5_bootstrap_node=self.enr_uri)
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node1, self.node2])
+
+    @pytest.fixture(scope="function")
+    def setup_optional_relay_nodes(self, request):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        if ADDITIONAL_NODES:
+            nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")]
+        else:
+            pytest.skip("ADDITIONAL_NODES is empty, cannot run test")
+        for index, node in enumerate(nodes):
+            node = WakuNode(node, f"node{index + 3}_{request.cls.test_id}")
+            node.start(relay="true", discv5_bootstrap_node=self.enr_uri)
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)
+
+    @pytest.fixture(scope="function")
+    def subscribe_main_relay_nodes(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic])
+
+    @pytest.fixture(scope="function")
+    def subscribe_optional_relay_nodes(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.ensure_relay_subscriptions_on_nodes(self.optional_nodes, [self.test_pubsub_topic])
+
+    @pytest.fixture(scope="function")
+    def relay_warm_up(self):
+        try:
+            self.wait_for_published_message_to_reach_relay_peer()
+            logger.info("WARM UP successful!!")
+        except Exception as ex:
+            raise TimeoutError(f"WARM UP FAILED WITH: {ex}")
+
+    # Refactor candidate
+    @allure.step
+    def setup_first_relay_node(self, **kwargs):
+        self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}")
+        self.node1.start(relay="true", **kwargs)
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        self.main_nodes.extend([self.node1])
+
+    # Refactor candidate
+    @allure.step
+    def setup_second_relay_node(self, **kwargs):
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(
+            relay="true",
+            discv5_bootstrap_node=self.enr_uri,
+            **kwargs,
+        )
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node2])
+
+    @allure.step
+    def setup_third_relay_node(self, **kwargs):
+        self.node3 = WakuNode(NODE_1, f"node3_{self.test_id}")
+        self.node3.start(
+            relay="true",
+            discv5_bootstrap_node=self.enr_uri,
+            **kwargs,
+        )
+        self.add_node_peer(self.node3, [self.multiaddr_with_id])
+        self.optional_nodes.extend([self.node3])
+
+    # this method should be used only for the tests that use the relay_warm_up fixture
+    # otherwise use wait_for_published_message_to_reach_relay_peer
+    @allure.step
+    def check_published_message_reaches_relay_peer(self, message=None, pubsub_topic=None, message_propagation_delay=0.1, sender=None, peer_list=None):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if not sender:
+            sender = self.node1
+        if not peer_list:
+            peer_list = self.main_nodes + self.optional_nodes
+
+        sender.send_relay_message(message, pubsub_topic)
+        delay(message_propagation_delay)
+        for index, peer in enumerate(peer_list):
+            logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the published message")
+            get_messages_response = peer.get_relay_messages(pubsub_topic)
+            assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages"
+            assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}"
+            waku_message = WakuMessage(get_messages_response)
+            waku_message.assert_received_message(message)
+
+    @allure.step
+    def check_publish_without_relay_subscription(self, pubsub_topic):
+        try:
+            self.node1.send_relay_message(self.create_message(), pubsub_topic)
+            raise AssertionError("Publish with no subscription worked!!!")
+        except Exception as ex:
+            assert "Bad Request" in str(ex) or "Internal Server Error" in str(ex)
+
+    # we need much bigger timeout in CI because we run tests in parallel there and the machine itself is slower
+    @allure.step
+    def wait_for_published_message_to_reach_relay_peer(
+        self, timeout_duration=120, time_between_retries=1, pubsub_topic=None, sender=None, peer_list=None
+    ):
+        @retry(stop=stop_after_delay(timeout_duration), wait=wait_fixed(time_between_retries), reraise=True)
+        def publish_and_check_relay_peer():
+            message = {"payload": to_base64(self.test_payload), "contentTopic": self.test_content_topic, "timestamp": int(time() * 1e9)}
+            self.check_published_message_reaches_relay_peer(message, pubsub_topic=pubsub_topic, sender=sender, peer_list=peer_list)
+
+        publish_and_check_relay_peer()
+
+    @allure.step
+    def ensure_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list):
+        for node in node_list:
+            node.set_relay_subscriptions(pubsub_topic_list)
+
+    @allure.step
+    def delete_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list):
+        for node in node_list:
+            node.delete_relay_subscriptions(pubsub_topic_list)
+
+    @allure.step
+    @retry(stop=stop_after_delay(120), wait=wait_fixed(1), reraise=True)
+    def subscribe_and_publish_with_retry(self, node_list, pubsub_topic_list):
+        self.ensure_relay_subscriptions_on_nodes(node_list, pubsub_topic_list)
+        self.check_published_message_reaches_relay_peer()
+
+    @allure.step
+    def setup_main_nodes(self, **kwargs):
+        self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}")
+        self.node1.start(relay="true", **kwargs)
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs)
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node1, self.node2])
+
+    @allure.step
+    def setup_optional_nodes(self, **kwargs):
+        if ADDITIONAL_NODES:
+            nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")]
+        else:
+            pytest.skip("ADDITIONAL_NODES is empty, cannot run test")
+        for index, node in enumerate(nodes):
+            node = WakuNode(node, f"node{index + 3}_{self.test_id}")
+            node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs)
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)

tests-e2e/src/steps/rln.py

@@ -0,0 +1,203 @@
+import os
+import inspect
+import random
+import string
+
+import pytest
+import allure
+
+from src.steps.common import StepsCommon
+from src.test_data import PUBSUB_TOPICS_RLN
+from src.env_vars import DEFAULT_NWAKU, RLN_CREDENTIALS, NODE_1, NODE_2, ADDITIONAL_NODES
+from src.libs.common import gen_step_id, delay
+from src.libs.custom_logger import get_custom_logger
+from src.node.waku_node import WakuNode, rln_credential_store_ready
+
+logger = get_custom_logger(__name__)
+
+
+class StepsRLN(StepsCommon):
+    test_pubsub_topic = PUBSUB_TOPICS_RLN[0]
+    test_content_topic = "/test/1/waku-rln-relay/proto"
+    test_payload = "RLN relay works!!"
+
+    main_nodes = []
+    optional_nodes = []
+    multiaddr_list = []
+    lightpush_nodes = []
+    keystore_prefixes = []
+    rln_membership_indexes = []
+
+    @allure.step
+    def generate_keystore_prefixes(self, count=2):
+        new_prefixes = []
+        for _ in range(count):
+            new_prefixes.append("".join(random.choices(string.ascii_lowercase, k=4)))
+
+        return new_prefixes
+
+    @allure.step
+    def register_rln_relay_nodes(self, count, orig_prefixes):
+        if count > 0:
+            self.keystore_prefixes = self.generate_keystore_prefixes(count)
+            self.rln_membership_indexes = []
+            for i, prefix in enumerate(self.keystore_prefixes):
+                membership_index = self.register_rln_single_node(prefix=prefix, rln_creds_source=RLN_CREDENTIALS, rln_creds_id=f"{i+1}")
+                self.rln_membership_indexes.append(membership_index)
+        else:
+            self.keystore_prefixes = orig_prefixes.get("keystore_prefixes", [])
+            self.rln_membership_indexes = orig_prefixes.get("rln_membership_indexes", [])
+
+        return {
+            "keystore_prefixes": self.keystore_prefixes,
+            "rln_membership_indexes": self.rln_membership_indexes,
+        }
+
+    @allure.step
+    def setup_main_rln_relay_nodes(self, **kwargs):
+        self.setup_first_rln_relay_node(**kwargs)
+        self.setup_second_rln_relay_node(**kwargs)
+
+    @allure.step
+    def setup_first_rln_relay_node(self, **kwargs):
+        self.node1 = WakuNode(NODE_1, f"node1_{self.test_id}")
+        self.node1.start(
+            relay="true",
+            rln_creds_source=RLN_CREDENTIALS,
+            rln_creds_id="1",
+            rln_relay_membership_index=self.resolve_rln_membership_index(0, **kwargs),
+            rln_keystore_prefix=self.keystore_prefixes[0],
+            **kwargs,
+        )
+        self.enr_uri = self.node1.get_enr_uri()
+        self.multiaddr_with_id = self.node1.get_multiaddr_with_id()
+        self.main_nodes.extend([self.node1])
+
+        self.multiaddr_list.extend([self.node1.get_multiaddr_with_id()])
+
+    @allure.step
+    def setup_second_rln_relay_node(self, **kwargs):
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(
+            relay="true",
+            discv5_bootstrap_node=self.enr_uri,
+            rln_creds_source=RLN_CREDENTIALS,
+            rln_creds_id="2",
+            rln_relay_membership_index=self.resolve_rln_membership_index(1, **kwargs),
+            rln_keystore_prefix=self.keystore_prefixes[1],
+            **kwargs,
+        )
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_nodes.extend([self.node2])
+
+    @allure.step
+    def setup_optional_rln_relay_nodes(self, **kwargs):
+        if ADDITIONAL_NODES:
+            nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")]
+        else:
+            pytest.skip("ADDITIONAL_NODES is empty, cannot run test")
+        if len(nodes) > 3:
+            logger.debug("More than 3 nodes are not supported for RLN tests, using first 3")
+            nodes = nodes[:3]
+        for index, node in enumerate(nodes):
+            node = WakuNode(node, f"node{index + 3}_{self.test_id}")
+            node.start(
+                relay="true",
+                discv5_bootstrap_node=self.enr_uri,
+                rln_creds_source=RLN_CREDENTIALS,
+                rln_creds_id=f"{index + 3}",
+                rln_relay_membership_index=self.resolve_rln_membership_index(index + 2, **kwargs),
+                rln_keystore_prefix=self.keystore_prefixes[index + 2],
+                **kwargs,
+            )
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)
+
+    @allure.step
+    def setup_second_rln_lightpush_node(self, relay="true", **kwargs):
+        self.light_push_node2 = WakuNode(NODE_2, f"lightpush_node2_{self.test_id}")
+        self.light_push_node2.start(
+            relay=relay,
+            discv5_bootstrap_node=self.enr_uri,
+            lightpush="true",
+            lightpushnode=self.multiaddr_list[0],
+            rln_creds_source=RLN_CREDENTIALS,
+            rln_creds_id="2",
+            rln_relay_membership_index=self.resolve_rln_membership_index(1, **kwargs),
+            rln_keystore_prefix=self.keystore_prefixes[1],
+            **kwargs,
+        )
+        if relay == "true":
+            self.main_nodes.extend([self.light_push_node2])
+        self.lightpush_nodes.extend([self.light_push_node2])
+        self.add_node_peer(self.light_push_node2, self.multiaddr_list)
+
+    @allure.step
+    def register_rln_single_node(self, prefix="", **kwargs):
+        logger.debug("Registering RLN credentials for single node")
+        self.node = WakuNode(DEFAULT_NWAKU, f"node_{gen_step_id()}")
+        return self.node.register_rln(rln_keystore_prefix=prefix, rln_creds_source=kwargs["rln_creds_source"], rln_creds_id=kwargs["rln_creds_id"])
+
+    @allure.step
+    def resolve_rln_membership_index(self, index, **kwargs):
+        explicit_index = kwargs.get("rln_relay_membership_index")
+        if explicit_index is not None:
+            return str(explicit_index)
+
+        if len(self.rln_membership_indexes) > index and self.rln_membership_indexes[index] is not None:
+            inferred_index = str(self.rln_membership_indexes[index])
+            logger.debug(f"Using inferred RLN membership index for position {index}: {inferred_index}")
+            return inferred_index
+
+        raise ValueError(
+            f"RLN membership index for position {index} is not available. "
+            "Register credentials and persist rln_membership_indexes together with keystore_prefixes before node startup."
+        )
+
+    @allure.step
+    def check_rln_registration(self, prefix, key_id):
+        cwd = os.getcwd()
+        creds_file_path = f"{cwd}/keystore_{prefix}_{key_id}/keystore.json"
+        try:
+            rln_credential_store_ready(creds_file_path)
+        except Exception as ex:
+            logger.error(f"Credentials at {creds_file_path} not available: {ex}")
+            raise
+
+    @allure.step
+    def publish_message(self, message=None, pubsub_topic=None, sender=None, use_lightpush=False):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if not sender:
+            sender = self.node1
+
+        if use_lightpush:
+            payload = self.create_payload(pubsub_topic, message)
+            sender.send_light_push_message(payload)
+        else:
+            sender.send_relay_message(message, pubsub_topic)
+
+    @allure.step
+    def ensure_relay_subscriptions_on_nodes(self, node_list, pubsub_topic_list):
+        for node in node_list:
+            node.set_relay_subscriptions(pubsub_topic_list)
+
+    @allure.step
+    def subscribe_main_relay_nodes(self):
+        self.ensure_relay_subscriptions_on_nodes(self.main_nodes, [self.test_pubsub_topic])
+
+    @allure.step
+    def subscribe_optional_relay_nodes(self):
+        self.ensure_relay_subscriptions_on_nodes(self.optional_nodes, [self.test_pubsub_topic])
+
+    @allure.step
+    def create_payload(self, pubsub_topic=None, message=None, **kwargs):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        payload = {"pubsubTopic": pubsub_topic, "message": message}
+        payload.update(kwargs)
+        return payload

tests-e2e/src/steps/sharding.py

@@ -0,0 +1,212 @@
+import inspect
+from uuid import uuid4
+from src.libs.custom_logger import get_custom_logger
+from time import time
+import pytest
+import allure
+from src.libs.common import to_base64, delay
+from src.node.waku_message import WakuMessage
+from src.env_vars import (
+    DEFAULT_NWAKU,
+    NODE_2,
+    ADDITIONAL_NODES,
+)
+from src.node.waku_node import WakuNode
+from src.steps.common import StepsCommon
+from src.steps.relay import StepsRelay
+
+logger = get_custom_logger(__name__)
+
+
+class StepsSharding(StepsRelay):
+    test_content_topic = "/myapp/1/latest/proto"
+    test_pubsub_topic = "/waku/2/rs/199/0"
+    test_payload = "Sharding works!!"
+    auto_cluster = 199
+    num_shards_in_network = 8
+
+    @pytest.fixture(scope="function", autouse=True)
+    def sharding_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.main_nodes = []
+        self.optional_nodes = []
+        self.main_filter_nodes = []
+        self.optional_filter_nodes = []
+
+    @allure.step
+    def setup_first_relay_node_with_filter(self, **kwargs):
+        self.setup_first_relay_node(filter="true", **kwargs)
+
+    @allure.step
+    def setup_second_node_as_filter(self, **kwargs):
+        self.node2 = WakuNode(NODE_2, f"node2_{self.test_id}")
+        self.node2.start(relay="false", discv5_bootstrap_node=self.enr_uri, filternode=self.multiaddr_with_id, **kwargs)
+        self.add_node_peer(self.node2, [self.multiaddr_with_id])
+        self.main_filter_nodes.extend([self.node2])
+
+    @allure.step
+    def setup_main_relay_nodes(self, **kwargs):
+        self.setup_first_relay_node_with_filter(**kwargs)
+        self.setup_second_relay_node(**kwargs)
+
+    @allure.step
+    def setup_optional_relay_nodes(self, **kwargs):
+        if ADDITIONAL_NODES:
+            nodes = [node.strip() for node in ADDITIONAL_NODES.split(",")]
+        else:
+            pytest.skip("ADDITIONAL_NODES is empty, cannot run test")
+        for index, node in enumerate(nodes):
+            node = WakuNode(node, f"node{index + 3}_{self.test_id}")
+            node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs)
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)
+
+    @allure.step
+    def setup_nwaku_relay_nodes(self, num_nodes, **kwargs):
+        for index in range(num_nodes):
+            node = WakuNode(DEFAULT_NWAKU, f"node{index + 3}_{self.test_id}")
+            node.start(relay="true", discv5_bootstrap_node=self.enr_uri, **kwargs)
+            self.add_node_peer(node, [self.multiaddr_with_id])
+            self.optional_nodes.append(node)
+
+    @allure.step
+    def subscribe_relay_node(self, node, content_topics, pubsub_topics):
+        if content_topics:
+            node.set_relay_auto_subscriptions(content_topics)
+        elif pubsub_topics:
+            node.set_relay_subscriptions(pubsub_topics)
+        else:
+            raise AttributeError("content_topics or pubsub_topics need to be passed")
+
+    @allure.step
+    def subscribe_first_relay_node(self, content_topics=None, pubsub_topics=None):
+        self.subscribe_relay_node(self.node1, content_topics, pubsub_topics)
+
+    @allure.step
+    def subscribe_second_relay_node(self, content_topics=None, pubsub_topics=None):
+        self.subscribe_relay_node(self.node2, content_topics, pubsub_topics)
+
+    @allure.step
+    def subscribe_main_relay_nodes(self, content_topics=None, pubsub_topics=None):
+        for node in self.main_nodes:
+            self.subscribe_relay_node(node, content_topics, pubsub_topics)
+
+    @allure.step
+    def subscribe_optional_relay_nodes(self, content_topics=None, pubsub_topics=None):
+        for node in self.optional_nodes:
+            self.subscribe_relay_node(node, content_topics, pubsub_topics)
+
+    @allure.step
+    def unsubscribe_relay_node(self, node, content_topics, pubsub_topics):
+        if content_topics:
+            node.delete_relay_auto_subscriptions(content_topics)
+        elif pubsub_topics:
+            node.delete_relay_subscriptions(pubsub_topics)
+        else:
+            raise AttributeError("content_topics or pubsub_topics need to be passed")
+
+    @allure.step
+    def unsubscribe_first_relay_node(self, content_topics=None, pubsub_topics=None):
+        self.unsubscribe_relay_node(self.node1, content_topics, pubsub_topics)
+
+    @allure.step
+    def unsubscribe_second_relay_node(self, content_topics=None, pubsub_topics=None):
+        self.unsubscribe_relay_node(self.node2, content_topics, pubsub_topics)
+
+    @allure.step
+    def unsubscribe_main_relay_nodes(self, content_topics=None, pubsub_topics=None):
+        for node in self.main_nodes:
+            self.unsubscribe_relay_node(node, content_topics, pubsub_topics)
+
+    @allure.step
+    def unsubscribe_optional_relay_nodes(self, content_topics=None, pubsub_topics=None):
+        for node in self.optional_nodes:
+            self.unsubscribe_relay_node(node, content_topics, pubsub_topics)
+
+    @allure.step
+    def subscribe_filter_node(self, node, content_topics=None, pubsub_topic=None):
+        subscription = {"requestId": str(uuid4()), "contentFilters": content_topics, "pubsubTopic": pubsub_topic}
+        node.set_filter_subscriptions(subscription)
+
+    @allure.step
+    def relay_message(self, node, message, pubsub_topic=None):
+        if pubsub_topic:
+            node.send_relay_message(message, pubsub_topic)
+        else:
+            node.send_relay_auto_message(message)
+
+    @allure.step
+    def retrieve_relay_message(self, node, content_topic=None, pubsub_topic=None):
+        if content_topic:
+            return node.get_relay_auto_messages(content_topic)
+        elif pubsub_topic:
+            return node.get_relay_messages(pubsub_topic)
+        else:
+            raise AttributeError("content_topic or pubsub_topic needs to be passed")
+
+    @allure.step
+    def check_published_message_reaches_relay_peer(self, content_topic=None, pubsub_topic=None, sender=None, peer_list=None):
+        message = self.create_message(contentTopic=content_topic) if content_topic else self.create_message()
+        if not sender:
+            sender = self.node1
+        if not peer_list:
+            peer_list = self.main_nodes + self.optional_nodes
+
+        self.relay_message(sender, message, pubsub_topic)
+        delay(0.1)
+        for index, peer in enumerate(peer_list):
+            logger.debug(f"Checking that peer NODE_{index + 1}:{peer.image} can find the published message")
+            get_messages_response = self.retrieve_relay_message(peer, content_topic, pubsub_topic)
+            assert get_messages_response, f"Peer NODE_{index + 1}:{peer.image} couldn't find any messages"
+            assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}"
+            waku_message = WakuMessage(get_messages_response)
+            waku_message.assert_received_message(message)
+
+    @allure.step
+    def get_filter_messages(self, content_topic, pubsub_topic=None, node=None):
+        if node is None:
+            node = self.node2
+        if node.is_nwaku():
+            return node.get_filter_messages(content_topic)
+        else:
+            raise NotImplementedError("Not implemented for this node type")
+
+    @allure.step
+    def check_published_message_reaches_filter_peer(self, content_topic=None, pubsub_topic=None, sender=None, peer_list=None):
+        message = self.create_message(contentTopic=content_topic) if content_topic else self.create_message()
+        if not sender:
+            sender = self.node1
+        if not peer_list:
+            peer_list = self.main_filter_nodes + self.optional_filter_nodes
+
+        self.relay_message(sender, message, pubsub_topic)
+        delay(0.1)
+        for index, peer in enumerate(peer_list):
+            logger.debug(f"Checking that peer NODE_{index + 2}:{peer.image} can find the published message")
+            get_messages_response = self.get_filter_messages(message["contentTopic"], pubsub_topic=pubsub_topic, node=peer)
+            assert get_messages_response, f"Peer NODE_{index + 2}:{peer.image} couldn't find any messages"
+            assert len(get_messages_response) == 1, f"Expected 1 message but got {len(get_messages_response)}"
+            waku_message = WakuMessage(get_messages_response)
+            waku_message.assert_received_message(message)
+
+    @allure.step
+    def check_published_message_doesnt_reach_relay_peer(self, pubsub_topic=None, content_topic=None):
+        try:
+            self.check_published_message_reaches_relay_peer(pubsub_topic=pubsub_topic, content_topic=content_topic)
+            raise AssertionError("Retrieving messages on not subscribed content topic worked!!!")
+        except Exception as ex:
+            error_message = str(ex)
+            expected_errors = [
+                "Not Found",
+                "NoPeersToPublish",
+                "Failed to publish: publish failed in relay: NoPeersToPublish",
+            ]
+            assert any(expected in error_message for expected in expected_errors), error_message
+
+    @allure.step
+    def check_publish_fails_on_not_subscribed_pubsub_topic(self, pubsub_topic):
+        try:
+            self.check_published_message_reaches_relay_peer(pubsub_topic=pubsub_topic)
+            raise AssertionError("Publishing messages on unsubscribed shard worked!!!")
+        except Exception as ex:
+            assert "Failed to publish: Node not subscribed to topic" in str(ex), "Expected Bad Request because the node is not subscribed"

tests-e2e/src/steps/store.py

@@ -0,0 +1,382 @@
+import inspect
+
+import requests
+
+from src.libs.custom_logger import get_custom_logger
+import pytest
+import allure
+from src.libs.common import delay
+from src.node.store_response import StoreResponse
+from src.node.waku_message import WakuMessage
+from src.env_vars import (
+    ADDITIONAL_NODES,
+    NODE_1,
+    NODE_2,
+)
+from src.node.waku_node import WakuNode
+from src.steps.common import StepsCommon
+from src.test_data import VALID_PUBSUB_TOPICS
+from tenacity import retry, stop_after_delay, wait_fixed
+
+logger = get_custom_logger(__name__)
+
+
+class StepsStore(StepsCommon):
+    test_content_topic = "/myapp/1/latest/proto"
+    test_pubsub_topic = VALID_PUBSUB_TOPICS[0]
+    test_payload = "Store works!!"
+
+    @pytest.fixture(scope="function", autouse=True)
+    def store_setup(self):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.main_publishing_nodes = []
+        self.store_nodes = []
+        self.optional_nodes = []
+        self.multiaddr_list = []
+
+    @pytest.fixture(scope="function", autouse=False)
+    def node_setup(self, store_setup):
+        logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+        self.setup_first_publishing_node(store="true", relay="true")
+        self.setup_first_store_node(store="true", relay="true")
+        self.subscribe_to_pubsub_topics_via_relay(node=self.main_publishing_nodes)
+
+    @allure.step
+    def start_publishing_node(self, image, node_index, **kwargs):
+        node = WakuNode(image, f"publishing_node{node_index}_{self.test_id}")
+        node.start(**kwargs)
+        if kwargs["relay"] == "true":
+            self.main_publishing_nodes.extend([node])
+        if kwargs["store"] == "true":
+            self.store_nodes.extend([node])
+        self.add_node_peer(node, self.multiaddr_list)
+        self.multiaddr_list.extend([node.get_multiaddr_with_id()])
+        return node
+
+    @allure.step
+    def setup_store_node(self, image, node_index, **kwargs):
+        node = WakuNode(image, f"store_node{node_index}_{self.test_id}")
+        node.start(discv5_bootstrap_node=self.enr_uri, storenode=self.multiaddr_list[0], **kwargs)
+        if kwargs["relay"] == "true":
+            self.main_publishing_nodes.extend([node])
+        self.store_nodes.extend([node])
+        self.add_node_peer(node, self.multiaddr_list)
+        return node
+
+    @allure.step
+    def setup_first_publishing_node(self, store="true", relay="true", **kwargs):
+        self.publishing_node1 = self.start_publishing_node(NODE_1, node_index=1, store=store, relay=relay, **kwargs)
+        self.enr_uri = self.publishing_node1.get_enr_uri()
+
+    @allure.step
+    def setup_second_publishing_node(self, store, relay, **kwargs):
+        self.publishing_node2 = self.start_publishing_node(NODE_1, node_index=2, store=store, relay=relay, **kwargs)
+
+    @allure.step
+    def setup_additional_publishing_nodes(self, node_list=ADDITIONAL_NODES, **kwargs):
+        if node_list:
+            nodes = [node.strip() for node in node_list.split(",") if node]
+        else:
+            pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test")
+        for index, node in enumerate(nodes):
+            self.start_publishing_node(node, node_index=index + 2, store="true", relay="true", **kwargs)
+
+    @allure.step
+    def setup_first_store_node(self, store="true", relay="true", **kwargs):
+        self.store_node1 = self.setup_store_node(NODE_2, node_index=1, store=store, relay=relay, **kwargs)
+
+    @allure.step
+    def setup_second_store_node(self, store="true", relay="false", **kwargs):
+        self.store_node2 = self.setup_store_node(NODE_2, node_index=2, store=store, relay=relay, **kwargs)
+
+    @allure.step
+    def setup_additional_store_nodes(self, node_list=ADDITIONAL_NODES, **kwargs):
+        if node_list:
+            nodes = [node.strip() for node in node_list.split(",") if node]
+        else:
+            pytest.skip("ADDITIONAL_NODES/node_list is empty, cannot run test")
+        self.additional_store_nodes = []
+        for index, node in enumerate(nodes):
+            node = self.setup_store_node(node, node_index=index + 2, store="true", relay="false", **kwargs)
+            self.additional_store_nodes.append(node)
+
+    @allure.step
+    def subscribe_to_pubsub_topics_via_relay(self, node=None, pubsub_topics=None):
+        if pubsub_topics is None:
+            pubsub_topics = [self.test_pubsub_topic]
+        if not node:
+            node = self.main_publishing_nodes
+        if isinstance(node, list):
+            for node in node:
+                node.set_relay_subscriptions(pubsub_topics)
+        else:
+            node.set_relay_subscriptions(pubsub_topics)
+
+    @allure.step
+    def subscribe_to_pubsub_topics_via_filter(self, node, pubsub_topic=None, content_topic=None):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if content_topic is None:
+            content_topic = [self.test_content_topic]
+        subscription = {"requestId": "1", "contentFilters": content_topic, "pubsubTopic": pubsub_topic}
+        node.set_filter_subscriptions(subscription)
+
+    @allure.step
+    def publish_message(self, via="relay", pubsub_topic=None, message=None, message_propagation_delay=0.2, sender=None):
+        self.message = self.create_message() if message is None else message
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if not sender:
+            sender = self.publishing_node1
+        if via == "relay":
+            logger.debug("Relaying message")
+            sender.send_relay_message(self.message, pubsub_topic)
+        elif via == "lightpush":
+            payload = self.create_payload(pubsub_topic, self.message)
+            sender.send_light_push_message(payload)
+        delay(message_propagation_delay)
+        return self.message
+
+    @retry(stop=stop_after_delay(30), wait=wait_fixed(1), reraise=True)
+    @allure.step
+    def get_messages_from_store_with_retry(self, node):
+        return self.get_messages_from_store(node, page_size=5)
+
+    @allure.step
+    def get_messages_from_store(
+        self,
+        node=None,
+        peer_addr=None,
+        include_data=None,
+        pubsub_topic=None,
+        content_topics=None,
+        start_time=None,
+        end_time=None,
+        hashes=None,
+        cursor=None,
+        page_size=None,
+        ascending="true",
+        store_v="v3",
+        **kwargs,
+    ):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        store_response = node.get_store_messages(
+            peer_addr=peer_addr,
+            include_data=include_data,
+            pubsub_topic=pubsub_topic,
+            content_topics=content_topics,
+            start_time=start_time,
+            end_time=end_time,
+            hashes=hashes,
+            cursor=cursor,
+            page_size=page_size,
+            ascending=ascending,
+            store_v=store_v,
+            **kwargs,
+        )
+        store_response = StoreResponse(store_response, node)
+        assert store_response.request_id is not None, "Request id is missing"
+        assert store_response.status_code, "Status code is missing"
+        assert store_response.status_desc, "Status desc is missing"
+        return store_response
+
+    @allure.step
+    def check_published_message_is_stored(
+        self,
+        store_node=None,
+        peer_addr=None,
+        include_data=None,
+        pubsub_topic=None,
+        content_topics=None,
+        start_time=None,
+        end_time=None,
+        hashes=None,
+        cursor=None,
+        page_size=None,
+        ascending=None,
+        store_v="v3",
+        messages_to_check=None,
+        **kwargs,
+    ):
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if messages_to_check is None:
+            messages_to_check = [self.message]
+        if store_node is None:
+            store_node = self.store_nodes
+        elif not isinstance(store_node, list):
+            store_node = [store_node]
+        for node in store_node:
+            logger.debug(f"Checking that peer {node.image} can find the stored messages")
+            self.store_response = self.get_messages_from_store(
+                node=node,
+                peer_addr=peer_addr,
+                include_data=include_data,
+                pubsub_topic=pubsub_topic,
+                content_topics=content_topics,
+                start_time=start_time,
+                end_time=end_time,
+                hashes=hashes,
+                cursor=cursor,
+                page_size=page_size,
+                ascending=ascending,
+                store_v=store_v,
+                **kwargs,
+            )
+
+            logger.debug(f"messages length is {len(self.store_response.messages)}")
+            assert self.store_response.messages, f"Peer {node.image} couldn't find any messages. Actual response: {self.store_response.resp_json}"
+            assert len(self.store_response.messages) >= len(
+                messages_to_check
+            ), f"Expected at least {len(messages_to_check)} messages but got {len(self.store_response.messages)}"
+
+            # Determine the range of indices for message comparison
+            if len(messages_to_check) == 1:
+                indices = [-1]  # Use the last message in the store response for a single message
+            else:
+                indices = range(len(messages_to_check))  # Use corresponding indices for multiple messages
+
+            # Iterate through messages_to_check and their respective indices
+            for idx, message_to_check in zip(indices, messages_to_check):
+                if store_v == "v1":
+                    waku_message = WakuMessage([self.store_response.messages[idx]])
+                    waku_message.assert_received_message(message_to_check)
+                else:
+                    hash_type = "hex" if node.is_nwaku() else "base64"
+                    expected_hash = self.compute_message_hash(pubsub_topic, message_to_check, hash_type=hash_type)
+                    actual_hash = self.store_response.message_hash(idx)
+                    assert (
+                        expected_hash == actual_hash
+                    ), f"Message hash at index {idx} returned by store doesn't match the computed message hash {expected_hash}. Actual hash: {actual_hash}"
+
+    @allure.step
+    def check_sent_message_is_stored(
+        self,
+        expected_hashes,
+        store_node=None,
+        peer_addr=None,
+        include_data=None,
+        pubsub_topic=None,
+        content_topics=None,
+        start_time=None,
+        end_time=None,
+        hashes=None,
+        cursor=None,
+        page_size=None,
+        ascending=None,
+        store_v="v3",
+        **kwargs,
+    ):
+        """Verify that messages with the given hashes are present in the store."""
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        if isinstance(expected_hashes, str):
+            expected_hashes = [expected_hashes]
+        if store_node is None:
+            store_node = self.store_nodes
+        elif not isinstance(store_node, list):
+            store_node = [store_node]
+
+        for node in store_node:
+            logger.debug(f"Checking that peer {node.image} can find the stored messages by hash")
+            self.store_response = self.get_messages_from_store(
+                node=node,
+                peer_addr=peer_addr,
+                include_data=include_data,
+                pubsub_topic=pubsub_topic,
+                content_topics=content_topics,
+                start_time=start_time,
+                end_time=end_time,
+                hashes=hashes,
+                cursor=cursor,
+                page_size=page_size,
+                ascending=ascending,
+                store_v=store_v,
+                **kwargs,
+            )
+
+            logger.debug(f"messages length is {len(self.store_response.messages)}")
+            assert self.store_response.messages, f"Peer {node.image} couldn't find any messages. " f"Actual response: {self.store_response.resp_json}"
+            assert len(self.store_response.messages) >= len(expected_hashes), (
+                f"Expected at least {len(expected_hashes)} messages " f"but got {len(self.store_response.messages)}"
+            )
+
+            actual_hashes = [self.store_response.message_hash(i) for i in range(len(self.store_response.messages))]
+            for expected_hash in expected_hashes:
+                assert expected_hash in actual_hashes, f"Expected hash {expected_hash} not found in store. " f"Actual hashes: {actual_hashes}"
+
+    @allure.step
+    def check_store_returns_empty_response(self, pubsub_topic=None):
+        if not pubsub_topic:
+            pubsub_topic = self.test_pubsub_topic
+        try:
+            self.check_published_message_is_stored(pubsubTopic=pubsub_topic, page_size=5, ascending="true")
+        except Exception as ex:
+            assert "couldn't find any messages" in str(ex)
+
+    @allure.step
+    def create_payload(self, pubsub_topic=None, message=None, **kwargs):
+        if message is None:
+            message = self.create_message()
+        if pubsub_topic is None:
+            pubsub_topic = self.test_pubsub_topic
+        payload = {"pubsubTopic": pubsub_topic, "message": message}
+        payload.update(kwargs)
+        return payload
+
+    @allure.step
+    def get_store_messages_with_errors(
+        self,
+        node=None,
+        peer_addr=None,
+        include_data=None,
+        pubsub_topic=None,
+        content_topics=None,
+        start_time=None,
+        end_time=None,
+        hashes=None,
+        cursor=None,
+        page_size=None,
+        ascending="true",
+        store_v="v3",
+        **kwargs,
+    ):
+        """
+        This method calls the original get_store_messages and returns the actual
+        error response from the service, if present.
+        """
+        try:
+            # Call the original get_store_messages method
+            store_response = node.get_store_messages(
+                peer_addr=peer_addr,
+                include_data=include_data,
+                pubsub_topic=pubsub_topic,
+                content_topics=content_topics,
+                start_time=start_time,
+                end_time=end_time,
+                hashes=hashes,
+                cursor=cursor,
+                page_size=page_size,
+                ascending=ascending,
+                store_v=store_v,
+                **kwargs,
+            )
+
+            # Check if the response has a status code >= 400, indicating an error
+            if store_response.status_code >= 400:
+                # Return the status code and the plain text error message directly
+                return {"status_code": store_response.status_code, "error_message": store_response.text}  # Handling plain text response
+
+            # Otherwise, return the successful response as JSON
+            response_json = store_response.json()
+            response_json["status_code"] = store_response.status_code
+            return response_json
+
+        except requests.exceptions.HTTPError as http_err:
+            # Handle HTTP errors separately
+            return {"status_code": http_err.response.status_code, "error_message": http_err.response.text}
+
+        except Exception as e:
+            # Handle unexpected errors and return as 500
+            return {"status_code": 500, "error_message": str(e)}

tests-e2e/src/test_data.py

@@ -0,0 +1,356 @@
+from time import time
+from datetime import datetime, timedelta
+
+NOW = datetime.now()
+
+SAMPLE_INPUTS = [
+    {"description": "A simple string", "value": "Hello World!"},
+    {"description": "An integer", "value": "1234567890"},
+    {"description": "A dictionary", "value": '{"key": "value"}'},
+    {"description": "Chinese characters", "value": "这是一些中文"},
+    {"description": "Emojis", "value": "🚀🌟✨"},
+    {"description": "Lorem ipsum text", "value": "Lorem ipsum dolor sit amet"},
+    {"description": "HTML content", "value": "<html><body>Hello</body></html>"},
+    {"description": "Cyrillic characters", "value": "\u041f\u0440\u0438\u0432\u0435\u0442"},
+    {"description": "Base64 encoded string", "value": "Base64==dGVzdA=="},
+    {"description": "Binary data", "value": "d29ya2luZyB3aXRoIGJpbmFyeSBkYXRh: \x50\x51"},
+    {"description": "Special characters with whitespace", "value": "\t\nSpecial\tCharacters\n"},
+    {"description": "Boolean false as a string", "value": "False"},
+    {"description": "A float number", "value": "3.1415926535"},
+    {"description": "A list", "value": "[1, 2, 3, 4, 5]"},
+    {"description": "Hexadecimal number as a string", "value": "0xDEADBEEF"},
+    {"description": "Email format", "value": "user@example.com"},
+    {"description": "URL format", "value": "http://example.com"},
+    {"description": "Date and time in ISO format", "value": "2023-11-01T12:00:00Z"},
+    {"description": "String with escaped quotes", "value": '"Escaped" \\"quotes\\"'},
+    {"description": "A regular expression", "value": "Regular expression: ^[a-z0-9_-]{3,16}$"},
+    {"description": "A very long string", "value": "x" * 1000},
+    {"description": "A JSON string", "value": '{"name": "John", "age": 30, "city": "New York"}'},
+    {"description": "A Unix path", "value": "/usr/local/bin"},
+    {"description": "A Windows path", "value": "C:\\Windows\\System32"},
+    {"description": "An SQL query", "value": "SELECT * FROM users WHERE id = 1;"},
+    {"description": "JavaScript code snippet", "value": "function test() { console.log('Hello World'); }"},
+    {"description": "A CSS snippet", "value": "body { background-color: #fff; }"},
+    {"description": "A Python one-liner", "value": "print('Hello World')"},
+    {"description": "An IP address", "value": "192.168.1.1"},
+    {"description": "A domain name", "value": "www.example.com"},
+    {"description": "A user agent string", "value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"},
+    {"description": "A credit card number", "value": "1234-5678-9012-3456"},
+    {"description": "A phone number", "value": "+1234567890"},
+    {"description": "A UUID", "value": "123e4567-e89b-12d3-a456-426614174000"},
+    {"description": "A hashtag", "value": "#helloWorld"},
+    {"description": "A Twitter handle", "value": "@username"},
+    {"description": "A password", "value": "P@ssw0rd!"},
+    {"description": "A date in common format", "value": "01/11/2023"},
+    {"description": "A time string", "value": "12:00:00"},
+    {"description": "A mathematical equation", "value": "E = mc^2"},
+]
+
+INVALID_PAYLOADS = [
+    {"description": "Empty string", "value": ""},
+    {"description": "Unecoded text", "value": "Hello World!"},
+    {"description": "A dictionary", "value": {"key": "YWFh"}},
+    {"description": "An integer", "value": 1234567890},
+    {"description": "A list", "value": ["YWFh"]},
+    {"description": "A bool", "value": True},
+]
+
+INVALID_CONTENT_TOPICS = [
+    {"description": "Empty string", "value": ""},
+    {"description": "A dictionary", "value": {"key": "YWFh"}},
+    {"description": "An integer", "value": 1234567890},
+    {"description": "A list", "value": ["YWFh"]},
+    {"description": "A bool", "value": True},
+]
+
+CONTENT_TOPICS_DIFFERENT_SHARDS = [
+    "/myapp/1/latest/proto",  # resolves to shard 0
+    "/waku/2/content/test.js",  # resolves to shard 1
+    "/app/22/sometopic/someencoding",  # resolves to shard 2
+    "/toychat/2/huilong/proto",  # resolves to shard 3
+    "/statusim/1/community/cbor",  # resolves to shard 4
+    "/app/27/sometopic/someencoding",  # resolves to shard 5
+    "/app/29/sometopic/someencoding",  # resolves to shard 6
+    "/app/20/sometopic/someencoding",  # resolves to shard 7
+]
+
+CONTENT_TOPICS_SHARD_0 = [
+    "/newsService/1.0/weekly/protobuf",
+    "/newsService/1.0/alerts/xml",
+    "/newsService/1.0/updates/json",
+    "/newsService/2.0/alerts/json",
+    "/newsService/2.0/summaries/xml",
+    "/newsService/2.0/highlights/yaml",
+    "/newsService/3.0/weekly/json",
+    "/newsService/3.0/summaries/xml",
+]
+
+CONTENT_TOPICS_SHARD_7 = [
+    "/newsService/2.0/alerts/yaml",
+    "/newsService/2.0/highlights/xml",
+    "/newsService/3.0/daily/protobuf",
+    "/newsService/3.0/alerts/xml",
+    "/newsService/3.0/updates/protobuf",
+    "/newsService/3.0/reviews/xml",
+    "/newsService/4.0/alerts/yaml",
+    "/newsService/4.0/updates/yaml",
+]
+
+DEFAULT_CLUSTER_ID = "198"
+VALID_PUBSUB_TOPICS = [
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/9",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/25",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1000",
+]
+
+PUBSUB_TOPICS_STORE = [
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/1",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/2",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/3",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/4",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/5",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/6",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/7",
+    f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/8",
+]
+
+INVALID_PUBSUB_TOPICS = ["/test/2/rs/0/1", "/waku/3/rs/0/1", "/waku/2/test/0/1", "/waku/2/rs/3/b", "/waku/2/rs/0"]
+
+PUBSUB_TOPICS_DIFFERENT_CLUSTERS = [
+    "/waku/2/rs/2/0",
+    "/waku/2/rs/2/1",
+    "/waku/2/rs/5/0",
+    "/waku/2/rs/5/1",
+    "/waku/2/rs/5/999",
+    "/waku/2/rs/8/0",
+    "/waku/2/rs/999/999",
+]
+
+PUBSUB_TOPICS_SAME_CLUSTER = [
+    "/waku/2/rs/199/0",
+    "/waku/2/rs/199/1",
+    "/waku/2/rs/199/2",
+    "/waku/2/rs/199/3",
+    "/waku/2/rs/199/4",
+    "/waku/2/rs/199/5",
+    "/waku/2/rs/199/6",
+    "/waku/2/rs/199/7",
+]
+
+PUBSUB_TOPICS_WRONG_FORMAT = [
+    {"description": "A dictionary", "value": {"key": "YWFh"}},
+    {"description": "An integer", "value": 1234567890},
+    {"description": "A list", "value": ["YWFh"]},
+    {"description": "A bool", "value": True},
+]
+
+SAMPLE_TIMESTAMPS = [
+    {"description": "Now", "value": int(time() * 1e9), "valid_for": ["nwaku"]},
+    {
+        "description": "Far future",
+        "value": int((NOW + timedelta(days=365 * 10)).timestamp() * 1e9),
+        "valid_for": ["nwaku"],
+    },  # 10 years from now
+    {"description": "Recent past", "value": int((NOW - timedelta(hours=1)).timestamp() * 1e9), "valid_for": ["nwaku"]},  # 1 hour ago
+    {"description": "Near future", "value": int((NOW + timedelta(hours=1)).timestamp() * 1e9), "valid_for": ["nwaku"]},  # 1 hour ahead
+    {"description": "Positive number", "value": 1, "valid_for": ["nwaku"]},
+    {"description": "Negative number", "value": -1, "valid_for": ["nwaku"]},
+    {"description": "DST change", "value": int(datetime(2020, 3, 8, 2, 0, 0).timestamp() * 1e9), "valid_for": ["nwaku"]},  # DST starts
+    {"description": "Timestamp as string number", "value": str(int(time() * 1e9)), "valid_for": []},
+    {"description": "Invalid large number", "value": 2**63, "valid_for": []},
+    {"description": "Float number", "value": float(time() * 1e9), "valid_for": []},
+    {"description": "Array instead of timestamp", "value": [int(time() * 1e9)], "valid_for": []},
+    {"description": "Object instead of timestamp", "value": {"time": int(time() * 1e9)}, "valid_for": []},
+    {"description": "ISO 8601 timestamp", "value": "2023-12-26T10:58:51", "valid_for": []},
+    {"description": "Missing", "value": None, "valid_for": []},
+]
+
+PUBSUB_TOPICS_RLN = [f"/waku/2/rs/{DEFAULT_CLUSTER_ID}/0"]
+
+LOG_ERROR_KEYWORDS = [
+    "crash",
+    "fatal",
+    "panic",
+    "abort",
+    "segfault",
+    "corrupt",
+    "terminated",
+    "unhandled",
+    "stacktrace",
+    "deadlock",
+    "SIGSEGV",
+    "SIGABRT",
+    "stack overflow",
+    "index out of bounds",
+    "nil pointer dereference",
+    "goroutine exit",
+    "nil pointer",
+    "runtime error",
+    "goexit",
+    "race condition",
+    "double free",
+]
+
+METRICS_WITH_INITIAL_VALUE_ZERO = [
+    "libp2p_peers",
+    "libp2p_failed_upgrades_incoming_total",
+    "libp2p_failed_upgrades_outgoing_total",
+    "libp2p_total_dial_attempts_total",
+    "libp2p_successful_dials_total",
+    "libp2p_failed_dials_total",
+    "waku_rln_messages_total_total",
+    "waku_rln_spam_messages_total_total",
+    "waku_rln_proof_verification_total_total",
+    "waku_rln_number_registered_memberships",
+    "waku_rln_proof_verification_duration_seconds",
+    "waku_rln_proof_generation_duration_seconds",
+    "waku_rln_instance_creation_duration_seconds",
+    "waku_rln_membership_insertion_duration_seconds",
+    "waku_rln_membership_credentials_import_duration_seconds",
+    "waku_rln_remaining_proofs_per_epoch",
+    "waku_rln_total_generated_proofs",
+    "libp2p_pubsub_sig_verify_success_total",
+    "libp2p_pubsub_sig_verify_failure_total",
+    "libp2p_pubsub_disconnects_over_non_priority_queue_limit_total",
+    "libp2p_pubsub_peers",
+    "libp2p_pubsub_unsubscriptions_total",
+    "libp2p_pubsub_validation_success_total",
+    "libp2p_pubsub_validation_failure_total",
+    "libp2p_pubsub_validation_ignore_total",
+    "libp2p_pubsub_broadcast_iwant_total",
+    "libp2p_pubsub_received_iwant_total",
+    "libp2p_gossipsub_cache_window_size",
+    'libp2p_gossipsub_peers_per_topic_mesh{topic="other"}',
+    'libp2p_gossipsub_peers_per_topic_fanout{topic="other"}',
+    'libp2p_gossipsub_peers_per_topic_gossipsub{topic="other"}',
+    "libp2p_gossipsub_low_peers_topics",
+    "libp2p_gossipsub_healthy_peers_topics",
+    "libp2p_gossipsub_failed_publish_total",
+    "libp2p_gossipsub_invalid_topic_subscription_total",
+    "libp2p_gossipsub_duplicate_during_validation_total",
+    "libp2p_gossipsub_idontwant_saved_messages_total",
+    "libp2p_gossipsub_duplicate_total",
+    "libp2p_gossipsub_received_total",
+    "libp2p_rendezvous_register_total",
+    "libp2p_rendezvous_discover_total",
+    "libp2p_rendezvous_registered",
+    "libp2p_rendezvous_namespaces",
+    "waku_peer_store_size",
+    "waku_total_unique_peers",
+    "presto_server_missing_requests_count",
+    "presto_server_invalid_requests_count",
+    'waku_archive_messages{type="stored"}',
+    "waku_archive_insert_duration_seconds_sum",
+    "waku_archive_insert_duration_seconds_count",
+    'waku_archive_insert_duration_seconds_bucket{le="0.005"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.01"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.025"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.05"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.075"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.1"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.25"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.5"}',
+    'waku_archive_insert_duration_seconds_bucket{le="0.75"}',
+    'waku_archive_insert_duration_seconds_bucket{le="1.0"}',
+    'waku_archive_insert_duration_seconds_bucket{le="2.5"}',
+    'waku_archive_insert_duration_seconds_bucket{le="5.0"}',
+    'waku_archive_insert_duration_seconds_bucket{le="7.5"}',
+    'waku_archive_insert_duration_seconds_bucket{le="10.0"}',
+    'waku_archive_insert_duration_seconds_bucket{le="+Inf"}',
+    "waku_archive_query_duration_seconds_sum",
+    "waku_archive_query_duration_seconds_count",
+    'waku_archive_query_duration_seconds_bucket{le="0.005"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.01"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.025"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.05"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.075"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.1"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.25"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.5"}',
+    'waku_archive_query_duration_seconds_bucket{le="0.75"}',
+    'waku_archive_query_duration_seconds_bucket{le="1.0"}',
+    'waku_archive_query_duration_seconds_bucket{le="2.5"}',
+    'waku_archive_query_duration_seconds_bucket{le="5.0"}',
+    'waku_archive_query_duration_seconds_bucket{le="7.5"}',
+    'waku_archive_query_duration_seconds_bucket{le="10.0"}',
+    'waku_archive_query_duration_seconds_bucket{le="+Inf"}',
+    "waku_filter_subscriptions",
+    "waku_filter_handle_message_duration_seconds_sum",
+    "waku_filter_handle_message_duration_seconds_count",
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.005"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.01"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.025"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.05"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.075"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.1"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.25"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.5"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="0.75"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="1.0"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="2.5"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="5.0"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="7.5"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="10.0"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="+Inf"}',
+    "discovery_session_lru_cache_hits_total",
+    "discovery_session_lru_cache_misses_total",
+    "discovery_session_decrypt_failures_total",
+    "discovery_unsolicited_messages_total",
+    "discovery_enr_auto_update_total",
+    "waku_px_peers_received_total",
+    "waku_px_peers_received_unknown",
+    "waku_px_peers_sent_total",
+    "waku_px_peers",
+    "waku_histogram_message_size_sum",
+    "waku_histogram_message_size_count",
+    'waku_histogram_message_size_bucket{le="0.0"}',
+    'waku_histogram_message_size_bucket{le="1.0"}',
+    'waku_histogram_message_size_bucket{le="3.0"}',
+    'waku_histogram_message_size_bucket{le="5.0"}',
+    'waku_histogram_message_size_bucket{le="15.0"}',
+    'waku_histogram_message_size_bucket{le="50.0"}',
+    'waku_histogram_message_size_bucket{le="75.0"}',
+    'waku_histogram_message_size_bucket{le="100.0"}',
+    'waku_histogram_message_size_bucket{le="125.0"}',
+    'waku_histogram_message_size_bucket{le="150.0"}',
+    'waku_histogram_message_size_bucket{le="500.0"}',
+    'waku_histogram_message_size_bucket{le="700.0"}',
+    'waku_histogram_message_size_bucket{le="1000.0"}',
+    'waku_histogram_message_size_bucket{le="+Inf"}',
+    "waku_lightpush_peers",
+    "waku_filter_peers",
+    "waku_store_peers",
+    "waku_px_peers",
+    "waku_dnsdisc_discovered",
+    "rendezvousPeerFoundTotal_total",
+    "reconciliation_roundtrips_sum",
+    "reconciliation_roundtrips_count",
+    "reconciliation_differences_sum",
+    "reconciliation_differences_count",
+    'reconciliation_roundtrips_bucket{le="1.0"}',
+    'reconciliation_roundtrips_bucket{le="2.0"}',
+    'reconciliation_roundtrips_bucket{le="3.0"}',
+    'reconciliation_roundtrips_bucket{le="5.0"}',
+    'reconciliation_roundtrips_bucket{le="8.0"}',
+    'reconciliation_roundtrips_bucket{le="13.0"}',
+    'reconciliation_roundtrips_bucket{le="+Inf"}',
+    'reconciliation_differences_bucket{le="0.0"}',
+    'reconciliation_differences_bucket{le="10.0"}',
+    'reconciliation_differences_bucket{le="50.0"}',
+    'reconciliation_differences_bucket{le="100.0"}',
+    'reconciliation_differences_bucket{le="500.0"}',
+    'reconciliation_differences_bucket{le="1000.0"}',
+    'reconciliation_differences_bucket{le="5000.0"}',
+    'reconciliation_differences_bucket{le="+Inf"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="15.0"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="20.0"}',
+    'waku_filter_handle_message_duration_seconds_bucket{le="30.0"}',
+    "total_messages_cached",
+    "waku_store_queries_total",
+    "mix_pool_size",
+    "libp2p_gossipsub_imreceiving_saved_messages_total",
+    "postgres_payload_size_bytes",
+]

tests-e2e/tests/conftest.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import inspect
+import glob
+from src.libs.custom_logger import get_custom_logger
+import os
+import pytest
+from datetime import datetime
+from time import time
+from uuid import uuid4
+from src.libs.common import attach_allure_file
+import src.env_vars as env_vars
+from src.data_storage import DS
+from src.postgres_setup import start_postgres, stop_postgres
+
+logger = get_custom_logger(__name__)
+
+
+# See https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
+@pytest.hookimpl(hookwrapper=True, tryfirst=True)
+def pytest_runtest_makereport(item):
+    outcome = yield
+    rep = outcome.get_result()
+    if rep.when == "call":
+        setattr(item, "rep_call", rep)
+        return rep
+    return None
+
+
+@pytest.fixture(scope="session", autouse=True)
+def set_allure_env_variables():
+    yield
+    if os.path.isdir("allure-results") and not os.path.isfile(os.path.join("allure-results", "environment.properties")):
+        logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
+        with open(os.path.join("allure-results", "environment.properties"), "w") as outfile:
+            for attribute_name in dir(env_vars):
+                if attribute_name.isupper():
+                    attribute_value = getattr(env_vars, attribute_name)
+                    outfile.write(f"{attribute_name}={attribute_value}\n")
+
+
+@pytest.fixture(scope="function", autouse=False)
+def start_postgres_container():
+    pg_container = start_postgres()
+    yield
+    stop_postgres(pg_container)
+
+
+@pytest.fixture(scope="function", autouse=True)
+def test_id(request):
+    # setting up an unique test id to be used where needed
+    logger.debug(f"Running fixture setup: {inspect.currentframe().f_code.co_name}")
+    request.cls.test_id = f"{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}__{str(uuid4())}"
+
+
+@pytest.fixture(scope="function", autouse=True)
+def test_setup(request, test_id):
+    logger.debug(f"Running test: {request.node.name} with id: {request.cls.test_id}")
+    yield
+    logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
+    for file in glob.glob(os.path.join(env_vars.DOCKER_LOG_DIR, "*")):
+        if os.path.getmtime(file) < time() - 3600:
+            logger.debug(f"Deleting old log file: {file}")
+            try:
+                os.remove(file)
+            except:
+                logger.error("Could not delete file")
+
+
+@pytest.fixture(scope="function", autouse=True)
+def attach_logs_on_fail(request):
+    yield
+    if env_vars.RUNNING_IN_CI and hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
+        logger.debug("Test failed, attempting to attach logs to the allure reports")
+        for file in glob.glob(os.path.join(env_vars.DOCKER_LOG_DIR, "*" + request.cls.test_id + "*")):
+            attach_allure_file(file)
+
+
+@pytest.fixture(scope="function", autouse=True)
+def close_open_nodes(attach_logs_on_fail):
+    DS.waku_nodes = []
+    yield
+    logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
+    crashed_containers = []
+    for node in DS.waku_nodes:
+        try:
+            node.stop()
+        except Exception as ex:
+            if "No such container" in str(ex):
+                crashed_containers.append(node.image)
+            logger.error(f"Failed to stop container because of error {ex}")
+    assert not crashed_containers, f"Containers {crashed_containers} crashed during the test!!!"
+
+
+@pytest.fixture(scope="function", autouse=True)
+def check_waku_log_errors():
+    yield
+    logger.debug(f"Running fixture teardown: {inspect.currentframe().f_code.co_name}")
+    for node in DS.waku_nodes:
+        node.check_waku_log_errors()

tests-e2e/tests/wrappers_tests/conftest.py

@@ -0,0 +1,35 @@
+import socket
+import pytest
+from src.test_data import DEFAULT_CLUSTER_ID
+
+
+def free_port():
+    """Return a currently-unbound TCP/UDP port from the OS."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("", 0))
+        return s.getsockname()[1]
+
+
+def build_node_config(**overrides):
+    config = {
+        "logLevel": "DEBUG",
+        "listenAddress": "0.0.0.0",
+        "tcpPort": free_port(),
+        "discv5UdpPort": free_port(),
+        "restPort": free_port(),
+        "restAddress": "0.0.0.0",
+        "clusterId": DEFAULT_CLUSTER_ID,
+        "relay": True,
+        "store": True,
+        "filter": False,
+        "lightpush": False,
+        "peerExchange": False,
+        "discv5Discovery": False,
+    }
+    config.update(overrides)
+    return config
+
+
+@pytest.fixture
+def node_config():
+    return build_node_config()

tests-e2e/tests/wrappers_tests/test_basic_life_cycle.py

@@ -0,0 +1,26 @@
+import pytest
+from src.node.wrappers_manager import WrapperManager
+
+
+@pytest.mark.smoke
+class TestLogosDeliveryLifecycle:
+    def _create_start_node(self, node_config):
+        result = WrapperManager.create_and_start(config=node_config)
+        assert result.is_ok(), f"Failed to create and start node: {result.err()}"
+        return result.ok_value
+
+    def test_create_start_and_stop_node(self, node_config):
+        node = self._create_start_node(node_config)
+
+        stop_result = node.stop_and_destroy()
+        assert stop_result.is_ok(), f"Failed to stop and destroy node: {stop_result.err()}"
+
+    def test_stop_node_without_destroy(self, node_config):
+        node = self._create_start_node(node_config)
+
+        try:
+            stop_result = node.stop_node()
+            assert stop_result.is_ok(), f"Failed to stop node: {stop_result.err()}"
+        finally:
+            destroy_result = node.destroy()
+            assert destroy_result.is_ok(), f"Failed to destroy node: {destroy_result.err()}"

tests-e2e/tests/wrappers_tests/test_send_e2e_part2.py

@@ -0,0 +1,699 @@
+import base64
+import pytest
+from src.steps.common import StepsCommon
+from src.libs.common import delay, to_base64
+from src.libs.custom_logger import get_custom_logger
+from src.node.wrappers_manager import WrapperManager
+from src.node.wrapper_helpers import (
+    EventCollector,
+    assert_event_invariants,
+    create_message_bindings,
+    get_node_multiaddr,
+    wait_for_connected,
+    wait_for_propagated,
+    wait_for_sent,
+    wait_for_error,
+)
+from tests.wrappers_tests.conftest import build_node_config
+
+logger = get_custom_logger(__name__)
+
+PROPAGATED_TIMEOUT_S = 30.0
+SENT_TIMEOUT_S = 10.0
+NO_SENT_OBSERVATION_S = 5.0
+SENT_AFTER_STORE_TIMEOUT_S = 60.0
+OVERSIZED_PAYLOAD_BYTES = 200 * 1024
+RECOVERY_TIMEOUT_S = 45.0
+SERVICE_DOWN_SETTLE_S = 3.0
+
+# MaxTimeInCache from send_service.nim.
+MAX_TIME_IN_CACHE_S = 60.0
+# Extra slack to cover the background retry loop tick after the window expires.
+CACHE_EXPIRY_SLACK_S = 10.0
+ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S = MAX_TIME_IN_CACHE_S + CACHE_EXPIRY_SLACK_S
+RETRY_WINDOW_EXPIRED_MSG = "Unable to send within retry time window"
+
+
+class TestS02AutoSubscribeOnFirstSend(StepsCommon):
+    """
+    S02 — Auto-subscribe on first send.
+    Sender never calls subscribe_content_topic() before send().
+    The send API must auto-subscribe to the content topic used in the message.
+    Expected: send() returns Ok(RequestId), message_propagated arrives.
+    """
+
+    def test_s02_send_without_explicit_subscribe(self, node_config):
+        sender_collector = EventCollector()
+
+        node_config.update(
+            {
+                "relay": True,
+                "store": False,
+                "lightpush": False,
+                "filter": False,
+                "discv5Discovery": False,
+                "numShardsInNetwork": 1,
+            }
+        )
+
+        sender_result = WrapperManager.create_and_start(
+            config=node_config,
+            event_cb=sender_collector.event_callback,
+        )
+        assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}"
+
+        with sender_result.ok_value as sender:
+            peer_config = {
+                **node_config,
+                "staticnodes": [get_node_multiaddr(sender)],
+                "portsShift": 1,
+            }
+
+            peer_result = WrapperManager.create_and_start(config=peer_config)
+            assert peer_result.is_ok(), f"Failed to start relay peer: {peer_result.err()}"
+
+            with peer_result.ok_value:
+                assert wait_for_connected(sender_collector) is not None, "Sender did not reach Connected/PartiallyConnected state"
+
+                message = create_message_bindings(
+                    payload=to_base64("S02 auto-subscribe test payload"),
+                    contentTopic="/test/1/s02-auto-subscribe/proto",
+                )
+
+                send_result = sender.send_message(message=message)
+                assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                request_id = send_result.ok_value
+                assert request_id, "send() returned an empty RequestId"
+
+                propagated = wait_for_propagated(
+                    collector=sender_collector,
+                    request_id=request_id,
+                    timeout_s=PROPAGATED_TIMEOUT_S,
+                )
+                assert propagated is not None, (
+                    f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}"
+                )
+                assert propagated["requestId"] == request_id
+
+                error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                assert error is None, f"Unexpected message_error event: {error}"
+
+
+class TestS06CoreSenderRelayOnly(StepsCommon):
+    """
+    S06 — Core sender with relay peers only, no store.
+    Sender has local relay enabled and is connected to one relay peer.
+    Expected: send() returns Ok(RequestId), message_propagated event arrives,
+    no message_sent (store disabled), no message_error.
+    """
+
+    def test_s06_relay_propagation_without_store(self, node_config):
+        sender_collector = EventCollector()
+
+        node_config.update(
+            {
+                "relay": True,
+                "store": False,
+                "lightpush": False,
+                "filter": False,
+                "discv5Discovery": False,
+                "numShardsInNetwork": 1,
+                "reliabilityEnabled": True,
+            }
+        )
+
+        sender_result = WrapperManager.create_and_start(
+            config=node_config,
+            event_cb=sender_collector.event_callback,
+        )
+        assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}"
+
+        with sender_result.ok_value as sender:
+            peer_config = {
+                **node_config,
+                "staticnodes": [get_node_multiaddr(sender)],
+                "portsShift": 1,
+            }
+
+            peer_result = WrapperManager.create_and_start(config=peer_config)
+            assert peer_result.is_ok(), f"Failed to start relay peer: {peer_result.err()}"
+
+            with peer_result.ok_value:
+                assert wait_for_connected(sender_collector) is not None, "Sender did not reach Connected/PartiallyConnected state"
+
+                message = create_message_bindings(
+                    payload=to_base64("S06 relay-only test payload"),
+                    contentTopic="/test/1/s06-relay-only/proto",
+                )
+
+                send_result = sender.send_message(message=message)
+                assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                request_id = send_result.ok_value
+                assert request_id, "send() returned an empty RequestId"
+
+                propagated = wait_for_propagated(
+                    collector=sender_collector,
+                    request_id=request_id,
+                    timeout_s=PROPAGATED_TIMEOUT_S,
+                )
+                assert propagated is not None, (
+                    f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}"
+                )
+                assert propagated["requestId"] == request_id
+
+                error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                assert error is None, f"Unexpected message_error event: {error}"
+
+                sent = wait_for_sent(sender_collector, request_id, timeout_s=0)
+                assert sent is None, f"Unexpected message_sent event (store is disabled): {sent}"
+
+                assert_event_invariants(sender_collector, request_id)
+
+
+class TestS07CoreSenderRelayAndStore(StepsCommon):
+    """
+    S07 — Core sender with relay peers and store peer, reliability enabled.
+    Sender relays message to a store-capable peer; delivery service validates
+    the message reached the store via p2p reliability check.
+    Expected: Propagated, then Sent.
+    """
+
+    def test_s07_relay_propagation_with_store_validation(self, node_config):
+        sender_collector = EventCollector()
+
+        node_config.update(
+            {
+                "relay": True,
+                "store": False,
+                "lightpush": False,
+                "filter": False,
+                "discv5Discovery": False,
+                "numShardsInNetwork": 1,
+                "reliabilityEnabled": True,
+            }
+        )
+
+        sender_result = WrapperManager.create_and_start(
+            config=node_config,
+            event_cb=sender_collector.event_callback,
+        )
+        assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}"
+
+        with sender_result.ok_value as sender:
+            peer_config = {
+                **node_config,
+                "staticnodes": [get_node_multiaddr(sender)],
+                "portsShift": 1,
+                "store": True,
+            }
+
+            peer_result = WrapperManager.create_and_start(config=peer_config)
+            assert peer_result.is_ok(), f"Failed to start store peer: {peer_result.err()}"
+
+            with peer_result.ok_value:
+                message = create_message_bindings(
+                    payload=to_base64("S07 relay+store test payload"),
+                    contentTopic="/test/1/s07-relay-store/proto",
+                )
+
+                send_result = sender.send_message(message=message)
+                assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                request_id = send_result.ok_value
+                assert request_id, "send() returned an empty RequestId"
+
+                propagated = wait_for_propagated(
+                    collector=sender_collector,
+                    request_id=request_id,
+                    timeout_s=PROPAGATED_TIMEOUT_S,
+                )
+                assert propagated is not None, (
+                    f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}"
+                )
+                assert propagated["requestId"] == request_id
+
+                sent = wait_for_sent(
+                    collector=sender_collector,
+                    request_id=request_id,
+                    timeout_s=SENT_TIMEOUT_S,
+                )
+                assert sent is not None, (
+                    f"No message_sent event within {SENT_TIMEOUT_S}s after propagation. " f"Collected events: {sender_collector.events}"
+                )
+                assert sent["requestId"] == request_id
+
+                error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                assert error is None, f"Unexpected message_error event: {error}"
+
+                assert_event_invariants(sender_collector, request_id)
+
+
+class TestRelayToLightpushFallback(StepsCommon):
+    """S08/S09 — Relay-to-lightpush fallback.
+
+    Sender has relay enabled but zero gossipsub relay peers.
+    A lightpush peer is reachable via lightpushnode (no staticnodes).
+    Relay fails with NO_PEERS_TO_RELAY, lightpush fallback succeeds
+    in the same processing pass.
+
+    Topology:
+        [Service]    relay=True, lightpush=True
+        [RelayPeer]  relay=True, staticnodes=[service]  (gives service gossipsub mesh)
+        [Sender]     relay=True, lightpush=True, lightpushnode=service
+                     (no staticnodes → zero gossipsub relay peers → fallback)
+    """
+
+    def test_s08_relay_fallback_to_lightpush(self, node_config):
+        """S08: no store peer → Propagated only."""
+        sender_collector = EventCollector()
+
+        node_config.update(
+            {
+                "relay": True,
+                "lightpush": True,
+                "store": False,
+                "filter": False,
+                "discv5Discovery": False,
+                "numShardsInNetwork": 1,
+            }
+        )
+
+        service_result = WrapperManager.create_and_start(config=node_config)
+        assert service_result.is_ok(), f"Failed to start service: {service_result.err()}"
+
+        with service_result.ok_value as service:
+            service_addr = get_node_multiaddr(service)
+
+            relay_config = {
+                **node_config,
+                "lightpush": False,
+                "staticnodes": [service_addr],
+                "portsShift": 1,
+            }
+            relay_result = WrapperManager.create_and_start(config=relay_config)
+            assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}"
+
+            with relay_result.ok_value:
+                sender_config = {
+                    **node_config,
+                    "lightpushnode": service_addr,
+                    "portsShift": 2,
+                }
+                sender_result = WrapperManager.create_and_start(
+                    config=sender_config,
+                    event_cb=sender_collector.event_callback,
+                )
+                assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}"
+
+                with sender_result.ok_value as sender:
+                    message = create_message_bindings()
+                    send_result = sender.send_message(message=message)
+                    assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                    request_id = send_result.ok_value
+                    assert request_id, "send() returned an empty RequestId"
+
+                    propagated = wait_for_propagated(
+                        collector=sender_collector,
+                        request_id=request_id,
+                        timeout_s=PROPAGATED_TIMEOUT_S,
+                    )
+                    assert propagated is not None, (
+                        f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}"
+                    )
+                    assert propagated["requestId"] == request_id
+
+                    error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                    assert error is None, f"Unexpected message_error event: {error}"
+
+                    sent = wait_for_sent(sender_collector, request_id, timeout_s=0)
+                    assert sent is None, f"Unexpected message_sent event (no store peer): {sent}"
+
+                    assert_event_invariants(sender_collector, request_id)
+
+    def test_s09_relay_fallback_to_lightpush_with_store_validation(self, node_config):
+        """S09: S08 + store peer + reliability → Propagated, then Sent."""
+        sender_collector = EventCollector()
+
+        node_config.update(
+            {
+                "relay": True,
+                "lightpush": True,
+                "store": True,
+                "filter": False,
+                "discv5Discovery": False,
+                "numShardsInNetwork": 1,
+            }
+        )
+
+        service_result = WrapperManager.create_and_start(config=node_config)
+        assert service_result.is_ok(), f"Failed to start service: {service_result.err()}"
+
+        with service_result.ok_value as service:
+            service_addr = get_node_multiaddr(service)
+
+            relay_config = {
+                **node_config,
+                "lightpush": False,
+                "store": False,
+                "staticnodes": [service_addr],
+                "portsShift": 1,
+            }
+            relay_result = WrapperManager.create_and_start(config=relay_config)
+            assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}"
+
+            with relay_result.ok_value:
+                sender_config = {**node_config, "reliabilityEnabled": True, "storenode": service_addr, "portsShift": 2, "store": False}
+                sender_result = WrapperManager.create_and_start(
+                    config=sender_config,
+                    event_cb=sender_collector.event_callback,
+                )
+                assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}"
+
+                with sender_result.ok_value as sender:
+                    message = create_message_bindings()
+                    send_result = sender.send_message(message=message)
+                    assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                    request_id = send_result.ok_value
+                    assert request_id, "send() returned an empty RequestId"
+
+                    propagated = wait_for_propagated(
+                        collector=sender_collector,
+                        request_id=request_id,
+                        timeout_s=PROPAGATED_TIMEOUT_S,
+                    )
+                    assert propagated is not None, (
+                        f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}"
+                    )
+                    assert propagated["requestId"] == request_id
+
+                    sent = wait_for_sent(
+                        collector=sender_collector,
+                        request_id=request_id,
+                        timeout_s=SENT_AFTER_STORE_TIMEOUT_S,
+                    )
+                    assert sent is not None, (
+                        f"No message_sent event within {SENT_AFTER_STORE_TIMEOUT_S}s "
+                        f"after propagation. Collected events: {sender_collector.events}"
+                    )
+                    assert sent["requestId"] == request_id
+
+                    error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                    assert error is None, f"Unexpected message_error event: {error}"
+
+                    assert_event_invariants(sender_collector, request_id)
+
+
+class TestS10EdgeSenderLightpushOnly(StepsCommon):
+    """
+    S10 — Edge sender with lightpush path only, no store peer.
+    Edge sender has no local relay; it publishes via a lightpush service node.
+    Expected: Propagated only (no Sent, no Error).
+    """
+
+    def test_s10_edge_lightpush_propagation(self, node_config):
+        sender_collector = EventCollector()
+
+        common = {
+            "store": False,
+            "filter": False,
+            "discv5Discovery": False,
+            "numShardsInNetwork": 1,
+        }
+
+        service_config = build_node_config(relay=True, lightpush=True, **common)
+
+        service_result = WrapperManager.create_and_start(config=service_config)
+        assert service_result.is_ok(), f"Failed to start service node: {service_result.err()}"
+
+        with service_result.ok_value as service_node:
+            service_multiaddr = get_node_multiaddr(service_node)
+
+            relay_config = build_node_config(
+                relay=True,
+                staticnodes=[service_multiaddr],
+                **common,
+            )
+
+            relay_result = WrapperManager.create_and_start(config=relay_config)
+            assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}"
+
+            with relay_result.ok_value:
+                edge_config = build_node_config(
+                    mode="Edge",
+                    relay=False,
+                    staticnodes=[service_multiaddr],
+                    **common,
+                )
+
+                edge_result = WrapperManager.create_and_start(
+                    config=edge_config,
+                    event_cb=sender_collector.event_callback,
+                )
+                assert edge_result.is_ok(), f"Failed to start edge sender: {edge_result.err()}"
+
+                with edge_result.ok_value as edge_sender:
+                    message = create_message_bindings(
+                        payload=to_base64("S10 edge lightpush test payload"),
+                        contentTopic="/test/1/s10-edge-lightpush/proto",
+                    )
+
+                    send_result = edge_sender.send_message(message=message)
+                    assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                    request_id = send_result.ok_value
+                    assert request_id, "send() returned an empty RequestId"
+
+                    propagated = wait_for_propagated(
+                        collector=sender_collector,
+                        request_id=request_id,
+                        timeout_s=PROPAGATED_TIMEOUT_S,
+                    )
+                    assert propagated is not None, (
+                        f"No message_propagated event within {PROPAGATED_TIMEOUT_S}s. " f"Collected events: {sender_collector.events}"
+                    )
+                    assert propagated["requestId"] == request_id
+
+                    sent = wait_for_sent(sender_collector, request_id, timeout_s=NO_SENT_OBSERVATION_S)
+                    assert sent is None, f"Unexpected message_sent event (no store peer): {sent}"
+
+                    error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                    assert error is None, f"Unexpected message_error event: {error}"
+
+                    assert_event_invariants(sender_collector, request_id)
+
+
+class TestS12IsolatedSenderNoPeers(StepsCommon):
+    """
+    S12 — Isolated sender, no peers.
+    Sender has relay enabled but zero relay peers and zero lightpush peers.
+    Expected: send() returns Ok(RequestId), but eventually a message_error
+    event arrives (no route to propagate).
+    """
+
+    def test_s12_send_with_no_peers_produces_error(self, node_config):
+        sender_collector = EventCollector()
+
+        node_config.update(
+            {
+                "relay": True,
+                "store": False,
+                "lightpush": False,
+                "filter": False,
+                "discv5Discovery": False,
+                "numShardsInNetwork": 1,
+            }
+        )
+
+        sender_result = WrapperManager.create_and_start(
+            config=node_config,
+            event_cb=sender_collector.event_callback,
+        )
+        assert sender_result.is_ok(), f"Failed to start sender: {sender_result.err()}"
+
+        with sender_result.ok_value as sender:
+            message = create_message_bindings(
+                payload=to_base64("S12 isolated sender payload"),
+                contentTopic="/test/1/s12-isolated/proto",
+            )
+
+            send_result = sender.send_message(message=message)
+            assert send_result.is_ok(), f"send() must return Ok(RequestId) even with no peers, got: {send_result.err()}"
+
+            request_id = send_result.ok_value
+            assert request_id, "send() returned an empty RequestId"
+
+            error = wait_for_error(
+                collector=sender_collector,
+                request_id=request_id,
+                timeout_s=ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S,
+            )
+            assert error is not None, (
+                f"No message_error event within {ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S}s "
+                f"(MaxTimeInCache={MAX_TIME_IN_CACHE_S}s + slack) for isolated sender. "
+                f"Collected events: {sender_collector.events}"
+            )
+            assert error["requestId"] == request_id
+
+            propagated = wait_for_propagated(sender_collector, request_id, timeout_s=0)
+            assert propagated is None, f"Unexpected message_propagated event for isolated sender: {propagated}"
+
+
+class TestS14LightpushNonRetryableError(StepsCommon):
+    """
+    S14 — Lightpush non-retryable error via oversized message.
+    Edge sender publishes a message exceeding DefaultMaxWakuMessageSize (150KiB)
+    through a lightpush service node. The server validates message size and
+    returns INVALID_MESSAGE (420), a non-retryable error.
+    Expected: send() returns Ok(RequestId), then message_error event.
+    """
+
+    def test_s14_oversized_message_triggers_error(self):
+        sender_collector = EventCollector()
+
+        common = {
+            "store": False,
+            "filter": False,
+            "discv5Discovery": False,
+            "numShardsInNetwork": 1,
+        }
+
+        service_config = build_node_config(relay=True, lightpush=True, **common)
+        service_result = WrapperManager.create_and_start(config=service_config)
+        assert service_result.is_ok(), f"Failed to start service: {service_result.err()}"
+
+        with service_result.ok_value as service:
+            service_multiaddr = get_node_multiaddr(service)
+
+            edge_config = build_node_config(
+                mode="Edge",
+                relay=False,
+                staticnodes=[service_multiaddr],
+                **common,
+            )
+            edge_result = WrapperManager.create_and_start(
+                config=edge_config,
+                event_cb=sender_collector.event_callback,
+            )
+            assert edge_result.is_ok(), f"Failed to start edge sender: {edge_result.err()}"
+
+            with edge_result.ok_value as edge_sender:
+                oversized_payload = base64.b64encode(b"x" * OVERSIZED_PAYLOAD_BYTES).decode()
+                message = create_message_bindings(
+                    payload=oversized_payload,
+                    contentTopic="/test/1/s14-oversized/proto",
+                )
+
+                send_result = edge_sender.send_message(message=message)
+                assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+
+                request_id = send_result.ok_value
+                assert request_id, "send() returned an empty RequestId"
+
+                error = wait_for_error(
+                    collector=sender_collector,
+                    request_id=request_id,
+                    timeout_s=PROPAGATED_TIMEOUT_S,
+                )
+                assert error is not None, (
+                    f"No message_error event within {PROPAGATED_TIMEOUT_S}s "
+                    f"after sending oversized message. "
+                    f"Collected events: {sender_collector.events}"
+                )
+                assert error["requestId"] == request_id
+                logger.info(f"S14 received error event: {error}")
+
+                error_msg = error.get("error", "").lower()
+                assert "size exceeded" in error_msg, f"Error message doesn't indicate size violation: {error}"
+
+                propagated = wait_for_propagated(sender_collector, request_id, timeout_s=0)
+                assert propagated is None, f"Unexpected message_propagated for an invalid message: {propagated}"
+
+                assert_event_invariants(sender_collector, request_id)
+
+
+class TestS15LightpushRetryableErrorRecovery(StepsCommon):
+    """
+    S15 — Lightpush retryable error + recovery.
+    Edge sender publishes via a lightpush service node that has NO relay peers.
+    The service accepts the lightpush request but returns NO_PEERS_TO_RELAY —
+    a retryable error (explicitly listed in the S15 spec).  The message enters
+    the retry loop.  A relay peer then joins the service node, and the next
+    retry succeeds.
+    Expected: send() returns Ok(RequestId), then eventually Propagated.
+    """
+
+    def test_s15_lightpush_retryable_error_then_recovery(self):
+        sender_collector = EventCollector()
+
+        common = {
+            "store": False,
+            "filter": False,
+            "discv5Discovery": False,
+            "numShardsInNetwork": 1,
+        }
+
+        service_config = build_node_config(relay=True, lightpush=True, **common)
+        service_result = WrapperManager.create_and_start(config=service_config)
+        assert service_result.is_ok(), f"Failed to start service: {service_result.err()}"
+
+        with service_result.ok_value as service:
+            service_multiaddr = get_node_multiaddr(service)
+
+            edge_config = build_node_config(
+                mode="Edge",
+                relay=False,
+                staticnodes=[service_multiaddr],
+                **common,
+            )
+            edge_result = WrapperManager.create_and_start(
+                config=edge_config,
+                event_cb=sender_collector.event_callback,
+            )
+            assert edge_result.is_ok(), f"Failed to start edge sender: {edge_result.err()}"
+
+            with edge_result.ok_value as edge_sender:
+                msg = create_message_bindings(
+                    payload=to_base64("S15 retryable error recovery"),
+                    contentTopic="/test/1/s15-recovery/proto",
+                )
+                send_result = edge_sender.send_message(message=msg)
+                assert send_result.is_ok(), f"send() failed: {send_result.err()}"
+                request_id = send_result.ok_value
+                assert request_id, "send() returned an empty RequestId"
+
+                delay(SERVICE_DOWN_SETTLE_S)
+
+                early_propagated = wait_for_propagated(sender_collector, request_id, timeout_s=0)
+                assert early_propagated is None, (
+                    f"message_propagated arrived before relay peer joined — " f"retryable error path was not exercised: {early_propagated}"
+                )
+
+                relay_config = build_node_config(
+                    relay=True,
+                    staticnodes=[service_multiaddr],
+                    **common,
+                )
+                relay_result = WrapperManager.create_and_start(config=relay_config)
+                assert relay_result.is_ok(), f"Failed to start relay peer: {relay_result.err()}"
+
+                with relay_result.ok_value:
+                    propagated = wait_for_propagated(
+                        collector=sender_collector,
+                        request_id=request_id,
+                        timeout_s=RECOVERY_TIMEOUT_S,
+                    )
+                    assert propagated is not None, (
+                        f"No message_propagated within {RECOVERY_TIMEOUT_S}s "
+                        f"after relay peer joined. "
+                        f"Collected events: {sender_collector.events}"
+                    )
+                    assert propagated["requestId"] == request_id
+
+                    error = wait_for_error(sender_collector, request_id, timeout_s=0)
+                    assert error is None, f"Unexpected message_error after recovery: {error}"
+
+                    assert_event_invariants(sender_collector, request_id)

tests-e2e/vendor/logos-delivery-python-bindings/waku/wrapper.py

@@ -0,0 +1,392 @@
+import json
+import threading
+
+from cffi import FFI
+from pathlib import Path
+from result import Result, Ok, Err
+
+ffi = FFI()
+
+ffi.cdef(
+    """
+typedef void (*FFICallBack)(int callerRet, const char *msg, size_t len, void *userData);
+
+void *logosdelivery_create_node(
+    const char *configJson,
+    FFICallBack callback,
+    void *userData
+);
+
+int logosdelivery_start_node(
+    void *ctx,
+    FFICallBack callback,
+    void *userData
+);
+
+int logosdelivery_stop_node(
+    void *ctx,
+    FFICallBack callback,
+    void *userData
+);
+
+void logosdelivery_set_event_callback(
+    void *ctx,
+    FFICallBack callback,
+    void *userData
+);
+
+int logosdelivery_destroy(
+    void *ctx,
+    FFICallBack callback,
+    void *userData
+);
+
+int logosdelivery_subscribe(
+    void *ctx,
+    FFICallBack callback,
+    void *userData,
+    const char *contentTopic
+);
+
+int logosdelivery_unsubscribe(
+    void *ctx,
+    FFICallBack callback,
+    void *userData,
+    const char *contentTopic
+);
+
+int logosdelivery_send(
+    void *ctx,
+    FFICallBack callback,
+    void *userData,
+    const char *messageJson
+);
+
+int logosdelivery_get_available_node_info_ids(
+    void *ctx,
+    FFICallBack callback,
+    void *userData
+);
+
+int logosdelivery_get_node_info(
+    void *ctx,
+    FFICallBack callback,
+    void *userData,
+    const char *nodeInfoId
+);
+
+int logosdelivery_get_available_configs(
+    void *ctx,
+    FFICallBack callback,
+    void *userData
+);
+"""
+)
+
+_repo_root = Path(__file__).resolve().parents[1]
+lib = ffi.dlopen(str(_repo_root / "lib" / "liblogosdelivery.so"))
+
+CallbackType = ffi.callback("void(int, const char*, size_t, void*)")
+
+
+def _new_cb_state():
+    return {
+        "done": threading.Event(),
+        "ret": None,
+        "msg": b"",
+    }
+
+
+def _wait_cb_raw(
+    state,
+    op_name: str,
+    timeout_s: float = 20.0,
+) -> Result[tuple[int, bytes], str]:
+    ok = state["done"].wait(timeout_s)
+    if not ok:
+        return Err(f"{op_name}: timeout after {timeout_s}s")
+
+    if state["ret"] is None:
+        return Err(f"{op_name}: callback ret is None")
+
+    return Ok((state["ret"], state["msg"]))
+
+
+def _wait_cb_ok(state, op_name: str, timeout_s: float = 20.0) -> Result[int, str]:
+    wait_result = _wait_cb_raw(state, op_name, timeout_s)
+    if wait_result.is_err():
+        return Err(wait_result.err())
+
+    cb_ret, cb_msg = wait_result.ok_value
+    if cb_ret != 0:
+        return Err(f"callback failed in _wait_cb_ok: {op_name} (ret={cb_ret}) msg={cb_msg!r}")
+
+    return Ok(cb_ret)
+
+
+class NodeWrapper:
+    def __init__(self, ctx, config_buffer, event_cb_handler):
+        self.ctx = ctx
+        self._config_buffer = config_buffer
+        self._event_cb_handler = event_cb_handler
+
+    @staticmethod
+    def _make_waiting_cb(state):
+        def c_cb(ret, char_p, length, userData):
+            msg = ffi.buffer(char_p, length)[:] if char_p != ffi.NULL else b""
+
+            if not state["done"].is_set():
+                state["ret"] = int(ret)
+                state["msg"] = msg
+                state["done"].set()
+
+        return CallbackType(c_cb)
+
+    @staticmethod
+    def _make_event_cb(py_callback):
+        def c_cb(ret, char_p, length, userData):
+            msg = ffi.buffer(char_p, length)[:] if char_p != ffi.NULL else b""
+            py_callback(int(ret), msg)
+
+        return CallbackType(c_cb)
+
+    @classmethod
+    def create_node(
+        cls,
+        config: dict,
+        event_cb=None,
+        *,
+        timeout_s: float = 20.0,
+    ) -> Result["NodeWrapper", str]:
+        config_json = json.dumps(config, separators=(",", ":"), ensure_ascii=False)
+        config_buffer = ffi.new("char[]", config_json.encode("utf-8"))
+
+        state = _new_cb_state()
+        cb = cls._make_waiting_cb(state)
+
+        ctx = lib.logosdelivery_create_node(
+            config_buffer,
+            cb,
+            ffi.NULL,
+        )
+
+        if ctx == ffi.NULL:
+            return Err("create_node: ctx is NULL")
+
+        wait_result = _wait_cb_ok(state, "create_node", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+
+        event_cb_handler = None
+        if event_cb is not None:
+            event_cb_handler = cls._make_event_cb(event_cb)
+            lib.logosdelivery_set_event_callback(
+                ctx,
+                event_cb_handler,
+                ffi.NULL,
+            )
+
+        return Ok(cls(ctx, config_buffer, event_cb_handler))
+
+    @classmethod
+    def create_and_start(
+        cls,
+        config: dict,
+        event_cb=None,
+        *,
+        timeout_s: float = 20.0,
+    ) -> Result["NodeWrapper", str]:
+        node_result = cls.create_node(
+            config=config,
+            event_cb=event_cb,
+            timeout_s=timeout_s,
+        )
+        if node_result.is_err():
+            return Err(node_result.err())
+
+        node = node_result.ok_value
+
+        start_result = node.start_node(timeout_s=timeout_s)
+        if start_result.is_err():
+            return Err(start_result.err())
+
+        return Ok(node)
+
+    def start_node(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_start_node(self.ctx, cb, ffi.NULL)
+        if rc != 0:
+            return Err(f"start_node: immediate call failed (ret={rc})")
+
+        return _wait_cb_ok(state, "start_node", timeout_s)
+
+    def stop_node(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_stop_node(self.ctx, cb, ffi.NULL)
+        if rc != 0:
+            return Err(f"stop_node: immediate call failed (ret={rc})")
+
+        return _wait_cb_ok(state, "stop_node", timeout_s)
+
+    def destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_destroy(self.ctx, cb, ffi.NULL)
+        if rc != 0:
+            return Err(f"destroy: immediate call failed (ret={rc})")
+
+        wait_result = _wait_cb_ok(state, "destroy", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+
+        self.ctx = ffi.NULL
+        return wait_result
+
+    def stop_and_destroy(self, *, timeout_s: float = 20.0) -> Result[int, str]:
+        stop_result = self.stop_node(timeout_s=timeout_s)
+        if stop_result.is_err():
+            return Err(stop_result.err())
+
+        return self.destroy(timeout_s=timeout_s)
+
+    def subscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_subscribe(
+            self.ctx,
+            cb,
+            ffi.NULL,
+            content_topic.encode("utf-8"),
+        )
+        if rc != 0:
+            return Err(f"subscribe_content_topic: immediate call failed (ret={rc})")
+
+        return _wait_cb_ok(state, f"subscribe({content_topic})", timeout_s)
+
+    def unsubscribe_content_topic(self, content_topic: str, *, timeout_s: float = 20.0) -> Result[int, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_unsubscribe(
+            self.ctx,
+            cb,
+            ffi.NULL,
+            content_topic.encode("utf-8"),
+        )
+        if rc != 0:
+            return Err(f"unsubscribe_content_topic: immediate call failed (ret={rc})")
+
+        return _wait_cb_ok(state, f"unsubscribe({content_topic})", timeout_s)
+
+    def send_message(self, message: dict, *, timeout_s: float = 20.0) -> Result[str, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        message_json = json.dumps(message, separators=(",", ":"), ensure_ascii=False)
+
+        rc = lib.logosdelivery_send(
+            self.ctx,
+            cb,
+            ffi.NULL,
+            message_json.encode("utf-8"),
+        )
+        if rc != 0:
+            return Err(f"send_message: immediate call failed (ret={rc})")
+
+        wait_result = _wait_cb_raw(state, "send_message", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+
+        cb_ret, cb_msg = wait_result.ok_value
+        if cb_ret != 0:
+            return Err(f"send_message: callback failed (ret={cb_ret}) msg={cb_msg!r}")
+
+        request_id = cb_msg.decode("utf-8") if cb_msg else ""
+        return Ok(request_id)
+
+    def get_available_node_info_ids(self, *, timeout_s: float = 20.0) -> Result[list[str], str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_get_available_node_info_ids(self.ctx, cb, ffi.NULL)
+        if rc != 0:
+            return Err(f"get_available_node_info_ids: immediate call failed (ret={rc})")
+
+        wait_result = _wait_cb_raw(state, "get_available_node_info_ids", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+
+        cb_ret, cb_msg = wait_result.ok_value
+        if cb_ret != 0:
+            return Err(f"get_available_node_info_ids: callback failed (ret={cb_ret})")
+        if not cb_msg:
+            return Err("get_available_node_info_ids: empty response")
+
+        try:
+            return Ok(json.loads(cb_msg.decode("utf-8").strip().lstrip("@")))
+        except Exception as e:
+            return Err(f"get_available_node_info_ids: invalid response: {e}")
+
+    def get_node_info(self, node_info_id: str, *, timeout_s: float = 20.0) -> Result[dict, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_get_node_info(
+            self.ctx,
+            cb,
+            ffi.NULL,
+            node_info_id.encode("utf-8"),
+        )
+        if rc != 0:
+            return Err(f"get_node_info: immediate call failed (ret={rc})")
+
+        wait_result = _wait_cb_raw(state, "get_node_info", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+
+        cb_ret, cb_msg = wait_result.ok_value
+        if cb_ret != 0:
+            return Err(f"get_node_info: callback failed (ret={cb_ret}) msg={cb_msg!r}")
+
+        if not cb_msg:
+            return Err("get_node_info: empty response")
+
+        try:
+            result = json.loads(cb_msg.decode("utf-8"))
+        except Exception as e:
+            return Err(f"get_node_info: invalid json: {e}")
+
+        return Ok(result)
+
+    def get_available_configs(self, *, timeout_s: float = 20.0) -> Result[dict, str]:
+        state = _new_cb_state()
+        cb = self._make_waiting_cb(state)
+
+        rc = lib.logosdelivery_get_available_configs(self.ctx, cb, ffi.NULL)
+        if rc != 0:
+            return Err(f"get_available_configs: immediate call failed (ret={rc})")
+
+        wait_result = _wait_cb_raw(state, "get_available_configs", timeout_s)
+        if wait_result.is_err():
+            return Err(wait_result.err())
+
+        cb_ret, cb_msg = wait_result.ok_value
+        if cb_ret != 0:
+            return Err(f"get_available_configs: callback failed (ret={cb_ret}) msg={cb_msg!r}")
+
+        if not cb_msg:
+            return Err("get_available_configs: empty response")
+
+        try:
+            result = json.loads(cb_msg.decode("utf-8"))
+        except Exception as e:
+            return Err(f"get_available_configs: invalid json: {e}")
+
+        return Ok(result)
+