2026-04-20 12:33:42 +01:00
|
|
|
from __future__ import annotations
|
|
|
|
|
|
2026-04-14 19:43:20 +02:00
|
|
|
import json
|
|
|
|
|
import threading
|
|
|
|
|
import time
|
2026-04-20 12:33:42 +01:00
|
|
|
from typing import Optional
|
2026-04-21 13:48:24 +02:00
|
|
|
from src.libs.common import to_base64
|
|
|
|
|
|
|
|
|
|
DEFAULT_CONTENT_TOPIC = "/test/1/default/proto"
|
|
|
|
|
DEFAULT_PAYLOAD = to_base64("test payload")
|
|
|
|
|
EVENT_PROPAGATED = "message_propagated"
|
|
|
|
|
EVENT_SENT = "message_sent"
|
|
|
|
|
EVENT_ERROR = "message_error"
|
|
|
|
|
|
2026-05-07 12:31:41 +02:00
|
|
|
# MaxTimeInCache from send_service.nim.
|
|
|
|
|
MAX_TIME_IN_CACHE_S = 60.0
|
|
|
|
|
# Extra slack to cover the background retry loop tick after the window expires.
|
|
|
|
|
CACHE_EXPIRY_SLACK_S = 10.0
|
|
|
|
|
ERROR_AFTER_CACHE_EXPIRY_TIMEOUT_S = MAX_TIME_IN_CACHE_S + CACHE_EXPIRY_SLACK_S
|
|
|
|
|
RETRY_WINDOW_EXPIRED_MSG = "Unable to send within retry time window"
|
|
|
|
|
|
2026-04-14 19:43:20 +02:00
|
|
|
|
|
|
|
|
class EventCollector:
|
|
|
|
|
"""Thread-safe collector for async node events.
|
|
|
|
|
|
|
|
|
|
Pass `collector.event_callback` as the `event_cb` argument to
|
|
|
|
|
WrapperManager.create_and_start(). Every event fired by the library
|
|
|
|
|
is decoded from JSON and appended to `self.events`.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self._lock = threading.Lock()
|
|
|
|
|
self.events: list[dict] = []
|
|
|
|
|
|
|
|
|
|
def event_callback(self, ret: int, raw: bytes) -> None:
|
|
|
|
|
try:
|
|
|
|
|
payload = json.loads(raw.decode("utf-8"))
|
|
|
|
|
except Exception:
|
|
|
|
|
payload = {"_raw": raw.decode("utf-8", errors="replace"), "_ret": ret}
|
|
|
|
|
|
|
|
|
|
with self._lock:
|
|
|
|
|
self.events.append(payload)
|
|
|
|
|
|
|
|
|
|
def get_events_for_request(self, request_id: str) -> list[dict]:
|
|
|
|
|
with self._lock:
|
|
|
|
|
return [e for e in self.events if e.get("requestId") == request_id]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_propagated_event(event: dict) -> bool:
|
2026-04-20 12:33:42 +01:00
|
|
|
return event.get("eventType") == EVENT_PROPAGATED
|
2026-04-14 19:43:20 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_sent_event(event: dict) -> bool:
|
2026-04-20 12:33:42 +01:00
|
|
|
return event.get("eventType") == EVENT_SENT
|
2026-04-14 19:43:20 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_error_event(event: dict) -> bool:
|
2026-04-20 12:33:42 +01:00
|
|
|
return event.get("eventType") == EVENT_ERROR
|
2026-04-14 19:43:20 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def wait_for_event(
|
|
|
|
|
collector: EventCollector,
|
|
|
|
|
request_id: str,
|
|
|
|
|
predicate,
|
|
|
|
|
timeout_s: float,
|
|
|
|
|
poll_interval_s: float = 0.5,
|
2026-04-20 12:33:42 +01:00
|
|
|
) -> Optional[dict]:
|
2026-04-14 19:43:20 +02:00
|
|
|
"""Poll until an event matching `predicate` arrives for `request_id`,
|
|
|
|
|
or until `timeout_s` elapses. Returns the matching event or None.
|
|
|
|
|
"""
|
|
|
|
|
deadline = time.monotonic() + timeout_s
|
|
|
|
|
|
2026-04-28 17:44:27 +01:00
|
|
|
while True:
|
2026-04-14 19:43:20 +02:00
|
|
|
for event in collector.get_events_for_request(request_id):
|
|
|
|
|
if predicate(event):
|
|
|
|
|
return event
|
2026-04-28 17:44:27 +01:00
|
|
|
if time.monotonic() >= deadline:
|
|
|
|
|
return None
|
2026-04-14 19:43:20 +02:00
|
|
|
time.sleep(poll_interval_s)
|
|
|
|
|
|
|
|
|
|
|
2026-04-20 12:33:42 +01:00
|
|
|
def wait_for_propagated(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]:
|
2026-04-14 19:43:20 +02:00
|
|
|
return wait_for_event(collector, request_id, is_propagated_event, timeout_s)
|
|
|
|
|
|
|
|
|
|
|
2026-04-20 12:33:42 +01:00
|
|
|
def wait_for_sent(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]:
|
2026-04-14 19:43:20 +02:00
|
|
|
return wait_for_event(collector, request_id, is_sent_event, timeout_s)
|
|
|
|
|
|
|
|
|
|
|
2026-04-20 12:33:42 +01:00
|
|
|
def wait_for_error(collector: EventCollector, request_id: str, timeout_s: float) -> Optional[dict]:
|
2026-04-14 19:43:20 +02:00
|
|
|
return wait_for_event(collector, request_id, is_error_event, timeout_s)
|
|
|
|
|
|
|
|
|
|
|
2026-05-07 12:31:41 +02:00
|
|
|
def assert_no_error(collector: EventCollector, request_id: str, context: str = "") -> None:
|
|
|
|
|
"""Assert that no message_error event is currently buffered for `request_id`."""
|
|
|
|
|
event = wait_for_error(collector, request_id, timeout_s=0)
|
|
|
|
|
suffix = f" ({context})" if context else ""
|
|
|
|
|
assert event is None, f"Unexpected message_error event{suffix}: {event}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def assert_no_sent(collector: EventCollector, request_id: str, context: str = "") -> None:
|
|
|
|
|
"""Assert that no message_sent event is currently buffered for `request_id`."""
|
|
|
|
|
event = wait_for_sent(collector, request_id, timeout_s=0)
|
|
|
|
|
suffix = f" ({context})" if context else ""
|
|
|
|
|
assert event is None, f"Unexpected message_sent event{suffix}: {event}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def assert_no_propagated(collector: EventCollector, request_id: str, context: str = "") -> None:
|
|
|
|
|
"""Assert that no message_propagated event is currently buffered for `request_id`."""
|
|
|
|
|
event = wait_for_propagated(collector, request_id, timeout_s=0)
|
|
|
|
|
suffix = f" ({context})" if context else ""
|
|
|
|
|
assert event is None, f"Unexpected message_propagated event{suffix}: {event}"
|
|
|
|
|
|
|
|
|
|
|
2026-04-30 16:15:50 +01:00
|
|
|
def wait_for_connected(
|
|
|
|
|
collector: EventCollector,
|
|
|
|
|
timeout_s: float = 10.0,
|
|
|
|
|
poll_interval_s: float = 0.3,
|
|
|
|
|
) -> Optional[dict]:
|
|
|
|
|
"""Wait until a connection_status_change event with PartiallyConnected or Connected arrives."""
|
|
|
|
|
deadline = time.monotonic() + timeout_s
|
|
|
|
|
while time.monotonic() < deadline:
|
|
|
|
|
with collector._lock:
|
|
|
|
|
for event in collector.events:
|
|
|
|
|
if event.get("eventType") == "connection_status_change" and event.get("connectionStatus") in ("PartiallyConnected", "Connected"):
|
|
|
|
|
return event
|
|
|
|
|
time.sleep(poll_interval_s)
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
2026-04-28 17:44:27 +01:00
|
|
|
TERMINAL_EVENT_TYPES = {EVENT_PROPAGATED, EVENT_SENT, EVENT_ERROR}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def assert_event_invariants(collector: EventCollector, request_id: str) -> None:
|
|
|
|
|
"""Check per-request event invariants (issue #163):
|
|
|
|
|
- All events carry the correct requestId.
|
|
|
|
|
- No duplicate terminal events (Propagated, Sent, Error).
|
|
|
|
|
- Sent never appears before Propagated.
|
|
|
|
|
"""
|
|
|
|
|
events = collector.get_events_for_request(request_id)
|
|
|
|
|
assert events, f"No events found for request {request_id}"
|
|
|
|
|
|
|
|
|
|
counts: dict[str, int] = {}
|
|
|
|
|
first_index: dict[str, int] = {}
|
|
|
|
|
for i, event in enumerate(events):
|
|
|
|
|
assert event.get("requestId") == request_id, (
|
|
|
|
|
f"Event at index {i} has wrong requestId: " f"expected {request_id!r}, got {event.get('requestId')!r}"
|
|
|
|
|
)
|
|
|
|
|
event_type = event.get("eventType", "")
|
|
|
|
|
if event_type in TERMINAL_EVENT_TYPES:
|
|
|
|
|
counts[event_type] = counts.get(event_type, 0) + 1
|
|
|
|
|
if event_type not in first_index:
|
|
|
|
|
first_index[event_type] = i
|
|
|
|
|
|
|
|
|
|
for event_type, count in counts.items():
|
|
|
|
|
assert count == 1, f"Duplicate {event_type} events for request {request_id}: " f"got {count}, expected 1. Events: {events}"
|
|
|
|
|
|
|
|
|
|
if EVENT_SENT in first_index and EVENT_PROPAGATED in first_index:
|
|
|
|
|
assert first_index[EVENT_PROPAGATED] < first_index[EVENT_SENT], (
|
|
|
|
|
f"message_sent (index {first_index[EVENT_SENT]}) arrived before "
|
|
|
|
|
f"message_propagated (index {first_index[EVENT_PROPAGATED]}) "
|
|
|
|
|
f"for request {request_id}. Events: {events}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
2026-04-14 19:43:20 +02:00
|
|
|
def get_node_multiaddr(node) -> str:
|
2026-05-07 11:21:36 +02:00
|
|
|
"""Return the TCP multiaddr (with peer-id) from a WrapperManager node.
|
|
|
|
|
|
|
|
|
|
Asserts that the wrapper returned exactly one address. If the wrapper ever
|
|
|
|
|
starts returning multiple addresses (newline/comma-separated or a JSON
|
|
|
|
|
list), this fails loudly instead of silently passing a malformed string
|
|
|
|
|
downstream to staticnodes / add_peers.
|
|
|
|
|
"""
|
2026-04-20 12:33:42 +01:00
|
|
|
result = node.get_node_info_raw("MyMultiaddresses")
|
|
|
|
|
if result.is_err():
|
|
|
|
|
raise RuntimeError(f"get_node_info_raw failed: {result.err()}")
|
2026-04-14 19:43:20 +02:00
|
|
|
|
2026-04-20 12:33:42 +01:00
|
|
|
addr = result.ok_value.strip()
|
2026-05-07 11:21:36 +02:00
|
|
|
if not addr or not addr.startswith("/"):
|
2026-04-20 12:33:42 +01:00
|
|
|
raise RuntimeError(f"Unexpected multiaddr format: {addr!r}")
|
2026-05-07 11:21:36 +02:00
|
|
|
|
|
|
|
|
if "\n" in addr or "," in addr or addr.startswith("["):
|
|
|
|
|
raise AssertionError(f"Expected a single multiaddr from MyMultiaddresses, got multiple: {addr!r}")
|
2026-04-14 19:43:20 +02:00
|
|
|
|
2026-04-20 12:33:42 +01:00
|
|
|
return addr
|
2026-04-21 13:48:24 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_message_bindings(**overrides) -> dict:
|
|
|
|
|
envelope = {
|
|
|
|
|
"contentTopic": DEFAULT_CONTENT_TOPIC,
|
|
|
|
|
"payload": DEFAULT_PAYLOAD,
|
|
|
|
|
"ephemeral": False,
|
|
|
|
|
}
|
|
|
|
|
envelope.update(overrides)
|
|
|
|
|
return envelope
|
2026-05-07 12:31:41 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def assert_no_unknown_request_ids(collector: EventCollector, issued_request_ids) -> None:
|
|
|
|
|
"""Cross-association guard: every event carrying a requestId must belong
|
|
|
|
|
to one of the request ids we issued. Catches events that get attached to
|
|
|
|
|
the wrong request id under concurrency.
|
|
|
|
|
"""
|
|
|
|
|
issued = set(issued_request_ids)
|
|
|
|
|
for event in collector.events:
|
|
|
|
|
event_request_id = event.get("requestId")
|
|
|
|
|
if event_request_id is None:
|
|
|
|
|
continue
|
|
|
|
|
assert event_request_id in issued, f"Event carries an unknown requestId={event_request_id!r}, " f"not in issued set {issued}. Event: {event}"
|