dump message storage in json

This commit is contained in:
Youngjoon Lee 2024-12-20 18:59:20 +09:00
parent 265ff88381
commit 16d9bc18e3
No known key found for this signature in database
GPG Key ID: 303963A54A81DD4D
2 changed files with 27 additions and 17 deletions

View File

@ -3,6 +3,7 @@ import json
import os import os
import subprocess import subprocess
from collections import OrderedDict from collections import OrderedDict
from dataclasses import asdict
import latency import latency
import mixlog import mixlog
@ -113,9 +114,15 @@ print(
) )
for idx, log_path in enumerate(log_paths): for idx, log_path in enumerate(log_paths):
network_diameter = topology_result(log_path)["diameter"] network_diameter = topology_result(log_path)["diameter"]
latency_res = latency.compute_results( message_stroage = latency.parse_record_stream(mixlog.get_input_stream(log_path))
latency.parse_record_stream(mixlog.get_input_stream(log_path)), STEP_DURATION_MS with open(f"{log_dir}/msgs-{idx}.json", "w") as file:
) json.dump(
{msg_id: asdict(msg) for msg_id, msg in message_stroage.items()},
file,
indent=2,
)
latency_res = latency.compute_results(message_stroage, STEP_DURATION_MS)
msg_count = latency_res["total_complete_messages"] msg_count = latency_res["total_complete_messages"]
min_latency = float(latency_res["min_latency_ms"]) / 1000.0 min_latency = float(latency_res["min_latency_ms"]) / 1000.0
min_latency_msg_id = latency_res["min_latency_message_id"] min_latency_msg_id = latency_res["min_latency_message_id"]

View File

@ -3,21 +3,22 @@ import argparse
import json import json
import statistics import statistics
from collections.abc import Iterable from collections.abc import Iterable
from dataclasses import dataclass, field
from typing import Dict, Optional from typing import Dict, Optional
import mixlog import mixlog
@dataclass
class Event: class Event:
def __init__(self, step_id: int, node_id: int): step_id: int
self.step_id = step_id node_id: int
self.node_id = node_id
@dataclass
class Latency: class Latency:
def __init__(self, start_event: Event): start_event: Event
self.start_event = start_event end_event: Optional[Event] = None
self.end_event = None
def finish(self, event: Event): def finish(self, event: Event):
assert self.end_event is None assert self.end_event is None
@ -34,12 +35,12 @@ class Latency:
return self.end_event.step_id - self.start_event.step_id return self.end_event.step_id - self.start_event.step_id
@dataclass
class Message: class Message:
def __init__(self, message_id: str, msg_gen_event: Event): id: str
self.id = message_id total_latency: Latency
self.total_latency = Latency(msg_gen_event) persistent_transmission_latencies: list[Latency] = field(default_factory=list)
self.persistent_transmission_latencies: list[Latency] = [] blend_latencies: list[Latency] = field(default_factory=list)
self.blend_latencies: list[Latency] = []
def __hash__(self): def __hash__(self):
return self.id return self.id
@ -147,20 +148,22 @@ def parse_record_stream(record_stream: Iterable[tuple[str, dict]]) -> MessageSto
for topic, record in record_stream: for topic, record in record_stream:
if topic in ("DataMessageGenerated", "CoverMessageGenerated"): if topic in ("DataMessageGenerated", "CoverMessageGenerated"):
payload_id = record["payload_id"] payload_id = record["payload_id"]
storage[payload_id] = Message(payload_id, event_from_record(record)) storage[payload_id] = Message(
payload_id, Latency(event_from_record(record))
)
elif topic == "MessageFullyUnwrapped": elif topic == "MessageFullyUnwrapped":
storage[record["payload_id"]].fully_unwrapped(event_from_record(record)) storage[record["payload_id"]].fully_unwrapped(event_from_record(record))
elif topic == "PersistentTransmissionScheduled": elif topic == "PersistentTransmissionScheduled":
storage[record["payload_id"]].persistent_transmission_scheduled( storage[record["payload_id"]].persistent_transmission_scheduled(
event_from_record(record) event_from_record(record)
) )
elif topic == "PersistentTransmissionReleased": elif topic == "MessageReleasedFromPersistentTransmission":
storage[record["payload_id"]].persistent_transmission_released( storage[record["payload_id"]].persistent_transmission_released(
event_from_record(record) event_from_record(record)
) )
elif topic == "BlendScheduled": elif topic == "BlendScheduled":
storage[record["payload_id"]].blend_scheduled(event_from_record(record)) storage[record["payload_id"]].blend_scheduled(event_from_record(record))
elif topic == "BlendReleased": elif topic == "MessageReleasedFromBlend":
storage[record["payload_id"]].blend_released(event_from_record(record)) storage[record["payload_id"]].blend_released(event_from_record(record))
return storage return storage