feat: add structured experiment iteration logs

benchmarks/core/experiments/experiments.py

@@ -133,29 +133,3 @@ class BoundExperiment(Experiment, Generic[TExperiment]):
 
     def run(self):
         self.env.run(self.experiment)
-
-
-class IteratedExperiment(Experiment, Generic[TExperiment]):
-    """An :class:`IteratedExperiment` will run a sequence of :class:`Experiment`s."""
-
-    def __init__(
-        self, experiments: Iterable[TExperiment], raise_when_failures: bool = True
-    ):
-        self.successful_runs = 0
-        self.failed_runs = 0
-        self.raise_when_failures = raise_when_failures
-        self.experiments = experiments
-
-    def run(self):
-        for experiment in self.experiments:
-            try:
-                experiment.run()
-                self.successful_runs += 1
-            except Exception:
-                self.failed_runs += 1
-                logger.exception("Error running experiment repetition")
-
-        if self.failed_runs > 0 and self.raise_when_failures:
-            raise RuntimeError(
-                "One or more experiments with an iterated experiment have failed."
-            )

benchmarks/core/experiments/iterated_experiment.py

@@ -0,0 +1,56 @@
+import logging
+import time
+from typing import Iterable
+
+from typing_extensions import Generic
+
+from benchmarks.core.experiments.experiments import Experiment, TExperiment
+from benchmarks.logging.logging import ExperimentStatus
+
+logger = logging.getLogger(__name__)
+
+
+class IteratedExperiment(Experiment, Generic[TExperiment]):
+    """An :class:`IteratedExperiment` will run a sequence of :class:`Experiment`s."""
+
+    def __init__(
+        self,
+        experiments: Iterable[TExperiment],
+        experiment_set_id: str = "unnamed",
+        raise_when_failures: bool = True,
+    ):
+        self.experiment_set_id = experiment_set_id
+        self.successful_runs = 0
+        self.failed_runs = 0
+        self.raise_when_failures = raise_when_failures
+        self.experiments = experiments
+
+    def run(self):
+        for i, experiment in enumerate(self.experiments):
+            start = time.time()
+            try:
+                experiment.run()
+                self.successful_runs += 1
+                logger.info(
+                    ExperimentStatus(
+                        name=self.experiment_set_id,
+                        repetition=i,
+                        duration=time.time() - start,
+                    )
+                )
+            except Exception as ex:
+                self.failed_runs += 1
+                logger.exception("Error running experiment repetition")
+                logger.info(
+                    ExperimentStatus(
+                        name=self.experiment_set_id,
+                        repetition=i,
+                        duration=time.time() - start,
+                        error=str(ex),
+                    )
+                )
+
+        if self.failed_runs > 0 and self.raise_when_failures:
+            raise RuntimeError(
+                "One or more experiments with an iterated experiment have failed."
+            )

benchmarks/deluge/config.py

@@ -8,11 +8,11 @@ from torrentool.torrent import Torrent
 from urllib3.util import parse_url
 
 from benchmarks.core.experiments.experiments import (
-    IteratedExperiment,
     ExperimentEnvironment,
     BoundExperiment,
     ExperimentBuilder,
 )
+from benchmarks.core.experiments.iterated_experiment import IteratedExperiment
 from benchmarks.core.experiments.static_experiment import StaticDisseminationExperiment
 from benchmarks.core.pydantic import Host
 from benchmarks.core.utils import sample
@@ -62,6 +62,9 @@ DelugeDisseminationExperiment = IteratedExperiment[
 
 
 class DelugeExperimentConfig(ExperimentBuilder[DelugeDisseminationExperiment]):
+    experiment_set_id: str = Field(
+        description="Identifies the group of experiment repetitions", default="unnamed"
+    )
     seeder_sets: int = Field(
         gt=0, default=1, description="Number of distinct seeder sets to experiment with"
     )
@@ -138,4 +141,6 @@ class DelugeExperimentConfig(ExperimentBuilder[DelugeDisseminationExperiment]):
                         )
                     )
 
-        return IteratedExperiment(repetitions())
+        return IteratedExperiment(
+            repetitions(), experiment_set_id=self.experiment_set_id
+        )

benchmarks/logging/logging.py

@@ -225,10 +225,18 @@ class RequestEvent(Event):
     type: RequestEventType
 
 
+class ExperimentStatus(Event):
+    name: str
+    repetition: int
+    duration: float
+    error: Optional[str] = None
+
+
 def basic_log_parser() -> LogParser:
     """Constructs a basic log parser which can understand some common log entry types."""
     parser = LogParser()
     parser.register(Event)
     parser.register(Metric)
     parser.register(RequestEvent)
+    parser.register(ExperimentStatus)
     return parser

experiments.k8s.yaml

@@ -1,4 +1,5 @@
 deluge_experiment:
+  experiment_set_id: ${EXPERIMENT_SET_ID}
   seeder_sets: ${SEEDER_SETS}
   seeders: ${SEEDERS}
   tracker_announce_url: ${TRACKER_ANNOUNCE_URL}

experiments.local.yaml

@@ -1,6 +1,7 @@
 # You can use this configuration to run the experiments locally with the provided
 # Docker compose environment.
 deluge_experiment:
+  experiment_set_id: local
   seeders: 1
   tracker_announce_url: ${TRACKER_ANNOUNCE_URL:-http://127.0.0.1:8000/announce}
   file_size: 52428800

k8s/charts/deluge/templates/testrunner-job.yaml

@@ -46,6 +46,8 @@ spec:
               value: "deluge-nodes-{{ include "experiment.fullId" . }}"
             - name: DELUGE_SERVICE
               value: "deluge-nodes-service-{{ include "experiment.fullId" . }}"
+            - name: EXPERIMENT_SET_ID
+              value: {{ include "experiment.fullId" . | quote }}
             - name: NAMESPACE
               valueFrom:
                 fieldRef: