From ba1b93d77c6f3b1e83ca36b626c5deea7b714e49 Mon Sep 17 00:00:00 2001 From: gmega Date: Mon, 27 Jan 2025 17:26:09 -0300 Subject: [PATCH] feat: add structured experiment iteration logs --- benchmarks/core/experiments/experiments.py | 26 --------- .../core/experiments/iterated_experiment.py | 56 +++++++++++++++++++ benchmarks/deluge/config.py | 9 ++- benchmarks/logging/logging.py | 8 +++ experiments.k8s.yaml | 1 + experiments.local.yaml | 1 + .../deluge/templates/testrunner-job.yaml | 2 + 7 files changed, 75 insertions(+), 28 deletions(-) create mode 100644 benchmarks/core/experiments/iterated_experiment.py diff --git a/benchmarks/core/experiments/experiments.py b/benchmarks/core/experiments/experiments.py index 2be2850..c5d8c90 100644 --- a/benchmarks/core/experiments/experiments.py +++ b/benchmarks/core/experiments/experiments.py @@ -133,29 +133,3 @@ class BoundExperiment(Experiment, Generic[TExperiment]): def run(self): self.env.run(self.experiment) - - -class IteratedExperiment(Experiment, Generic[TExperiment]): - """An :class:`IteratedExperiment` will run a sequence of :class:`Experiment`s.""" - - def __init__( - self, experiments: Iterable[TExperiment], raise_when_failures: bool = True - ): - self.successful_runs = 0 - self.failed_runs = 0 - self.raise_when_failures = raise_when_failures - self.experiments = experiments - - def run(self): - for experiment in self.experiments: - try: - experiment.run() - self.successful_runs += 1 - except Exception: - self.failed_runs += 1 - logger.exception("Error running experiment repetition") - - if self.failed_runs > 0 and self.raise_when_failures: - raise RuntimeError( - "One or more experiments with an iterated experiment have failed." - ) diff --git a/benchmarks/core/experiments/iterated_experiment.py b/benchmarks/core/experiments/iterated_experiment.py new file mode 100644 index 0000000..31b0891 --- /dev/null +++ b/benchmarks/core/experiments/iterated_experiment.py @@ -0,0 +1,56 @@ +import logging +import time +from typing import Iterable + +from typing_extensions import Generic + +from benchmarks.core.experiments.experiments import Experiment, TExperiment +from benchmarks.logging.logging import ExperimentStatus + +logger = logging.getLogger(__name__) + + +class IteratedExperiment(Experiment, Generic[TExperiment]): + """An :class:`IteratedExperiment` will run a sequence of :class:`Experiment`s.""" + + def __init__( + self, + experiments: Iterable[TExperiment], + experiment_set_id: str = "unnamed", + raise_when_failures: bool = True, + ): + self.experiment_set_id = experiment_set_id + self.successful_runs = 0 + self.failed_runs = 0 + self.raise_when_failures = raise_when_failures + self.experiments = experiments + + def run(self): + for i, experiment in enumerate(self.experiments): + start = time.time() + try: + experiment.run() + self.successful_runs += 1 + logger.info( + ExperimentStatus( + name=self.experiment_set_id, + repetition=i, + duration=time.time() - start, + ) + ) + except Exception as ex: + self.failed_runs += 1 + logger.exception("Error running experiment repetition") + logger.info( + ExperimentStatus( + name=self.experiment_set_id, + repetition=i, + duration=time.time() - start, + error=str(ex), + ) + ) + + if self.failed_runs > 0 and self.raise_when_failures: + raise RuntimeError( + "One or more experiments with an iterated experiment have failed." + ) diff --git a/benchmarks/deluge/config.py b/benchmarks/deluge/config.py index 3456b04..a5d8586 100644 --- a/benchmarks/deluge/config.py +++ b/benchmarks/deluge/config.py @@ -8,11 +8,11 @@ from torrentool.torrent import Torrent from urllib3.util import parse_url from benchmarks.core.experiments.experiments import ( - IteratedExperiment, ExperimentEnvironment, BoundExperiment, ExperimentBuilder, ) +from benchmarks.core.experiments.iterated_experiment import IteratedExperiment from benchmarks.core.experiments.static_experiment import StaticDisseminationExperiment from benchmarks.core.pydantic import Host from benchmarks.core.utils import sample @@ -62,6 +62,9 @@ DelugeDisseminationExperiment = IteratedExperiment[ class DelugeExperimentConfig(ExperimentBuilder[DelugeDisseminationExperiment]): + experiment_set_id: str = Field( + description="Identifies the group of experiment repetitions", default="unnamed" + ) seeder_sets: int = Field( gt=0, default=1, description="Number of distinct seeder sets to experiment with" ) @@ -138,4 +141,6 @@ class DelugeExperimentConfig(ExperimentBuilder[DelugeDisseminationExperiment]): ) ) - return IteratedExperiment(repetitions()) + return IteratedExperiment( + repetitions(), experiment_set_id=self.experiment_set_id + ) diff --git a/benchmarks/logging/logging.py b/benchmarks/logging/logging.py index 1ec3e12..1b9a984 100644 --- a/benchmarks/logging/logging.py +++ b/benchmarks/logging/logging.py @@ -225,10 +225,18 @@ class RequestEvent(Event): type: RequestEventType +class ExperimentStatus(Event): + name: str + repetition: int + duration: float + error: Optional[str] = None + + def basic_log_parser() -> LogParser: """Constructs a basic log parser which can understand some common log entry types.""" parser = LogParser() parser.register(Event) parser.register(Metric) parser.register(RequestEvent) + parser.register(ExperimentStatus) return parser diff --git a/experiments.k8s.yaml b/experiments.k8s.yaml index cabc320..64b862c 100644 --- a/experiments.k8s.yaml +++ b/experiments.k8s.yaml @@ -1,4 +1,5 @@ deluge_experiment: + experiment_set_id: ${EXPERIMENT_SET_ID} seeder_sets: ${SEEDER_SETS} seeders: ${SEEDERS} tracker_announce_url: ${TRACKER_ANNOUNCE_URL} diff --git a/experiments.local.yaml b/experiments.local.yaml index 201bd15..d7ff6b5 100644 --- a/experiments.local.yaml +++ b/experiments.local.yaml @@ -1,6 +1,7 @@ # You can use this configuration to run the experiments locally with the provided # Docker compose environment. deluge_experiment: + experiment_set_id: local seeders: 1 tracker_announce_url: ${TRACKER_ANNOUNCE_URL:-http://127.0.0.1:8000/announce} file_size: 52428800 diff --git a/k8s/charts/deluge/templates/testrunner-job.yaml b/k8s/charts/deluge/templates/testrunner-job.yaml index 40305ae..0bae8aa 100644 --- a/k8s/charts/deluge/templates/testrunner-job.yaml +++ b/k8s/charts/deluge/templates/testrunner-job.yaml @@ -46,6 +46,8 @@ spec: value: "deluge-nodes-{{ include "experiment.fullId" . }}" - name: DELUGE_SERVICE value: "deluge-nodes-service-{{ include "experiment.fullId" . }}" + - name: EXPERIMENT_SET_ID + value: {{ include "experiment.fullId" . | quote }} - name: NAMESPACE valueFrom: fieldRef: