add CLI for running experiments

benchmarks/cli.py

@@ -0,0 +1,55 @@
+import sys
+from pathlib import Path
+
+import typer
+from pydantic_core import ValidationError
+
+from benchmarks.core.config import ConfigParser
+from benchmarks.deluge.config import DelugeExperimentConfig
+
+parser = ConfigParser()
+parser.register(DelugeExperimentConfig)
+
+app = typer.Typer()
+
+
+def _parse_config(config: Path):
+    if not config.exists():
+        print(f'Config file {config} does not exist.')
+        sys.exit(-1)
+
+    with config.open(encoding='utf-8') as config:
+        try:
+            return parser.parse(config)
+        except ValidationError as e:
+            print(f'There were errors parsing the config file.')
+            for error in e.errors():
+                print(f' - {error["loc"]}: {error["msg"]} {error["input"]}')
+            sys.exit(-1)
+
+
+@app.command()
+def list(config: Path):
+    """
+    Lists the experiments available in CONFIG.
+    """
+    parsed = _parse_config(config)
+    print(f'Available experiments in {config}:')
+    for experiment in parsed.keys():
+        print(f'  - {experiment}')
+
+
+@app.command()
+def run(config: Path, experiment: str):
+    """
+    Runs the experiment with name EXPERIMENT.
+    """
+    parsed = _parse_config(config)
+    if experiment not in parsed:
+        print(f'Experiment {experiment} not found in {config}.')
+        sys.exit(-1)
+    parsed[experiment].run()
+
+
+if __name__ == '__main__':
+    app()

benchmarks/core/config.py

@@ -1,22 +1,28 @@
 """Basic utilities for structuring experiment configurations based on Pydantic schemas."""
-
+import os
 import re
 from abc import abstractmethod
-from typing import Annotated
+from io import TextIOBase
+from typing import Annotated, Type, Dict, TextIO
 
+import yaml
 from pydantic import BaseModel, IPvAnyAddress, AfterValidator
-from typing_extensions import Generic
+from typing_extensions import Generic, overload
 
-from benchmarks.core.experiments.experiments import TExperiment
+from benchmarks.core.experiments.experiments import TExperiment, Experiment
 
 
 def drop_config_suffix(name: str) -> str:
     return name[:-6] if name.endswith('Config') else name
 
 
+def to_snake_case(name: str) -> str:
+    return re.sub(r'(?<!^)(?=[A-Z])', '_', name).lower()
+
+
 class ConfigModel(BaseModel):
     model_config = {
-        'alias_generator': drop_config_suffix
+        'alias_generator': lambda x: to_snake_case(drop_config_suffix(x))
     }
 
 
@@ -38,9 +44,43 @@ class Host(BaseModel):
     address: IPvAnyAddress | DomainName
 
 
-class ExperimentBuilder(Generic[TExperiment], ConfigModel):
+class ExperimentBuilder(ConfigModel, Generic[TExperiment]):
     """:class:`ExperimentBuilders` can build real :class:`Experiment`s out of :class:`ConfigModel`s. """
 
     @abstractmethod
     def build(self) -> TExperiment:
         pass
+
+
+class ConfigParser:
+    """
+    :class:`ConfigParser` is a utility class to parse configuration files into :class:`ExperimentBuilder`s.
+    Currently, each :class:`ExperimentBuilder` can appear at most once in the config file.
+    """
+
+    def __init__(self):
+        self.root_tags = {}
+
+    def register(self, root: Type[ExperimentBuilder[Experiment]]):
+        name = root.__name__
+        alias = root.model_config.get('alias_generator', lambda x: x)(name)
+        self.root_tags[alias] = root
+
+    @overload
+    def parse(self, data: dict) -> Dict[str, ExperimentBuilder[Experiment]]:
+        ...
+
+    @overload
+    def parse(self, data: TextIO) -> Dict[str, ExperimentBuilder[Experiment]]:
+        ...
+
+    def parse(self, data: dict | TextIO) -> Dict[str, ExperimentBuilder[Experiment]]:
+        if isinstance(data, TextIOBase):
+            entries = yaml.safe_load(os.path.expandvars(data.read()))
+        else:
+            entries = data
+
+        return {
+            tag: self.root_tags[tag].model_validate(config)
+            for tag, config in entries.items()
+        }

benchmarks/core/tests/test_config.py

@@ -1,9 +1,13 @@
+import os
+from io import StringIO
 from ipaddress import IPv4Address, IPv6Address
+from typing import cast
 
 import pytest
-from pydantic import ValidationError
+import yaml
+from pydantic import ValidationError, BaseModel
 
-from benchmarks.core.config import Host, DomainName
+from benchmarks.core.config import Host, DomainName, ConfigParser, ConfigModel
 
 
 def test_should_parse_ipv4_address():
@@ -47,3 +51,51 @@ def test_should_fail_invalid_names():
     for invalid_name in invalid_names:
         with pytest.raises(ValidationError):
             Host(address=invalid_name)
+
+
+class Root1(ConfigModel):
+    index: int
+
+
+class Root2(ConfigModel):
+    name: str
+
+
+def test_should_parse_multiple_roots():
+    config_file = StringIO("""
+    root1:
+      index: 1
+    
+    root2: 
+      name: "root2"
+    """)
+
+    parser = ConfigParser()
+
+    parser.register(Root1)
+    parser.register(Root2)
+
+    conf = parser.parse(yaml.safe_load(config_file))
+
+    assert cast(Root1, conf['root1']).index == 1
+    assert cast(Root2, conf['root2']).name == 'root2'
+
+
+def test_should_expand_env_vars_when_fed_a_config_file():
+    config_file = StringIO("""
+    root1:
+      index: ${BTB_MY_INDEX}
+    root2:
+      name: "My name is ${BTB_NAME}"
+    """)
+
+    os.environ['BTB_MY_INDEX'] = '10'
+    os.environ['BTB_NAME'] = 'John Doe'
+
+    parser = ConfigParser()
+    parser.register(Root1)
+    parser.register(Root2)
+
+    conf = parser.parse(config_file)
+    assert cast(Root1, conf['root1']).index == 10
+    assert cast(Root2, conf['root2']).name == 'My name is John Doe'

poetry.lock

@@ -11,6 +11,20 @@ files = [
     {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
 ]
 
+[[package]]
+name = "click"
+version = "8.1.7"
+description = "Composable command line interface toolkit"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"},
+    {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -44,6 +58,41 @@ files = [
     {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
 ]
 
+[[package]]
+name = "markdown-it-py"
+version = "3.0.0"
+description = "Python port of markdown-it. Markdown parsing, done right!"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
+    {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
+]
+
+[package.dependencies]
+mdurl = ">=0.1,<1.0"
+
+[package.extras]
+benchmarking = ["psutil", "pytest", "pytest-benchmark"]
+code-style = ["pre-commit (>=3.0,<4.0)"]
+compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
+linkify = ["linkify-it-py (>=1,<3)"]
+plugins = ["mdit-py-plugins"]
+profiling = ["gprof2dot"]
+rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+description = "Markdown URL utilities"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
+    {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
+]
+
 [[package]]
 name = "mypy"
 version = "1.13.0"
@@ -281,6 +330,20 @@ files = [
 [package.dependencies]
 typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
+[[package]]
+name = "pygments"
+version = "2.18.0"
+description = "Pygments is a syntax highlighting package written in Python."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"},
+    {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"},
+]
+
+[package.extras]
+windows-terminal = ["colorama (>=0.4.6)"]
+
 [[package]]
 name = "pytest"
 version = "8.3.3"
@@ -363,6 +426,35 @@ files = [
     {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"},
 ]
 
+[[package]]
+name = "rich"
+version = "13.9.4"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+optional = false
+python-versions = ">=3.8.0"
+files = [
+    {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"},
+    {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"},
+]
+
+[package.dependencies]
+markdown-it-py = ">=2.2.0"
+pygments = ">=2.13.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<9)"]
+
+[[package]]
+name = "shellingham"
+version = "1.5.4"
+description = "Tool to Detect Surrounding Shell"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"},
+    {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"},
+]
+
 [[package]]
 name = "torrentool"
 version = "1.2.0"
@@ -377,6 +469,23 @@ files = [
 [package.extras]
 cli = ["click"]
 
+[[package]]
+name = "typer"
+version = "0.13.1"
+description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "typer-0.13.1-py3-none-any.whl", hash = "sha256:5b59580fd925e89463a29d363e0a43245ec02765bde9fb77d39e5d0f29dd7157"},
+    {file = "typer-0.13.1.tar.gz", hash = "sha256:9d444cb96cc268ce6f8b94e13b4335084cef4c079998a9f4851a90229a3bd25c"},
+]
+
+[package.dependencies]
+click = ">=8.0.0"
+rich = ">=10.11.0"
+shellingham = ">=1.3.0"
+typing-extensions = ">=3.7.4.3"
+
 [[package]]
 name = "types-pyyaml"
 version = "6.0.12.20240917"
@@ -402,4 +511,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "c10ab6006a3097ae8fcbac02448e98cf18f61146ab311979e1e9d5e735e2369d"
+content-hash = "f3c2c0480921a5d481f0e8de78329998a8fddae7ccecc6cb9cb9a01e3f6137c5"

pyproject.toml

@@ -14,6 +14,7 @@ pathvalidate = "^3.2.1"
 torrentool = "^1.2.0"
 pydantic = "^2.10.2"
 pyyaml = "^6.0.2"
+typer = "^0.13.1"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.3"