feat: add constrained parameter expansion

benchmarks/k8s/parameter_expander.py

@@ -0,0 +1,101 @@
+import itertools
+import json
+import sys
+from json import JSONDecodeError
+from typing import Dict, Any, List, Tuple
+
+
+def expand(parameters: Dict[str, Any], run_id: bool = False) -> List[Dict[str, Any]]:
+    simple = {}
+    constrained = {}
+    fixed = {}
+
+    for k, v in parameters.items():
+        if not isinstance(v, list):
+            fixed[k] = v
+            continue
+
+        if k.startswith("constrained__"):
+            constrained[k] = v
+        else:
+            simple[k] = v
+
+    simple_expansion = _expand_simple(simple)
+    constrained_expansion = _expand_constrained(constrained)
+
+    if not constrained_expansion:
+        final_expansion = [dict(item, **fixed) for item in simple_expansion]
+    else:
+        final_expansion = [
+            dict(simple + constrained, **fixed) for simple, constrained in
+            itertools.product(simple_expansion, constrained_expansion)
+        ]
+
+    if run_id:
+        for i, item in enumerate(final_expansion, start=1):
+            item["runId"] = i
+
+    return final_expansion
+
+
+def _expand_simple(expandable: Dict[str, List[Any]]) -> List[List[Tuple[str, List[Any]]]]:
+    keys = expandable.keys()
+    return [
+        list(zip(keys, list(value_set)))
+        for value_set in itertools.product(*expandable.values())
+    ]
+
+
+def _expand_constrained(constrained: Dict[str, List[Any]]) -> List[List[Tuple[str, List[Any]]]]:
+    return [
+        expansion
+        for k, v in constrained.items()
+        for expansion in _expand_single_constraint(k, v)
+    ]
+
+
+def _expand_single_constraint(combined_key: str,
+                              values: List[List[Any]]) -> List[List[Tuple[str, List[Any]]]]:
+    keys = combined_key[len('constrained__'):].split('_')
+    if len(keys) < 2:
+        raise ValueError(f'Invalid combined key {combined_key}')
+
+    normalized_values = [_normalize_values(value_set) for value_set in values]
+
+    return [
+        expansion
+        for value_sets in normalized_values
+        for expansion in _expand_simple(dict(zip(keys, value_sets)))
+    ]
+
+
+def _normalize_values(values: List[Any | List[Any]]) -> List[List[Any]]:
+    return [
+        value if isinstance(value, list) else [value]
+        for value in values
+    ]
+
+
+def normalize_argo_params(argo_params: List[Dict[str, Any]]) -> Dict[str, Any]:
+    for param in argo_params:
+        try:
+            param["value"] = json.loads(param["value"])
+        except JSONDecodeError:
+            pass
+
+    return {param["name"]: param["value"] for param in argo_params}
+
+
+if __name__ == "__main__":
+
+    if len(sys.argv) < 2:
+        print(f"Usage: {sys.argv[0]} '<json_string>'")
+        sys.exit(1)
+
+    try:
+        params = normalize_argo_params(json.loads(sys.argv[1]))
+        print(json.dumps(expand(params, run_id=True)))
+    except JSONDecodeError as err:
+        print("Error decoding JSON: ", err)
+        print("Input:", sys.argv[1])
+        sys.exit(1)

benchmarks/k8s/parameter_matrix.py

@@ -1,40 +0,0 @@
-import itertools
-import json
-import sys
-from json import JSONDecodeError
-from typing import Dict, Any, List
-
-
-class ParameterMatrix:
-    def __init__(self, parameters: Dict[str, Any]):
-        self.parameters = parameters
-
-    def expand(self, run_id: bool = False) -> List[Dict[str, Any]]:
-        expandable = {k: v for k, v in self.parameters.items() if isinstance(v, list)}
-        fixed = {k: v for k, v in self.parameters.items() if k not in expandable}
-        expansion = [
-            dict(zip(expandable.keys(), values), **fixed)
-            for values in itertools.product(*expandable.values())
-        ]
-
-        if run_id:
-            for i, item in enumerate(expansion, start=1):
-                item["runId"] = i
-
-        return expansion
-
-
-if __name__ == "__main__":
-
-    if len(sys.argv) < 2:
-        print(f"Usage: {sys.argv[0]} '<json_string>'")
-        sys.exit(1)
-
-    try:
-        matrix_str = json.loads(sys.argv[1])
-    except JSONDecodeError as err:
-        print(f"Error decoding JSON: ", err)
-        print("Input:", sys.argv[1])
-        sys.exit(1)
-
-    print(json.dumps(ParameterMatrix(matrix_str).expand()))

benchmarks/k8s/tests/test_parameter_expander.py

@@ -0,0 +1,86 @@
+import json
+
+from benchmarks.k8s import parameter_expander as expander
+from benchmarks.k8s.parameter_expander import normalize_argo_params
+
+
+def test_should_expand_simple_parameter_lists():
+    matrix = {
+        "a": [1, 2],
+        "b": [3, 4],
+        "c": "foo",
+        "d": 5
+    }
+
+    assert expander.expand(matrix) == [
+        {"a": 1, "b": 3, "c": "foo", "d": 5},
+        {"a": 1, "b": 4, "c": "foo", "d": 5},
+        {"a": 2, "b": 3, "c": "foo", "d": 5},
+        {"a": 2, "b": 4, "c": "foo", "d": 5},
+    ]
+
+
+def test_should_add_run_id_when_requested():
+    matrix = {
+        "a": [1, 2],
+        "b": [3, 4],
+        "c": "foo",
+        "d": 5
+    }
+
+    assert expander.expand(matrix, run_id=True) == [
+        {"a": 1, "b": 3, "c": "foo", "d": 5, "runId": 1},
+        {"a": 1, "b": 4, "c": "foo", "d": 5, "runId": 2},
+        {"a": 2, "b": 3, "c": "foo", "d": 5, "runId": 3},
+        {"a": 2, "b": 4, "c": "foo", "d": 5, "runId": 4},
+    ]
+
+
+def test_should_expand_constrained_parameter_pairs():
+    matrix = {
+        "constrained__att1_att2": [
+            [1, [2, 3]],
+            [[4, 5], 6]
+        ],
+        "b": [1, 2]
+    }
+
+    assert expander.expand(matrix) == [
+        {"att1": 1, "att2": 2, "b": 1},
+        {"att1": 1, "att2": 3, "b": 1},
+        {"att1": 4, "att2": 6, "b": 1},
+        {"att1": 5, "att2": 6, "b": 1},
+        {"att1": 1, "att2": 2, "b": 2},
+        {"att1": 1, "att2": 3, "b": 2},
+        {"att1": 4, "att2": 6, "b": 2},
+        {"att1": 5, "att2": 6, "b": 2},
+    ]
+
+
+def test_should_normalize_simple_argo_parameter_list():
+    argo_params = json.loads('[{"name":"repetitions","value":"1"},{"name":"fileSize","value":"100MB"},'
+                             '{"name":"networkSize","value":"5"},{"name":"seeders","value":"1"},'
+                             '{"name":"seederSets","value":"1"},{"name":"maxExperimentDuration","value":"72h"}]')
+
+    assert normalize_argo_params(argo_params) == {
+        "repetitions": 1,
+        "fileSize": "100MB",
+        "networkSize": 5,
+        "seeders": 1,
+        "seederSets": 1,
+        "maxExperimentDuration": "72h",
+    }
+
+
+def test_should_find_and_pre_expand_lists_encoded_as_strings():
+    argo_params = [
+        {"name": "a", "value": "[1, 2]"},
+        {"name": "b", "value": "[1, [2, 3]]"},
+        {"name": "c", "value": "foo"},
+    ]
+
+    assert normalize_argo_params(argo_params) == {
+        "a": [1, 2],
+        "b": [1, [2, 3]],
+        "c": "foo",
+    }

benchmarks/k8s/tests/test_parameter_matrix.py

@@ -1,36 +0,0 @@
-from benchmarks.k8s.parameter_matrix import ParameterMatrix
-
-
-def test_should_expand_simple_parameter_lists():
-    matrix = ParameterMatrix(
-        {
-            "a": [1, 2],
-            "b": [3, 4],
-            "c": "foo",
-            "d": 5
-        }
-    )
-
-    assert matrix.expand() == [
-        {"a": 1, "b": 3, "c": "foo", "d": 5},
-        {"a": 1, "b": 4, "c": "foo", "d": 5},
-        {"a": 2, "b": 3, "c": "foo", "d": 5},
-        {"a": 2, "b": 4, "c": "foo", "d": 5},
-    ]
-
-def test_should_add_run_id_when_requested():
-    matrix = ParameterMatrix(
-        {
-            "a": [1, 2],
-            "b": [3, 4],
-            "c": "foo",
-            "d": 5
-        }
-    )
-
-    assert matrix.expand(run_id=True) == [
-        {"a": 1, "b": 3, "c": "foo", "d": 5, "runId": 1},
-        {"a": 1, "b": 4, "c": "foo", "d": 5, "runId": 2},
-        {"a": 2, "b": 3, "c": "foo", "d": 5, "runId": 3},
-        {"a": 2, "b": 4, "c": "foo", "d": 5, "runId": 4},
-    ]

docker/bittorrent-benchmarks-workflows.Dockerfile

@@ -15,4 +15,4 @@ WORKDIR /opt/bittorrent-benchmarks
 
 COPY ./k8s ./k8s
 COPY ./docker ./docker
-COPY ./benchmarks/k8s/parameter_matrix.py .
+COPY ./benchmarks/k8s/parameter_expander.py .