diff --git a/benchmarks/k8s/parameter_expander.py b/benchmarks/k8s/parameter_expander.py new file mode 100644 index 0000000..4cf0a77 --- /dev/null +++ b/benchmarks/k8s/parameter_expander.py @@ -0,0 +1,101 @@ +import itertools +import json +import sys +from json import JSONDecodeError +from typing import Dict, Any, List, Tuple + + +def expand(parameters: Dict[str, Any], run_id: bool = False) -> List[Dict[str, Any]]: + simple = {} + constrained = {} + fixed = {} + + for k, v in parameters.items(): + if not isinstance(v, list): + fixed[k] = v + continue + + if k.startswith("constrained__"): + constrained[k] = v + else: + simple[k] = v + + simple_expansion = _expand_simple(simple) + constrained_expansion = _expand_constrained(constrained) + + if not constrained_expansion: + final_expansion = [dict(item, **fixed) for item in simple_expansion] + else: + final_expansion = [ + dict(simple + constrained, **fixed) for simple, constrained in + itertools.product(simple_expansion, constrained_expansion) + ] + + if run_id: + for i, item in enumerate(final_expansion, start=1): + item["runId"] = i + + return final_expansion + + +def _expand_simple(expandable: Dict[str, List[Any]]) -> List[List[Tuple[str, List[Any]]]]: + keys = expandable.keys() + return [ + list(zip(keys, list(value_set))) + for value_set in itertools.product(*expandable.values()) + ] + + +def _expand_constrained(constrained: Dict[str, List[Any]]) -> List[List[Tuple[str, List[Any]]]]: + return [ + expansion + for k, v in constrained.items() + for expansion in _expand_single_constraint(k, v) + ] + + +def _expand_single_constraint(combined_key: str, + values: List[List[Any]]) -> List[List[Tuple[str, List[Any]]]]: + keys = combined_key[len('constrained__'):].split('_') + if len(keys) < 2: + raise ValueError(f'Invalid combined key {combined_key}') + + normalized_values = [_normalize_values(value_set) for value_set in values] + + return [ + expansion + for value_sets in normalized_values + for expansion in _expand_simple(dict(zip(keys, value_sets))) + ] + + +def _normalize_values(values: List[Any | List[Any]]) -> List[List[Any]]: + return [ + value if isinstance(value, list) else [value] + for value in values + ] + + +def normalize_argo_params(argo_params: List[Dict[str, Any]]) -> Dict[str, Any]: + for param in argo_params: + try: + param["value"] = json.loads(param["value"]) + except JSONDecodeError: + pass + + return {param["name"]: param["value"] for param in argo_params} + + +if __name__ == "__main__": + + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ''") + sys.exit(1) + + try: + params = normalize_argo_params(json.loads(sys.argv[1])) + print(json.dumps(expand(params, run_id=True))) + except JSONDecodeError as err: + print("Error decoding JSON: ", err) + print("Input:", sys.argv[1]) + sys.exit(1) diff --git a/benchmarks/k8s/parameter_matrix.py b/benchmarks/k8s/parameter_matrix.py deleted file mode 100644 index da273a5..0000000 --- a/benchmarks/k8s/parameter_matrix.py +++ /dev/null @@ -1,40 +0,0 @@ -import itertools -import json -import sys -from json import JSONDecodeError -from typing import Dict, Any, List - - -class ParameterMatrix: - def __init__(self, parameters: Dict[str, Any]): - self.parameters = parameters - - def expand(self, run_id: bool = False) -> List[Dict[str, Any]]: - expandable = {k: v for k, v in self.parameters.items() if isinstance(v, list)} - fixed = {k: v for k, v in self.parameters.items() if k not in expandable} - expansion = [ - dict(zip(expandable.keys(), values), **fixed) - for values in itertools.product(*expandable.values()) - ] - - if run_id: - for i, item in enumerate(expansion, start=1): - item["runId"] = i - - return expansion - - -if __name__ == "__main__": - - if len(sys.argv) < 2: - print(f"Usage: {sys.argv[0]} ''") - sys.exit(1) - - try: - matrix_str = json.loads(sys.argv[1]) - except JSONDecodeError as err: - print(f"Error decoding JSON: ", err) - print("Input:", sys.argv[1]) - sys.exit(1) - - print(json.dumps(ParameterMatrix(matrix_str).expand())) diff --git a/benchmarks/k8s/tests/test_parameter_expander.py b/benchmarks/k8s/tests/test_parameter_expander.py new file mode 100644 index 0000000..e688eb8 --- /dev/null +++ b/benchmarks/k8s/tests/test_parameter_expander.py @@ -0,0 +1,86 @@ +import json + +from benchmarks.k8s import parameter_expander as expander +from benchmarks.k8s.parameter_expander import normalize_argo_params + + +def test_should_expand_simple_parameter_lists(): + matrix = { + "a": [1, 2], + "b": [3, 4], + "c": "foo", + "d": 5 + } + + assert expander.expand(matrix) == [ + {"a": 1, "b": 3, "c": "foo", "d": 5}, + {"a": 1, "b": 4, "c": "foo", "d": 5}, + {"a": 2, "b": 3, "c": "foo", "d": 5}, + {"a": 2, "b": 4, "c": "foo", "d": 5}, + ] + + +def test_should_add_run_id_when_requested(): + matrix = { + "a": [1, 2], + "b": [3, 4], + "c": "foo", + "d": 5 + } + + assert expander.expand(matrix, run_id=True) == [ + {"a": 1, "b": 3, "c": "foo", "d": 5, "runId": 1}, + {"a": 1, "b": 4, "c": "foo", "d": 5, "runId": 2}, + {"a": 2, "b": 3, "c": "foo", "d": 5, "runId": 3}, + {"a": 2, "b": 4, "c": "foo", "d": 5, "runId": 4}, + ] + + +def test_should_expand_constrained_parameter_pairs(): + matrix = { + "constrained__att1_att2": [ + [1, [2, 3]], + [[4, 5], 6] + ], + "b": [1, 2] + } + + assert expander.expand(matrix) == [ + {"att1": 1, "att2": 2, "b": 1}, + {"att1": 1, "att2": 3, "b": 1}, + {"att1": 4, "att2": 6, "b": 1}, + {"att1": 5, "att2": 6, "b": 1}, + {"att1": 1, "att2": 2, "b": 2}, + {"att1": 1, "att2": 3, "b": 2}, + {"att1": 4, "att2": 6, "b": 2}, + {"att1": 5, "att2": 6, "b": 2}, + ] + + +def test_should_normalize_simple_argo_parameter_list(): + argo_params = json.loads('[{"name":"repetitions","value":"1"},{"name":"fileSize","value":"100MB"},' + '{"name":"networkSize","value":"5"},{"name":"seeders","value":"1"},' + '{"name":"seederSets","value":"1"},{"name":"maxExperimentDuration","value":"72h"}]') + + assert normalize_argo_params(argo_params) == { + "repetitions": 1, + "fileSize": "100MB", + "networkSize": 5, + "seeders": 1, + "seederSets": 1, + "maxExperimentDuration": "72h", + } + + +def test_should_find_and_pre_expand_lists_encoded_as_strings(): + argo_params = [ + {"name": "a", "value": "[1, 2]"}, + {"name": "b", "value": "[1, [2, 3]]"}, + {"name": "c", "value": "foo"}, + ] + + assert normalize_argo_params(argo_params) == { + "a": [1, 2], + "b": [1, [2, 3]], + "c": "foo", + } diff --git a/benchmarks/k8s/tests/test_parameter_matrix.py b/benchmarks/k8s/tests/test_parameter_matrix.py deleted file mode 100644 index 806363d..0000000 --- a/benchmarks/k8s/tests/test_parameter_matrix.py +++ /dev/null @@ -1,36 +0,0 @@ -from benchmarks.k8s.parameter_matrix import ParameterMatrix - - -def test_should_expand_simple_parameter_lists(): - matrix = ParameterMatrix( - { - "a": [1, 2], - "b": [3, 4], - "c": "foo", - "d": 5 - } - ) - - assert matrix.expand() == [ - {"a": 1, "b": 3, "c": "foo", "d": 5}, - {"a": 1, "b": 4, "c": "foo", "d": 5}, - {"a": 2, "b": 3, "c": "foo", "d": 5}, - {"a": 2, "b": 4, "c": "foo", "d": 5}, - ] - -def test_should_add_run_id_when_requested(): - matrix = ParameterMatrix( - { - "a": [1, 2], - "b": [3, 4], - "c": "foo", - "d": 5 - } - ) - - assert matrix.expand(run_id=True) == [ - {"a": 1, "b": 3, "c": "foo", "d": 5, "runId": 1}, - {"a": 1, "b": 4, "c": "foo", "d": 5, "runId": 2}, - {"a": 2, "b": 3, "c": "foo", "d": 5, "runId": 3}, - {"a": 2, "b": 4, "c": "foo", "d": 5, "runId": 4}, - ] \ No newline at end of file diff --git a/docker/bittorrent-benchmarks-workflows.Dockerfile b/docker/bittorrent-benchmarks-workflows.Dockerfile index ec0986c..34272cd 100644 --- a/docker/bittorrent-benchmarks-workflows.Dockerfile +++ b/docker/bittorrent-benchmarks-workflows.Dockerfile @@ -15,4 +15,4 @@ WORKDIR /opt/bittorrent-benchmarks COPY ./k8s ./k8s COPY ./docker ./docker -COPY ./benchmarks/k8s/parameter_matrix.py . +COPY ./benchmarks/k8s/parameter_expander.py .