From 79d0fa037f95a0c9cf29fed6db405ed817aed351 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 7 May 2021 05:30:26 +0200 Subject: [PATCH] updated config util --- setup.py | 1 + .../pyspec/eth2spec/config/config_util.py | 98 ++++++++++++++----- tests/core/pyspec/eth2spec/test/conftest.py | 12 ++- .../pyspec/eth2spec/test/helpers/constants.py | 6 +- .../pyspec/eth2spec/test/helpers/typing.py | 2 +- 5 files changed, 90 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 0bdf04b55..811a750d1 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ ALTAIR = 'altair' MERGE = 'merge' CONFIG_LOADER = ''' +PRESET_BASE = 'mainnet' apply_constants_config(globals()) ''' diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index 917cf3a60..01b272f17 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -1,9 +1,11 @@ import os from pathlib import Path -from typing import Dict, Any - +from copy import deepcopy +from contextlib import ExitStack +from typing import Dict, Iterable, Union, BinaryIO, TextIO, Literal, Any from ruamel.yaml import YAML +# This holds the full config (both runtime config and compile-time preset), for specs to initialize config: Dict[str, Any] = {} @@ -23,38 +25,88 @@ def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = print(f"WARNING: unknown config key: '{k}' with value: '{v}'") -# Load presets from a file, and then prepares the global config setting. This does not apply the config. +# Load YAML configuration from a file path or input, or pick the default 'mainnet' and 'minimal' configs. +# This prepares the global config memory. This does not apply the config. # To apply the config, reload the spec module (it will re-initialize with the config taken from here). -def prepare_config(configs_path: str, config_name: str) -> None: +def prepare_config(config_path: Union[Path, BinaryIO, TextIO, Literal['mainnet'], Literal['minimal']]) -> None: + # Load the configuration, and try in-memory defaults. + if config_path == 'mainnet': + conf_data = deepcopy(mainnet_config_data) + elif config_path == 'minimal': + conf_data = deepcopy(minimal_config_data) + else: + conf_data = load_config_file(config_path) + # Check the configured preset + base = conf_data['PRESET_BASE'] + if base not in ('minimal', 'mainnet'): + raise Exception(f"unknown PRESET_BASE: {base}") + # Apply configuration if everything checks out global config - config = load_config_file(configs_path, config_name) + config = deepcopy(mainnet_preset_data if base == 'mainnet' else minimal_preset_data) + config.update(conf_data) -def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: +def parse_config_vars(conf: Dict[str, Any]) -> Dict[str, Any]: """ - Loads the given preset - :param presets_name: The name of the presets. (lowercase snake_case) - :return: Dictionary, mapping of constant-name -> constant-value + Parses a dict of basic str/int/list types into more detailed python types """ - present_dir = Path(configs_dir) / presets_name - _, _, config_files = next(os.walk(present_dir)) - config_files.sort() - loaded_config = {} - for config_file_name in config_files: - yaml = YAML(typ='base') - path = present_dir / config_file_name - loaded = yaml.load(path) - loaded_config.update(loaded) - assert loaded_config != {} - out: Dict[str, Any] = dict() - for k, v in loaded_config.items(): + for k, v in conf.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str out[k] = [int(item) if item.isdigit() else item for item in v] elif isinstance(v, str) and v.startswith("0x"): out[k] = bytes.fromhex(v[2:]) - else: + elif k != 'PRESET_BASE': out[k] = int(v) - out['CONFIG_NAME'] = presets_name + else: + out[k] = v return out + + +def load_preset(preset_files: Iterable[Union[Path, BinaryIO, TextIO]]) -> Dict[str, Any]: + """ + Loads the a directory of preset files, merges the result into one preset. + """ + preset = {} + for fork_file in preset_files: + yaml = YAML(typ='base') + fork_preset: dict = yaml.load(fork_file) + if fork_preset is None: # for empty YAML files + continue + if not set(fork_preset.keys()).isdisjoint(preset.keys()): + duplicates = set(fork_preset.keys()).intersection(set(preset.keys())) + raise Exception(f"duplicate config var(s) in preset files: {', '.join(duplicates)}") + preset.update(fork_preset) + assert preset != {} + return parse_config_vars(preset) + + +def load_config_file(config_path: Union[Path, BinaryIO, TextIO]) -> Dict[str, Any]: + """ + Loads the given configuration file. + """ + yaml = YAML(typ='base') + config_data = yaml.load(config_path) + return parse_config_vars(config_data) + + +# Can't load these with pkg_resources, because the files are not in a package (requires `__init__.py`). +mainnet_preset_data: Dict[str, Any] +minimal_preset_data: Dict[str, Any] +mainnet_config_data: Dict[str, Any] +minimal_config_data: Dict[str, Any] +loaded_defaults = False + +def load_defaults(spec_configs_path: Path) -> None: + global mainnet_preset_data, minimal_preset_data, mainnet_config_data, minimal_config_data + + _, _, mainnet_preset_file_names = next(os.walk(spec_configs_path / 'mainnet_preset')) + mainnet_preset_data = load_preset([spec_configs_path / 'mainnet_preset' / p for p in mainnet_preset_file_names]) + _, _, minimal_preset_file_names = next(os.walk(spec_configs_path / 'minimal_preset')) + minimal_preset_data = load_preset([spec_configs_path / 'minimal_preset' / p for p in minimal_preset_file_names]) + mainnet_config_data = load_config_file(spec_configs_path / 'mainnet_config.yaml') + minimal_config_data = load_config_file(spec_configs_path / 'minimal_config.yaml') + + global loaded_defaults + loaded_defaults = True diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index b961ee082..ca7516a2e 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -1,3 +1,4 @@ +from pathlib import Path from eth2spec.config import config_util from eth2spec.test import context from eth2spec.utils import bls as bls_utils @@ -42,8 +43,15 @@ def pytest_addoption(parser): @fixture(autouse=True) def config(request): - config_name = request.config.getoption("--config") - config_util.prepare_config('../../../configs/', config_name) + if not config_util.loaded_defaults: + config_util.load_defaults(Path("../../../configs")) + + config_flag_value = request.config.getoption("--config") + if config_flag_value in ('minimal', 'mainnet'): + config_util.prepare_config(config_flag_value) + else: + # absolute network config path, e.g. run tests with testnet config + config_util.prepare_config(Path(config_flag_value)) # now that the presets are loaded, reload the specs to apply them context.reload_specs() diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index d8f2a37ba..2f7cb403a 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -1,4 +1,4 @@ -from .typing import SpecForkName, ConfigName +from .typing import SpecForkName, PresetBaseName # @@ -28,7 +28,7 @@ FORKS_BEFORE_MERGE = (PHASE0,) # # Config # -MAINNET = ConfigName('mainnet') -MINIMAL = ConfigName('minimal') +MAINNET = PresetBaseName('mainnet') +MINIMAL = PresetBaseName('minimal') ALL_CONFIGS = (MINIMAL, MAINNET) diff --git a/tests/core/pyspec/eth2spec/test/helpers/typing.py b/tests/core/pyspec/eth2spec/test/helpers/typing.py index 04578f64c..19657a8f7 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/typing.py +++ b/tests/core/pyspec/eth2spec/test/helpers/typing.py @@ -1,4 +1,4 @@ from typing import NewType SpecForkName = NewType("SpecForkName", str) -ConfigName = NewType("ConfigName", str) +PresetBaseName = NewType("PresetBaseName", str)