updated config util

setup.py

@@ -31,6 +31,7 @@ ALTAIR = 'altair'
 MERGE = 'merge'
 
 CONFIG_LOADER = '''
+PRESET_BASE = 'mainnet'
 apply_constants_config(globals())
 '''
 

tests/core/pyspec/eth2spec/config/config_util.py

@@ -1,9 +1,11 @@
 import os
 from pathlib import Path
-from typing import Dict, Any
-
+from copy import deepcopy
+from contextlib import ExitStack
+from typing import Dict, Iterable, Union, BinaryIO, TextIO, Literal, Any
 from ruamel.yaml import YAML
 
+# This holds the full config (both runtime config and compile-time preset), for specs to initialize
 config: Dict[str, Any] = {}
 
 
@@ -23,38 +25,88 @@ def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool =
                 print(f"WARNING: unknown config key: '{k}' with value: '{v}'")
 
 
-# Load presets from a file, and then prepares the global config setting. This does not apply the config.
+# Load YAML configuration from a file path or input, or pick the default 'mainnet' and 'minimal' configs.
+# This prepares the global config memory. This does not apply the config.
 # To apply the config, reload the spec module (it will re-initialize with the config taken from here).
-def prepare_config(configs_path: str, config_name: str) -> None:
+def prepare_config(config_path: Union[Path, BinaryIO, TextIO, Literal['mainnet'], Literal['minimal']]) -> None:
+    # Load the configuration, and try in-memory defaults.
+    if config_path == 'mainnet':
+        conf_data = deepcopy(mainnet_config_data)
+    elif config_path == 'minimal':
+        conf_data = deepcopy(minimal_config_data)
+    else:
+        conf_data = load_config_file(config_path)
+    # Check the configured preset
+    base = conf_data['PRESET_BASE']
+    if base not in ('minimal', 'mainnet'):
+        raise Exception(f"unknown PRESET_BASE: {base}")
+    # Apply configuration if everything checks out
     global config
-    config = load_config_file(configs_path, config_name)
+    config = deepcopy(mainnet_preset_data if base == 'mainnet' else minimal_preset_data)
+    config.update(conf_data)
 
 
-def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]:
+def parse_config_vars(conf: Dict[str, Any]) -> Dict[str, Any]:
     """
-    Loads the given preset
-    :param presets_name: The name of the presets. (lowercase snake_case)
-    :return: Dictionary, mapping of constant-name -> constant-value
+    Parses a dict of basic str/int/list types into more detailed python types
     """
-    present_dir = Path(configs_dir) / presets_name
-    _, _, config_files = next(os.walk(present_dir))
-    config_files.sort()
-    loaded_config = {}
-    for config_file_name in config_files:
-        yaml = YAML(typ='base')
-        path = present_dir / config_file_name
-        loaded = yaml.load(path)
-        loaded_config.update(loaded)
-    assert loaded_config != {}
-
     out: Dict[str, Any] = dict()
-    for k, v in loaded_config.items():
+    for k, v in conf.items():
         if isinstance(v, list):
             # Clean up integer values. YAML parser renders lists of ints as list of str
             out[k] = [int(item) if item.isdigit() else item for item in v]
         elif isinstance(v, str) and v.startswith("0x"):
             out[k] = bytes.fromhex(v[2:])
-        else:
+        elif k != 'PRESET_BASE':
             out[k] = int(v)
-    out['CONFIG_NAME'] = presets_name
+        else:
+            out[k] = v
     return out
+
+
+def load_preset(preset_files: Iterable[Union[Path, BinaryIO, TextIO]]) -> Dict[str, Any]:
+    """
+    Loads the a directory of preset files, merges the result into one preset.
+    """
+    preset = {}
+    for fork_file in preset_files:
+        yaml = YAML(typ='base')
+        fork_preset: dict = yaml.load(fork_file)
+        if fork_preset is None:  # for empty YAML files
+            continue
+        if not set(fork_preset.keys()).isdisjoint(preset.keys()):
+            duplicates = set(fork_preset.keys()).intersection(set(preset.keys()))
+            raise Exception(f"duplicate config var(s) in preset files: {', '.join(duplicates)}")
+        preset.update(fork_preset)
+    assert preset != {}
+    return parse_config_vars(preset)
+
+
+def load_config_file(config_path: Union[Path, BinaryIO, TextIO]) -> Dict[str, Any]:
+    """
+    Loads the given configuration file.
+    """
+    yaml = YAML(typ='base')
+    config_data = yaml.load(config_path)
+    return parse_config_vars(config_data)
+
+
+# Can't load these with pkg_resources, because the files are not in a package (requires `__init__.py`).
+mainnet_preset_data: Dict[str, Any]
+minimal_preset_data: Dict[str, Any]
+mainnet_config_data: Dict[str, Any]
+minimal_config_data: Dict[str, Any]
+loaded_defaults = False
+
+def load_defaults(spec_configs_path: Path) -> None:
+    global mainnet_preset_data, minimal_preset_data, mainnet_config_data, minimal_config_data
+
+    _, _, mainnet_preset_file_names = next(os.walk(spec_configs_path / 'mainnet_preset'))
+    mainnet_preset_data = load_preset([spec_configs_path / 'mainnet_preset' / p for p in mainnet_preset_file_names])
+    _, _, minimal_preset_file_names = next(os.walk(spec_configs_path / 'minimal_preset'))
+    minimal_preset_data = load_preset([spec_configs_path / 'minimal_preset' / p for p in minimal_preset_file_names])
+    mainnet_config_data = load_config_file(spec_configs_path / 'mainnet_config.yaml')
+    minimal_config_data = load_config_file(spec_configs_path / 'minimal_config.yaml')
+
+    global loaded_defaults
+    loaded_defaults = True

tests/core/pyspec/eth2spec/test/conftest.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from eth2spec.config import config_util
 from eth2spec.test import context
 from eth2spec.utils import bls as bls_utils
@@ -42,8 +43,15 @@ def pytest_addoption(parser):
 
 @fixture(autouse=True)
 def config(request):
-    config_name = request.config.getoption("--config")
-    config_util.prepare_config('../../../configs/', config_name)
+    if not config_util.loaded_defaults:
+        config_util.load_defaults(Path("../../../configs"))
+
+    config_flag_value = request.config.getoption("--config")
+    if config_flag_value in ('minimal', 'mainnet'):
+        config_util.prepare_config(config_flag_value)
+    else:
+        # absolute network config path, e.g. run tests with testnet config
+        config_util.prepare_config(Path(config_flag_value))
     # now that the presets are loaded, reload the specs to apply them
     context.reload_specs()
 

tests/core/pyspec/eth2spec/test/helpers/constants.py

@@ -1,4 +1,4 @@
-from .typing import SpecForkName, ConfigName
+from .typing import SpecForkName, PresetBaseName
 
 
 #
@@ -28,7 +28,7 @@ FORKS_BEFORE_MERGE = (PHASE0,)
 #
 # Config
 #
-MAINNET = ConfigName('mainnet')
-MINIMAL = ConfigName('minimal')
+MAINNET = PresetBaseName('mainnet')
+MINIMAL = PresetBaseName('minimal')
 
 ALL_CONFIGS = (MINIMAL, MAINNET)

tests/core/pyspec/eth2spec/test/helpers/typing.py

@@ -1,4 +1,4 @@
 from typing import NewType
 
 SpecForkName = NewType("SpecForkName", str)
-ConfigName = NewType("ConfigName", str)
+PresetBaseName = NewType("PresetBaseName", str)