status-im
/
eth2.0-specs
mirror of https://github.com/status-im/eth2.0-specs.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

config as dataclass

This commit is contained in:
protolambda 2021-05-18 12:48:42 +02:00
parent 6f68913e11
commit ccc6679e21
No known key found for this signature in database
GPG Key ID: EC89FDBB2B4C7623
3 changed files with 26 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
setup.py
View File

@ -40,11 +40,6 @@ PHASE0 = 'phase0'
ALTAIR = 'altair'
MERGE = 'merge'
CONFIG_LOADER = '''
PRESET_BASE = 'mainnet'
apply_constants_config(globals())
'''
# The helper functions that are used when defining constants
CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = '''
def ceillog2(x: int) -> uint64:
@ -333,7 +328,6 @@ from typing import (
Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar
)
from eth2spec.config.config_util import apply_constants_config
from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes
from eth2spec.utils.ssz.ssz_typing import (
View, boolean, Container, List, Vector, uint8, uint32, uint64,
@ -347,7 +341,6 @@ from eth2spec.utils.hash_function import hash
def preparations(cls) -> str:
return '''
SSZObject = TypeVar('SSZObject', bound=View)
CONFIG_NAME = 'mainnet'
'''
@classmethod
@ -598,18 +591,30 @@ def objects_to_spec(spec_object: SpecObject, builder: SpecBuilder, ordered_class
del spec_object.functions[k]
functions_spec = '\n\n\n'.join(spec_object.functions.values())
# Access global dict of config vars for runtime configurables
for name in spec_object.config_vars.keys():
functions_spec = functions_spec.replace(name, 'config.' + name)
def format_config_var(name: str, vardef: VariableDefinition) -> str:
out = f'{name}={vardef.type_name}({vardef.value}),'
if vardef.comment is not None:
out += f' # {vardef.comment}'
return out
config_spec = '@dataclass\nclass Configuration(object):\n'
config_spec += '\n'.join(f' {k}: {v.type_name}' for k, v in spec_object.config_vars.items())
config_spec += '\n\n\nconfig = Configuration(\n'
config_spec += '\n'.join(' ' + format_config_var(k, v) for k, v in spec_object.config_vars.items())
config_spec += '\n)\n'
def format_constant(name: str, vardef: VariableDefinition) -> str:
if not hasattr(vardef, 'value'):
print(vardef)
raise Exception("oh no")
out = f'{name} = {vardef.type_name}({vardef.value})'
if vardef.comment is not None:
out += f'# {vardef.comment}'
out += f' # {vardef.comment}'
return out
constant_vars_spec = '# Constant vars \n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items())
preset_vars_spec = '# Preset vars \n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items())
config_vars_spec = '# Config vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.config_vars.items()) # TODO make config reloading easier.
ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants()))
ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants()))
@ -626,8 +631,7 @@ def objects_to_spec(spec_object: SpecObject, builder: SpecBuilder, ordered_class
+ ('\n\n' + ssz_dep_constants if ssz_dep_constants != '' else '')
+ '\n\n' + constant_vars_spec
+ '\n\n' + preset_vars_spec
+ '\n\n' + config_vars_spec
+ '\n\n' + CONFIG_LOADER
+ '\n\n\n' + config_spec
+ '\n\n' + ordered_class_objects_spec
+ ('\n\n\n' + protocols_spec if protocols_spec != '' else '')
+ '\n\n\n' + functions_spec

19
tests/core/pyspec/eth2spec/config/README.md
View File

@ -6,22 +6,15 @@ For configuration, see [Configs documentation](../../../../../configs/README.md)
```python
from eth2spec.config import config_util
from eth2spec.phase0 import spec
from importlib import reload
from eth2spec.phase0.mainnet import as spec
from pathlib import Path
# To load the presets and configurations
# To load the default configurations
config_util.load_defaults(Path("eth2.0-specs/configs")) # change path to point to equivalent of specs `configs` dir.
# After loading the defaults, a config can be chosen: 'mainnet', 'minimal', or custom network config
config_util.prepare_config('minimal')
# Alternatively, load a custom testnet config:
config_util.prepare_config('my_config.yaml')
# reload spec to make loaded config effective
reload(spec)
# After loading the defaults, a config can be chosen: 'mainnet', 'minimal', or custom network config (by file path)
spec.config = spec.Configuration(**config_util.load_config_file('mytestnet.yaml'))
```
Note: previously the testnet config files included both preset and runtime-configuration data.
The new config loader is compatible with this: just run `prepare_config` without loading preset defaults,
and omit the `PRESET_BASE` from the config.
WARNING: this overwrites globals, make sure to prevent accidental collisions with other usage of the same imported specs package.
The new config loader is compatible with this: all config vars are loaded from the file,
but those that have become presets will be ignored.

56
tests/core/pyspec/eth2spec/config/config_util.py
View File

@ -1,52 +1,7 @@
import os
from pathlib import Path
from copy import deepcopy
from typing import Dict, Iterable, Union, BinaryIO, TextIO, Literal, Any
from typing import Dict, Iterable, Union, BinaryIO, TextIO, Any
from ruamel.yaml import YAML
# This holds the full config (both runtime config and compile-time preset), for specs to initialize
config: Dict[str, Any] = {}
# Access to overwrite spec constants based on configuration
# This is called by the spec module after declaring its globals, and applies the loaded presets.
def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = False) -> None:
global config
for k, v in config.items():
# the spec should have default values for everything, if not, the config key is invalid.
if k in spec_globals:
# Keep the same type as the default value indicates (which may be an SSZ basic type subclass, e.g. 'Gwei')
spec_globals[k] = spec_globals[k].__class__(v)
else:
# Note: The phase 0 spec will not warn if Altair or later config values are applied.
# During debugging you can enable explicit warnings.
if warn_if_unknown:
print(f"WARNING: unknown config key: '{k}' with value: '{v}'")
# Load YAML configuration from a file path or input, or pick the default 'mainnet' and 'minimal' configs.
# This prepares the global config memory. This does not apply the config.
# To apply the config, reload the spec module (it will re-initialize with the config taken from here).
def prepare_config(config_path: Union[Path, BinaryIO, TextIO, Literal['mainnet'], Literal['minimal']]) -> None:
# Load the configuration, and try in-memory defaults.
if config_path == 'mainnet':
conf_data = deepcopy(mainnet_config_data)
elif config_path == 'minimal':
conf_data = deepcopy(minimal_config_data)
else:
conf_data = load_config_file(config_path)
# Apply configuration if everything checks out
global config
if 'PRESET_BASE' in conf_data:
# Check the configured preset
base = conf_data['PRESET_BASE']
if base not in ('minimal', 'mainnet'):
raise Exception(f"unknown PRESET_BASE: {base}")
config = deepcopy(mainnet_preset_data if base == 'mainnet' else minimal_preset_data)
config.update(conf_data)
else:
config = conf_data
def parse_config_vars(conf: Dict[str, Any]) -> Dict[str, Any]:
"""
@ -93,21 +48,14 @@ def load_config_file(config_path: Union[Path, BinaryIO, TextIO]) -> Dict[str, An
return parse_config_vars(config_data)
# Can't load these with pkg_resources, because the files are not in a package (requires `__init__.py`).
mainnet_preset_data: Dict[str, Any]
minimal_preset_data: Dict[str, Any]
mainnet_config_data: Dict[str, Any]
minimal_config_data: Dict[str, Any]
loaded_defaults = False
def load_defaults(spec_configs_path: Path) -> None:
global mainnet_preset_data, minimal_preset_data, mainnet_config_data, minimal_config_data
global mainnet_config_data, minimal_config_data
_, _, mainnet_preset_file_names = next(os.walk(spec_configs_path / 'mainnet_preset'))
mainnet_preset_data = load_preset([spec_configs_path / 'mainnet_preset' / p for p in mainnet_preset_file_names])
_, _, minimal_preset_file_names = next(os.walk(spec_configs_path / 'minimal_preset'))
minimal_preset_data = load_preset([spec_configs_path / 'minimal_preset' / p for p in minimal_preset_file_names])
mainnet_config_data = load_config_file(spec_configs_path / 'mainnet_config.yaml')
minimal_config_data = load_config_file(spec_configs_path / 'minimal_config.yaml')