From 658ede2191e7da57a37a9c12f5c21cc4bc4c7020 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 20:34:51 +0800 Subject: [PATCH 01/13] Refactor pyspec builder with `SpecAdjustment` classes --- setup.py | 339 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 206 insertions(+), 133 deletions(-) diff --git a/setup.py b/setup.py index 514f75c50..65b04fcf2 100644 --- a/setup.py +++ b/setup.py @@ -6,15 +6,35 @@ from distutils.util import convert_path import os import re from typing import Dict, NamedTuple, List +from abc import ABC, abstractmethod + FUNCTION_REGEX = r'^def [\w_]*' - # Definitions in context.py PHASE0 = 'phase0' ALTAIR = 'altair' MERGE = 'merge' +CONFIG_LOADER = ''' +apply_constants_config(globals()) +''' + +# The helper functions that are used when defining constants +CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS = ''' +def ceillog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"ceillog2 accepts only positive values, x={x}") + return uint64((x - 1).bit_length()) + + +def floorlog2(x: int) -> uint64: + if x < 1: + raise ValueError(f"floorlog2 accepts only positive values, x={x}") + return uint64(x.bit_length() - 1) +''' + + class SpecObject(NamedTuple): functions: Dict[str, str] custom_types: Dict[str, str] @@ -111,11 +131,55 @@ def get_spec(file_name: str) -> SpecObject: ) -CONFIG_LOADER = ''' -apply_constants_config(globals()) -''' +class SpecAdjustment(ABC): + @classmethod + @abstractmethod + def imports_and_predefinitions(cls) -> str: + """ + Importing functions and defining special types/constants for building pyspec. + """ + raise NotImplementedError() -PHASE0_IMPORTS = '''from eth2spec.config.config_util import apply_constants_config + @classmethod + @abstractmethod + def sundry_functions(cls) -> str: + """ + The functions that are (1) defined abstractly in specs or (2) adjusted for getting better performance. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + """ + The constants that are required for SSZ objects. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: + """ + The constants that are required for custom types. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def invariant_checks(cls) -> str: + """ + The invariant checks + """ + raise NotImplementedError() + + +# +# Phase0SpecAdjustment +# +class Phase0SpecAdjustment(SpecAdjustment): + @classmethod + def imports_and_predefinitions(cls) -> str: + return '''from eth2spec.config.config_util import apply_constants_config from typing import ( Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar ) @@ -141,82 +205,9 @@ SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' -ALTAIR_IMPORTS = '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union -) - -from dataclasses import ( - dataclass, - field, -) - -from lru import LRU - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, - Path, -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -# Whenever altair is loaded, make sure we have the latest phase0 -from importlib import reload -reload(phase0) - - -SSZVariableName = str -GeneralizedIndex = NewType('GeneralizedIndex', int) -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' -''' - -MERGE_IMPORTS = '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - -from dataclasses import ( - dataclass, - field, -) - -from lru import LRU - -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, - Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist, - ByteList, ByteVector -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' -''' - -SUNDRY_CONSTANTS_FUNCTIONS = ''' -def ceillog2(x: int) -> uint64: - if x < 1: - raise ValueError(f"ceillog2 accepts only positive values, x={x}") - return uint64((x - 1).bit_length()) - - -def floorlog2(x: int) -> uint64: - if x < 1: - raise ValueError(f"floorlog2 accepts only positive values, x={x}") - return uint64(x.bit_length() - 1) -''' -PHASE0_SUNDRY_FUNCTIONS = ''' + @classmethod + def sundry_functions(cls) -> str: + return ''' def get_eth1_data(block: Eth1Block) -> Eth1Data: """ A stub function return mocking Eth1Data. @@ -287,9 +278,62 @@ get_attesting_indices = cache_this( ), _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + return {} -ALTAIR_SUNDRY_FUNCTIONS = ''' + @classmethod + def hardcoded_custom_type_dep_constants(cls) -> Dict[str, str]: + return {} + @classmethod + def invariant_checks(cls) -> str: + return '' + + +# +# AltairSpecAdjustment +# +class AltairSpecAdjustment(Phase0SpecAdjustment): + @classmethod + def imports_and_predefinitions(cls) -> str: + return '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union +) + +from dataclasses import ( + dataclass, + field, +) + +from lru import LRU + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + Path, +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +# Whenever altair is loaded, make sure we have the latest phase0 +from importlib import reload +reload(phase0) + + +SSZVariableName = str +GeneralizedIndex = NewType('GeneralizedIndex', int) +SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' +''' + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + ''' def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: ssz_path = Path(ssz_class) for item in path: @@ -297,7 +341,59 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariable return GeneralizedIndex(ssz_path.gindex())''' -MERGE_SUNDRY_FUNCTIONS = """ + @classmethod + def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: + constants = { + 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', + 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', + } + return {**super().hardcoded_ssz_dep_constants(), **constants} + + @classmethod + def invariant_checks(cls) -> str: + return ''' +assert ( + TIMELY_HEAD_WEIGHT + TIMELY_SOURCE_WEIGHT + TIMELY_TARGET_WEIGHT + SYNC_REWARD_WEIGHT + PROPOSER_WEIGHT +) == WEIGHT_DENOMINATOR''' + + +# +# MergeSpecAdjustment +# +class MergeSpecAdjustment(Phase0SpecAdjustment): + @classmethod + def imports_and_predefinitions(cls): + return '''from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) + +from dataclasses import ( + dataclass, + field, +) + +from lru import LRU + +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes +from eth2spec.utils.ssz.ssz_typing import ( + View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, + Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist, + ByteList, ByteVector +) +from eth2spec.utils import bls + +from eth2spec.utils.hash_function import hash + +SSZObject = TypeVar('SSZObject', bound=View) + +CONFIG_NAME = 'mainnet' +''' + + @classmethod + def sundry_functions(cls) -> str: + return super().sundry_functions() + '\n\n' + """ ExecutionState = Any @@ -321,22 +417,18 @@ def produce_execution_payload(parent_hash: Bytes32) -> ExecutionPayload: pass""" -# The constants that depend on SSZ objects -# Will verify the value at the end of the spec -ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS = { - 'FINALIZED_ROOT_INDEX': 'GeneralizedIndex(105)', - 'NEXT_SYNC_COMMITTEE_INDEX': 'GeneralizedIndex(55)', -} + @classmethod + def hardcoded_custom_type_dep_constants(cls) -> str: + constants = { + 'MAX_BYTES_PER_OPAQUE_TRANSACTION': 'uint64(2**20)', + } + return {**super().hardcoded_custom_type_dep_constants(), **constants} -ALTAIR_INVAIANT_CHECKS = ''' -assert ( - TIMELY_HEAD_WEIGHT + TIMELY_SOURCE_WEIGHT + TIMELY_TARGET_WEIGHT + SYNC_REWARD_WEIGHT + PROPOSER_WEIGHT -) == WEIGHT_DENOMINATOR''' - - -MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS = { - 'MAX_BYTES_PER_OPAQUE_TRANSACTION': 'uint64(2**20)', +spec_adjustments = { + PHASE0: Phase0SpecAdjustment, + ALTAIR: AltairSpecAdjustment, + MERGE: MergeSpecAdjustment, } @@ -352,7 +444,7 @@ def is_merge(fork): return fork == MERGE -def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str: +def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: str, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ @@ -382,41 +474,29 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl spec_object.constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) ordered_class_objects_spec = '\n\n'.join(ordered_class_objects.values()) - - if is_altair(fork): - altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - - if is_merge(fork): - merge_custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS[x]), MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS)) - - + ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_ssz_dep_constants()[x]), adjustment.hardcoded_ssz_dep_constants())) + ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), adjustment.hardcoded_ssz_dep_constants())) + custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_custom_type_dep_constants()[x]), adjustment.hardcoded_custom_type_dep_constants())) spec = ( - imports + adjustment.imports_and_predefinitions() + '\n\n' + f"fork = \'{fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` - + ('\n\n' + merge_custom_type_dep_constants + '\n' if is_merge(fork) else '') + + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + '\n\n' + new_type_definitions - + '\n' + SUNDRY_CONSTANTS_FUNCTIONS + + '\n' + CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS # The constants that some SSZ containers require. Need to be defined before `constants_spec` - + ('\n\n' + altair_ssz_dep_constants if is_altair(fork) else '') + + ('\n\n' + ssz_dep_constants if ssz_dep_constants != '' else '') + '\n\n' + constants_spec + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec - # Functions to make pyspec work - + '\n' + PHASE0_SUNDRY_FUNCTIONS - + ('\n' + ALTAIR_SUNDRY_FUNCTIONS if is_altair(fork) else '') - + ('\n' + MERGE_SUNDRY_FUNCTIONS if is_merge(fork) else '') + + '\n' + adjustment.sundry_functions() + # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are + # as same as the spec definition. + + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') + + ('\n' + adjustment.invariant_checks() if adjustment.invariant_checks() != '' else '') + + '\n' ) - - # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are - # as same as the spec definition. - if is_altair(fork): - altair_ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS)) - spec += '\n\n\n' + altair_ssz_dep_constants_verification - spec += '\n' + ALTAIR_INVAIANT_CHECKS - - spec += '\n' return spec @@ -496,13 +576,6 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: ) -fork_imports = { - 'phase0': PHASE0_IMPORTS, - 'altair': ALTAIR_IMPORTS, - 'merge': MERGE_IMPORTS, -} - - def build_spec(fork: str, source_files: List[str]) -> str: all_specs = [get_spec(spec) for spec in source_files] @@ -513,7 +586,7 @@ def build_spec(fork: str, source_files: List[str]) -> str: class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses} dependency_order_class_objects(class_objects, spec_object.custom_types) - return objects_to_spec(spec_object, fork_imports[fork], fork, class_objects) + return objects_to_spec(spec_object, spec_adjustments[fork], fork, class_objects) class PySpecCommand(Command): @@ -611,7 +684,7 @@ class BuildPyCommand(build_py): self.run_command('pyspec') def run(self): - for spec_fork in fork_imports: + for spec_fork in spec_adjustments: self.run_pyspec_cmd(spec_fork=spec_fork) super(BuildPyCommand, self).run() @@ -639,7 +712,7 @@ class PyspecDevCommand(Command): def run(self): print("running build_py command") - for spec_fork in fork_imports: + for spec_fork in spec_adjustments: self.run_pyspec_cmd(spec_fork=spec_fork) commands = { From 1ffa436836c9117a0157290ff40b86478ddefa66 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 21:28:58 +0800 Subject: [PATCH 02/13] Update `imports_and_predefinitions` --- setup.py | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index 65b04fcf2..5498f2803 100644 --- a/setup.py +++ b/setup.py @@ -179,25 +179,22 @@ class SpecAdjustment(ABC): class Phase0SpecAdjustment(SpecAdjustment): @classmethod def imports_and_predefinitions(cls) -> str: - return '''from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - + return '''from lru import LRU from dataclasses import ( dataclass, field, ) +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) -from lru import LRU - +from eth2spec.config.config_util import apply_constants_config from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils import bls - from eth2spec.utils.hash_function import hash SSZObject = TypeVar('SSZObject', bound=View) @@ -297,19 +294,18 @@ get_attesting_indices = cache_this( class AltairSpecAdjustment(Phase0SpecAdjustment): @classmethod def imports_and_predefinitions(cls) -> str: - return '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union -) - + return '''from lru import LRU from dataclasses import ( dataclass, field, ) +from typing import ( + Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union +) -from lru import LRU +from eth2spec.config.config_util import apply_constants_config +from eth2spec.phase0 import spec as phase0 from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, @@ -320,7 +316,7 @@ from eth2spec.utils import bls from eth2spec.utils.hash_function import hash -# Whenever altair is loaded, make sure we have the latest phase0 +# Whenever this spec version is loaded, make sure we have the latest phase0 from importlib import reload reload(phase0) @@ -363,19 +359,17 @@ assert ( class MergeSpecAdjustment(Phase0SpecAdjustment): @classmethod def imports_and_predefinitions(cls): - return '''from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - + return '''from lru import LRU from dataclasses import ( dataclass, field, ) +from typing import ( + Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar +) -from lru import LRU - +from eth2spec.phase0 import spec as phase0 +from eth2spec.config.config_util import apply_constants_config from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, @@ -383,9 +377,13 @@ from eth2spec.utils.ssz.ssz_typing import ( ByteList, ByteVector ) from eth2spec.utils import bls - from eth2spec.utils.hash_function import hash + +# Whenever this spec version is loaded, make sure we have the latest phase0 +from importlib import reload +reload(phase0) + SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' From ceb352be12fbca77e710574c0961db237b67652c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 22:17:01 +0800 Subject: [PATCH 03/13] Refactor `imports_and_predefinitions` into `imports` and `preparations` --- setup.py | 101 ++++++++++++++++++++++--------------------------------- 1 file changed, 40 insertions(+), 61 deletions(-) diff --git a/setup.py b/setup.py index 5498f2803..8201df574 100644 --- a/setup.py +++ b/setup.py @@ -134,9 +134,17 @@ def get_spec(file_name: str) -> SpecObject: class SpecAdjustment(ABC): @classmethod @abstractmethod - def imports_and_predefinitions(cls) -> str: + def imports(cls) -> str: """ - Importing functions and defining special types/constants for building pyspec. + Import objects from other libaries. + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def preparations(cls) -> str: + """ + Define special types/constants for building pyspec or call functions. """ raise NotImplementedError() @@ -178,7 +186,7 @@ class SpecAdjustment(ABC): # class Phase0SpecAdjustment(SpecAdjustment): @classmethod - def imports_and_predefinitions(cls) -> str: + def imports(cls) -> str: return '''from lru import LRU from dataclasses import ( dataclass, @@ -192,12 +200,15 @@ from eth2spec.config.config_util import apply_constants_config from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, -) + Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) +from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 from eth2spec.utils import bls from eth2spec.utils.hash_function import hash +''' -SSZObject = TypeVar('SSZObject', bound=View) + @classmethod + def preparations(cls) -> str: + return '''SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' @@ -293,40 +304,25 @@ get_attesting_indices = cache_this( # class AltairSpecAdjustment(Phase0SpecAdjustment): @classmethod - def imports_and_predefinitions(cls) -> str: - return '''from lru import LRU -from dataclasses import ( - dataclass, - field, -) -from typing import ( - Any, Dict, Set, Sequence, NewType, Tuple, TypeVar, Callable, Optional, Union -) - - -from eth2spec.config.config_util import apply_constants_config -from eth2spec.phase0 import spec as phase0 -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, - Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, - Path, -) -from eth2spec.utils import bls - -from eth2spec.utils.hash_function import hash - -# Whenever this spec version is loaded, make sure we have the latest phase0 + def imports(cls) -> str: + return super().imports() + '\n' + ''' +from typing import NewType, Union from importlib import reload -reload(phase0) +from eth2spec.phase0 import spec as phase0 +from eth2spec.utils.ssz.ssz_typing import Path +''' + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' +# Whenever this spec version is loaded, make sure we have the latest phase0 +reload(phase0) SSZVariableName = str GeneralizedIndex = NewType('GeneralizedIndex', int) -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' ''' + @classmethod def sundry_functions(cls) -> str: return super().sundry_functions() + '\n\n' + ''' @@ -358,35 +354,17 @@ assert ( # class MergeSpecAdjustment(Phase0SpecAdjustment): @classmethod - def imports_and_predefinitions(cls): - return '''from lru import LRU -from dataclasses import ( - dataclass, - field, -) -from typing import ( - Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar -) - + def imports(cls): + return super().imports() + '\n' + ''' from eth2spec.phase0 import spec as phase0 -from eth2spec.config.config_util import apply_constants_config -from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes -from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, - Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist, - ByteList, ByteVector -) -from eth2spec.utils import bls -from eth2spec.utils.hash_function import hash - - -# Whenever this spec version is loaded, make sure we have the latest phase0 +from eth2spec.utils.ssz.ssz_typing import Bytes20, ByteList, ByteVector, uint256 from importlib import reload +''' + + @classmethod + def preparations(cls): + return super().preparations() + '\n' + ''' reload(phase0) - -SSZObject = TypeVar('SSZObject', bound=View) - -CONFIG_NAME = 'mainnet' ''' @classmethod @@ -476,7 +454,8 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: s ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), adjustment.hardcoded_ssz_dep_constants())) custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_custom_type_dep_constants()[x]), adjustment.hardcoded_custom_type_dep_constants())) spec = ( - adjustment.imports_and_predefinitions() + adjustment.imports() + + adjustment.preparations() + '\n\n' + f"fork = \'{fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') From 3320ebb8654b419d6b38129162ad2accf783572c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 9 Apr 2021 22:26:32 +0800 Subject: [PATCH 04/13] Fix typo and add `SpecAdjustment.fork` property --- setup.py | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 8201df574..2101eebc4 100644 --- a/setup.py +++ b/setup.py @@ -132,11 +132,16 @@ def get_spec(file_name: str) -> SpecObject: class SpecAdjustment(ABC): + @property + @abstractmethod + def fork(self) -> str: + raise NotImplementedError() + @classmethod @abstractmethod def imports(cls) -> str: """ - Import objects from other libaries. + Import objects from other libraries. """ raise NotImplementedError() @@ -185,6 +190,8 @@ class SpecAdjustment(ABC): # Phase0SpecAdjustment # class Phase0SpecAdjustment(SpecAdjustment): + fork: str = PHASE0 + @classmethod def imports(cls) -> str: return '''from lru import LRU @@ -208,8 +215,8 @@ from eth2spec.utils.hash_function import hash @classmethod def preparations(cls) -> str: - return '''SSZObject = TypeVar('SSZObject', bound=View) - + return ''' +SSZObject = TypeVar('SSZObject', bound=View) CONFIG_NAME = 'mainnet' ''' @@ -303,6 +310,8 @@ get_attesting_indices = cache_this( # AltairSpecAdjustment # class AltairSpecAdjustment(Phase0SpecAdjustment): + fork: str = ALTAIR + @classmethod def imports(cls) -> str: return super().imports() + '\n' + ''' @@ -353,6 +362,8 @@ assert ( # MergeSpecAdjustment # class MergeSpecAdjustment(Phase0SpecAdjustment): + fork: str = MERGE + @classmethod def imports(cls): return super().imports() + '\n' + ''' @@ -402,25 +413,12 @@ def produce_execution_payload(parent_hash: Bytes32) -> ExecutionPayload: spec_adjustments = { - PHASE0: Phase0SpecAdjustment, - ALTAIR: AltairSpecAdjustment, - MERGE: MergeSpecAdjustment, + adjustment.fork: adjustment + for adjustment in (Phase0SpecAdjustment, AltairSpecAdjustment, MergeSpecAdjustment) } -def is_phase0(fork): - return fork == PHASE0 - - -def is_altair(fork): - return fork == ALTAIR - - -def is_merge(fork): - return fork == MERGE - - -def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: str, ordered_class_objects: Dict[str, str]) -> str: +def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ @@ -456,7 +454,7 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, fork: s spec = ( adjustment.imports() + adjustment.preparations() - + '\n\n' + f"fork = \'{fork}\'\n" + + '\n\n' + f"fork = \'{adjustment.fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + '\n\n' + new_type_definitions @@ -563,7 +561,7 @@ def build_spec(fork: str, source_files: List[str]) -> str: class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses} dependency_order_class_objects(class_objects, spec_object.custom_types) - return objects_to_spec(spec_object, spec_adjustments[fork], fork, class_objects) + return objects_to_spec(spec_object, spec_adjustments[fork], class_objects) class PySpecCommand(Command): @@ -595,14 +593,14 @@ class PySpecCommand(Command): if len(self.md_doc_paths) == 0: print("no paths were specified, using default markdown file paths for pyspec" " build (spec fork: %s)" % self.spec_fork) - if is_phase0(self.spec_fork): + if self.spec_fork == PHASE0: self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md """ - elif is_altair(self.spec_fork): + elif self.spec_fork == ALTAIR: self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md @@ -613,7 +611,7 @@ class PySpecCommand(Command): specs/altair/validator.md specs/altair/sync-protocol.md """ - elif is_merge(self.spec_fork): + elif self.spec_fork == MERGE: self.md_doc_paths = """ specs/phase0/beacon-chain.md specs/phase0/fork-choice.md From 81a83898cf29e25ce536f2fadcf19c0362ab893e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 15 Apr 2021 12:19:51 -0500 Subject: [PATCH 05/13] add committee progress tests for non genesis case --- .../test_process_sync_committee_updates.py | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index 7ba2645a2..c5bd9443e 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -1,7 +1,11 @@ from eth2spec.test.context import ( spec_state_test, + spec_test, with_all_phases_except, with_configs, + with_custom_state, + single_phase, + misc_balances, ) from eth2spec.test.helpers.constants import ( PHASE0, @@ -13,20 +17,12 @@ from eth2spec.test.helpers.epoch_processing import ( ) -@with_all_phases_except([PHASE0]) -@spec_state_test -@with_configs([MINIMAL], reason="too slow") -def test_sync_committees_progress(spec, state): - current_epoch = spec.get_current_epoch(state) - # NOTE: if not in the genesis epoch, period math below needs to be - # adjusted relative to the current epoch - assert current_epoch == 0 - +def run_sync_committees_progress_test(spec, state): first_sync_committee = state.current_sync_committee second_sync_committee = state.next_sync_committee - slot_at_end_of_current_period = spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - 1 - transition_to(spec, state, slot_at_end_of_current_period) + end_slot_of_current_period = state.slot + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - 1 + transition_to(spec, state, end_slot_of_current_period) # Ensure assignments have not changed: assert state.current_sync_committee == first_sync_committee @@ -36,7 +32,39 @@ def test_sync_committees_progress(spec, state): # Can compute the third committee having computed final balances in the last epoch # of this `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` - third_sync_committee = spec.get_sync_committee(state, 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD) + current_epoch = spec.get_current_epoch(state) + third_sync_committee = spec.get_sync_committee(state, current_epoch + 2 * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD) assert state.current_sync_committee == second_sync_committee assert state.next_sync_committee == third_sync_committee + + +@with_all_phases_except([PHASE0]) +@spec_state_test +@with_configs([MINIMAL], reason="too slow") +def test_sync_committees_progress_genesis(spec, state): + # Genesis epoch period has an exceptional case + spec.get_current_epoch(state) + assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH + + yield from run_sync_committees_progress_test(spec, state) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +@with_configs([MINIMAL], reason="too slow") +def test_sync_committees_progress_not_genesis(spec, state): + # Transition out of the genesis epoch period to test non-exceptional case + start_slot_of_next_period = state.slot + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + transition_to(spec, state, start_slot_of_next_period) + + yield from run_sync_committees_progress_test(spec, state) + + +@with_all_phases_except([PHASE0]) +@with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) +@spec_test +@single_phase +@with_configs([MINIMAL], reason="too slow") +def test_sync_committees_progress_misc_balances(spec, state): + yield from run_sync_committees_progress_test(spec, state) From 5349645a8f012c38dc7abe2e060a593c4ab409d0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 16 Apr 2021 11:29:10 +0800 Subject: [PATCH 06/13] Rename `SpecAdjustment` to `SpecBuilder` and add `build_spec` interface --- setup.py | 59 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index c6e9f55ae..66a1b7cfe 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils import dir_util from distutils.util import convert_path import os import re -from typing import Dict, NamedTuple, List +from typing import Dict, NamedTuple, List, Sequence from abc import ABC, abstractmethod @@ -131,7 +131,7 @@ def get_spec(file_name: str) -> SpecObject: ) -class SpecAdjustment(ABC): +class SpecBuilder(ABC): @property @abstractmethod def fork(self) -> str: @@ -185,11 +185,16 @@ class SpecAdjustment(ABC): """ raise NotImplementedError() + @classmethod + @abstractmethod + def build_spec(cls, source_files: List[str]) -> str: + raise NotImplementedError() + # -# Phase0SpecAdjustment +# Phase0SpecBuilder # -class Phase0SpecAdjustment(SpecAdjustment): +class Phase0SpecBuilder(SpecBuilder): fork: str = PHASE0 @classmethod @@ -305,11 +310,15 @@ get_attesting_indices = cache_this( def invariant_checks(cls) -> str: return '' + @classmethod + def build_spec(cls, source_files: Sequence[str]) -> str: + return _build_spec(cls.fork, source_files) + # -# AltairSpecAdjustment +# AltairSpecBuilder # -class AltairSpecAdjustment(Phase0SpecAdjustment): +class AltairSpecBuilder(Phase0SpecBuilder): fork: str = ALTAIR @classmethod @@ -359,9 +368,9 @@ assert ( # -# MergeSpecAdjustment +# MergeSpecBuilder # -class MergeSpecAdjustment(Phase0SpecAdjustment): +class MergeSpecBuilder(Phase0SpecBuilder): fork: str = MERGE @classmethod @@ -412,13 +421,13 @@ def produce_execution_payload(parent_hash: Hash32, timestamp: uint64) -> Executi return {**super().hardcoded_custom_type_dep_constants(), **constants} -spec_adjustments = { - adjustment.fork: adjustment - for adjustment in (Phase0SpecAdjustment, AltairSpecAdjustment, MergeSpecAdjustment) +spec_builders = { + builder.fork: builder + for builder in (Phase0SpecBuilder, AltairSpecBuilder, MergeSpecBuilder) } -def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered_class_objects: Dict[str, str]) -> str: +def objects_to_spec(spec_object: SpecObject, builder: SpecBuilder, ordered_class_objects: Dict[str, str]) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ @@ -448,13 +457,13 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered spec_object.constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, spec_object.constants[x]), spec_object.constants)) ordered_class_objects_spec = '\n\n'.join(ordered_class_objects.values()) - ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_ssz_dep_constants()[x]), adjustment.hardcoded_ssz_dep_constants())) - ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), adjustment.hardcoded_ssz_dep_constants())) - custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, adjustment.hardcoded_custom_type_dep_constants()[x]), adjustment.hardcoded_custom_type_dep_constants())) + ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants())) + ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants())) + custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants()[x]), builder.hardcoded_custom_type_dep_constants())) spec = ( - adjustment.imports() - + adjustment.preparations() - + '\n\n' + f"fork = \'{adjustment.fork}\'\n" + builder.imports() + + builder.preparations() + + '\n\n' + f"fork = \'{builder.fork}\'\n" # The constants that some SSZ containers require. Need to be defined before `new_type_definitions` + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '') + '\n\n' + new_type_definitions @@ -465,11 +474,11 @@ def objects_to_spec(spec_object: SpecObject, adjustment: SpecAdjustment, ordered + '\n\n' + CONFIG_LOADER + '\n\n' + ordered_class_objects_spec + '\n\n' + functions_spec - + '\n' + adjustment.sundry_functions() + + '\n' + builder.sundry_functions() # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') - + ('\n' + adjustment.invariant_checks() if adjustment.invariant_checks() != '' else '') + + ('\n' + builder.invariant_checks() if builder.invariant_checks() != '' else '') + '\n' ) return spec @@ -551,7 +560,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: ) -def build_spec(fork: str, source_files: List[str]) -> str: +def _build_spec(fork: str, source_files: Sequence[str]) -> str: all_specs = [get_spec(spec) for spec in source_files] spec_object = all_specs[0] @@ -561,7 +570,7 @@ def build_spec(fork: str, source_files: List[str]) -> str: class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses} dependency_order_class_objects(class_objects, spec_object.custom_types) - return objects_to_spec(spec_object, spec_adjustments[fork], class_objects) + return objects_to_spec(spec_object, spec_builders[fork], class_objects) class PySpecCommand(Command): @@ -631,7 +640,7 @@ class PySpecCommand(Command): raise Exception('Pyspec markdown input file "%s" does not exist.' % filename) def run(self): - spec_str = build_spec(self.spec_fork, self.parsed_md_doc_paths) + spec_str = spec_builders[self.spec_fork].build_spec(self.parsed_md_doc_paths) if self.dry_run: self.announce('dry run successfully prepared contents for spec.' f' out dir: "{self.out_dir}", spec fork: "{self.spec_fork}"') @@ -659,7 +668,7 @@ class BuildPyCommand(build_py): self.run_command('pyspec') def run(self): - for spec_fork in spec_adjustments: + for spec_fork in spec_builders: self.run_pyspec_cmd(spec_fork=spec_fork) super(BuildPyCommand, self).run() @@ -687,7 +696,7 @@ class PyspecDevCommand(Command): def run(self): print("running build_py command") - for spec_fork in spec_adjustments: + for spec_fork in spec_builders: self.run_pyspec_cmd(spec_fork=spec_fork) commands = { From 7167c5a9d9b05ed1b57e356206930a15bce9c5e7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 16 Apr 2021 11:32:27 -0500 Subject: [PATCH 07/13] generate sync committee update tests with always_bls --- .../test_process_sync_committee_updates.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index c5bd9443e..e88d3ef1a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -1,4 +1,5 @@ from eth2spec.test.context import ( + always_bls, spec_state_test, spec_test, with_all_phases_except, @@ -17,6 +18,11 @@ from eth2spec.test.helpers.epoch_processing import ( ) +# +# Note: +# Calculating sync committees requires pubkey aggregation, thus all tests are generated with `always_bls` +# + def run_sync_committees_progress_test(spec, state): first_sync_committee = state.current_sync_committee second_sync_committee = state.next_sync_committee @@ -41,6 +47,7 @@ def run_sync_committees_progress_test(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@always_bls @with_configs([MINIMAL], reason="too slow") def test_sync_committees_progress_genesis(spec, state): # Genesis epoch period has an exceptional case @@ -52,6 +59,7 @@ def test_sync_committees_progress_genesis(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test +@always_bls @with_configs([MINIMAL], reason="too slow") def test_sync_committees_progress_not_genesis(spec, state): # Transition out of the genesis epoch period to test non-exceptional case @@ -65,6 +73,7 @@ def test_sync_committees_progress_not_genesis(spec, state): @with_custom_state(balances_fn=misc_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE) @spec_test @single_phase +@always_bls @with_configs([MINIMAL], reason="too slow") def test_sync_committees_progress_misc_balances(spec, state): yield from run_sync_committees_progress_test(spec, state) From 55f2cc6e414f9ea26bfe5856f329d84b522dbb39 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 16 Apr 2021 11:42:26 -0500 Subject: [PATCH 08/13] address @ralexstokes PR comments --- .../test_process_sync_committee_updates.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py index e88d3ef1a..8ffc51507 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_sync_committee_updates.py @@ -27,7 +27,11 @@ def run_sync_committees_progress_test(spec, state): first_sync_committee = state.current_sync_committee second_sync_committee = state.next_sync_committee - end_slot_of_current_period = state.slot + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - 1 + current_period = spec.get_current_epoch(state) // spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + next_period = current_period + 1 + next_period_start_epoch = next_period * spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD + next_period_start_slot = next_period_start_epoch * spec.SLOTS_PER_EPOCH + end_slot_of_current_period = next_period_start_slot - 1 transition_to(spec, state, end_slot_of_current_period) # Ensure assignments have not changed: @@ -51,7 +55,6 @@ def run_sync_committees_progress_test(spec, state): @with_configs([MINIMAL], reason="too slow") def test_sync_committees_progress_genesis(spec, state): # Genesis epoch period has an exceptional case - spec.get_current_epoch(state) assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH yield from run_sync_committees_progress_test(spec, state) @@ -63,8 +66,9 @@ def test_sync_committees_progress_genesis(spec, state): @with_configs([MINIMAL], reason="too slow") def test_sync_committees_progress_not_genesis(spec, state): # Transition out of the genesis epoch period to test non-exceptional case - start_slot_of_next_period = state.slot + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - transition_to(spec, state, start_slot_of_next_period) + assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH + slot_in_next_period = state.slot + spec.EPOCHS_PER_SYNC_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + transition_to(spec, state, slot_in_next_period) yield from run_sync_committees_progress_test(spec, state) From c0f12315220258006ef171addc78f7cafa1afff3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 19 Apr 2021 15:24:06 +1000 Subject: [PATCH 09/13] Rename MAX_APPLICATION_TRANSACTIONS --- specs/merge/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/merge/beacon-chain.md b/specs/merge/beacon-chain.md index a508cb670..b9c443962 100644 --- a/specs/merge/beacon-chain.md +++ b/specs/merge/beacon-chain.md @@ -61,7 +61,7 @@ We define the following Python custom types for type hinting and readability: | Name | Value | | - | - | | `MAX_BYTES_PER_OPAQUE_TRANSACTION` | `uint64(2**20)` (= 1,048,576) | -| `MAX_APPLICATION_TRANSACTIONS` | `uint64(2**14)` (= 16,384) | +| `MAX_EXECUTION_TRANSACTIONS` | `uint64(2**14)` (= 16,384) | | `BYTES_PER_LOGS_BLOOM` | `uint64(2**8)` (= 256) | ## Containers @@ -108,7 +108,7 @@ class ExecutionPayload(Container): timestamp: uint64 receipt_root: Bytes32 logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] - transactions: List[OpaqueTransaction, MAX_APPLICATION_TRANSACTIONS] + transactions: List[OpaqueTransaction, MAX_EXECUTION_TRANSACTIONS] ``` #### `ExecutionPayloadHeader` From 9e4f5c18796a8097fd5f4995122df337a19f55be Mon Sep 17 00:00:00 2001 From: "Sam.An" <56215891+sammiee5311@users.noreply.github.com> Date: Mon, 19 Apr 2021 22:41:45 +0900 Subject: [PATCH 10/13] changed alphabet string to string module. changed alphabet string to string module. --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 70cf5176e..0fb56e708 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ from distutils import dir_util from distutils.util import convert_path import os import re +import string from typing import Dict, NamedTuple, List FUNCTION_REGEX = r'^def [\w_]*' @@ -89,10 +90,10 @@ def get_spec(file_name: str) -> SpecObject: if '`' in row[i]: row[i] = row[i][:row[i].find('`')] is_constant_def = True - if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + if row[0][0] not in string.ascii_uppercase + '_': is_constant_def = False for c in row[0]: - if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + if c not in string.ascii_uppercase + '_' + string.digits: is_constant_def = False if is_constant_def: if row[1].startswith('get_generalized_index'): From b49869a78420610ba57daf5f96dc9a7ee31d08b8 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 19 Apr 2021 21:19:38 -0700 Subject: [PATCH 11/13] Use `process_execution_payload` in sharding doc --- specs/sharding/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index e02229838..bbcc57211 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -466,7 +466,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in Sharding] - process_application_payload(state, block.body) # [New in Merge] + process_execution_payload(state, block.body) # [New in Merge] ``` #### Operations From a83a2f1bc7456c1db0ed3bce9fa84745405918cd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 20 Apr 2021 20:18:00 +0800 Subject: [PATCH 12/13] Add note for ByteList and ByteVector --- ssz/simple-serialize.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ssz/simple-serialize.md b/ssz/simple-serialize.md index d36e444ec..d97b8ea1c 100644 --- a/ssz/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -75,7 +75,8 @@ For convenience we alias: * `bit` to `boolean` * `byte` to `uint8` (this is a basic type) -* `BytesN` to `Vector[byte, N]` (this is *not* a basic type) +* `BytesN` and `ByteVector[N]` to `Vector[byte, N]` (this is *not* a basic type) +* `ByteList[N]` to `List[byte, N]` * `null`: `{}` ### Default values From 66e1a2858f9fbebf5e00539d1a34b78025673d37 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 21 Apr 2021 00:24:44 +0800 Subject: [PATCH 13/13] Use `ALTAIR_FORK_EPOCH` instead of `ALTAIR_FORK_SLOT` --- configs/README.md | 2 +- configs/mainnet/altair.yaml | 2 +- configs/mainnet/merge.yaml | 2 +- configs/mainnet/sharding.yaml | 2 +- configs/minimal/altair.yaml | 2 +- configs/minimal/merge.yaml | 2 +- configs/minimal/sharding.yaml | 2 +- specs/altair/fork.md | 6 +++--- specs/das/fork-choice.md | 2 +- tests/core/pyspec/eth2spec/test/helpers/state.py | 5 +++-- 10 files changed, 14 insertions(+), 13 deletions(-) diff --git a/configs/README.md b/configs/README.md index 15529e590..2cf4e3f60 100644 --- a/configs/README.md +++ b/configs/README.md @@ -15,7 +15,7 @@ Over time, the need to sync an older state may be deprecated. In this case, the prefix on the new constant may be removed, and the old constant will keep a special name before completely being removed. A previous iteration of forking made use of "timelines", but this collides with the definitions used in the spec (constants for special forking slots, etc.), and was not integrated sufficiently in any of the spec tools or implementations. -Instead, the config essentially doubles as fork definition now, e.g. changing the value for `ALTAIR_FORK_SLOT` changes the fork. +Instead, the config essentially doubles as fork definition now, e.g. changing the value for `ALTAIR_FORK_EPOCH` changes the fork. Another reason to prefer forking through constants is the ability to program a forking moment based on context, instead of being limited to a static slot number. diff --git a/configs/mainnet/altair.yaml b/configs/mainnet/altair.yaml index 44490f982..3cd4b8419 100644 --- a/configs/mainnet/altair.yaml +++ b/configs/mainnet/altair.yaml @@ -38,7 +38,7 @@ DOMAIN_CONTRIBUTION_AND_PROOF: 0x09000000 # 0x01000000 ALTAIR_FORK_VERSION: 0x01000000 # TBD -ALTAIR_FORK_SLOT: 18446744073709551615 +ALTAIR_FORK_EPOCH: 18446744073709551615 # Sync protocol diff --git a/configs/mainnet/merge.yaml b/configs/mainnet/merge.yaml index b4667f5b5..4e012ac05 100644 --- a/configs/mainnet/merge.yaml +++ b/configs/mainnet/merge.yaml @@ -4,4 +4,4 @@ # --------------------------------------------------------------- MERGE_FORK_VERSION: 0x02000000 # TBD, temporarily max uint64 value: 2**64 - 1 -MERGE_FORK_SLOT: 18446744073709551615 +MERGE_FORK_EPOCH: 18446744073709551615 diff --git a/configs/mainnet/sharding.yaml b/configs/mainnet/sharding.yaml index 4773aa5f5..b3c22c354 100644 --- a/configs/mainnet/sharding.yaml +++ b/configs/mainnet/sharding.yaml @@ -4,7 +4,7 @@ # --------------------------------------------------------------- SHARDING_FORK_VERSION: 0x03000000 # TBD, temporarily max uint64 value: 2**64 - 1 -SHARDING_FORK_SLOT: 18446744073709551615 +SHARDING_FORK_EPOCH: 18446744073709551615 # Beacon-chain diff --git a/configs/minimal/altair.yaml b/configs/minimal/altair.yaml index 10bdf318b..f9b30eea2 100644 --- a/configs/minimal/altair.yaml +++ b/configs/minimal/altair.yaml @@ -38,7 +38,7 @@ DOMAIN_CONTRIBUTION_AND_PROOF: 0x09000000 # [customized] Highest byte set to 0x01 to avoid collisions with mainnet versioning ALTAIR_FORK_VERSION: 0x01000001 # [customized] -ALTAIR_FORK_SLOT: 18446744073709551615 +ALTAIR_FORK_EPOCH: 18446744073709551615 # Sync protocol diff --git a/configs/minimal/merge.yaml b/configs/minimal/merge.yaml index 394595d02..3b50cd5ca 100644 --- a/configs/minimal/merge.yaml +++ b/configs/minimal/merge.yaml @@ -4,4 +4,4 @@ # --------------------------------------------------------------- MERGE_FORK_VERSION: 0x02000001 # TBD, temporarily max uint64 value: 2**64 - 1 -MERGE_FORK_SLOT: 18446744073709551615 +MERGE_FORK_EPOCH: 18446744073709551615 diff --git a/configs/minimal/sharding.yaml b/configs/minimal/sharding.yaml index f32e2827d..c6ca8b560 100644 --- a/configs/minimal/sharding.yaml +++ b/configs/minimal/sharding.yaml @@ -4,7 +4,7 @@ # --------------------------------------------------------------- SHARDING_FORK_VERSION: 0x03000001 # TBD, temporarily max uint64 value: 2**64 - 1 -MERGE_FORK_SLOT: 18446744073709551615 +MERGE_FORK_EPOCH: 18446744073709551615 # Beacon-chain diff --git a/specs/altair/fork.md b/specs/altair/fork.md index b51466c1e..739584309 100644 --- a/specs/altair/fork.md +++ b/specs/altair/fork.md @@ -26,17 +26,17 @@ Warning: this configuration is not definitive. | Name | Value | | - | - | | `ALTAIR_FORK_VERSION` | `Version('0x01000000')` | -| `ALTAIR_FORK_SLOT` | `Slot(18446744073709551615)` **TBD** | +| `ALTAIR_FORK_EPOCH` | `Epoch(18446744073709551615)` **TBD** | ## Fork to Altair ### Fork trigger -TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `ALTAIR_FORK_SLOT`, where `ALTAIR_FORK_SLOT % SLOTS_PER_EPOCH == 0`. +TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at epoch `ALTAIR_FORK_EPOCH`. ### Upgrading the state -After `process_slots` of Phase 0 finishes, if `state.slot == ALTAIR_FORK_SLOT`, an irregular state change is made to upgrade to Altair. +After `process_slots` of Phase 0 finishes, if `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == ALTAIR_FORK_EPOCH`, an irregular state change is made to upgrade to Altair. ```python def upgrade_to_altair(pre: phase0.BeaconState) -> BeaconState: diff --git a/specs/das/fork-choice.md b/specs/das/fork-choice.md index 329c033a7..ec9f3ab59 100644 --- a/specs/das/fork-choice.md +++ b/specs/das/fork-choice.md @@ -37,7 +37,7 @@ def get_new_dependencies(state: BeaconState) -> Set[DataCommitment]: ```python def get_all_dependencies(store: Store, block: BeaconBlock) -> Set[DataCommitment]: - if block.slot < SHARDING_FORK_SLOT: + if compute_epoch_at_slot(block.slot) < SHARDING_FORK_EPOCH: return set() else: latest = get_new_dependencies(store.block_states[hash_tree_root(block)]) diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 8980053f5..d61df7610 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -42,9 +42,10 @@ def transition_to_slot_via_block(spec, state, slot): def transition_to_valid_shard_slot(spec, state): """ - Transition to slot `spec.SHARDING_FORK_SLOT + 1` and fork at `spec.SHARDING_FORK_SLOT`. + Transition to slot `compute_epoch_at_slot(spec.SHARDING_FORK_EPOCH) + 1` + and fork at `compute_epoch_at_slot(spec.SHARDING_FORK_EPOCH)`. """ - transition_to(spec, state, spec.SHARDING_FORK_SLOT) + transition_to(spec, state, spec.compute_epoch_at_slot(spec.SHARDING_FORK_EPOCH)) next_slot(spec, state)