Merge pull request #2316 from ethereum/merge-exe

Make executable merge spec pass CI linter checks

.gitignore

@@ -17,6 +17,7 @@ eth2.0-spec-tests/
 # Dynamically built from Markdown spec
 tests/core/pyspec/eth2spec/phase0/
 tests/core/pyspec/eth2spec/altair/
+tests/core/pyspec/eth2spec/merge/
 
 # coverage reports
 .htmlcov

Makefile

@@ -55,6 +55,7 @@ partial_clean:
 	rm -rf $(DEPOSIT_CONTRACT_TESTER_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_DIR)/phase0
 	rm -rf $(PY_SPEC_DIR)/altair
+	rm -rf $(PY_SPEC_DIR)/merge
 	rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT)
 	rm -rf $(PY_SPEC_DIR)/.coverage
 	rm -rf $(PY_SPEC_DIR)/test-reports
@@ -119,7 +120,7 @@ codespell:
 lint: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
-	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair
+	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair -p eth2spec.merge
 
 lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \

setup.py

@@ -13,7 +13,7 @@ FUNCTION_REGEX = r'^def [\w_]*'
 # Definitions in context.py
 PHASE0 = 'phase0'
 ALTAIR = 'altair'
-
+MERGE = 'merge'
 
 class SpecObject(NamedTuple):
     functions: Dict[str, str]
@@ -99,7 +99,7 @@ def get_spec(file_name: str) -> SpecObject:
                             ssz_dep_constants[row[0]] = row[1]
                         else:
                             constants[row[0]] = row[1].replace('**TBD**', '2**32')
-                    elif row[1].startswith('uint') or row[1].startswith('Bytes'):
+                    elif row[1].startswith('uint') or row[1].startswith('Bytes') or row[1].startswith('ByteList'):
                         custom_types[row[0]] = row[1]
     return SpecObject(
         functions=functions,
@@ -176,6 +176,34 @@ SSZObject = TypeVar('SSZObject', bound=View)
 CONFIG_NAME = 'mainnet'
 '''
 
+MERGE_IMPORTS = '''from eth2spec.phase0 import spec as phase0
+from eth2spec.config.config_util import apply_constants_config
+from typing import (
+    Any, Callable, Dict, Set, Sequence, Tuple, Optional, TypeVar
+)
+
+from dataclasses import (
+    dataclass,
+    field,
+)
+
+from lru import LRU
+
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes
+from eth2spec.utils.ssz.ssz_typing import (
+    View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256,
+    Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, Bitlist,
+    ByteList, ByteVector
+)
+from eth2spec.utils import bls
+
+from eth2spec.utils.hash_function import hash
+
+SSZObject = TypeVar('SSZObject', bound=View)
+
+CONFIG_NAME = 'mainnet'
+'''
+
 SUNDRY_CONSTANTS_FUNCTIONS = '''
 def ceillog2(x: int) -> uint64:
     if x < 1:
@@ -269,6 +297,30 @@ def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariable
     return GeneralizedIndex(ssz_path.gindex())'''
 
 
+MERGE_SUNDRY_FUNCTIONS = """
+ExecutionState = Any
+
+
+def get_pow_block(hash: Bytes32) -> PowBlock:
+    pass
+
+
+def get_execution_state(execution_state_root: Bytes32) -> ExecutionState:
+    pass
+
+
+def get_pow_chain_head() -> PowBlock:
+    pass
+
+
+def execution_state_transition(execution_state: ExecutionState, execution_payload: ExecutionPayload) -> None:
+    pass
+
+
+def produce_execution_payload(parent_hash: Bytes32) -> ExecutionPayload:
+    pass"""
+
+
 # The constants that depend on SSZ objects
 # Will verify the value at the end of the spec
 ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS = {
@@ -283,6 +335,11 @@ assert (
 ) == WEIGHT_DENOMINATOR'''
 
 
+MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS = {
+    'MAX_BYTES_PER_OPAQUE_TRANSACTION': 'uint64(2**20)',
+}
+
+
 def is_phase0(fork):
     return fork == PHASE0
 
@@ -291,6 +348,10 @@ def is_altair(fork):
     return fork == ALTAIR
 
 
+def is_merge(fork):
+    return fork == MERGE
+
+
 def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_class_objects: Dict[str, str]) -> str:
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
@@ -300,6 +361,15 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl
             [
                 f"class {key}({value}):\n    pass\n"
                 for key, value in spec_object.custom_types.items()
+                if not value.startswith('ByteList')
+            ]
+        )
+        + ('\n\n' if len([key for key, value in spec_object.custom_types.items() if value.startswith('ByteList')]) > 0 else '')
+        + '\n\n'.join(
+            [
+                f"{key} = {value}\n"
+                for key, value in spec_object.custom_types.items()
+                if value.startswith('ByteList')
             ]
         )
     )
@@ -316,9 +386,15 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl
     if is_altair(fork):
         altair_ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS[x]), ALTAIR_HARDCODED_SSZ_DEP_CONSTANTS))
 
+    if is_merge(fork):
+        merge_custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS[x]), MERGE_HARDCODED_CUSTOM_TYPE_DEP_CONSTANTS))
+
+    
     spec = (
             imports
             + '\n\n' + f"fork = \'{fork}\'\n"
+            # The constants that some SSZ containers require. Need to be defined before `new_type_definitions`
+            + ('\n\n' + merge_custom_type_dep_constants  + '\n' if is_merge(fork) else '')
             + '\n\n' + new_type_definitions
             + '\n' + SUNDRY_CONSTANTS_FUNCTIONS
             # The constants that some SSZ containers require. Need to be defined before `constants_spec`
@@ -330,6 +406,7 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str, ordered_cl
             # Functions to make pyspec work
             + '\n' + PHASE0_SUNDRY_FUNCTIONS
             + ('\n' + ALTAIR_SUNDRY_FUNCTIONS if is_altair(fork) else '')
+            + ('\n' + MERGE_SUNDRY_FUNCTIONS if is_merge(fork) else '')
     )
 
     # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
@@ -357,7 +434,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st
 
 ignored_dependencies = [
     'bit', 'boolean', 'Vector', 'List', 'Container', 'BLSPubkey', 'BLSSignature',
-    'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector',
+    'Bytes1', 'Bytes4', 'Bytes20', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector',
     'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
     'bytes', 'byte', 'ByteList', 'ByteVector',
     'Dict', 'dict', 'field', 'ceillog2', 'floorlog2',
@@ -422,6 +499,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
 fork_imports = {
     'phase0': PHASE0_IMPORTS,
     'altair': ALTAIR_IMPORTS,
+    'merge': MERGE_IMPORTS,
 }
 
 
@@ -485,6 +563,16 @@ class PySpecCommand(Command):
                     specs/altair/validator.md
                     specs/altair/sync-protocol.md
                 """
+            elif is_merge(self.spec_fork):
+                self.md_doc_paths = """
+                    specs/phase0/beacon-chain.md
+                    specs/phase0/fork-choice.md
+                    specs/phase0/validator.md
+                    specs/phase0/weak-subjectivity.md
+                    specs/merge/beacon-chain.md
+                    specs/merge/fork-choice.md
+                    specs/merge/validator.md
+                """
             else:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)
 

specs/merge/beacon-chain.md

@@ -182,7 +182,6 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody) -> None
     """
     Note: This function is designed to be able to be run in parallel with the other `process_block` sub-functions
     """
-
     # Pre-merge, skip processing
     if not is_transition_completed(state) and not is_transition_block(state, body):
         return

tests/core/pyspec/eth2spec/test/context.py

@@ -2,6 +2,7 @@ import pytest
 
 from eth2spec.phase0 import spec as spec_phase0
 from eth2spec.altair import spec as spec_altair
+from eth2spec.merge import spec as spec_merge
 from eth2spec.utils import bls
 
 from .exceptions import SkippedTest
@@ -19,6 +20,7 @@ from importlib import reload
 def reload_specs():
     reload(spec_phase0)
     reload(spec_altair)
+    reload(spec_merge)
 
 
 # Some of the Spec module functionality is exposed here to deal with phase-specific changes.
@@ -61,9 +63,14 @@ class SpecAltair(Spec):
     ...
 
 
+class SpecMerge(Spec):
+    ...
+
+
 class SpecForks(TypedDict, total=False):
     PHASE0: SpecPhase0
     ALTAIR: SpecAltair
+    MERGE: SpecMerge
 
 
 def _prepare_state(balances_fn: Callable[[Any], Sequence[int]], threshold_fn: Callable[[Any], int],

tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py

@@ -6,3 +6,6 @@ from remerkleable.basic import boolean, bit, uint, byte, uint8, uint16, uint32,
 from remerkleable.bitfields import Bitvector, Bitlist
 from remerkleable.byte_arrays import ByteVector, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, ByteList
 from remerkleable.core import BasicView, View, Path
+
+
+Bytes20 = ByteVector[20]  # type: ignore