Adds Insert-into-label functionality

scripts/build_spec.py

@@ -1,4 +1,3 @@
-import sys
 import re
 import function_puller
 from argparse import ArgumentParser
@@ -108,7 +107,7 @@ def apply_constants_preset(preset: Dict[str, Any]):
 '''
 
 
-def objects_to_spec(functions, constants, ssz_objects, imports):
+def objects_to_spec(functions, constants, ssz_objects, imports, inserts={}):
     new_type_definitions = '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()])
     functions_spec = '\n\n'.join(functions.values())
     constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants))
@@ -119,7 +118,7 @@ def objects_to_spec(functions, constants, ssz_objects, imports):
         'def init_SSZ_types():\n    global_vars = globals()\n' 
         + ssz_objects_reinitialization_spec
     )
-    return (
+    spec = (
         imports
         + '\n' + new_type_definitions
         + '\n\n' + constants_spec
@@ -129,11 +128,15 @@ def objects_to_spec(functions, constants, ssz_objects, imports):
         + '\n\n' + ssz_objects_reinitialization_spec
         + '\n'
     )
+    # Handle @inserts
+    for key, value in inserts.items():
+        spec = re.sub('[ ]*# %s\\n' % key, value, spec)
+    return spec
+
 
 def combine_functions(old_funcitons, new_functions):
     for key, value in new_functions.items():
         old_funcitons[key] = value
-    # TODO: Add insert functionality
     return old_funcitons
 
 
@@ -167,8 +170,12 @@ def combine_ssz_objects(old_objects, new_objects):
     return old_objects
 
 
+# inserts are handeled the same way as functions
+combine_inserts = combine_functions
+
+
 def build_phase0_spec(sourcefile, outfile=None):
-    functions, constants, ssz_objects = function_puller.get_spec(sourcefile)
+    functions, constants, ssz_objects, _ = function_puller.get_spec(sourcefile)
     spec = objects_to_spec(functions, constants, ssz_objects, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
@@ -178,12 +185,13 @@ def build_phase0_spec(sourcefile, outfile=None):
 
 
 def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None):
-    phase0_functions, phase0_constants, phase0_ssz_objects = function_puller.get_spec(phase0_sourcefile)
-    phase1_functions, phase1_constants, phase1_ssz_objects = function_puller.get_spec(phase1_sourcefile)
+    phase0_functions, phase0_constants, phase0_ssz_objects, phase0_inserts = function_puller.get_spec(phase0_sourcefile)
+    phase1_functions, phase1_constants, phase1_ssz_objects, phase1_inserts = function_puller.get_spec(phase1_sourcefile)
     functions = combine_functions(phase0_functions, phase1_functions)
     constants = combine_constants(phase0_constants, phase1_constants)
     ssz_objects = combine_ssz_objects(phase0_ssz_objects, phase1_ssz_objects)
-    spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS)
+    inserts = combine_inserts(phase0_inserts, phase1_inserts)
+    spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS, inserts)
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)

scripts/function_puller.py

@@ -1,19 +1,22 @@
-import sys
 import re
-from typing import List
+from typing import Dict, Tuple
 
 
 FUNCTION_REGEX = r'^def [\w_]*'
+BEGIN_INSERT_REGEX = r'# begin insert '
+END_INSERT_REGEX = r'# end insert'
 
 
-def get_spec(file_name: str):
-    code_lines = []
-    pulling_from = None # line number of start of latest object
-    current_name = None # most recent section title
+def get_spec(file_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]:
+    pulling_from = None  # line number of start of latest object
+    current_name = None  # most recent section title
+    insert_name = None  # stores the label of the current insert object
     functions = {}
     constants = {}
     ssz_objects = {}
+    inserts = {}
     function_matcher = re.compile(FUNCTION_REGEX)
+    inserts_matcher = re.compile(BEGIN_INSERT_REGEX)
     for linenum, line in enumerate(open(file_name).readlines()):
         line = line.rstrip()
         if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
@@ -23,12 +26,19 @@ def get_spec(file_name: str):
             pulling_from = linenum + 1
         elif line[:3] == '```':
             pulling_from = None
+        elif inserts_matcher.match(line) is not None:
+            insert_name = re.search(r'@[\w]*', line).group(0)
+        elif insert_name is not None:
+            if re.match(END_INSERT_REGEX, line) is not None:
+                insert_name = None
+            else:
+                inserts[insert_name] = inserts.get(insert_name, '') + line + '\n'
         else:
             # Handle function definitions
             if pulling_from is not None:
-                match = function_matcher.match(line)
-                if match is not None:
-                    current_name = match.group(0)
+                func_match = function_matcher.match(line)
+                if func_match is not None:
+                    current_name = func_match.group(0)
                 if function_matcher.match(current_name) is None:
                     ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n'
                 else:
@@ -49,4 +59,4 @@ def get_spec(file_name: str):
                             eligible = False
                     if eligible:
                         constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890')
-    return functions, constants, ssz_objects
+    return functions, constants, ssz_objects, inserts

specs/core/0_beacon-chain.md

@@ -1280,8 +1280,11 @@ def process_epoch(state: BeaconState) -> None:
     process_crosslinks(state)
     process_rewards_and_penalties(state)
     process_registry_updates(state)
+    # @process_reveal_deadlines
+    # @process_challenge_deadlines
     process_slashings(state)
     process_final_updates(state)
+    # @after_process_final_updates
 ```
 
 #### Helper functions

specs/core/1_custody-game.md

@@ -674,6 +674,9 @@ def process_bit_challenge_response(state: BeaconState,
 Run `process_reveal_deadlines(state)` immediately after `process_registry_updates(state)`:
 
 ```python
+# begin insert @process_reveal_deadlines
+    process_reveal_deadlines(state)
+# end insert @process_reveal_deadlines
 def process_reveal_deadlines(state: BeaconState) -> None:
     for index, validator in enumerate(state.validator_registry):
         if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD)
@@ -684,6 +687,9 @@ def process_reveal_deadlines(state: BeaconState) -> None:
 Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`:
 
 ```python
+# begin insert @process_challenge_deadlines
+    process_challenge_deadlines(state)
+# end insert @process_challenge_deadlines
 def process_challenge_deadlines(state: BeaconState) -> None:
     for challenge in state.custody_chunk_challenge_records:
         if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE:
@@ -701,12 +707,15 @@ def process_challenge_deadlines(state: BeaconState) -> None:
 Append this to `process_final_updates(state)`:
 
 ```python
+# begin insert @after_process_final_updates
+    after_process_final_updates(state)
+# end insert @after_process_final_updates
 def after_process_final_updates(state: BeaconState) -> None:
     current_epoch = get_current_epoch(state)
     # Clean up exposed RANDAO key reveals
     state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = []
     # Reset withdrawable epochs if challenge records are empty
-    records = state.custody_chunk_challenge_records + state.bit_challenge_records
+    records = state.custody_chunk_challenge_records + state.custody_bit_challenge_records
     validator_indices_in_records = set(
         [record.challenger_index for record in records] + [record.responder_index for record in records]
     )