Adds Docstrings and type-checking to spec builder.

scripts/build_spec.py

@@ -1,6 +1,13 @@
 import re
-import function_puller
+from function_puller import (
+    get_spec,
+    SpecObject,
+)
 from argparse import ArgumentParser
+from typing import (
+    Dict,
+    Optional,
+)
 
 
 PHASE0_IMPORTS = '''from typing import (
@@ -107,8 +114,16 @@ def apply_constants_preset(preset: Dict[str, Any]):
 '''
 
 
-def objects_to_spec(functions, constants, ssz_objects, inserts, imports):
+def objects_to_spec(functions: Dict[str, str],
-    new_type_definitions = '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()])
+                    constants: Dict[str, str],
+                    ssz_objects: Dict[str, str],
+                    inserts: Dict[str, str],
+                    imports: Dict[str, str]) -> str:
+    """
+    Given all the objects that constitute a spec, combine them into a single pyfile.
+    """
+    new_type_definitions = \
+        '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()])
     functions_spec = '\n\n'.join(functions.values())
     constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants))
     ssz_objects_instantiation_spec = '\n'.join(map(
@@ -139,19 +154,22 @@ def objects_to_spec(functions, constants, ssz_objects, inserts, imports):
     return spec
 
 
-def combine_functions(old_funcitons, new_functions):
+def combine_functions(old_funcitons: Dict[str, str], new_functions: Dict[str, str]) -> Dict[str, str]:
     for key, value in new_functions.items():
         old_funcitons[key] = value
     return old_funcitons
 
 
-def combine_constants(old_constants, new_constants):
+def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, str]) -> Dict[str, str]:
     for key, value in new_constants.items():
         old_constants[key] = value
     return old_constants
 
 
-def dependency_order_ssz_objects(objects):
+def dependency_order_ssz_objects(objects: Dict[str, str]) -> Dict[str, str]:
+    """
+    Determines which SSZ Object is depenedent on which other and orders them appropriately
+    """
     items = list(objects.items())
     for key, value in items:
         dependencies = re.findall(r'(: [\[]*[A-Z][a-z][\w]+)', value)
@@ -164,7 +182,11 @@ def dependency_order_ssz_objects(objects):
                 objects[item] = objects.pop(item)
 
 
-def combine_ssz_objects(old_objects, new_objects):
+def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]) -> Dict[str, str]:
+    """
+    Thakes in old spec and new spec ssz objects, combines them,
+    and returns the newer versions of the objects in dependency order.
+    """
     for key, value in new_objects.items():
         # remove leading "{" and trailing "\n}"
         old_objects[key] = old_objects.get(key, '')[1:-3]
@@ -179,7 +201,10 @@ def combine_ssz_objects(old_objects, new_objects):
 combine_inserts = combine_functions
 
 
-def combine_spec_objects(spec0, spec1):
+def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
+    """
+    Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function.
+    """
     functions0, constants0, ssz_objects0, inserts0 = spec0
     functions1, constants1, ssz_objects1, inserts1 = spec1
     functions = combine_functions(functions0, functions1)
@@ -189,8 +214,8 @@ def combine_spec_objects(spec0, spec1):
     return functions, constants, ssz_objects, inserts
 
 
-def build_phase0_spec(sourcefile, outfile=None):
+def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]:
-    functions, constants, ssz_objects, inserts = function_puller.get_spec(sourcefile)
+    functions, constants, ssz_objects, inserts = get_spec(sourcefile)
     spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
@@ -199,10 +224,13 @@ def build_phase0_spec(sourcefile, outfile=None):
         return spec
 
 
-def build_phase1_spec(phase0_sourcefile, phase1_custody_sourcefile, phase1_shard_sourcefile, outfile=None):
+def build_phase1_spec(phase0_sourcefile: str,
-    phase0_spec = function_puller.get_spec(phase0_sourcefile)
+                      phase1_custody_sourcefile: str,
-    phase1_custody = function_puller.get_spec(phase1_custody_sourcefile)
+                      phase1_shard_sourcefile: str,
-    phase1_shard_data = function_puller.get_spec(phase1_shard_sourcefile)
+                      outfile: str=None) -> Optional[str]:
+    phase0_spec = get_spec(phase0_sourcefile)
+    phase1_custody = get_spec(phase1_custody_sourcefile)
+    phase1_shard_data = get_spec(phase1_shard_sourcefile)
     spec_objects = phase0_spec
     for value in [phase1_custody, phase1_shard_data]:
         spec_objects = combine_spec_objects(spec_objects, value)

scripts/function_puller.py

@@ -1,13 +1,25 @@
 import re
-from typing import Dict, Tuple
+from typing import Dict, Tuple, NewType
 
 
 FUNCTION_REGEX = r'^def [\w_]*'
 BEGIN_INSERT_REGEX = r'# begin insert '
 END_INSERT_REGEX = r'# end insert'
 
+SpecObject = NewType('SpecObjects', Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]])
 
-def get_spec(file_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]:
+
+def get_spec(file_name: str) -> SpecObject:
+    """
+    Takes in the file name of a spec.md file, opens it and returns the following objects:
+    functions = {function_name: function_code}
+    constants= {constant_name: constant_code}
+    ssz_objects= {object_name: object}
+    inserts= {insert_tag: code to be inserted}
+
+    Note: This function makes heavy use of the inherent ordering of dicts,
+    if this is not supported by your python version, it will not work.
+    """
     pulling_from = None  # line number of start of latest object
     current_name = None  # most recent section title
     insert_name = None  # stores the label of the current insert object
@@ -27,21 +39,23 @@ def get_spec(file_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str,
         elif line[:3] == '```':
             pulling_from = None
         elif inserts_matcher.match(line) is not None:
+            # Find @insert names
             insert_name = re.search(r'@[\w]*', line).group(0)
         elif insert_name is not None:
+            # In insert mode, either the next line is more code, or the end of the insert
             if re.match(END_INSERT_REGEX, line) is not None:
                 insert_name = None
             else:
                 inserts[insert_name] = inserts.get(insert_name, '') + line + '\n'
         else:
-            # Handle function definitions
+            # Handle function definitions & ssz_objects
             if pulling_from is not None:
                 func_match = function_matcher.match(line)
                 if func_match is not None:
                     current_name = func_match.group(0)
-                if function_matcher.match(current_name) is None:
+                if function_matcher.match(current_name) is None:  # The current line is an SSZ Object
                     ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n'
-                else:
+                else:  # The current line is code
                     functions[current_name] = functions.get(current_name, '') + line + '\n'
             # Handle constant table entries
             elif pulling_from is None and len(line) > 0 and line[0] == '|':