status-im
/
eth2.0-specs
mirror of https://github.com/status-im/eth2.0-specs.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

new spec buildign stratergy

This commit is contained in:
Carl Beekhuizen 2019-05-16 16:36:35 +02:00
parent 49d708e44b
commit 14ea7d9f40
No known key found for this signature in database
GPG Key ID: D05CA176D0020646
5 changed files with 199 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -67,11 +67,11 @@ phase0: $(PY_SPEC_PHASE_0_TARGETS)
# "make phase1" to create pyspec for phase1 # "make phase1" to create pyspec for phase1
phase1: $(PY_SPEC_PHASE_1_TARGETS) phase1: $(PY_SPEC_PHASE_1_TARGETS)
$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS) $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@
$(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $@
CURRENT_DIR = ${CURDIR} CURRENT_DIR = ${CURDIR}

0
scripts/phase0/__init__.py → scripts/__init__.py
View File

196
scripts/build_spec.py Normal file
View File

@ -0,0 +1,196 @@
import sys
import re
import function_puller
from argparse import ArgumentParser
from typing import Tuple, List
def split_retain_delimiter(regex_pattern: str, text: str) -> List[str]:
'''
Splits a string based on regex, but down not remove the matched text
'''
find_pattern = r'%s.*?(?=%s|$)' % (regex_pattern, regex_pattern)
return re.findall(find_pattern, text, re.DOTALL)
def inserter(oldfile: str, newfile: str) -> Tuple[str, str]:
'''
Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows:
# begin insert @LabelName
def foo(bar):
return bar
# end insert @LabelName
'''
new_insert_objects = re.split(r"(# begin insert |# end insert @[a-zA-Z0-9_]*\n)", newfile)
# Retrieve label from insert objects
def get_labeled_object(labeled_text):
label = re.match(r"@[a-zA-Z0-9_]*\n", labeled_text)
if label is not None:
label = label.group(0)
labeled_text = re.sub(label, '', labeled_text)
return {'label': label, 'text': labeled_text}
new_insert_objects = map(get_labeled_object, new_insert_objects)
# Find and replace labels
newfile = ""
for item in new_insert_objects:
if item['label'] is not None:
oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile)
if insertions == 0:
newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label']))
elif re.match(r"(# begin insert |# end insert )", item['text']) is None:
newfile += item['text']
return oldfile, newfile
def merger(oldfile:str, newfile:str) -> str:
'''
Seeks out functions and objects in new and old files.
Replaces old objects with new ones if they exist.
'''
old_objects = split_retain_delimiter('\n[a-zA-Z]', oldfile)
new_objects = split_retain_delimiter('\n[a-zA-Z]', newfile)
object_regex = r"\n[#@a-zA-Z_0-9]+[\sa-zA-Z_0-9]*[(){}=:'" "]*"
old_object_tuples = list(map(lambda x: [re.match(object_regex, x).group(0),x], old_objects))
for new_item in new_objects:
found_old = False
for old_item in old_object_tuples:
if old_item[0] == re.match(object_regex, new_item).group(0):
print(old_item[0])
old_item[1] = new_item
found_old = True
break
if not found_old:
old_object_tuples += [[re.match(object_regex, new_item).group(0), new_item]]
return ''.join(elem for elem in map(lambda x: x[1], old_object_tuples))
def build_phase0_spec(sourcefile, outfile=None):
code_lines = []
code_lines.append("""
from typing import (
Any,
Dict,
List,
NewType,
Tuple,
)
from eth2spec.utils.minimal_ssz import (
SSZType,
hash_tree_root,
signing_root,
)
from eth2spec.utils.bls_stub import (
bls_aggregate_pubkeys,
bls_verify,
bls_verify_multiple,
)
from eth2spec.utils.hash_function import hash
# stub, will get overwritten by real var
SLOTS_PER_EPOCH = 64
Slot = NewType('Slot', int) # uint64
Epoch = NewType('Epoch', int) # uint64
Shard = NewType('Shard', int) # uint64
ValidatorIndex = NewType('ValidatorIndex', int) # uint64
Gwei = NewType('Gwei', int) # uint64
Bytes32 = NewType('Bytes32', bytes) # bytes32
BLSPubkey = NewType('BLSPubkey', bytes) # bytes48
BLSSignature = NewType('BLSSignature', bytes) # bytes96
Store = None
""")
code_lines += function_puller.get_spec(sourcefile)
code_lines.append("""
# Monkey patch validator compute committee code
_compute_committee = compute_committee
committee_cache = {}
def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]:
param_hash = (hash_tree_root(indices), seed, index, count)
if param_hash in committee_cache:
return committee_cache[param_hash]
else:
ret = _compute_committee(indices, seed, index, count)
committee_cache[param_hash] = ret
return ret
# Monkey patch hash cache
_hash = hash
hash_cache = {}
def hash(x):
if x in hash_cache:
return hash_cache[x]
else:
ret = _hash(x)
hash_cache[x] = ret
return ret
# Access to overwrite spec constants based on configuration
def apply_constants_preset(preset: Dict[str, Any]):
global_vars = globals()
for k, v in preset.items():
global_vars[k] = v
# Deal with derived constants
global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
# Initialize SSZ types again, to account for changed lengths
init_SSZ_types()
""")
if outfile is not None:
with open(outfile, 'w') as out:
out.write("\n".join(code_lines))
else:
return "\n".join(code_lines)
def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None):
phase0_code = build_phase0_spec(phase0_sourcefile)
phase1_code = build_phase0_spec(phase1_sourcefile)
phase0_code, phase1_code = inserter(phase0_code, phase1_code)
phase1_code = merger(phase0_code, phase1_code)
if outfile is not None:
with open(outfile, 'w') as out:
out.write(phase1_code)
else:
return phase1_code
if __name__ == '__main__':
description = '''
Build the specs from the md docs.
If building phase 0:
1st argument is input spec.md
2nd argument is output spec.py
If building phase 1:
1st argument is input spec_phase0.md
2nd argument is input spec_phase1.md
3rd argument is output spec.py
'''
parser = ArgumentParser(description=description)
parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #")
parser.add_argument(dest="files", help="Input and output files", nargs="+")
args = parser.parse_args()
if args.phase == 0:
build_phase0_spec(*args.files)
elif args.phase == 1:
if len(args.files) == 3:
build_phase1_spec(*args.files)
else:
print(" Phase 1 requires an output as well as 2 input files (phase0.md and phase1.md)")
else:
print("Invalid phase: {0}".format(args.phase))

0
scripts/phase0/function_puller.py → scripts/function_puller.py
View File

143
scripts/phase0/build_spec.py
View File

@ -1,143 +0,0 @@
import sys
import function_puller
from optparse import OptionParser
def build_phase0_spec(sourcefile, outfile):
code_lines = []
code_lines.append("""
from typing import (
Any,
Dict,
List,
NewType,
Tuple,
)
from eth2spec.utils.minimal_ssz import (
SSZType,
SSZTypeExtension,
hash_tree_root,
signing_root,
)
from eth2spec.utils.bls_stub import (
bls_aggregate_pubkeys,
bls_verify,
bls_verify_multiple,
)
from eth2spec.utils.hash_function import hash
# stub, will get overwritten by real var
SLOTS_PER_EPOCH = 64
Slot = NewType('Slot', int) # uint64
Epoch = NewType('Epoch', int) # uint64
Shard = NewType('Shard', int) # uint64
ValidatorIndex = NewType('ValidatorIndex', int) # uint64
Gwei = NewType('Gwei', int) # uint64
Bytes32 = NewType('Bytes32', bytes) # bytes32
BLSPubkey = NewType('BLSPubkey', bytes) # bytes48
BLSSignature = NewType('BLSSignature', bytes) # bytes96
Store = None
""")
code_lines += function_puller.get_spec(sourcefile)
code_lines.append("""
# Monkey patch validator compute committee code
_compute_committee = compute_committee
committee_cache = {}
def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]:
param_hash = (hash_tree_root(indices), seed, index, count)
if param_hash in committee_cache:
return committee_cache[param_hash]
else:
ret = _compute_committee(indices, seed, index, count)
committee_cache[param_hash] = ret
return ret
# Monkey patch hash cache
_hash = hash
hash_cache = {}
def hash(x):
if x in hash_cache:
return hash_cache[x]
else:
ret = _hash(x)
hash_cache[x] = ret
return ret
# Access to overwrite spec constants based on configuration
def apply_constants_preset(preset: Dict[str, Any]):
global_vars = globals()
for k, v in preset.items():
global_vars[k] = v
# Deal with derived constants
global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
# Initialize SSZ types again, to account for changed lengths
init_SSZ_types()
""")
with open(outfile, 'w') as out:
out.write("\n".join(code_lines))
def build_phase1_spec(sourcefile, outfile):
code_lines = []
code_lines.append("""
from eth2spec.phase0.spec import *
from eth2spec.phase0.spec import apply_constants_preset as apply_constants_preset_phase0
""")
code_lines += function_puller.get_spec(sourcefile)
code_lines.append("""
# Access to overwrite spec constants based on configuration
def apply_constants_preset(preset: Dict[str, Any]):
apply_constants_preset_phase0(preset)
global_vars = globals()
for k, v in preset.items():
global_vars[k] = v
# Deal with derived constants
global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
# Initialize SSZ types again, to account for changed lengths
init_SSZ_types()
""")
with open(outfile, 'w') as out:
out.write("\n".join(code_lines))
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("-p", "--phase", dest="phase", type="int", default=0,
help="Build for phase #")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
if options.phase == 0:
build_phase0_spec(args[0], args[1])
print(args[0])
print(args[1])
elif options.phase == 1:
build_phase1_spec(args[0], args[1])
else:
print("Invalid phase: {0}".format(options["phase"]))