new spec buildign stratergy
This commit is contained in:
parent
49d708e44b
commit
14ea7d9f40
6
Makefile
6
Makefile
|
@ -67,11 +67,11 @@ phase0: $(PY_SPEC_PHASE_0_TARGETS)
|
||||||
# "make phase1" to create pyspec for phase1
|
# "make phase1" to create pyspec for phase1
|
||||||
phase1: $(PY_SPEC_PHASE_1_TARGETS)
|
phase1: $(PY_SPEC_PHASE_1_TARGETS)
|
||||||
|
|
||||||
$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS)
|
$(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
|
||||||
python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@
|
python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@
|
||||||
|
|
||||||
$(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
|
$(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
|
||||||
python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@
|
python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $@
|
||||||
|
|
||||||
CURRENT_DIR = ${CURDIR}
|
CURRENT_DIR = ${CURDIR}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,196 @@
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
import function_puller
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from typing import Tuple, List
|
||||||
|
|
||||||
|
|
||||||
|
def split_retain_delimiter(regex_pattern: str, text: str) -> List[str]:
|
||||||
|
'''
|
||||||
|
Splits a string based on regex, but down not remove the matched text
|
||||||
|
'''
|
||||||
|
find_pattern = r'%s.*?(?=%s|$)' % (regex_pattern, regex_pattern)
|
||||||
|
return re.findall(find_pattern, text, re.DOTALL)
|
||||||
|
|
||||||
|
|
||||||
|
def inserter(oldfile: str, newfile: str) -> Tuple[str, str]:
|
||||||
|
'''
|
||||||
|
Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows:
|
||||||
|
# begin insert @LabelName
|
||||||
|
def foo(bar):
|
||||||
|
return bar
|
||||||
|
# end insert @LabelName
|
||||||
|
'''
|
||||||
|
new_insert_objects = re.split(r"(# begin insert |# end insert @[a-zA-Z0-9_]*\n)", newfile)
|
||||||
|
# Retrieve label from insert objects
|
||||||
|
def get_labeled_object(labeled_text):
|
||||||
|
label = re.match(r"@[a-zA-Z0-9_]*\n", labeled_text)
|
||||||
|
if label is not None:
|
||||||
|
label = label.group(0)
|
||||||
|
labeled_text = re.sub(label, '', labeled_text)
|
||||||
|
return {'label': label, 'text': labeled_text}
|
||||||
|
new_insert_objects = map(get_labeled_object, new_insert_objects)
|
||||||
|
# Find and replace labels
|
||||||
|
newfile = ""
|
||||||
|
for item in new_insert_objects:
|
||||||
|
if item['label'] is not None:
|
||||||
|
oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile)
|
||||||
|
if insertions == 0:
|
||||||
|
newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label']))
|
||||||
|
elif re.match(r"(# begin insert |# end insert )", item['text']) is None:
|
||||||
|
newfile += item['text']
|
||||||
|
return oldfile, newfile
|
||||||
|
|
||||||
|
|
||||||
|
def merger(oldfile:str, newfile:str) -> str:
|
||||||
|
'''
|
||||||
|
Seeks out functions and objects in new and old files.
|
||||||
|
Replaces old objects with new ones if they exist.
|
||||||
|
'''
|
||||||
|
old_objects = split_retain_delimiter('\n[a-zA-Z]', oldfile)
|
||||||
|
new_objects = split_retain_delimiter('\n[a-zA-Z]', newfile)
|
||||||
|
object_regex = r"\n[#@a-zA-Z_0-9]+[\sa-zA-Z_0-9]*[(){}=:'" "]*"
|
||||||
|
old_object_tuples = list(map(lambda x: [re.match(object_regex, x).group(0),x], old_objects))
|
||||||
|
for new_item in new_objects:
|
||||||
|
found_old = False
|
||||||
|
for old_item in old_object_tuples:
|
||||||
|
if old_item[0] == re.match(object_regex, new_item).group(0):
|
||||||
|
print(old_item[0])
|
||||||
|
old_item[1] = new_item
|
||||||
|
found_old = True
|
||||||
|
break
|
||||||
|
if not found_old:
|
||||||
|
old_object_tuples += [[re.match(object_regex, new_item).group(0), new_item]]
|
||||||
|
return ''.join(elem for elem in map(lambda x: x[1], old_object_tuples))
|
||||||
|
|
||||||
|
|
||||||
|
def build_phase0_spec(sourcefile, outfile=None):
|
||||||
|
code_lines = []
|
||||||
|
code_lines.append("""
|
||||||
|
|
||||||
|
from typing import (
|
||||||
|
Any,
|
||||||
|
Dict,
|
||||||
|
List,
|
||||||
|
NewType,
|
||||||
|
Tuple,
|
||||||
|
)
|
||||||
|
from eth2spec.utils.minimal_ssz import (
|
||||||
|
SSZType,
|
||||||
|
hash_tree_root,
|
||||||
|
signing_root,
|
||||||
|
)
|
||||||
|
from eth2spec.utils.bls_stub import (
|
||||||
|
bls_aggregate_pubkeys,
|
||||||
|
bls_verify,
|
||||||
|
bls_verify_multiple,
|
||||||
|
)
|
||||||
|
from eth2spec.utils.hash_function import hash
|
||||||
|
|
||||||
|
|
||||||
|
# stub, will get overwritten by real var
|
||||||
|
SLOTS_PER_EPOCH = 64
|
||||||
|
|
||||||
|
Slot = NewType('Slot', int) # uint64
|
||||||
|
Epoch = NewType('Epoch', int) # uint64
|
||||||
|
Shard = NewType('Shard', int) # uint64
|
||||||
|
ValidatorIndex = NewType('ValidatorIndex', int) # uint64
|
||||||
|
Gwei = NewType('Gwei', int) # uint64
|
||||||
|
Bytes32 = NewType('Bytes32', bytes) # bytes32
|
||||||
|
BLSPubkey = NewType('BLSPubkey', bytes) # bytes48
|
||||||
|
BLSSignature = NewType('BLSSignature', bytes) # bytes96
|
||||||
|
Store = None
|
||||||
|
""")
|
||||||
|
|
||||||
|
code_lines += function_puller.get_spec(sourcefile)
|
||||||
|
|
||||||
|
code_lines.append("""
|
||||||
|
# Monkey patch validator compute committee code
|
||||||
|
_compute_committee = compute_committee
|
||||||
|
committee_cache = {}
|
||||||
|
|
||||||
|
|
||||||
|
def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]:
|
||||||
|
param_hash = (hash_tree_root(indices), seed, index, count)
|
||||||
|
|
||||||
|
if param_hash in committee_cache:
|
||||||
|
return committee_cache[param_hash]
|
||||||
|
else:
|
||||||
|
ret = _compute_committee(indices, seed, index, count)
|
||||||
|
committee_cache[param_hash] = ret
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
# Monkey patch hash cache
|
||||||
|
_hash = hash
|
||||||
|
hash_cache = {}
|
||||||
|
|
||||||
|
|
||||||
|
def hash(x):
|
||||||
|
if x in hash_cache:
|
||||||
|
return hash_cache[x]
|
||||||
|
else:
|
||||||
|
ret = _hash(x)
|
||||||
|
hash_cache[x] = ret
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
# Access to overwrite spec constants based on configuration
|
||||||
|
def apply_constants_preset(preset: Dict[str, Any]):
|
||||||
|
global_vars = globals()
|
||||||
|
for k, v in preset.items():
|
||||||
|
global_vars[k] = v
|
||||||
|
|
||||||
|
# Deal with derived constants
|
||||||
|
global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
|
||||||
|
|
||||||
|
# Initialize SSZ types again, to account for changed lengths
|
||||||
|
init_SSZ_types()
|
||||||
|
""")
|
||||||
|
|
||||||
|
if outfile is not None:
|
||||||
|
with open(outfile, 'w') as out:
|
||||||
|
out.write("\n".join(code_lines))
|
||||||
|
else:
|
||||||
|
return "\n".join(code_lines)
|
||||||
|
|
||||||
|
|
||||||
|
def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None):
|
||||||
|
phase0_code = build_phase0_spec(phase0_sourcefile)
|
||||||
|
phase1_code = build_phase0_spec(phase1_sourcefile)
|
||||||
|
phase0_code, phase1_code = inserter(phase0_code, phase1_code)
|
||||||
|
phase1_code = merger(phase0_code, phase1_code)
|
||||||
|
|
||||||
|
if outfile is not None:
|
||||||
|
with open(outfile, 'w') as out:
|
||||||
|
out.write(phase1_code)
|
||||||
|
else:
|
||||||
|
return phase1_code
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
description = '''
|
||||||
|
Build the specs from the md docs.
|
||||||
|
If building phase 0:
|
||||||
|
1st argument is input spec.md
|
||||||
|
2nd argument is output spec.py
|
||||||
|
|
||||||
|
If building phase 1:
|
||||||
|
1st argument is input spec_phase0.md
|
||||||
|
2nd argument is input spec_phase1.md
|
||||||
|
3rd argument is output spec.py
|
||||||
|
'''
|
||||||
|
parser = ArgumentParser(description=description)
|
||||||
|
parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #")
|
||||||
|
parser.add_argument(dest="files", help="Input and output files", nargs="+")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.phase == 0:
|
||||||
|
build_phase0_spec(*args.files)
|
||||||
|
elif args.phase == 1:
|
||||||
|
if len(args.files) == 3:
|
||||||
|
build_phase1_spec(*args.files)
|
||||||
|
else:
|
||||||
|
print(" Phase 1 requires an output as well as 2 input files (phase0.md and phase1.md)")
|
||||||
|
else:
|
||||||
|
print("Invalid phase: {0}".format(args.phase))
|
|
@ -1,143 +0,0 @@
|
||||||
import sys
|
|
||||||
import function_puller
|
|
||||||
from optparse import OptionParser
|
|
||||||
|
|
||||||
def build_phase0_spec(sourcefile, outfile):
|
|
||||||
code_lines = []
|
|
||||||
code_lines.append("""
|
|
||||||
|
|
||||||
from typing import (
|
|
||||||
Any,
|
|
||||||
Dict,
|
|
||||||
List,
|
|
||||||
NewType,
|
|
||||||
Tuple,
|
|
||||||
)
|
|
||||||
from eth2spec.utils.minimal_ssz import (
|
|
||||||
SSZType,
|
|
||||||
SSZTypeExtension,
|
|
||||||
hash_tree_root,
|
|
||||||
signing_root,
|
|
||||||
)
|
|
||||||
from eth2spec.utils.bls_stub import (
|
|
||||||
bls_aggregate_pubkeys,
|
|
||||||
bls_verify,
|
|
||||||
bls_verify_multiple,
|
|
||||||
)
|
|
||||||
from eth2spec.utils.hash_function import hash
|
|
||||||
|
|
||||||
|
|
||||||
# stub, will get overwritten by real var
|
|
||||||
SLOTS_PER_EPOCH = 64
|
|
||||||
|
|
||||||
Slot = NewType('Slot', int) # uint64
|
|
||||||
Epoch = NewType('Epoch', int) # uint64
|
|
||||||
Shard = NewType('Shard', int) # uint64
|
|
||||||
ValidatorIndex = NewType('ValidatorIndex', int) # uint64
|
|
||||||
Gwei = NewType('Gwei', int) # uint64
|
|
||||||
Bytes32 = NewType('Bytes32', bytes) # bytes32
|
|
||||||
BLSPubkey = NewType('BLSPubkey', bytes) # bytes48
|
|
||||||
BLSSignature = NewType('BLSSignature', bytes) # bytes96
|
|
||||||
Store = None
|
|
||||||
""")
|
|
||||||
|
|
||||||
code_lines += function_puller.get_spec(sourcefile)
|
|
||||||
|
|
||||||
code_lines.append("""
|
|
||||||
# Monkey patch validator compute committee code
|
|
||||||
_compute_committee = compute_committee
|
|
||||||
committee_cache = {}
|
|
||||||
|
|
||||||
|
|
||||||
def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]:
|
|
||||||
param_hash = (hash_tree_root(indices), seed, index, count)
|
|
||||||
|
|
||||||
if param_hash in committee_cache:
|
|
||||||
return committee_cache[param_hash]
|
|
||||||
else:
|
|
||||||
ret = _compute_committee(indices, seed, index, count)
|
|
||||||
committee_cache[param_hash] = ret
|
|
||||||
return ret
|
|
||||||
|
|
||||||
|
|
||||||
# Monkey patch hash cache
|
|
||||||
_hash = hash
|
|
||||||
hash_cache = {}
|
|
||||||
|
|
||||||
|
|
||||||
def hash(x):
|
|
||||||
if x in hash_cache:
|
|
||||||
return hash_cache[x]
|
|
||||||
else:
|
|
||||||
ret = _hash(x)
|
|
||||||
hash_cache[x] = ret
|
|
||||||
return ret
|
|
||||||
|
|
||||||
|
|
||||||
# Access to overwrite spec constants based on configuration
|
|
||||||
def apply_constants_preset(preset: Dict[str, Any]):
|
|
||||||
global_vars = globals()
|
|
||||||
for k, v in preset.items():
|
|
||||||
global_vars[k] = v
|
|
||||||
|
|
||||||
# Deal with derived constants
|
|
||||||
global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
|
|
||||||
|
|
||||||
# Initialize SSZ types again, to account for changed lengths
|
|
||||||
init_SSZ_types()
|
|
||||||
""")
|
|
||||||
|
|
||||||
with open(outfile, 'w') as out:
|
|
||||||
out.write("\n".join(code_lines))
|
|
||||||
|
|
||||||
|
|
||||||
def build_phase1_spec(sourcefile, outfile):
|
|
||||||
code_lines = []
|
|
||||||
code_lines.append("""
|
|
||||||
from eth2spec.phase0.spec import *
|
|
||||||
from eth2spec.phase0.spec import apply_constants_preset as apply_constants_preset_phase0
|
|
||||||
|
|
||||||
""")
|
|
||||||
|
|
||||||
code_lines += function_puller.get_spec(sourcefile)
|
|
||||||
|
|
||||||
code_lines.append("""
|
|
||||||
# Access to overwrite spec constants based on configuration
|
|
||||||
def apply_constants_preset(preset: Dict[str, Any]):
|
|
||||||
|
|
||||||
apply_constants_preset_phase0(preset)
|
|
||||||
|
|
||||||
global_vars = globals()
|
|
||||||
for k, v in preset.items():
|
|
||||||
global_vars[k] = v
|
|
||||||
|
|
||||||
# Deal with derived constants
|
|
||||||
global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT)
|
|
||||||
|
|
||||||
# Initialize SSZ types again, to account for changed lengths
|
|
||||||
init_SSZ_types()
|
|
||||||
""")
|
|
||||||
|
|
||||||
with open(outfile, 'w') as out:
|
|
||||||
out.write("\n".join(code_lines))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
parser = OptionParser()
|
|
||||||
parser.add_option("-p", "--phase", dest="phase", type="int", default=0,
|
|
||||||
help="Build for phase #")
|
|
||||||
|
|
||||||
(options, args) = parser.parse_args()
|
|
||||||
|
|
||||||
if len(args) < 2:
|
|
||||||
parser.print_help()
|
|
||||||
|
|
||||||
if options.phase == 0:
|
|
||||||
build_phase0_spec(args[0], args[1])
|
|
||||||
print(args[0])
|
|
||||||
print(args[1])
|
|
||||||
elif options.phase == 1:
|
|
||||||
build_phase1_spec(args[0], args[1])
|
|
||||||
else:
|
|
||||||
print("Invalid phase: {0}".format(options["phase"]))
|
|
||||||
|
|
Loading…
Reference in New Issue