From cfed4bb41ce30fb25b6ae2f36dc3439d4ec24256 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 10:50:07 +0200 Subject: [PATCH] Adds missing functions --- scripts/build_spec.py | 101 ++++++------------ scripts/function_puller.py | 4 - specs/core/1_custody-game.md | 20 ++-- .../pyspec/eth2spec/utils/minimal_ssz.py | 10 ++ 4 files changed, 50 insertions(+), 85 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index cdcbc2456..f244b91e1 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -5,7 +5,7 @@ from argparse import ArgumentParser from typing import Tuple, List -IMPORTS = '''from typing import ( +PHASE0_IMPORTS = '''from typing import ( Any, Dict, List, @@ -27,6 +27,31 @@ from eth2spec.utils.bls_stub import ( from eth2spec.utils.hash_function import hash ''' +PHASE1_IMPORTS = '''from typing import ( + Any, + Dict, + List, + NewType, + Tuple, +) + +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, + type_of, + empty, +) + +from eth2spec.utils.bls_stub import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) + +from eth2spec.utils.hash_function import hash +import math +''' NEW_TYPE_DEFINITIONS = ''' Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 @@ -82,74 +107,8 @@ def apply_constants_preset(preset: Dict[str, Any]): init_SSZ_types() ''' -def split_and_label(regex_pattern: str, text: str) -> List[str]: - ''' - Splits a string based on regex, but down not remove the matched text. - It subsequently labels the matches with their match - ''' - find_pattern = r'''%s.*?(?=%s|$)''' % (regex_pattern, regex_pattern) - matches = re.findall(find_pattern, text, re.DOTALL) - return list(map(lambda x: [re.match(regex_pattern, x).group(0), x], matches)) - -def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: - ''' - Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows: - # begin insert @LabelName - def foo(bar): - return bar - # end insert @LabelName - ''' - new_insert_objects = re.split(r"(# begin insert |# end insert @[\w\d_]*\n)", newfile) - # Retrieve label from insert objects - def get_labeled_object(labeled_text): - label = re.match(r"@[\w\d_]*\n", labeled_text) - if label is not None: - label = label.group(0) - labeled_text = re.sub(label, '', labeled_text) - return {'label': label, 'text': labeled_text} - new_insert_objects = map(get_labeled_object, new_insert_objects) - # Find and replace labels - newfile = "" - for item in new_insert_objects: - if item['label'] is not None: - oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile) - if insertions == 0: - newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label'])) - elif re.match(r"(# begin insert |# end insert )", item['text']) is None: - newfile += item['text'] - return oldfile, newfile - - -def merger(oldfile:str, newfile:str) -> str: - ''' - Seeks out functions and objects in new and old files. - Replaces old objects with new ones if they exist. - ''' - object_regex = r'''(?:\n[@\w]+[\s\w]*[='" "\.\w]*)|(?:\s{4}global_vars\["\w+"\])''' - ssz_object_regex = r'''(?:\w+|\s{4}global_vars\["\w+"\]) = SSZType\(\{\n''' - old_objects = split_and_label(object_regex, oldfile) - new_objects = split_and_label(object_regex, newfile) - for new_item in new_objects: - found_old = False - for index, old_item in enumerate(old_objects): - if old_item[0] == new_item[0]: - ssz_object_match = re.match(ssz_object_regex, new_item[1]) - if ssz_object_match is not None: - new_item[1] = re.sub(ssz_object_regex, '', new_item[1]) - old_item[1] = re.sub(r'\n\w*\}\)', '', old_item[1]) - old_item[1] += new_item[1] - else: - old_item[1] = new_item[1] - found_old = True - old_objects[index] = old_item - break - if not found_old: - old_objects.append(new_item) - return ''.join(elem for elem in map(lambda x: x[1], old_objects)) - - -def objects_to_spec(functions, constants, ssz_objects): +def objects_to_spec(functions, constants, ssz_objects, imports): functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) ssz_objects_instantiation_spec = '\n'.join(map(lambda x: '%s = SSZType(%s)' % (x, ssz_objects[x][:-1]), ssz_objects)) @@ -160,7 +119,7 @@ def objects_to_spec(functions, constants, ssz_objects): + ssz_objects_reinitialization_spec ) return ( - IMPORTS + imports + '\n' + NEW_TYPE_DEFINITIONS + '\n' + constants_spec + '\n' + ssz_objects_instantiation_spec @@ -194,7 +153,7 @@ def combine_ssz_objects(old_objects, new_objects): def build_phase0_spec(sourcefile, outfile=None): functions, constants, ssz_objects = function_puller.get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects) + spec = objects_to_spec(functions, constants, ssz_objects, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -208,7 +167,7 @@ def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): functions = combine_functions(phase0_functions, phase1_functions) constants = combine_constants(phase0_constants, phase1_constants) ssz_objects = combine_functions(phase0_ssz_objects, phase1_ssz_objects) - spec = objects_to_spec(functions, constants, ssz_objects) + spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 720f07502..dc0c30998 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -15,7 +15,6 @@ def get_spec(file_name: str): constants = {} ssz_objects = defaultdict(str) function_matcher = re.compile(FUNCTION_REGEX) - # type_defs = [] for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -26,9 +25,6 @@ def get_spec(file_name: str): elif line[:3] == '```': pulling_from = None else: - # # Handle SSZObjects - # if pulling_from == linenum and line == '{': - # code_lines.append('%s = SSZType({' % current_name) # Handle function definitions if pulling_from is not None: match = function_matcher.match(line) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index df4f495be..adc121160 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -31,7 +31,7 @@ - [`BeaconState`](#beaconstate) - [`BeaconBlockBody`](#beaconblockbody) - [Helpers](#helpers) - - [`typeof`](#typeof) + - [`type_of`](#type_of) - [`empty`](#empty) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) @@ -259,13 +259,13 @@ Add the following fields to the end of the specified container objects. Fields w ## Helpers -### `typeof` +### `type_of` -The `typeof` function accepts and SSZ object as a single input and returns the corresponding SSZ type. +The `type_of` function accepts an SSZ object as a single input and returns the corresponding SSZ type. ### `empty` -The `empty` function accepts and SSZ type as input and returns an object of that type with all fields initialized to default values. +The `empty` function accepts an SSZ type as input and returns an object of that type with all fields initialized to default values. ### `get_crosslink_chunk_count` @@ -327,7 +327,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ```python def replace_empty_or_append(list: List[Any], new_element: Any) -> int: for i in range(len(list)): - if list[i] == empty(typeof(new_element)): + if list[i] == empty(type_of(new_element)): list[i] = new_element return i list.append(new_element) @@ -492,7 +492,7 @@ def process_chunk_challenge(state: BeaconState, record.chunk_index != challenge.chunk_index ) # Verify depth - depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation))) + depth = math.ceil(math.log2(get_custody_chunk_count(challenge.attestation))) assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -580,7 +580,7 @@ def process_bit_challenge(state: BeaconState, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, data_root=challenge.attestation.data.crosslink.data_root, chunk_count=chunk_count, - chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), + chunk_bits_merkle_root=hash_tree_root(challenge.chunk_bits), responder_key=challenge.responder_key, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) @@ -646,7 +646,7 @@ def process_bit_challenge_response(state: BeaconState, assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.data_branch, - depth=math.log2(next_power_of_two(challenge.chunk_count)), + depth=math.ceil(math.log2(challenge.chunk_count)), index=response.chunk_index, root=challenge.data_root, ) @@ -654,13 +654,13 @@ def process_bit_challenge_response(state: BeaconState, assert verify_merkle_branch( leaf=response.chunk_bits_leaf, branch=response.chunk_bits_branch, - depth=math.log2(next_power_of_two(challenge.chunk_count) // 256), + depth=math.ceil(math.log2(challenge.chunk_count)) >> 8, index=response.chunk_index // 256, root=challenge.chunk_bits_merkle_root ) # Verify the chunk bit does not match the challenge chunk bit assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) + != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index f7784b6de..63516780e 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -58,6 +58,16 @@ class Vector(): return self.length +def type_of(obj): + return obj.__class__ + + +def empty(obj): + for field in obj.fields: + field = get_zero_value(field) + return obj + + def is_basic(typ): # if not a string, it is a complex, and cannot be basic if not isinstance(typ, str):