From 5ddfe34f0c3363589b402e88abde2c0aa1b05225 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:51:38 +0200 Subject: [PATCH 01/68] Simplified SSZ impl --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 13 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 407 ++++-------------- 2 files changed, 102 insertions(+), 318 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b3c877d48..c88cfed1f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,9 +1,8 @@ -from ..merkle_minimal import merkleize_chunks, hash -from eth2spec.utils.ssz.ssz_typing import ( +from ..merkle_minimal import merkleize_chunks, hash, ZERO_BYTES32 +from .ssz_typing import ( is_uint_type, is_bool_type, is_container_type, is_list_kind, is_vector_kind, - read_vector_elem_type, read_elem_type, - uint_byte_size, + read_elem_type, infer_input_type, get_zero_value, ) @@ -20,7 +19,7 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint_type(typ): - return value.to_bytes(uint_byte_size(typ), 'little') + return value.to_bytes(typ.byte_len, 'little') elif is_bool_type(typ): if value: return b'\x01' @@ -140,6 +139,8 @@ def get_typed_values(obj, typ=None): else: raise Exception("Invalid type") +def item_length(typ): + return 1 if typ == bool else typ.byte_len if is_uint_type(typ) else 32 @infer_input_type def hash_tree_root(obj, typ=None): @@ -150,6 +151,8 @@ def hash_tree_root(obj, typ=None): fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): + full_chunk_length = (item_length(read_elem_type(typ)) * typ.length + 31) // 32 + leaves += [ZERO_BYTES32] * (full_chunk_length - len(obj)) return mix_in_length(merkleize_chunks(leaves), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index f870336e8..dbc3f9523 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -6,74 +6,38 @@ from typing_inspect import get_origin # SSZ integers # ----------------------------- - class uint(int): byte_len = 0 def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") + if value.byte_len and value.bit_length() > value.byte_len: + raise ValueError("value out of bounds for uint{}".format(value.byte_len)) return super().__new__(cls, value) class uint8(uint): byte_len = 1 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 8: - raise ValueError("value out of bounds for uint8") - return super().__new__(cls, value) - - # Alias for uint8 byte = NewType('byte', uint8) - class uint16(uint): byte_len = 2 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 16: - raise ValueError("value out of bounds for uint16") - return super().__new__(cls, value) - - class uint32(uint): byte_len = 4 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 32: - raise ValueError("value out of bounds for uint16") - return super().__new__(cls, value) - - class uint64(uint): byte_len = 8 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 64: - raise ValueError("value out of bounds for uint64") - return super().__new__(cls, value) - - class uint128(uint): byte_len = 16 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 128: - raise ValueError("value out of bounds for uint128") - return super().__new__(cls, value) - - class uint256(uint): byte_len = 32 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 256: - raise ValueError("value out of bounds for uint256") - return super().__new__(cls, value) - - def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. @@ -84,21 +48,6 @@ def is_uint_type(typ): return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) - -def uint_byte_size(typ): - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ - - if isinstance(typ, type): - if issubclass(typ, uint): - return typ.byte_len - elif issubclass(typ, int): - # Default to uint64 - return 8 - else: - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) - - # SSZ Container base class # ----------------------------- @@ -166,45 +115,19 @@ class Container(object): return list(cls.__annotations__.values()) -# SSZ vector -# ----------------------------- - - -def _is_vector_instance_of(a, b): - # Other must not be a BytesN - if issubclass(b, bytes): - return False - elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector (b) is not an instance of Vector[X, Y] (a) - return False - elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): - # Vector[X, Y] (b) is an instance of Vector (a) - return True +def get_zero_value(typ): + if typ == int: + return 0 else: - # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) - return a.elem_type == b.elem_type and a.length == b.length + return typ.default() - -def _is_equal_vector_type(a, b): - # Other must not be a BytesN - if issubclass(b, bytes): - return False - elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector == Vector - return True - else: - # Vector != Vector[X, Y] - return False - elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector[X, Y] != Vector - return False +def type_check(typ, value): + if typ == int or typ == uint64: + return isinstance(value, int) else: - # Vector[X, Y] == Vector[X, Y] - return a.elem_type == b.elem_type and a.length == b.length + return typ.value_check(value) - -class VectorMeta(type): +class AbstractListMeta(type): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) if 'elem_type' in attrs and 'length' in attrs: @@ -214,239 +137,115 @@ class VectorMeta(type): def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: - raise Exception("Vector must be instantiated with two args: elem type and length") - o = self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) - o._name = 'Vector' + raise Exception("List must be instantiated with two args: elem type and length") + o = self.__class__(self.__name__, (self,), {'elem_type': params[0], 'length': params[1]}) + o._name = 'AbstractList' return o - def __subclasscheck__(self, sub): - return _is_vector_instance_of(self, sub) + def __instancecheck__(self, obj): + if obj.__class__.__name__ != self.__name__: + return False + if hasattr(self, 'elem_type') and obj.__class__.elem_type != self.elem_type: + return False + if hasattr(self, 'length') and obj.__class__.length != self.length: + return False + return True - def __instancecheck__(self, other): - return _is_vector_instance_of(self, other.__class__) +class ValueCheckError(Exception): + pass - def __eq__(self, other): - return _is_equal_vector_type(self, other) +class AbstractList(metaclass=AbstractListMeta): + def __init__(self, *args): + items = self.extract_args(args) + + if not self.value_check(items): + raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) + self.items = items + + def value_check(self, value): + for v in value: + if not type_check(self.__class__.elem_type, v): + return False + return True - def __ne__(self, other): - return not _is_equal_vector_type(self, other) + def extract_args(self, args): + return list(args) if len(args) > 0 else self.default() - def __hash__(self): - return hash(self.__class__) + def default(self): + raise Exception("Not implemented") + def __getitem__(self, i): + return self.items[i] -class Vector(metaclass=VectorMeta): - - def __init__(self, *args: Iterable): - cls = self.__class__ - if not hasattr(cls, 'elem_type'): - raise TypeError("Type Vector without elem_type data cannot be instantiated") - elif not hasattr(cls, 'length'): - raise TypeError("Type Vector without length data cannot be instantiated") - - if len(args) != cls.length: - if len(args) == 0: - args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] - else: - raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) - - self.items = list(args) - - # cannot check non-type objects, or parametrized types - if isinstance(cls.elem_type, type) and not hasattr(cls.elem_type, '__args__'): - for i, item in enumerate(self.items): - if not issubclass(cls.elem_type, type(item)): - raise TypeError("Typed vector cannot hold differently typed value" - " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) - - def serialize(self): - from .ssz_impl import serialize - return serialize(self, self.__class__) - - def hash_tree_root(self): - from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) - - def __repr__(self): - return repr({'length': self.__class__.length, 'items': self.items}) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) + def __setitem__(self, k, v): + self.items[k] = v def __len__(self): return len(self.items) - def __eq__(self, other): - return self.hash_tree_root() == other.hash_tree_root() + def __repr__(self): + return repr(self.items) - -# SSZ BytesN -# ----------------------------- - - -def _is_bytes_n_instance_of(a, b): - # Other has to be a Bytes derivative class to be a BytesN - if not issubclass(b, bytes): - return False - elif not hasattr(b, 'length'): - # BytesN (b) is not an instance of BytesN[X] (a) - return False - elif not hasattr(a, 'length'): - # BytesN[X] (b) is an instance of BytesN (a) - return True - else: - # BytesN[X] (a) is an instance of BytesN[X] (b) - return a.length == b.length - - -def _is_equal_bytes_n_type(a, b): - # Other has to be a Bytes derivative class to be a BytesN - if not issubclass(b, bytes): - return False - elif not hasattr(a, 'length'): - if not hasattr(b, 'length'): - # BytesN == BytesN - return True - else: - # BytesN != BytesN[X] - return False - elif not hasattr(b, 'length'): - # BytesN[X] != BytesN - return False - else: - # BytesN[X] == BytesN[X] - return a.length == b.length - - -class BytesNMeta(type): - def __new__(cls, class_name, parents, attrs): - out = type.__new__(cls, class_name, parents, attrs) - if 'length' in attrs: - setattr(out, 'length', attrs['length']) - out._name = 'BytesN' - out.elem_type = byte - return out - - def __getitem__(self, n): - return self.__class__(self.__name__, (BytesN,), {'length': n}) - - def __subclasscheck__(self, sub): - return _is_bytes_n_instance_of(self, sub) - - def __instancecheck__(self, other): - return _is_bytes_n_instance_of(self, other.__class__) + def __iter__(self): + return iter(self.items) def __eq__(self, other): - return _is_equal_bytes_n_type(self, other) + return self.items == other.items - def __ne__(self, other): - return not _is_equal_bytes_n_type(self, other) +class List(AbstractList, metaclass=AbstractListMeta): + def value_check(self, value): + return len(value) <= self.__class__.length and super().value_check(value) - def __hash__(self): - return hash(self.__class__) - - -def parse_bytes(val): - if val is None: - return None - elif isinstance(val, str): - # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) - return None - elif isinstance(val, bytes): - return val - elif isinstance(val, int): - return bytes([val]) - elif isinstance(val, (list, GeneratorType)): - return bytes(val) - else: - return None - - -class BytesN(bytes, metaclass=BytesNMeta): - def __new__(cls, *args): - if not hasattr(cls, 'length'): - return - bytesval = None - if len(args) == 1: - val: Union[bytes, int, str] = args[0] - bytesval = parse_bytes(val) - elif len(args) > 1: - # TODO: each int is 1 byte, check size, create bytesval - bytesval = bytes(args) - - if bytesval is None: - if cls.length == 0: - bytesval = b'' - else: - bytesval = b'\x00' * cls.length - if len(bytesval) != cls.length: - raise TypeError("BytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) - return super().__new__(cls, bytesval) - - def serialize(self): - from .ssz_impl import serialize - return serialize(self, self.__class__) - - def hash_tree_root(self): - from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) - - -class Bytes4(BytesN): - length = 4 - - -class Bytes32(BytesN): - length = 32 - - -class Bytes48(BytesN): - length = 48 - - -class Bytes96(BytesN): - length = 96 - - -# SSZ Defaults -# ----------------------------- -def get_zero_value(typ): - if is_uint_type(typ): - return uint64(0) - elif is_list_type(typ): + def default(self): return [] - elif is_bool_type(typ): - return False - elif is_vector_type(typ): - return typ() - elif is_bytesn_type(typ): - return typ() - elif is_bytes_type(typ): + +class Vector(AbstractList, metaclass=AbstractListMeta): + def value_check(self, value): + return len(value) == self.__class__.length and super().value_check(value) + + def default(self): + return [get_zero_value(self.__class__.elem_type) for _ in range(self.__class__.length)] + +class BytesMeta(AbstractListMeta): + def __getitem__(self, params): + if not isinstance(params, int): + raise Exception("Bytes must be instantiated with one arg: length") + o = self.__class__(self.__name__, (self,), {'length': params}) + o._name = 'Bytes' + return o + +def single_item_extractor(cls, args): + assert len(args) < 2 + return args[0] if len(args) > 0 else cls.default() + +class Bytes(AbstractList, metaclass=BytesMeta): + def value_check(self, value): + return len(value) <= self.__class__.length and isinstance(value, bytes) + + extract_args = single_item_extractor + + def default(self): return b'' - elif is_container_type(typ): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - else: - raise Exception("Type not supported: {}".format(typ)) + +class BytesN(AbstractList, metaclass=BytesMeta): + def value_check(self, value): + return len(value) == self.__class__.length and isinstance(value, bytes) + + extract_args = single_item_extractor + + def default(self): + return b'\x00' * self.__class__.length # Type helpers # ----------------------------- - def infer_type(obj): if is_uint_type(obj.__class__): return obj.__class__ elif isinstance(obj, int): return uint64 - elif isinstance(obj, list): - return List[infer_type(obj[0])] - elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): + elif isinstance(obj, (List, Vector, Container, bool, BytesN, Bytes)): return obj.__class__ else: raise Exception("Unknown type for {}".format(obj)) @@ -476,15 +275,14 @@ def is_list_type(typ): """ Check if the given type is a list. """ - return get_origin(typ) is List or get_origin(typ) is list + return isinstance(typ, type) and issubclass(typ, List) def is_bytes_type(typ): """ Check if the given type is a ``bytes``. """ - # Do not accept subclasses of bytes here, to avoid confusion with BytesN - return typ == bytes + return isinstance(typ, type) and issubclass(typ, Bytes) def is_bytesn_type(typ): @@ -526,22 +324,5 @@ T = TypeVar('T') L = TypeVar('L') -def read_list_elem_type(list_typ: Type[List[T]]) -> T: - if list_typ.__args__ is None or len(list_typ.__args__) != 1: - raise TypeError("Supplied list-type is invalid, no element type found.") - return list_typ.__args__[0] - - -def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: - return vector_typ.elem_type - - def read_elem_type(typ): - if typ == bytes or (isinstance(typ, type) and issubclass(typ, bytes)): # bytes or bytesN - return byte - elif is_list_type(typ): - return read_list_elem_type(typ) - elif is_vector_type(typ): - return read_vector_elem_type(typ) - else: - raise TypeError("Unexpected type: {}".format(typ)) + return typ.elem_type From 7c4232455c305b58143d9f62dfd29ebbaa2eb5e2 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Fri, 14 Jun 2019 14:07:25 -0400 Subject: [PATCH 02/68] Added get_container_type to get_zero_value --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index dbc3f9523..de54bbf05 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -118,6 +118,8 @@ class Container(object): def get_zero_value(typ): if typ == int: return 0 + elif is_container_type(typ): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: return typ.default() From 8919f628cbb9f17472716cb8cd850b963a7f37c7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 14 Jun 2019 18:32:30 -0400 Subject: [PATCH 03/68] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: Diederik Loerakker --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index c88cfed1f..b08a3d4e2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,4 +1,5 @@ -from ..merkle_minimal import merkleize_chunks, hash, ZERO_BYTES32 +from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 +from .hash_function import hash from .ssz_typing import ( is_uint_type, is_bool_type, is_container_type, is_list_kind, is_vector_kind, From d1ecfd510ec2738dd267994a1e2b305176bb4eda Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:53:32 +0200 Subject: [PATCH 04/68] typing improvements --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 297 ++++++++---------- 1 file changed, 128 insertions(+), 169 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index de54bbf05..9aafb5294 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,52 +1,56 @@ from types import GeneratorType -from typing import List, Iterable, TypeVar, Type, NewType -from typing import Union -from typing_inspect import get_origin + + +class DefaultingTypeMeta(type): + def default(cls): + raise Exception("Not implemented") # SSZ integers # ----------------------------- -class uint(int): + +class uint(int, metaclass=DefaultingTypeMeta): byte_len = 0 def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") - if value.byte_len and value.bit_length() > value.byte_len: - raise ValueError("value out of bounds for uint{}".format(value.byte_len)) + if cls.byte_len and (value.bit_length() >> 3) > cls.byte_len: + raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) return super().__new__(cls, value) + @classmethod + def default(cls): + return cls(0) + class uint8(uint): byte_len = 1 + # Alias for uint8 byte = NewType('byte', uint8) + class uint16(uint): byte_len = 2 + class uint32(uint): byte_len = 4 + class uint64(uint): byte_len = 8 + class uint128(uint): byte_len = 16 + class uint256(uint): byte_len = 32 -def is_uint_type(typ): - # All integers are uint in the scope of the spec here. - # Since we default to uint64. Bounds can be checked elsewhere. - # However, some are wrapped in a NewType - if hasattr(typ, '__supertype__'): - # get the type that the NewType is wrapping - typ = typ.__supertype__ - - return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) # SSZ Container base class # ----------------------------- @@ -59,7 +63,7 @@ class Container(object): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, get_zero_value(t)) + setattr(self, f, t.default()) else: setattr(self, f, kwargs[f]) @@ -83,9 +87,9 @@ class Container(object): return repr({field: getattr(self, field) for field in self.get_field_names()}) def __str__(self): - output = [] + output = [f'{self.__class__.__name__}'] for field in self.get_field_names(): - output.append(f'{field}: {getattr(self, field)}') + output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) def __eq__(self, other): @@ -114,67 +118,94 @@ class Container(object): # values of annotations are the types corresponding to the fields, not instance values. return list(cls.__annotations__.values()) + @classmethod + def default(cls): + return cls(**{f: t.default() for f, t in cls.get_fields()}) -def get_zero_value(typ): - if typ == int: - return 0 - elif is_container_type(typ): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - else: - return typ.default() -def type_check(typ, value): - if typ == int or typ == uint64: - return isinstance(value, int) - else: - return typ.value_check(value) +class ParamsBase: + _bare = True + + def __new__(cls, *args, **kwargs): + if cls._bare: + raise Exception("cannot init bare type without params") + return super().__new__(cls, **kwargs) + + +class ParamsMeta(DefaultingTypeMeta): -class AbstractListMeta(type): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) - if 'elem_type' in attrs and 'length' in attrs: - setattr(out, 'elem_type', attrs['elem_type']) - setattr(out, 'length', attrs['length']) + for k, v in attrs.items(): + setattr(out, k, v) return out def __getitem__(self, params): - if not isinstance(params, tuple) or len(params) != 2: - raise Exception("List must be instantiated with two args: elem type and length") - o = self.__class__(self.__name__, (self,), {'elem_type': params[0], 'length': params[1]}) - o._name = 'AbstractList' + o = self.__class__(self.__name__, (self,), self.attr_from_params(params)) + o._bare = False return o + def attr_from_params(self, p): + # single key params are valid too. Wrap them in a tuple. + params = p if isinstance(p, tuple) else (p,) + res = {} + i = 0 + for (name, typ) in self.__annotations__.items(): + param = params[i] + if hasattr(self.__class__, name): + res[name] = getattr(self.__class__, name) + else: + if not isinstance(param, typ): + raise TypeError( + "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) + res[name] = param + i += 1 + if len(params) != i: + raise TypeError("provided parameters {} mismatch required parameter count {}".format(params, i)) + return res + def __instancecheck__(self, obj): if obj.__class__.__name__ != self.__name__: return False - if hasattr(self, 'elem_type') and obj.__class__.elem_type != self.elem_type: - return False - if hasattr(self, 'length') and obj.__class__.length != self.length: - return False + for name, typ in self.__annotations__: + if hasattr(self, name) and hasattr(obj.__class__, name) \ + and getattr(obj.__class__, name) != getattr(self, name): + return False return True + class ValueCheckError(Exception): pass -class AbstractList(metaclass=AbstractListMeta): + +class AbstractListMeta(ParamsMeta): + elem_type: DefaultingTypeMeta + length: int + + +class AbstractList(ParamsBase, metaclass=AbstractListMeta): + def __init__(self, *args): - items = self.extract_args(args) - + items = self.extract_args(*args) + if not self.value_check(items): raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) self.items = items - - def value_check(self, value): - for v in value: - if not type_check(self.__class__.elem_type, v): - return False - return True - def extract_args(self, args): - return list(args) if len(args) > 0 else self.default() + @classmethod + def value_check(cls, value): + return all(isinstance(v, cls.elem_type) for v in value) - def default(self): - raise Exception("Not implemented") + @classmethod + def extract_args(cls, *args): + x = list(args) + if len(x) == 1 and isinstance(x[0], GeneratorType): + x = list(x[0]) + return x if len(x) > 0 else cls.default() + + def __str__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" def __getitem__(self, i): return self.items[i] @@ -194,137 +225,65 @@ class AbstractList(metaclass=AbstractListMeta): def __eq__(self, other): return self.items == other.items -class List(AbstractList, metaclass=AbstractListMeta): + +class List(AbstractList): def value_check(self, value): return len(value) <= self.__class__.length and super().value_check(value) - def default(self): - return [] + @classmethod + def default(cls): + return cls() + class Vector(AbstractList, metaclass=AbstractListMeta): def value_check(self, value): return len(value) == self.__class__.length and super().value_check(value) - def default(self): - return [get_zero_value(self.__class__.elem_type) for _ in range(self.__class__.length)] + @classmethod + def default(cls): + return [cls.elem_type.default() for _ in range(cls.length)] + class BytesMeta(AbstractListMeta): - def __getitem__(self, params): - if not isinstance(params, int): - raise Exception("Bytes must be instantiated with one arg: length") - o = self.__class__(self.__name__, (self,), {'length': params}) - o._name = 'Bytes' - return o + elem_type: DefaultingTypeMeta = byte + length: int -def single_item_extractor(cls, args): - assert len(args) < 2 - return args[0] if len(args) > 0 else cls.default() -class Bytes(AbstractList, metaclass=BytesMeta): +class BytesLike(AbstractList, metaclass=BytesMeta): + + @classmethod + def extract_args(cls, args): + if isinstance(args, bytes): + return args + elif isinstance(args, BytesLike): + return args.items + elif isinstance(args, GeneratorType): + return bytes(args) + else: + return bytes(args) + + @classmethod + def value_check(cls, value): + return len(value) == cls.length and isinstance(value, bytes) + + def __str__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.length}]: {self.items.hex()}" + + +class Bytes(BytesLike): + def value_check(self, value): return len(value) <= self.__class__.length and isinstance(value, bytes) - extract_args = single_item_extractor - - def default(self): + @classmethod + def default(cls): return b'' -class BytesN(AbstractList, metaclass=BytesMeta): - def value_check(self, value): - return len(value) == self.__class__.length and isinstance(value, bytes) - extract_args = single_item_extractor +class BytesN(BytesLike): - def default(self): - return b'\x00' * self.__class__.length + @classmethod + def default(cls): + return b'\x00' * cls.length - -# Type helpers -# ----------------------------- - -def infer_type(obj): - if is_uint_type(obj.__class__): - return obj.__class__ - elif isinstance(obj, int): - return uint64 - elif isinstance(obj, (List, Vector, Container, bool, BytesN, Bytes)): - return obj.__class__ - else: - raise Exception("Unknown type for {}".format(obj)) - - -def infer_input_type(fn): - """ - Decorator to run infer_type on the obj if typ argument is None - """ - def infer_helper(obj, typ=None, **kwargs): - if typ is None: - typ = infer_type(obj) - return fn(obj, typ=typ, **kwargs) - return infer_helper - - -def is_bool_type(typ): - """ - Check if the given type is a bool. - """ - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ - return isinstance(typ, type) and issubclass(typ, bool) - - -def is_list_type(typ): - """ - Check if the given type is a list. - """ - return isinstance(typ, type) and issubclass(typ, List) - - -def is_bytes_type(typ): - """ - Check if the given type is a ``bytes``. - """ - return isinstance(typ, type) and issubclass(typ, Bytes) - - -def is_bytesn_type(typ): - """ - Check if the given type is a BytesN. - """ - return isinstance(typ, type) and issubclass(typ, BytesN) - - -def is_list_kind(typ): - """ - Check if the given type is a kind of list. Can be bytes. - """ - return is_list_type(typ) or is_bytes_type(typ) - - -def is_vector_type(typ): - """ - Check if the given type is a vector. - """ - return isinstance(typ, type) and issubclass(typ, Vector) - - -def is_vector_kind(typ): - """ - Check if the given type is a kind of vector. Can be BytesN. - """ - return is_vector_type(typ) or is_bytesn_type(typ) - - -def is_container_type(typ): - """ - Check if the given type is a container. - """ - return isinstance(typ, type) and issubclass(typ, Container) - - -T = TypeVar('T') -L = TypeVar('L') - - -def read_elem_type(typ): - return typ.elem_type From b6cf809d9b686f090dc4e19a68fbf311cd247831 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:54:59 +0200 Subject: [PATCH 05/68] more improvements, and implement new space-efficient merkleization with padding support --- .../pyspec/eth2spec/utils/merkle_minimal.py | 36 +++++++--- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 66 +++++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 32 +++++---- 3 files changed, 86 insertions(+), 48 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index c508f0df2..ebfb4faf6 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -44,11 +44,31 @@ def next_power_of_two(v: int) -> int: return 1 << (v - 1).bit_length() -def merkleize_chunks(chunks): - tree = chunks[::] - margin = next_power_of_two(len(chunks)) - len(chunks) - tree.extend([ZERO_BYTES32] * margin) - tree = [ZERO_BYTES32] * len(tree) + tree - for i in range(len(tree) // 2 - 1, 0, -1): - tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) - return tree[1] +def merkleize_chunks(chunks, pad_to: int = None): + count = len(chunks) + depth = max(count - 1, 0).bit_length() + max_depth = max(depth, (pad_to - 1).bit_length()) + tmp = [None for _ in range(max_depth + 1)] + + def merge(h, i): + j = 0 + while True: + if i & (1 << j) == 0: + if i == count and j < depth: + h = hash(h + zerohashes[j]) + else: + break + else: + h = hash(tmp[j] + h) + j += 1 + tmp[j] = h + + for i in range(count): + merge(chunks[i], i) + + merge(zerohashes[0], count) + + for j in range(depth, max_depth): + tmp[j + 1] = hash(tmp[j] + zerohashes[j]) + + return tmp[max_depth] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b08a3d4e2..1a556bc7d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,11 +1,7 @@ from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 -from .hash_function import hash +from ..hash_function import hash from .ssz_typing import ( - is_uint_type, is_bool_type, is_container_type, - is_list_kind, is_vector_kind, - read_elem_type, - infer_input_type, - get_zero_value, + get_zero_value, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -15,13 +11,13 @@ BYTES_PER_LENGTH_OFFSET = 4 def is_basic_type(typ): - return is_uint_type(typ) or is_bool_type(typ) + return issubclass(typ, (bool, uint)) def serialize_basic(value, typ): - if is_uint_type(typ): + if issubclass(typ, uint): return value.to_bytes(typ.byte_len, 'little') - elif is_bool_type(typ): + elif issubclass(typ, bool): if value: return b'\x01' else: @@ -31,22 +27,34 @@ def serialize_basic(value, typ): def deserialize_basic(value, typ): - if is_uint_type(typ): + if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif is_bool_type(typ): + elif issubclass(typ, bool): assert value in (b'\x00', b'\x01') return True if value == b'\x01' else False else: raise Exception("Type not supported: {}".format(typ)) +def is_list_kind(typ): + return issubclass(typ, (List, Bytes)) + + +def is_vector_kind(typ): + return issubclass(typ, (Vector, BytesN)) + + +def is_container_type(typ): + return issubclass(typ, Container) + + def is_fixed_size(typ): if is_basic_type(typ): return True elif is_list_kind(typ): return False elif is_vector_kind(typ): - return is_fixed_size(read_vector_elem_type(typ)) + return is_fixed_size(typ.elem_type) elif is_container_type(typ): return all(is_fixed_size(t) for t in typ.get_field_types()) else: @@ -57,12 +65,11 @@ def is_empty(obj): return get_zero_value(type(obj)) == obj -@infer_input_type -def serialize(obj, typ=None): +def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [read_elem_type(typ)] * len(obj)) + return encode_series(obj, [typ.elem_type] * len(obj)) elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: @@ -126,40 +133,41 @@ def mix_in_length(root, length): def is_bottom_layer_kind(typ): return ( is_basic_type(typ) or - (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(read_elem_type(typ)) + (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(typ.elem_type) ) -@infer_input_type -def get_typed_values(obj, typ=None): +def get_typed_values(obj, typ): if is_container_type(typ): return obj.get_typed_values() elif is_list_kind(typ) or is_vector_kind(typ): - elem_type = read_elem_type(typ) - return list(zip(obj, [elem_type] * len(obj))) + return list(zip(obj, [typ.elem_type] * len(obj))) else: raise Exception("Invalid type") -def item_length(typ): - return 1 if typ == bool else typ.byte_len if is_uint_type(typ) else 32 -@infer_input_type -def hash_tree_root(obj, typ=None): +def item_length(typ): + if typ == bool: + return 1 + elif issubclass(typ, uint): + return typ.byte_len + else: + return 32 + + +def hash_tree_root(obj, typ): if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, typ.elem_type) leaves = chunkify(data) else: fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): - full_chunk_length = (item_length(read_elem_type(typ)) * typ.length + 31) // 32 - leaves += [ZERO_BYTES32] * (full_chunk_length - len(obj)) - return mix_in_length(merkleize_chunks(leaves), len(obj)) + return mix_in_length(merkleize_chunks(leaves, pad_to=typ.length), len(obj)) else: return merkleize_chunks(leaves) -@infer_input_type def signing_root(obj, typ): assert is_container_type(typ) # ignore last field diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 9aafb5294..30f71f87d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,10 +1,26 @@ +from typing import NewType, Union from types import GeneratorType +class ValueCheckError(Exception): + pass + + class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") +# Every type is subclassed and has a default() method, except bool. +TypeWithDefault = Union[DefaultingTypeMeta, bool] + + +def get_zero_value(typ: TypeWithDefault): + if issubclass(typ, bool): + return False + else: + return typ.default() + + # SSZ integers # ----------------------------- @@ -63,7 +79,7 @@ class Container(object): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, t.default()) + setattr(self, f, get_zero_value(t)) else: setattr(self, f, kwargs[f]) @@ -120,7 +136,7 @@ class Container(object): @classmethod def default(cls): - return cls(**{f: t.default() for f, t in cls.get_fields()}) + return cls(**{f: get_zero_value(t) for f, t in cls.get_fields()}) class ParamsBase: @@ -174,12 +190,8 @@ class ParamsMeta(DefaultingTypeMeta): return True -class ValueCheckError(Exception): - pass - - class AbstractListMeta(ParamsMeta): - elem_type: DefaultingTypeMeta + elem_type: TypeWithDefault length: int @@ -227,8 +239,6 @@ class AbstractList(ParamsBase, metaclass=AbstractListMeta): class List(AbstractList): - def value_check(self, value): - return len(value) <= self.__class__.length and super().value_check(value) @classmethod def default(cls): @@ -241,11 +251,11 @@ class Vector(AbstractList, metaclass=AbstractListMeta): @classmethod def default(cls): - return [cls.elem_type.default() for _ in range(cls.length)] + return [get_zero_value(cls.elem_type) for _ in range(cls.length)] class BytesMeta(AbstractListMeta): - elem_type: DefaultingTypeMeta = byte + elem_type: TypeWithDefault = byte length: int From cd5f59eb74cf8c398033ccd7e862d89c4d6ceffa Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:55:53 +0200 Subject: [PATCH 06/68] fix bytes value check, fix default-type checking --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 30f71f87d..6f578796d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -171,7 +171,10 @@ class ParamsMeta(DefaultingTypeMeta): if hasattr(self.__class__, name): res[name] = getattr(self.__class__, name) else: - if not isinstance(param, typ): + if typ == TypeWithDefault: + if not (isinstance(param, bool) or isinstance(param, DefaultingTypeMeta)): + raise TypeError("expected param {} as {} to have a type default".format(param, name, typ)) + elif not isinstance(param, typ): raise TypeError( "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) res[name] = param @@ -246,8 +249,10 @@ class List(AbstractList): class Vector(AbstractList, metaclass=AbstractListMeta): - def value_check(self, value): - return len(value) == self.__class__.length and super().value_check(value) + + @classmethod + def value_check(cls, value): + return len(value) == cls.length and super().value_check(value) @classmethod def default(cls): @@ -274,7 +279,7 @@ class BytesLike(AbstractList, metaclass=BytesMeta): @classmethod def value_check(cls, value): - return len(value) == cls.length and isinstance(value, bytes) + return isinstance(value, bytes) def __str__(self): cls = self.__class__ @@ -283,9 +288,6 @@ class BytesLike(AbstractList, metaclass=BytesMeta): class Bytes(BytesLike): - def value_check(self, value): - return len(value) <= self.__class__.length and isinstance(value, bytes) - @classmethod def default(cls): return b'' @@ -297,3 +299,7 @@ class BytesN(BytesLike): def default(cls): return b'\x00' * cls.length + @classmethod + def value_check(cls, value): + return len(value) == cls.length and super().value_check(value) + From 54a1fa9abec52a747ea47bf73c679a7b2f2bef11 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 15 Jun 2019 20:57:22 +0200 Subject: [PATCH 07/68] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: vbuterin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1a556bc7d..ed8dbf1e0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -153,7 +153,13 @@ def item_length(typ): return typ.byte_len else: return 32 - +def chunk_count(typ): + if is_basic_type(typ): + return 1 + elif is_list_kind(typ) or is_vector_kind(typ): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + else: + return len(typ.get_fields()) def hash_tree_root(obj, typ): if is_bottom_layer_kind(typ): From 3a9b1fb72c97e4512a935fb3c99c084ac411cae1 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 15 Jun 2019 20:57:30 +0200 Subject: [PATCH 08/68] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: vbuterin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index ed8dbf1e0..bcdef3988 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -169,7 +169,7 @@ def hash_tree_root(obj, typ): fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): - return mix_in_length(merkleize_chunks(leaves, pad_to=typ.length), len(obj)) + return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(typ)), len(obj)) else: return merkleize_chunks(leaves) From 82e7392b17f7d63f58a2745d00555bb8e479cf18 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 21:13:01 +0200 Subject: [PATCH 09/68] default method for container is recognized now --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6f578796d..7cf44f6b3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -73,7 +73,7 @@ class uint256(uint): # Note: importing ssz functionality locally, to avoid import loop -class Container(object): +class Container(object, metaclass=DefaultingTypeMeta): def __init__(self, **kwargs): cls = self.__class__ From 108410d8626a2db20bb7e44422180f98d41162c6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 22:12:59 +0200 Subject: [PATCH 10/68] Change byte to explict class instead of newtype --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7cf44f6b3..40901ad97 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -45,7 +45,8 @@ class uint8(uint): # Alias for uint8 -byte = NewType('byte', uint8) +class byte(uint8): + pass class uint16(uint): From 4ebdceaf129e40cd27c2b95ffbd36af0d72e9db6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:57:50 +0200 Subject: [PATCH 11/68] highly experimental typing --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 142 +++++++----------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 141 +++++++++++------ 2 files changed, 146 insertions(+), 137 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index bcdef3988..679574891 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ -from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 +from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - get_zero_value, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -10,79 +10,48 @@ from .ssz_typing import ( BYTES_PER_LENGTH_OFFSET = 4 -def is_basic_type(typ): - return issubclass(typ, (bool, uint)) - - -def serialize_basic(value, typ): - if issubclass(typ, uint): - return value.to_bytes(typ.byte_len, 'little') - elif issubclass(typ, bool): +def serialize_basic(value: SSZValue): + if isinstance(value, uint): + return value.to_bytes(value.type().byte_len, 'little') + elif isinstance(value, Bit): if value: return b'\x01' else: return b'\x00' else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {type(value)}") -def deserialize_basic(value, typ): +def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, bool): + elif issubclass(typ, Bit): assert value in (b'\x00', b'\x01') - return True if value == b'\x01' else False + return Bit(value == b'\x01') else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {typ}") -def is_list_kind(typ): - return issubclass(typ, (List, Bytes)) +def is_empty(obj: SSZValue): + return type(obj).default() == obj -def is_vector_kind(typ): - return issubclass(typ, (Vector, BytesN)) - - -def is_container_type(typ): - return issubclass(typ, Container) - - -def is_fixed_size(typ): - if is_basic_type(typ): - return True - elif is_list_kind(typ): - return False - elif is_vector_kind(typ): - return is_fixed_size(typ.elem_type) - elif is_container_type(typ): - return all(is_fixed_size(t) for t in typ.get_field_types()) +def serialize(obj: SSZValue): + if isinstance(obj, BasicValue): + return serialize_basic(obj) + elif isinstance(obj, Series): + return encode_series(obj) else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {type(obj)}") -def is_empty(obj): - return get_zero_value(type(obj)) == obj - - -def serialize(obj, typ): - if is_basic_type(typ): - return serialize_basic(obj, typ) - elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [typ.elem_type] * len(obj)) - elif is_container_type(typ): - return encode_series(obj.get_field_values(), typ.get_field_types()) - else: - raise Exception("Type not supported: {}".format(typ)) - - -def encode_series(values, types): +def encode_series(values: Series): # bytes and bytesN are already in the right format. if isinstance(values, bytes): return values # Recursively serialize - parts = [(is_fixed_size(types[i]), serialize(values[i], typ=types[i])) for i in range(len(values))] + parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] # Compute and check lengths fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET @@ -114,10 +83,10 @@ def encode_series(values, types): # ----------------------------- -def pack(values, subtype): +def pack(values: Series): if isinstance(values, bytes): return values - return b''.join([serialize_basic(value, subtype) for value in values]) + return b''.join([serialize_basic(value) for value in values]) def chunkify(bytez): @@ -130,52 +99,49 @@ def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) -def is_bottom_layer_kind(typ): +def is_bottom_layer_kind(typ: SSZType): return ( - is_basic_type(typ) or - (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(typ.elem_type) + issubclass(typ, BasicType) or + (issubclass(typ, Elements) and issubclass(typ.elem_type, BasicType)) ) -def get_typed_values(obj, typ): - if is_container_type(typ): - return obj.get_typed_values() - elif is_list_kind(typ) or is_vector_kind(typ): - return list(zip(obj, [typ.elem_type] * len(obj))) - else: - raise Exception("Invalid type") - - -def item_length(typ): - if typ == bool: - return 1 - elif issubclass(typ, uint): +def item_length(typ: SSZType) -> int: + if issubclass(typ, BasicType): return typ.byte_len else: return 32 -def chunk_count(typ): - if is_basic_type(typ): - return 1 - elif is_list_kind(typ) or is_vector_kind(typ): - return (typ.length * item_length(typ.elem_type) + 31) // 32 - else: - return len(typ.get_fields()) -def hash_tree_root(obj, typ): - if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, typ.elem_type) - leaves = chunkify(data) + +def chunk_count(typ: SSZType) -> int: + if issubclass(typ, BasicType): + return 1 + elif issubclass(typ, Elements): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + elif issubclass(typ, Container): + return len(typ.get_fields()) else: - fields = get_typed_values(obj, typ=typ) - leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] - if is_list_kind(typ): - return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(typ)), len(obj)) + raise Exception(f"Type not supported: {typ}") + + +def hash_tree_root(obj: SSZValue): + if isinstance(obj, Series): + if is_bottom_layer_kind(obj.type()): + leaves = chunkify(pack(obj)) + else: + leaves = [hash_tree_root(value) for value in obj] + elif isinstance(obj, BasicValue): + leaves = chunkify(serialize_basic(obj)) + else: + raise Exception(f"Type not supported: {obj.type()}") + + if isinstance(obj, (List, Bytes)): + return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) -def signing_root(obj, typ): - assert is_container_type(typ) +def signing_root(obj: Container): # ignore last field - leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] + leaves = [hash_tree_root(field) for field in obj[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 40901ad97..b79789f27 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import NewType, Union +from typing import Tuple, Dict, Iterator from types import GeneratorType @@ -10,24 +10,42 @@ class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") -# Every type is subclassed and has a default() method, except bool. -TypeWithDefault = Union[DefaultingTypeMeta, bool] + +class SSZType(DefaultingTypeMeta): + + def is_fixed_size(cls): + raise Exception("Not implemented") -def get_zero_value(typ: TypeWithDefault): - if issubclass(typ, bool): - return False - else: - return typ.default() +class SSZValue(object, metaclass=SSZType): + + def type(self): + return self.__class__ -# SSZ integers -# ----------------------------- - - -class uint(int, metaclass=DefaultingTypeMeta): +class BasicType(SSZType): byte_len = 0 + def is_fixed_size(cls): + return True + + +class BasicValue(int, SSZValue, metaclass=BasicType): + pass + + +class Bit(BasicValue): # can't subclass bool. + + @classmethod + def default(cls): + return cls(False) + + def __bool__(self): + return self > 0 + + +class uint(BasicValue, metaclass=BasicType): + def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") @@ -69,36 +87,39 @@ class uint256(uint): byte_len = 32 -# SSZ Container base class -# ----------------------------- +class Series(SSZValue): + + def __iter__(self) -> Iterator[SSZValue]: + raise Exception("Not implemented") + # Note: importing ssz functionality locally, to avoid import loop -class Container(object, metaclass=DefaultingTypeMeta): +class Container(Series, metaclass=SSZType): def __init__(self, **kwargs): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, get_zero_value(t)) + setattr(self, f, t.default()) else: setattr(self, f, kwargs[f]) def serialize(self): from .ssz_impl import serialize - return serialize(self, self.__class__) + return serialize(self) def hash_tree_root(self): from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) + return hash_tree_root(self) def signing_root(self): from .ssz_impl import signing_root - return signing_root(self, self.__class__) + return signing_root(self) - def get_field_values(self): + def get_field_values(self) -> Tuple[SSZValue, ...]: cls = self.__class__ - return [getattr(self, field) for field in cls.get_field_names()] + return tuple(getattr(self, field) for field in cls.get_field_names()) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -116,31 +137,38 @@ class Container(object, metaclass=DefaultingTypeMeta): return hash(self.hash_tree_root()) @classmethod - def get_fields_dict(cls): + def get_fields_dict(cls) -> Dict[str, SSZType]: return dict(cls.__annotations__) @classmethod - def get_fields(cls): - return list(dict(cls.__annotations__).items()) + def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + return tuple((f, SSZType(t)) for f, t in dict(cls.__annotations__).items()) def get_typed_values(self): - return list(zip(self.get_field_values(), self.get_field_types())) + return tuple(zip(self.get_field_values(), self.get_field_types())) @classmethod - def get_field_names(cls): - return list(cls.__annotations__.keys()) + def get_field_names(cls) -> Tuple[str]: + return tuple(cls.__annotations__.keys()) @classmethod - def get_field_types(cls): + def get_field_types(cls) -> Tuple[SSZType, ...]: # values of annotations are the types corresponding to the fields, not instance values. - return list(cls.__annotations__.values()) + return tuple(cls.__annotations__.values()) @classmethod def default(cls): - return cls(**{f: get_zero_value(t) for f, t in cls.get_fields()}) + return cls(**{f: t.default() for f, t in cls.get_fields()}) + + @classmethod + def is_fixed_size(cls): + return all(t.is_fixed_size() for t in cls.get_field_types()) + + def __iter__(self) -> Iterator[SSZValue]: + return iter(self.get_field_values()) -class ParamsBase: +class ParamsBase(Series): _bare = True def __new__(cls, *args, **kwargs): @@ -149,7 +177,7 @@ class ParamsBase: return super().__new__(cls, **kwargs) -class ParamsMeta(DefaultingTypeMeta): +class ParamsMeta(SSZType): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) @@ -168,14 +196,14 @@ class ParamsMeta(DefaultingTypeMeta): res = {} i = 0 for (name, typ) in self.__annotations__.items(): - param = params[i] if hasattr(self.__class__, name): res[name] = getattr(self.__class__, name) else: - if typ == TypeWithDefault: - if not (isinstance(param, bool) or isinstance(param, DefaultingTypeMeta)): - raise TypeError("expected param {} as {} to have a type default".format(param, name, typ)) - elif not isinstance(param, typ): + if i >= len(params): + i += 1 + continue + param = params[i] + if not isinstance(param, typ): raise TypeError( "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) res[name] = param @@ -194,12 +222,12 @@ class ParamsMeta(DefaultingTypeMeta): return True -class AbstractListMeta(ParamsMeta): - elem_type: TypeWithDefault +class Elements(ParamsMeta): + elem_type: SSZType length: int -class AbstractList(ParamsBase, metaclass=AbstractListMeta): +class ElementsBase(ParamsBase, metaclass=Elements): def __init__(self, *args): items = self.extract_args(*args) @@ -223,7 +251,7 @@ class AbstractList(ParamsBase, metaclass=AbstractListMeta): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" - def __getitem__(self, i): + def __getitem__(self, i) -> SSZValue: return self.items[i] def __setitem__(self, k, v): @@ -235,21 +263,25 @@ class AbstractList(ParamsBase, metaclass=AbstractListMeta): def __repr__(self): return repr(self.items) - def __iter__(self): + def __iter__(self) -> Iterator[SSZValue]: return iter(self.items) def __eq__(self, other): return self.items == other.items -class List(AbstractList): +class List(ElementsBase): @classmethod def default(cls): return cls() + @classmethod + def is_fixed_size(cls): + return False -class Vector(AbstractList, metaclass=AbstractListMeta): + +class Vector(ElementsBase): @classmethod def value_check(cls, value): @@ -257,15 +289,19 @@ class Vector(AbstractList, metaclass=AbstractListMeta): @classmethod def default(cls): - return [get_zero_value(cls.elem_type) for _ in range(cls.length)] + return [cls.elem_type.default() for _ in range(cls.length)] + + @classmethod + def is_fixed_size(cls): + return cls.elem_type.is_fixed_size() -class BytesMeta(AbstractListMeta): - elem_type: TypeWithDefault = byte +class BytesMeta(Elements): + elem_type: SSZType = byte length: int -class BytesLike(AbstractList, metaclass=BytesMeta): +class BytesLike(ElementsBase, metaclass=BytesMeta): @classmethod def extract_args(cls, args): @@ -293,6 +329,10 @@ class Bytes(BytesLike): def default(cls): return b'' + @classmethod + def is_fixed_size(cls): + return False + class BytesN(BytesLike): @@ -304,3 +344,6 @@ class BytesN(BytesLike): def value_check(cls, value): return len(value) == cls.length and super().value_check(value) + @classmethod + def is_fixed_size(cls): + return True From 97025c51ac399140445ee103f20ab488c2f3ca2f Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:03:11 +0200 Subject: [PATCH 12/68] start updating virtual sizes of lists --- specs/core/0_beacon-chain.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef9..ddc4c8f61 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -367,7 +367,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: bytes # Bit set for every attesting participant within a committee + aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] # Bit set for every attesting participant within a committee data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -434,9 +434,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: bytes + aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] data: AttestationData - custody_bitfield: bytes + custody_bitfield: Bytes[MAX_COMMITTEE_SIZE] signature: BLSSignature ``` @@ -480,12 +480,12 @@ class BeaconBlockBody(Container): eth1_data: Eth1Data # Eth1 data vote graffiti: Bytes32 # Arbitrary data # Operations - proposer_slashings: List[ProposerSlashing] - attester_slashings: List[AttesterSlashing] - attestations: List[Attestation] - deposits: List[Deposit] - voluntary_exits: List[VoluntaryExit] - transfers: List[Transfer] + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] + transfers: List[Transfer, MAX_TRANSFERS] ``` #### `BeaconBlock` From 08e6f32f3897889a62dcdf0887e4ba42403aa3a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 16 Jun 2019 03:21:58 +0200 Subject: [PATCH 13/68] typing improvements, type testing --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 21 ++- .../eth2spec/utils/ssz/test_ssz_typing.py | 131 ++++++++++++++++++ 2 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index b79789f27..5cab68aa8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -36,6 +36,11 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bit(BasicValue): # can't subclass bool. + def __new__(cls, value, *args, **kwargs): + if value < 0 or value > 1: + raise ValueError(f"value {value} out of bounds for bit") + return super().__new__(cls, value) + @classmethod def default(cls): return cls(False) @@ -49,7 +54,7 @@ class uint(BasicValue, metaclass=BasicType): def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") - if cls.byte_len and (value.bit_length() >> 3) > cls.byte_len: + if cls.byte_len and value.bit_length() > (cls.byte_len << 3): raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) return super().__new__(cls, value) @@ -142,7 +147,7 @@ class Container(Series, metaclass=SSZType): @classmethod def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: - return tuple((f, SSZType(t)) for f, t in dict(cls.__annotations__).items()) + return tuple((f, t) for f, t in cls.__annotations__.items()) def get_typed_values(self): return tuple(zip(self.get_field_values(), self.get_field_types())) @@ -190,6 +195,12 @@ class ParamsMeta(SSZType): o._bare = False return o + def __str__(self): + return f"{self.__name__}~{self.__class__.__name__}" + + def __repr__(self): + return self, self.__class__ + def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. params = p if isinstance(p, tuple) else (p,) @@ -215,7 +226,7 @@ class ParamsMeta(SSZType): def __instancecheck__(self, obj): if obj.__class__.__name__ != self.__name__: return False - for name, typ in self.__annotations__: + for name, typ in self.__annotations__.items(): if hasattr(self, name) and hasattr(obj.__class__, name) \ and getattr(obj.__class__, name) != getattr(self, name): return False @@ -233,7 +244,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): items = self.extract_args(*args) if not self.value_check(items): - raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) + raise ValueCheckError(f"Bad input for class {self.__class__}: {items}") self.items = items @classmethod @@ -245,7 +256,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): x = list(args) if len(x) == 1 and isinstance(x[0], GeneratorType): x = list(x[0]) - return x if len(x) > 0 else cls.default() + return x def __str__(self): cls = self.__class__ diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py new file mode 100644 index 000000000..a0705f8d3 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -0,0 +1,131 @@ +from .ssz_typing import ( + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, + uint, uint8, uint16, uint32, uint64, uint128, uint256 +) + + +def test_subclasses(): + for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + assert issubclass(u, uint) + assert issubclass(u, int) + assert issubclass(u, BasicValue) + assert issubclass(u, SSZValue) + assert isinstance(u, SSZType) + assert isinstance(u, BasicType) + assert issubclass(Bit, BasicValue) + assert isinstance(Bit, BasicType) + + for c in [Container, List, Vector, Bytes, BytesN]: + assert issubclass(c, Series) + assert issubclass(c, SSZValue) + assert isinstance(c, SSZType) + assert not issubclass(c, BasicValue) + assert not isinstance(c, BasicType) + + for c in [List, Vector, Bytes, BytesN]: + assert isinstance(c, Elements) + + +def test_basic_instances(): + for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + v = u(123) + assert isinstance(v, uint) + assert isinstance(v, int) + assert isinstance(v, BasicValue) + assert isinstance(v, SSZValue) + + assert isinstance(Bit(True), BasicValue) + assert isinstance(Bit(False), BasicValue) + + +def test_basic_value_bounds(): + max = { + Bit: 2**1, + uint8: 2**(8 * 1), + uint16: 2**(8 * 2), + uint32: 2**(8 * 4), + uint64: 2**(8 * 8), + uint128: 2**(8 * 16), + uint256: 2**(8 * 32), + } + for k, v in max.items(): + # this should work + assert k(v - 1) == v - 1 + # but we do not allow overflows + try: + k(v) + assert False + except ValueError: + pass + + for k, _ in max.items(): + # this should work + assert k(0) == 0 + # but we do not allow underflows + try: + k(-1) + assert False + except ValueError: + pass + + +def test_container(): + class Foo(Container): + a: uint8 + b: uint32 + + assert issubclass(Foo, Container) + assert issubclass(Foo, SSZValue) + assert issubclass(Foo, Series) + + assert Foo.is_fixed_size() + x = Foo(a=uint8(123), b=uint32(45)) + assert x.a == 123 + assert x.b == 45 + assert isinstance(x.a, uint8) + assert isinstance(x.b, uint32) + assert x.type().is_fixed_size() + + class Bar(Container): + a: uint8 + b: List[uint8, 1024] + + assert not Bar.is_fixed_size() + + y = Bar(a=uint8(123), b=List[uint8, 1024](uint8(1), uint8(2))) + assert y.a == 123 + assert len(y.b) == 2 + assert isinstance(y.a, uint8) + assert isinstance(y.b, List[uint8, 1024]) + assert not y.type().is_fixed_size() + assert y.b[0] == 1 + v: List = y.b + assert v.type().elem_type == uint8 + assert v.type().length == 1024 + + +def test_list(): + typ = List[uint64, 128] + assert issubclass(typ, List) + assert issubclass(typ, SSZValue) + assert issubclass(typ, Series) + assert isinstance(typ, Elements) + + assert not typ.is_fixed_size() + + assert len(typ()) == 0 # empty + assert len(typ(uint64(0))) == 1 # single arg + assert len(typ(uint64(i) for i in range(10))) == 10 # generator + assert len(typ(uint64(0), uint64(1), uint64(2))) == 3 # args + + v = typ(uint64(0)) + v[0] = uint64(123) + assert v[0] == 123 + assert isinstance(v[0], uint64) + + assert isinstance(v, List) + assert isinstance(v, List[uint64, 128]) + assert isinstance(v, typ) + assert isinstance(v, SSZValue) + assert isinstance(v, Series) + assert isinstance(v.type(), Elements) From 8bd2e878ef308c58da54f35bd8dd67c13756048c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 17 Jun 2019 01:39:39 +0200 Subject: [PATCH 14/68] bugfixes and typing improvements --- .../pyspec/eth2spec/utils/merkle_minimal.py | 10 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 18 +-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 69 ++++++---- .../eth2spec/utils/ssz/test_ssz_impl.py | 118 ++++++++++++++++++ .../eth2spec/utils/ssz/test_ssz_typing.py | 11 +- 5 files changed, 187 insertions(+), 39 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index ebfb4faf6..21583ee92 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -44,7 +44,7 @@ def next_power_of_two(v: int) -> int: return 1 << (v - 1).bit_length() -def merkleize_chunks(chunks, pad_to: int = None): +def merkleize_chunks(chunks, pad_to: int = 1): count = len(chunks) depth = max(count - 1, 0).bit_length() max_depth = max(depth, (pad_to - 1).bit_length()) @@ -55,7 +55,7 @@ def merkleize_chunks(chunks, pad_to: int = None): while True: if i & (1 << j) == 0: if i == count and j < depth: - h = hash(h + zerohashes[j]) + h = hash(h + zerohashes[j]) # keep going if we are complementing the void to the next power of 2 else: break else: @@ -63,11 +63,15 @@ def merkleize_chunks(chunks, pad_to: int = None): j += 1 tmp[j] = h + # merge in leaf by leaf. for i in range(count): merge(chunks[i], i) - merge(zerohashes[0], count) + # complement with 0 if empty, or if not the right power of 2 + if 1 << depth != count: + merge(zerohashes[0], count) + # the next power of two may be smaller than the ultimate virtual size, complement with zero-hashes at each depth. for j in range(depth, max_depth): tmp[j + 1] = hash(tmp[j] + zerohashes[j]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 679574891..1b59b276b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -47,8 +47,8 @@ def serialize(obj: SSZValue): def encode_series(values: Series): # bytes and bytesN are already in the right format. - if isinstance(values, bytes): - return values + if isinstance(values, (Bytes, BytesN)): + return values.items # Recursively serialize parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] @@ -84,8 +84,8 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, bytes): - return values + if isinstance(values, (Bytes, BytesN)): + return values.items return b''.join([serialize_basic(value) for value in values]) @@ -101,8 +101,8 @@ def mix_in_length(root, length): def is_bottom_layer_kind(typ: SSZType): return ( - issubclass(typ, BasicType) or - (issubclass(typ, Elements) and issubclass(typ.elem_type, BasicType)) + isinstance(typ, BasicType) or + (issubclass(typ, Elements) and isinstance(typ.elem_type, BasicType)) ) @@ -114,7 +114,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: - if issubclass(typ, BasicType): + if isinstance(typ, BasicType): return 1 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 @@ -133,7 +133,7 @@ def hash_tree_root(obj: SSZValue): elif isinstance(obj, BasicValue): leaves = chunkify(serialize_basic(obj)) else: - raise Exception(f"Type not supported: {obj.type()}") + raise Exception(f"Type not supported: {type(obj)}") if isinstance(obj, (List, Bytes)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5cab68aa8..7662971c4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -92,6 +92,22 @@ class uint256(uint): byte_len = 32 +def coerce_type_maybe(v, typ: SSZType): + v_typ = type(v) + # shortcut if it's already the type we are looking for + if v_typ == typ: + return v + elif isinstance(v, int): + return typ(v) + elif isinstance(v, (list, tuple)): + return typ(*v) + elif isinstance(v, GeneratorType): + return typ(v) + else: + # just return as-is, Value-checkers will take care of it not being coerced. + return v + + class Series(SSZValue): def __iter__(self) -> Iterator[SSZValue]: @@ -108,7 +124,11 @@ class Container(Series, metaclass=SSZType): if f not in kwargs: setattr(self, f, t.default()) else: - setattr(self, f, kwargs[f]) + value = coerce_type_maybe(kwargs[f], t) + if not isinstance(value, t): + raise ValueCheckError(f"Bad input for class {self.__class__}:" + f" field: {f} type: {t} value: {value} value type: {type(value)}") + setattr(self, f, value) def serialize(self): from .ssz_impl import serialize @@ -141,23 +161,22 @@ class Container(Series, metaclass=SSZType): def __hash__(self): return hash(self.hash_tree_root()) - @classmethod - def get_fields_dict(cls) -> Dict[str, SSZType]: - return dict(cls.__annotations__) - @classmethod def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () return tuple((f, t) for f, t in cls.__annotations__.items()) - def get_typed_values(self): - return tuple(zip(self.get_field_values(), self.get_field_types())) - @classmethod - def get_field_names(cls) -> Tuple[str]: + def get_field_names(cls) -> Tuple[str, ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () return tuple(cls.__annotations__.keys()) @classmethod def get_field_types(cls) -> Tuple[SSZType, ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () # values of annotations are the types corresponding to the fields, not instance values. return tuple(cls.__annotations__.values()) @@ -233,12 +252,12 @@ class ParamsMeta(SSZType): return True -class Elements(ParamsMeta): +class ElementsType(ParamsMeta): elem_type: SSZType length: int -class ElementsBase(ParamsBase, metaclass=Elements): +class Elements(ParamsBase, metaclass=ElementsType): def __init__(self, *args): items = self.extract_args(*args) @@ -256,6 +275,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): x = list(args) if len(x) == 1 and isinstance(x[0], GeneratorType): x = list(x[0]) + x = [coerce_type_maybe(v, cls.elem_type) for v in x] return x def __str__(self): @@ -281,7 +301,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): return self.items == other.items -class List(ElementsBase): +class List(Elements): @classmethod def default(cls): @@ -292,7 +312,7 @@ class List(ElementsBase): return False -class Vector(ElementsBase): +class Vector(Elements): @classmethod def value_check(cls, value): @@ -307,23 +327,26 @@ class Vector(ElementsBase): return cls.elem_type.is_fixed_size() -class BytesMeta(Elements): +class BytesType(ElementsType): elem_type: SSZType = byte length: int -class BytesLike(ElementsBase, metaclass=BytesMeta): +class BytesLike(Elements, metaclass=BytesType): @classmethod - def extract_args(cls, args): - if isinstance(args, bytes): - return args - elif isinstance(args, BytesLike): - return args.items - elif isinstance(args, GeneratorType): - return bytes(args) + def extract_args(cls, *args): + x = list(args) + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, BytesLike)): + x = x[0] + if isinstance(x, bytes): + return x + elif isinstance(x, BytesLike): + return x.items + elif isinstance(x, GeneratorType): + return bytes(x) else: - return bytes(args) + return bytes(x) @classmethod def value_check(cls, value): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py new file mode 100644 index 000000000..8dd04a86d --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -0,0 +1,118 @@ +from .ssz_impl import serialize, serialize_basic, encode_series, signing_root, hash_tree_root +from .ssz_typing import ( + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Bit, Container, List, Vector, Bytes, BytesN, + uint, uint8, uint16, uint32, uint64, uint128, uint256, byte +) + +import pytest + + +class EmptyTestStruct(Container): + pass + + +class SingleFieldTestStruct(Container): + A: byte + + +class SmallTestStruct(Container): + A: uint16 + B: uint16 + + +class FixedTestStruct(Container): + A: uint8 + B: uint64 + C: uint32 + + +class VarTestStruct(Container): + A: uint16 + B: List[uint16, 1024] + C: uint8 + + +class ComplexTestStruct(Container): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: Bytes[256] + E: VarTestStruct + F: Vector[FixedTestStruct, 4] + G: Vector[VarTestStruct, 2] + + +sig_test_data = [0 for i in range(96)] +for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): + sig_test_data[k] = v + +test_data = [ + ("bool F", Bit(False), "00"), + ("bool T", Bit(True), "01"), + ("uint8 00", uint8(0x00), "00"), + ("uint8 01", uint8(0x01), "01"), + ("uint8 ab", uint8(0xab), "ab"), + ("uint16 0000", uint16(0x0000), "0000"), + ("uint16 abcd", uint16(0xabcd), "cdab"), + ("uint32 00000000", uint32(0x00000000), "00000000"), + ("uint32 01234567", uint32(0x01234567), "67452301"), + ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301"), + ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301"), + ("uint32 01234567", uint32(0x01234567), "67452301"), + ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000"), + ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301"), + ("sig", BytesN[96](*sig_test_data), + "0100000000000000000000000000000000000000000000000000000000000000" + "0200000000000000000000000000000000000000000000000000000000000000" + "03000000000000000000000000000000000000000000000000000000000000ff"), + ("emptyTestStruct", EmptyTestStruct(), ""), + ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab"), + ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412"), + ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff"), + ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff"), + ("varTestStruct some", VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + "cdab07000000ff010002000300"), + ("complexTestStruct", + ComplexTestStruct( + A=0xaabb, + B=List[uint16, 128](0x1122, 0x3344), + C=0xff, + D=Bytes[256](b"foobar"), + E=VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + F=Vector[FixedTestStruct, 4]( + FixedTestStruct(A=0xcc, B=0x4242424242424242, C=0x13371337), + FixedTestStruct(A=0xdd, B=0x3333333333333333, C=0xabcdabcd), + FixedTestStruct(A=0xee, B=0x4444444444444444, C=0x00112233), + FixedTestStruct(A=0xff, B=0x5555555555555555, C=0x44556677)), + G=Vector[VarTestStruct, 2]( + VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff)), + ), + "bbaa" + "47000000" # offset of B, []uint16 + "ff" + "4b000000" # offset of foobar + "51000000" # offset of E + "cc424242424242424237133713" + "dd3333333333333333cdabcdab" + "ee444444444444444433221100" + "ff555555555555555577665544" + "5e000000" # pointer to G + "22114433" # contents of B + "666f6f626172" # foobar + "cdab07000000ff010002000300" # contents of E + "08000000" "15000000" # [start G]: local offsets of [2]varTestStruct + "cdab07000000ff010002000300" + "cdab07000000ff010002000300", + ) +] + + +@pytest.mark.parametrize("name, value, serialized", test_data) +def test_serialize(name, value, serialized): + assert serialize(value) == bytes.fromhex(serialized) + + +@pytest.mark.parametrize("name, value, _", test_data) +def test_hash_tree_root(name, value, _): + hash_tree_root(value) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index a0705f8d3..e59d29b91 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,5 +1,5 @@ from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint, uint8, uint16, uint32, uint64, uint128, uint256 ) @@ -23,7 +23,8 @@ def test_subclasses(): assert not isinstance(c, BasicType) for c in [List, Vector, Bytes, BytesN]: - assert isinstance(c, Elements) + assert issubclass(c, Elements) + assert isinstance(c, ElementsType) def test_basic_instances(): @@ -109,7 +110,8 @@ def test_list(): assert issubclass(typ, List) assert issubclass(typ, SSZValue) assert issubclass(typ, Series) - assert isinstance(typ, Elements) + assert issubclass(typ, Elements) + assert isinstance(typ, ElementsType) assert not typ.is_fixed_size() @@ -128,4 +130,5 @@ def test_list(): assert isinstance(v, typ) assert isinstance(v, SSZValue) assert isinstance(v, Series) - assert isinstance(v.type(), Elements) + assert issubclass(v.type(), Elements) + assert isinstance(v.type(), ElementsType) From 0a43003b426b581de664ff27bc0e5e766a960b29 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 17 Jun 2019 02:11:50 +0200 Subject: [PATCH 15/68] minor test improvements --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index e59d29b91..99416e333 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -93,8 +93,9 @@ def test_container(): assert not Bar.is_fixed_size() - y = Bar(a=uint8(123), b=List[uint8, 1024](uint8(1), uint8(2))) + y = Bar(a=123, b=List[uint8, 1024](uint8(1), uint8(2))) assert y.a == 123 + assert isinstance(y.a, uint8) assert len(y.b) == 2 assert isinstance(y.a, uint8) assert isinstance(y.b, List[uint8, 1024]) @@ -119,6 +120,8 @@ def test_list(): assert len(typ(uint64(0))) == 1 # single arg assert len(typ(uint64(i) for i in range(10))) == 10 # generator assert len(typ(uint64(0), uint64(1), uint64(2))) == 3 # args + assert isinstance(typ(1, 2, 3, 4, 5)[4], uint64) # coercion + assert isinstance(typ(i for i in range(10))[9], uint64) # coercion in generator v = typ(uint64(0)) v[0] = uint64(123) From b89183ae69faf82a1eec798928da7c358d759621 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:05:34 +0200 Subject: [PATCH 16/68] Update spec for new SSZ with list max length --- specs/core/0_beacon-chain.md | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ddc4c8f61..44c20ec84 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,6 +224,7 @@ These configurations are updated for releases, but may be out of sync during `de | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | +| `HISTORICAL_ROOTS_LENGTH` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | @@ -237,6 +238,12 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | + +* Assuming the maximum 16 deposits per slot, the validator registry will last for at least 13,065 years (but likely much much longer) ### Rewards and penalties @@ -357,8 +364,8 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): - custody_bit_0_indices: List[ValidatorIndex] # Indices with custody bit equal to 0 - custody_bit_1_indices: List[ValidatorIndex] # Indices with custody bit equal to 1 + custody_bit_0_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 0 + custody_bit_1_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 1 data: AttestationData signature: BLSSignature ``` @@ -513,14 +520,14 @@ class BeaconState(Container): latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash] + historical_roots: List[Hash, HISTORICAL_ROOTS_LENGTH] # Eth1 eth1_data: Eth1Data - eth1_data_votes: List[Eth1Data] + eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] eth1_deposit_index: uint64 # Registry - validators: List[Validator] - balances: List[Gwei] + validators: List[Validator, VALIDATOR_REGISTRY_SIZE] + balances: List[Gwei, VALIDATOR_REGISTRY_SIZE] # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] @@ -528,8 +535,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations - previous_epoch_attestations: List[PendingAttestation] - current_epoch_attestations: List[PendingAttestation] + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -984,8 +991,6 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 - # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted @@ -1624,6 +1629,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) +<<<<<<< HEAD all_operations = [ (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), @@ -1634,6 +1640,17 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ] # type: List[Tuple[List[Container], int, Callable]] for operations, max_operations, function in all_operations: assert len(operations) <= max_operations +======= + + for operations, max_operations, function in ( + (body.proposer_slashings, process_proposer_slashing), + (body.attester_slashings, process_attester_slashing), + (body.attestations, process_attestation), + (body.deposits, process_deposit), + (body.voluntary_exits, process_voluntary_exit), + (body.transfers, process_transfer), + ): +>>>>>>> f6a2345f... Update spec for new SSZ with list max length for operation in operations: function(state, operation) ``` From 4c2adcc5e643279c462ba3d25ce474ff4bb741f2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 17 Jun 2019 19:07:20 -0400 Subject: [PATCH 17/68] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 44c20ec84..8f2078956 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -243,8 +243,6 @@ These configurations are updated for releases, but may be out of sync during `de | `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | -* Assuming the maximum 16 deposits per slot, the validator registry will last for at least 13,065 years (but likely much much longer) - ### Rewards and penalties | Name | Value | @@ -991,6 +989,8 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 + # Verify max number of indices + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted From 73ba419d640920a1427c357e487625ba70ae87b4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:04:10 +0200 Subject: [PATCH 18/68] check virtual lengths, fix imports --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7662971c4..efcb8f207 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -268,7 +268,7 @@ class Elements(ParamsBase, metaclass=ElementsType): @classmethod def value_check(cls, value): - return all(isinstance(v, cls.elem_type) for v in value) + return all(isinstance(v, cls.elem_type) for v in value) and len(value) <= cls.length @classmethod def extract_args(cls, *args): @@ -316,6 +316,7 @@ class Vector(Elements): @classmethod def value_check(cls, value): + # check length limit strictly return len(value) == cls.length and super().value_check(value) @classmethod @@ -350,7 +351,8 @@ class BytesLike(Elements, metaclass=BytesType): @classmethod def value_check(cls, value): - return isinstance(value, bytes) + # check type and virtual length limit + return isinstance(value, bytes) and len(value) <= cls.length def __str__(self): cls = self.__class__ @@ -376,6 +378,7 @@ class BytesN(BytesLike): @classmethod def value_check(cls, value): + # check length limit strictly return len(value) == cls.length and super().value_check(value) @classmethod From 8c6ddd5233d51bc8937dc6a7e84d8b95f381fc80 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:18:00 +0200 Subject: [PATCH 19/68] container field coercion --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 12 +++++++++++- .../pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index efcb8f207..942095622 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Dict, Iterator +from typing import Tuple, Iterator from types import GeneratorType @@ -142,6 +142,16 @@ class Container(Series, metaclass=SSZType): from .ssz_impl import signing_root return signing_root(self) + def __setattr__(self, name, value): + if name not in self.__class__.__annotations__: + raise AttributeError("Cannot change non-existing SSZ-container attribute") + field_typ = self.__class__.__annotations__[name] + value = coerce_type_maybe(value, field_typ) + if not isinstance(value, field_typ): + raise ValueCheckError(f"Cannot set field of {self.__class__}:" + f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") + super().__setattr__(name, value) + def get_field_values(self) -> Tuple[SSZValue, ...]: cls = self.__class__ return tuple(getattr(self, field) for field in cls.get_field_names()) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 99416e333..291c4b955 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -105,6 +105,20 @@ def test_container(): assert v.type().elem_type == uint8 assert v.type().length == 1024 + y.a = 42 + y.a = uint16(255) + try: + y.a = uint16(256) + assert False + except ValueError: + pass + + try: + y.not_here = 5 + assert False + except AttributeError: + pass + def test_list(): typ = List[uint64, 128] From 4aefc078e9e6cd9a11757681c4cb55abc2b908dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:54:08 +0200 Subject: [PATCH 20/68] list-rework type fixes --- specs/core/0_beacon-chain.md | 23 +++------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++++---- .../eth2spec/utils/ssz/test_ssz_typing.py | 43 ++++++++++++++++++- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f2078956..a505fb451 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -241,7 +241,7 @@ These configurations are updated for releases, but may be out of sync during `de | `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | +| `VALIDATOR_REGISTRY_SIZE` | `2**40` (= 1,099,511,627,776) | | | ### Rewards and penalties @@ -372,7 +372,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] # Bit set for every attesting participant within a committee + aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -439,9 +439,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] + aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] data: AttestationData - custody_bitfield: Bytes[MAX_COMMITTEE_SIZE] + custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] signature: BLSSignature ``` @@ -1629,20 +1629,8 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) -<<<<<<< HEAD - all_operations = [ - (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), - (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), - (body.attestations, MAX_ATTESTATIONS, process_attestation), - (body.deposits, MAX_DEPOSITS, process_deposit), - (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), - (body.transfers, MAX_TRANSFERS, process_transfer), - ] # type: List[Tuple[List[Container], int, Callable]] - for operations, max_operations, function in all_operations: - assert len(operations) <= max_operations -======= - for operations, max_operations, function in ( + for operations, function in ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), @@ -1650,7 +1638,6 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), ): ->>>>>>> f6a2345f... Update spec for new SSZ with list max length for operation in operations: function(state, operation) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 942095622..5aadfae32 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -2,10 +2,6 @@ from typing import Tuple, Iterator from types import GeneratorType -class ValueCheckError(Exception): - pass - - class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") @@ -97,7 +93,7 @@ def coerce_type_maybe(v, typ: SSZType): # shortcut if it's already the type we are looking for if v_typ == typ: return v - elif isinstance(v, int): + elif isinstance(v, int) and not isinstance(v, uint): # do not coerce from one uintX to another uintY return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) @@ -126,7 +122,7 @@ class Container(Series, metaclass=SSZType): else: value = coerce_type_maybe(kwargs[f], t) if not isinstance(value, t): - raise ValueCheckError(f"Bad input for class {self.__class__}:" + raise ValueError(f"Bad input for class {self.__class__}:" f" field: {f} type: {t} value: {value} value type: {type(value)}") setattr(self, f, value) @@ -148,7 +144,7 @@ class Container(Series, metaclass=SSZType): field_typ = self.__class__.__annotations__[name] value = coerce_type_maybe(value, field_typ) if not isinstance(value, field_typ): - raise ValueCheckError(f"Cannot set field of {self.__class__}:" + raise ValueError(f"Cannot set field of {self.__class__}:" f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) @@ -273,7 +269,7 @@ class Elements(ParamsBase, metaclass=ElementsType): items = self.extract_args(*args) if not self.value_check(items): - raise ValueCheckError(f"Bad input for class {self.__class__}: {items}") + raise ValueError(f"Bad input for class {self.__class__}: {items}") self.items = items @classmethod @@ -296,6 +292,16 @@ class Elements(ParamsBase, metaclass=ElementsType): return self.items[i] def __setitem__(self, k, v): + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self.items): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") + typ = self.__class__.elem_type + v = coerce_type_maybe(v, typ) + if not isinstance(v, typ): + raise ValueError(f"Cannot set item in type {self.__class__}," + f" mismatched element type: {v} of {type(v)}, expected {typ}") self.items[k] = v def __len__(self): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 291c4b955..0f4a06c5f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -106,9 +106,14 @@ def test_container(): assert v.type().length == 1024 y.a = 42 - y.a = uint16(255) try: - y.a = uint16(256) + y.a = 256 # out of bounds + assert False + except ValueError: + pass + + try: + y.a = uint16(255) # within bounds, wrong type assert False except ValueError: pass @@ -149,3 +154,37 @@ def test_list(): assert isinstance(v, Series) assert issubclass(v.type(), Elements) assert isinstance(v.type(), ElementsType) + + foo = List[uint32, 128](0 for i in range(128)) + foo[0] = 123 + foo[1] = 654 + foo[127] = 222 + assert sum(foo) == 999 + try: + foo[3] = 2**32 # out of bounds + except ValueError: + pass + + try: + foo[3] = uint64(2**32 - 1) # within bounds, wrong type + assert False + except ValueError: + pass + + try: + foo[128] = 100 + assert False + except IndexError: + pass + + try: + foo[-1] = 100 # valid in normal python lists + assert False + except IndexError: + pass + + try: + foo[128] = 100 # out of bounds + assert False + except IndexError: + pass From 439e4d485938b07a9cc52b5e7745d052245569b4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:12:17 +0200 Subject: [PATCH 21/68] Build spec --- scripts/build_spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 3a1b22efd..61613376f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,8 +26,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, - Bytes4, Bytes32, Bytes48, Bytes96, + Bit, Container, List, Vector, Bytes, BytesN, uint64 ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -179,8 +178,9 @@ def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str """ items = list(objects.items()) for key, value in items: - dependencies = re.findall(r'(: [A-Z][\w[]*)', value) - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|Bytes\d+|bytes', '', x), dependencies) + dependencies = re.findall(r'(: [A-Z][\w\[]*)', value) + dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants + dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|BytesN\[.+\\]|Bytes\[.+\\]|Bytes\d*|bytes\d*', '', x), dependencies) for dep in dependencies: if dep in custom_types or len(dep) == 0: continue From c9747b634f1cca8994af1b97207769b222b4165e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:13:55 +0200 Subject: [PATCH 22/68] improve build spec, get clean dependencies list --- scripts/build_spec.py | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 61613376f..ca648a754 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,12 +12,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, - Callable, - Dict, - List, - Set, - Tuple, + Any, Callable, Iterable, Dict, Set, Tuple ) from eth2spec.utils.ssz.ssz_impl import ( @@ -33,18 +28,14 @@ from eth2spec.utils.bls import ( bls_verify, bls_verify_multiple, ) -# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation. from eth2spec.utils.hash_function import hash + + +Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, + Any, Callable, Dict, Optional, Set, Tuple, Iterable ) from eth2spec.utils.ssz.ssz_impl import ( @@ -54,8 +45,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, + Bit, Container, List, Vector, Bytes, BytesN, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -65,8 +55,10 @@ from eth2spec.utils.bls import ( ) from eth2spec.utils.hash_function import hash + + +Deltas = list ''' -BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' def get_ssz_type_by_name(name: str) -> Container: return globals()[name] @@ -172,18 +164,32 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st return old_constants +ignored_dependencies = [ + 'Bit', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', + 'bytes' # to be removed after updating spec doc +] + + def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: """ Determines which SSZ Object is depenedent on which other and orders them appropriately """ items = list(objects.items()) for key, value in items: - dependencies = re.findall(r'(: [A-Z][\w\[]*)', value) + dependencies = [] + for line in value.split('\n'): + if not re.match(r'\s+\w+: .+', line): + continue # skip whitespace etc. + line = line[line.index(':') + 1:] # strip of field name + if '#' in line: + line = line[:line.index('#')] # strip of comment + dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|BytesN\[.+\\]|Bytes\[.+\\]|Bytes\d*|bytes\d*', '', x), dependencies) + dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) + dependencies = filter(lambda x: x not in custom_types, dependencies) for dep in dependencies: - if dep in custom_types or len(dep) == 0: - continue key_list = list(objects.keys()) for item in [dep, key] + key_list[key_list.index(dep)+1:]: objects[item] = objects.pop(item) From 8344d50ae5ceaa20998eda8f8f063dc1f53e7f03 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:15:48 +0200 Subject: [PATCH 23/68] update beacon chain doc, use new types, avoid List --- specs/core/0_beacon-chain.md | 64 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a505fb451..7cbb9b67b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -291,6 +291,8 @@ We define the following Python custom types for type hinting and readability: | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | +`Deltas` is a non-SSZ type, a series of changes applied to balances, optimized by clients. + ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. @@ -315,7 +317,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: bool + slashed: Bit # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -355,7 +357,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: bool # Challengeable bit for the custody of crosslink data + custody_bit: Bit # Challengeable bit for the custody of crosslink data ``` #### `IndexedAttestation` @@ -649,11 +651,11 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Tuple[ValidatorIndex, ...]: """ Get active validator indices at ``epoch``. """ - return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] + return tuple(ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)) ``` ### `increase_balance` @@ -815,7 +817,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: List[Hash], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, proof: Tuple[Hash, ...], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -859,16 +861,17 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V ### `compute_committee` ```python -def compute_committee(indices: List[ValidatorIndex], seed: Hash, index: int, count: int) -> List[ValidatorIndex]: +def compute_committee(indices: Tuple[ValidatorIndex, ...], + seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return tuple(indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)) ``` ### `get_crosslink_committee` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> List[ValidatorIndex]: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: return compute_committee( indices=get_active_validator_indices(state, epoch), seed=generate_seed(state, epoch), @@ -882,13 +885,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: + bitfield: bytes) -> Tuple[ValidatorIndex, ...]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return tuple(sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])) ``` ### `int_to_bytes` @@ -908,7 +911,7 @@ def bytes_to_int(data: bytes) -> int: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, indices: Iterable[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ @@ -1134,7 +1137,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1151,7 +1154,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: # Process deposits state = BeaconState() for deposit in deposits: @@ -1173,10 +1176,10 @@ def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, - eth1_data=genesis_eth1_data, + eth1_data=eth1_data, latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) @@ -1270,21 +1273,21 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ``` ```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: - return [ +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: + return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) ] ``` ```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) @@ -1293,7 +1296,7 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[Pen ```python def get_unslashed_attesting_indices(state: BeaconState, - attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + attestations: Iterable[PendingAttestation]) -> Iterable[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) @@ -1301,14 +1304,14 @@ def get_unslashed_attesting_indices(state: BeaconState, ``` ```python -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: +def get_attesting_balance(state: BeaconState, attestations: Iterable[PendingAttestation]) -> Gwei: return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, - shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: + shard: Shard) -> Tuple[Crosslink, Iterable[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1397,11 +1400,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: +def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = [Gwei(0) for _ in range(len(state.validators))] - penalties = [Gwei(0) for _ in range(len(state.validators))] + rewards = Deltas(0 for _ in range(len(state.validators))) + penalties = Deltas(0 for _ in range(len(state.validators))) eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1448,9 +1451,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: ``` ```python -def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - rewards = [Gwei(0) for index in range(len(state.validators))] - penalties = [Gwei(0) for index in range(len(state.validators))] +def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: + rewards = Deltas(0 for _ in range(len(state.validators))) + penalties = Deltas(0 for _ in range(len(state.validators))) epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) @@ -1630,14 +1633,15 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) - for operations, function in ( + all_operations = ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ): + ) # type: Tuple[Tuple[List, Callable], ...] + for operations, function in all_operations: for operation in operations: function(state, operation) ``` From 4b4bf87e476769d56227bdd3dc2486a6886947fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 00:55:01 +0200 Subject: [PATCH 24/68] update shard doc, use new types, avoid List --- specs/core/1_shard-data-chains.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b83cd54f7..79e2478fe 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -154,7 +154,7 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> List[ValidatorIndex]: + slot: Slot) -> Tuple[ValidatorIndex, ...]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ @@ -175,10 +175,10 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return sorted(list(set( + return tuple(sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - ))) + )))) ``` ### `get_shard_proposer_index` @@ -284,7 +284,7 @@ Let: ```python def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, - valid_shard_blocks: List[ShardBlock], + valid_shard_blocks: Iterable[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -348,7 +348,7 @@ Let: * `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python -def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], +def is_valid_shard_attestation(valid_shard_blocks: Iterable[ShardBlock], beacon_state: BeaconState, candidate: ShardAttestation) -> bool: # Check shard block @@ -380,7 +380,7 @@ Let: def is_valid_beacon_attestation(shard: Shard, shard_blocks: List[ShardBlock], beacon_state: BeaconState, - valid_attestations: List[Attestation], + valid_attestations: Set[Attestation], candidate: Attestation) -> bool: # Check if attestation is already determined valid for _, attestation in enumerate(valid_attestations): From 5be0c57aade56b95c0d8b8b2d54d437cdc14ddb4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 01:03:04 +0200 Subject: [PATCH 25/68] fix linting + mypy --- scripts/build_spec.py | 6 +++--- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 13 +++++++++--- .../eth2spec/utils/ssz/test_ssz_impl.py | 6 +++--- .../eth2spec/utils/ssz/test_ssz_typing.py | 21 ++++++++++--------- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index ca648a754..20e90ee33 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -20,8 +20,8 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - # unused: uint8, uint16, uint32, uint128, uint256, - Bit, Container, List, Vector, Bytes, BytesN, uint64 + Bit, Container, List, Vector, Bytes, uint64, + Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -45,7 +45,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, BytesN, uint64, + Bit, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1b59b276b..fd17e29f9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint ) # SSZ Serialization diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5aadfae32..082e3ed30 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -123,7 +123,7 @@ class Container(Series, metaclass=SSZType): value = coerce_type_maybe(kwargs[f], t) if not isinstance(value, t): raise ValueError(f"Bad input for class {self.__class__}:" - f" field: {f} type: {t} value: {value} value type: {type(value)}") + f" field: {f} type: {t} value: {value} value type: {type(value)}") setattr(self, f, value) def serialize(self): @@ -145,7 +145,7 @@ class Container(Series, metaclass=SSZType): value = coerce_type_maybe(value, field_typ) if not isinstance(value, field_typ): raise ValueError(f"Cannot set field of {self.__class__}:" - f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") + f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) def get_field_values(self) -> Tuple[SSZValue, ...]: @@ -301,7 +301,7 @@ class Elements(ParamsBase, metaclass=ElementsType): v = coerce_type_maybe(v, typ) if not isinstance(v, typ): raise ValueError(f"Cannot set item in type {self.__class__}," - f" mismatched element type: {v} of {type(v)}, expected {typ}") + f" mismatched element type: {v} of {type(v)}, expected {typ}") self.items[k] = v def __len__(self): @@ -400,3 +400,10 @@ class BytesN(BytesLike): @classmethod def is_fixed_size(cls): return True + + +# Helpers for common BytesN types. +Bytes4 = BytesN[4] +Bytes32 = BytesN[32] +Bytes48 = BytesN[48] +Bytes96 = BytesN[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 8dd04a86d..ae0849098 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ -from .ssz_impl import serialize, serialize_basic, encode_series, signing_root, hash_tree_root +from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256, byte + Bit, Container, List, Vector, Bytes, BytesN, + uint8, uint16, uint32, uint64, byte ) import pytest diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 0f4a06c5f..f604b6468 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,5 +1,6 @@ from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, + Elements, Bit, Container, List, Vector, Bytes, BytesN, uint, uint8, uint16, uint32, uint64, uint128, uint256 ) @@ -41,13 +42,13 @@ def test_basic_instances(): def test_basic_value_bounds(): max = { - Bit: 2**1, - uint8: 2**(8 * 1), - uint16: 2**(8 * 2), - uint32: 2**(8 * 4), - uint64: 2**(8 * 8), - uint128: 2**(8 * 16), - uint256: 2**(8 * 32), + Bit: 2 ** 1, + uint8: 2 ** (8 * 1), + uint16: 2 ** (8 * 2), + uint32: 2 ** (8 * 4), + uint64: 2 ** (8 * 8), + uint128: 2 ** (8 * 16), + uint256: 2 ** (8 * 32), } for k, v in max.items(): # this should work @@ -161,12 +162,12 @@ def test_list(): foo[127] = 222 assert sum(foo) == 999 try: - foo[3] = 2**32 # out of bounds + foo[3] = 2 ** 32 # out of bounds except ValueError: pass try: - foo[3] = uint64(2**32 - 1) # within bounds, wrong type + foo[3] = uint64(2 ** 32 - 1) # within bounds, wrong type assert False except ValueError: pass From 6f46c1d837646bd6906845253c13f2e6ee1d2d72 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 01:12:51 +0200 Subject: [PATCH 26/68] fix typing in spec builder monkey patch --- scripts/build_spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 20e90ee33..7d42c218e 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -66,13 +66,13 @@ def get_ssz_type_by_name(name: str) -> Container: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], List[ValidatorIndex]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} -def compute_committee(indices: List[ValidatorIndex], # type: ignore +def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore seed: Hash, index: int, - count: int) -> List[ValidatorIndex]: + count: int) -> Tuple[ValidatorIndex, ...]: param_hash = (hash_tree_root(indices), seed, index, count) if param_hash not in committee_cache: From 6b82e3faa56707b0c1fd11cd82e2f8eb36037370 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:20:07 +0200 Subject: [PATCH 27/68] Modifications from Vitalik, to enable SSZ Partials --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 54 ++++++++----------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index fd17e29f9..a9c36649b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -107,7 +107,7 @@ def is_bottom_layer_kind(typ: SSZType): def item_length(typ: SSZType) -> int: - if issubclass(typ, BasicType): + if issubclass(typ, BasicValue): return typ.byte_len else: return 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 082e3ed30..981f30d9b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,6 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bit(BasicValue): # can't subclass bool. + byte_len = 1 def __new__(cls, value, *args, **kwargs): if value < 0 or value > 1: @@ -88,7 +89,7 @@ class uint256(uint): byte_len = 32 -def coerce_type_maybe(v, typ: SSZType): +def coerce_type_maybe(v, typ: SSZType, strict: bool = False): v_typ = type(v) # shortcut if it's already the type we are looking for if v_typ == typ: @@ -97,10 +98,14 @@ def coerce_type_maybe(v, typ: SSZType): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) + elif isinstance(v, bytes): + return typ(v) elif isinstance(v, GeneratorType): return typ(v) else: # just return as-is, Value-checkers will take care of it not being coerced. + if strict and not isinstance(v, typ): + raise ValueError("Type coercion of {} to {} failed".format(v, typ)) return v @@ -116,7 +121,7 @@ class Container(Series, metaclass=SSZType): def __init__(self, **kwargs): cls = self.__class__ - for f, t in cls.get_fields(): + for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, t.default()) else: @@ -148,16 +153,12 @@ class Container(Series, metaclass=SSZType): f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) - def get_field_values(self) -> Tuple[SSZValue, ...]: - cls = self.__class__ - return tuple(getattr(self, field) for field in cls.get_field_names()) - def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_field_names()}) + return repr({field: getattr(self, field) for field in self.get_fields()}) def __str__(self): output = [f'{self.__class__.__name__}'] - for field in self.get_field_names(): + for field in self.get_fields(): output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) @@ -168,23 +169,10 @@ class Container(Series, metaclass=SSZType): return hash(self.hash_tree_root()) @classmethod - def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + def get_fields(cls) -> Dict[str, SSZType]: if not hasattr(cls, '__annotations__'): # no container fields - return () - return tuple((f, t) for f, t in cls.__annotations__.items()) - - @classmethod - def get_field_names(cls) -> Tuple[str, ...]: - if not hasattr(cls, '__annotations__'): # no container fields - return () - return tuple(cls.__annotations__.keys()) - - @classmethod - def get_field_types(cls) -> Tuple[SSZType, ...]: - if not hasattr(cls, '__annotations__'): # no container fields - return () - # values of annotations are the types corresponding to the fields, not instance values. - return tuple(cls.__annotations__.values()) + return {} + return dict(cls.__annotations__) @classmethod def default(cls): @@ -195,7 +183,7 @@ class Container(Series, metaclass=SSZType): return all(t.is_fixed_size() for t in cls.get_field_types()) def __iter__(self) -> Iterator[SSZValue]: - return iter(self.get_field_values()) + return iter([getattr(self, field) for field in self.get_fields()]) class ParamsBase(Series): @@ -297,12 +285,16 @@ class Elements(ParamsBase, metaclass=ElementsType): if k > len(self.items): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") - typ = self.__class__.elem_type - v = coerce_type_maybe(v, typ) - if not isinstance(v, typ): - raise ValueError(f"Cannot set item in type {self.__class__}," - f" mismatched element type: {v} of {type(v)}, expected {typ}") - self.items[k] = v + self.items[k] = coerce_type_maybe(v, self.__class__.elem_type, strict=True) + + def append(self, v): + self.items.append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + + def pop(self): + if len(self.items) == 0: + raise IndexError("Pop from empty list") + else: + return self.items.pop() def __len__(self): return len(self.items) From 5048b9e87a1f260c4c307100e489407adcade2ac Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:14:13 +0200 Subject: [PATCH 28/68] temporary fix for phase-1 spec typing --- scripts/build_spec.py | 3 ++- specs/core/1_custody-game.md | 34 +++++++++++++++++++------------ specs/core/1_shard-data-chains.md | 24 ++++++++++++++-------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7d42c218e..612edbd00 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -35,7 +35,8 @@ from eth2spec.utils.hash_function import hash Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable + Any, Callable, Dict, Optional, Set, Tuple, Iterable, + List as TypingList ) from eth2spec.utils.ssz.ssz_impl import ( diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3fe132c07..99c85a034 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -113,6 +113,13 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | + +### TODO PLACEHOLDER + +| Name | Value | +| - | - | +| `PLACEHOLDER` | `2**32` | + ## Data structures ### Custody objects @@ -134,7 +141,7 @@ class CustodyBitChallenge(Container): attestation: Attestation challenger_index: ValidatorIndex responder_key: BLSSignature - chunk_bits: bytes + chunk_bits: Bytes[PLACEHOLDER] signature: BLSSignature ``` @@ -171,9 +178,9 @@ class CustodyBitChallengeRecord(Container): class CustodyResponse(Container): challenge_index: uint64 chunk_index: uint64 - chunk: Vector[bytes, BYTES_PER_CUSTODY_CHUNK] - data_branch: List[Bytes32] - chunk_bits_branch: List[Bytes32] + chunk: Vector[Bytes[PLACEHOLDER], BYTES_PER_CUSTODY_CHUNK] + data_branch: List[Bytes32, PLACEHOLDER] + chunk_bits_branch: List[Bytes32, PLACEHOLDER] chunk_bits_leaf: Bytes32 ``` @@ -226,24 +233,25 @@ class Validator(Container): ```python class BeaconState(Container): - custody_chunk_challenge_records: List[CustodyChunkChallengeRecord] - custody_bit_challenge_records: List[CustodyBitChallengeRecord] + custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, PLACEHOLDER] + custody_bit_challenge_records: List[CustodyBitChallengeRecord, PLACEHOLDER] custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - exposed_derived_secrets: Vector[List[ValidatorIndex], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + exposed_derived_secrets: Vector[List[ValidatorIndex, PLACEHOLDER], + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` #### `BeaconBlockBody` ```python class BeaconBlockBody(Container): - custody_chunk_challenges: List[CustodyChunkChallenge] - custody_bit_challenges: List[CustodyBitChallenge] - custody_responses: List[CustodyResponse] - custody_key_reveals: List[CustodyKeyReveal] - early_derived_secret_reveals: List[EarlyDerivedSecretReveal] + custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] + custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] + custody_responses: List[CustodyResponse, PLACEHOLDER] + custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] + early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] ``` ## Helpers @@ -310,7 +318,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: List[Any], new_element: Any) -> int: +def replace_empty_or_append(list: TypingList[Any], new_element: Any) -> int: for i in range(len(list)): if is_empty(list[i]): list[i] = new_element diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 79e2478fe..bdf6550ff 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -70,13 +70,19 @@ This document describes the shard data layer and the shard fork choice rule in P | `DOMAIN_SHARD_PROPOSER` | `128` | | `DOMAIN_SHARD_ATTESTER` | `129` | +### TODO PLACEHOLDER + +| Name | Value | +| - | - | +| `PLACEHOLDER` | `2**32` | + ## Data structures ### `ShardBlockBody` ```python class ShardBlockBody(Container): - data: Vector[bytes, BYTES_PER_SHARD_BLOCK_BODY] + data: Vector[Bytes[PLACEHOLDER], BYTES_PER_SHARD_BLOCK_BODY] ``` ### `ShardAttestation` @@ -87,7 +93,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: bytes + aggregation_bitfield: Bytes[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -101,7 +107,7 @@ class ShardBlock(Container): parent_root: Bytes32 data: ShardBlockBody state_root: Bytes32 - attestations: List[ShardAttestation] + attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -115,7 +121,7 @@ class ShardBlockHeader(Container): parent_root: Bytes32 body_root: Bytes32 state_root: Bytes32 - attestations: List[ShardAttestation] + attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -128,7 +134,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, - count: int) -> List[ValidatorIndex]: + count: int) -> Tuple[ValidatorIndex, ...]: """ Return committee for a period. Used to construct persistent committees. """ @@ -243,11 +249,11 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Iterable[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) - def pad_to_power_of_2(values: List[bytes]) -> List[bytes]: + def pad_to_power_of_2(values: TypingList[bytes]) -> TypingList[bytes]: while not is_power_of_two(len(values)): values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] return values @@ -282,7 +288,7 @@ Let: * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: List[BeaconBlock], +def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: Iterable[ShardBlock], candidate: ShardBlock) -> bool: @@ -378,7 +384,7 @@ Let: ```python def is_valid_beacon_attestation(shard: Shard, - shard_blocks: List[ShardBlock], + shard_blocks: TypingList[ShardBlock], beacon_state: BeaconState, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: From a33c67894ebd9593fea94d38fd6914a3725ffe04 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:37:22 +0200 Subject: [PATCH 29/68] update ssz testing/debug utils --- test_libs/pyspec/eth2spec/debug/decode.py | 44 +++---- test_libs/pyspec/eth2spec/debug/encode.py | 34 +++--- .../pyspec/eth2spec/debug/random_value.py | 112 ++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 5 +- 4 files changed, 84 insertions(+), 111 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 5ce116025..743479371 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,39 +1,29 @@ +from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, - is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, - read_vector_elem_type, read_list_elem_type, + SSZType, SSZValue, uint, Container, Bytes, List, Bit, Vector, BytesN ) -def decode(data, typ): - if is_uint_type(typ): - return data - elif is_bool_type(typ): - assert data in (True, False) - return data - elif is_list_type(typ): - elem_typ = read_list_elem_type(typ) - return [decode(element, elem_typ) for element in data] - elif is_vector_type(typ): - elem_typ = read_vector_elem_type(typ) - return Vector(decode(element, elem_typ) for element in data) - elif is_bytes_type(typ): - return bytes.fromhex(data[2:]) - elif is_bytesn_type(typ): - return BytesN(bytes.fromhex(data[2:])) - elif is_container_type(typ): +def decode(data: Any, typ: SSZType) -> SSZValue: + if issubclass(typ, (uint, Bit)): + return typ(data) + elif issubclass(typ, (List, Vector)): + return typ(decode(element, typ.elem_type) for element in data) + elif issubclass(typ, (Bytes, BytesN)): + return typ(bytes.fromhex(data[2:])) + elif issubclass(typ, Container): temp = {} - for field, subtype in typ.get_fields(): - temp[field] = decode(data[field], subtype) - if field + "_hash_tree_root" in data: - assert(data[field + "_hash_tree_root"][2:] == - hash_tree_root(temp[field], subtype).hex()) + for field_name, field_type in typ.get_fields().items(): + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) ret = typ(**temp) if "hash_tree_root" in data: - assert(data["hash_tree_root"][2:] == - hash_tree_root(ret, typ).hex()) + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) return ret else: raise Exception(f"Type not recognized: data={data}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 61dd87928..d264bd7ff 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,36 +1,30 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, is_vector_type, is_container_type, - read_elem_type, - uint + SSZValue, uint, Container, Bytes, BytesN, List, Vector, Bit ) -def encode(value, typ, include_hash_tree_roots=False): - if is_uint_type(typ): - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ +def encode(value: SSZValue, include_hash_tree_roots=False): + if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length - if issubclass(typ, uint) and typ.byte_len > 8: + if value.type().byte_len > 8: return str(value) return value - elif is_bool_type(typ): + elif isinstance(value, Bit): assert value in (True, False) return value - elif is_list_type(typ) or is_vector_type(typ): - elem_typ = read_elem_type(typ) - return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif isinstance(typ, type) and issubclass(typ, bytes): # both bytes and BytesN + elif isinstance(value, (List, Vector)): + return [encode(element, include_hash_tree_roots) for element in value] + elif isinstance(value, (Bytes, BytesN)): # both bytes and BytesN return '0x' + value.hex() - elif is_container_type(typ): + elif isinstance(value, Container): ret = {} - for field, subtype in typ.get_fields(): - field_value = getattr(value, field) - ret[field] = encode(field_value, subtype, include_hash_tree_roots) + for field_value, field_name in zip(value, value.get_fields().keys()): + ret[field_name] = encode(field_value, include_hash_tree_roots) if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(field_value, subtype).hex() + ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(field_value).hex() if include_hash_tree_roots: - ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() + ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() return ret else: - raise Exception(f"Type not recognized: value={value}, typ={typ}") + raise Exception(f"Type not recognized: value={value}, typ={value.type()}") diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 3edcc8808..bd40cb832 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -1,18 +1,13 @@ from random import Random -from typing import Any from enum import Enum -from eth2spec.utils.ssz.ssz_impl import is_basic_type - from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, - is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, - read_vector_elem_type, read_list_elem_type, - uint_byte_size + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bit, + Vector, BytesN ) # in bytes -UINT_SIZES = (1, 2, 4, 8, 16, 32) +UINT_BYTE_SIZES = (1, 2, 4, 8, 16, 32) random_mode_names = ("random", "zero", "max", "nil", "one", "lengthy") @@ -39,11 +34,11 @@ class RandomizationMode(Enum): def get_random_ssz_object(rng: Random, - typ: Any, + typ: SSZType, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, - chaos: bool) -> Any: + chaos: bool) -> SSZValue: """ Create an object for a given type, filled with random data. :param rng: The random number generator to use. @@ -56,33 +51,31 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if is_bytes_type(typ): + if issubclass(typ, Bytes): # Bytes array if mode == RandomizationMode.mode_nil_count: - return b'' + return typ(b'') elif mode == RandomizationMode.mode_max_count: - return get_random_bytes_list(rng, max_bytes_length) + return typ(get_random_bytes_list(rng, max_bytes_length)) elif mode == RandomizationMode.mode_one_count: - return get_random_bytes_list(rng, 1) + return typ(get_random_bytes_list(rng, 1)) elif mode == RandomizationMode.mode_zero: - return b'\x00' + return typ(b'\x00') elif mode == RandomizationMode.mode_max: - return b'\xff' + return typ(b'\xff') else: - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) - elif is_bytesn_type(typ): - # BytesN - length = typ.length + return typ(get_random_bytes_list(rng, rng.randint(0, max_bytes_length))) + elif issubclass(typ, BytesN): # Sanity, don't generate absurdly big random values # If a client is aiming to performance-test, they should create a benchmark suite. - assert length <= max_bytes_length + assert typ.length <= max_bytes_length if mode == RandomizationMode.mode_zero: - return b'\x00' * length + return typ(b'\x00' * typ.length) elif mode == RandomizationMode.mode_max: - return b'\xff' * length + return typ(b'\xff' * typ.length) else: - return get_random_bytes_list(rng, length) - elif is_basic_type(typ): + return typ(get_random_bytes_list(rng, typ.length)) + elif issubclass(typ, BasicValue): # Basic types if mode == RandomizationMode.mode_zero: return get_min_basic_value(typ) @@ -90,32 +83,28 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif is_vector_type(typ): - # Vector - elem_typ = read_vector_elem_type(typ) - return [ - get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + elif issubclass(typ, Vector): + return typ( + get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) - ] - elif is_list_type(typ): - # List - elem_typ = read_list_elem_type(typ) - length = rng.randint(0, max_list_length) + ) + elif issubclass(typ, List): + length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 elif mode == RandomizationMode.mode_max_count: length = max_list_length - return [ - get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + return typ( + get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) - ] - elif is_container_type(typ): + ) + elif issubclass(typ, Container): # Container return typ(**{ - field: - get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) - for field, subtype in typ.get_fields() + field_name: + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + for field_name, field_type in typ.get_fields().items() }) else: raise Exception(f"Type not recognized: typ={typ}") @@ -125,34 +114,31 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: return bytes(rng.getrandbits(8) for _ in range(length)) -def get_random_basic_value(rng: Random, typ) -> Any: - if is_bool_type(typ): - return rng.choice((True, False)) - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return rng.randint(0, 256**size - 1) +def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(rng.choice((True, False))) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(rng.randint(0, 256 ** typ.byte_len - 1)) else: raise ValueError(f"Not a basic type: typ={typ}") -def get_min_basic_value(typ) -> Any: - if is_bool_type(typ): - return False - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return 0 +def get_min_basic_value(typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(False) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(0) else: raise ValueError(f"Not a basic type: typ={typ}") -def get_max_basic_value(typ) -> Any: - if is_bool_type(typ): - return True - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return 256**size - 1 +def get_max_basic_value(typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(True) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(256 ** typ.byte_len - 1) else: raise ValueError(f"Not a basic type: typ={typ}") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 981f30d9b..ea4d85e82 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,7 +364,10 @@ class BytesLike(Elements, metaclass=BytesType): def __str__(self): cls = self.__class__ - return f"{cls.__name__}[{cls.length}]: {self.items.hex()}" + return f"{cls.__name__}[{cls.length}]: {self.hex()}" + + def hex(self) -> str: + return self.items.hex() class Bytes(BytesLike): From 7cdec746b47bacff0e2d4352cb4838dd6d28182b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:41:43 +0200 Subject: [PATCH 30/68] fix field iteration crash in ssz typing --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea4d85e82..51a790853 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -154,11 +154,11 @@ class Container(Series, metaclass=SSZType): super().__setattr__(name, value) def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_fields()}) + return repr({field: getattr(self, field) for field in self.get_fields().keys()}) def __str__(self): output = [f'{self.__class__.__name__}'] - for field in self.get_fields(): + for field in self.get_fields().keys(): output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) @@ -176,7 +176,7 @@ class Container(Series, metaclass=SSZType): @classmethod def default(cls): - return cls(**{f: t.default() for f, t in cls.get_fields()}) + return cls(**{f: t.default() for f, t in cls.get_fields().items()}) @classmethod def is_fixed_size(cls): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index f604b6468..6bb56f4e5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -76,6 +76,10 @@ def test_container(): a: uint8 b: uint32 + empty = Foo() + assert empty.a == uint8(0) + assert empty.b == uint32(0) + assert issubclass(Foo, Container) assert issubclass(Foo, SSZValue) assert issubclass(Foo, Series) From 4e747fb8879540012655e534e9d8fd214e47be87 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:25:22 +0200 Subject: [PATCH 31/68] fixes for class based ssz typing --- scripts/build_spec.py | 32 ++++++++--------- specs/core/0_beacon-chain.md | 8 +++-- .../pyspec/eth2spec/test/helpers/custody.py | 4 +-- .../pyspec/eth2spec/utils/merkle_minimal.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 +-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 36 ++++++++++--------- .../eth2spec/utils/ssz/test_ssz_typing.py | 20 ++++++++++- 7 files changed, 66 insertions(+), 41 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 612edbd00..d33ba6642 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -65,22 +65,6 @@ def get_ssz_type_by_name(name: str) -> Container: return globals()[name] -# Monkey patch validator compute committee code -_compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} - - -def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore - seed: Hash, - index: int, - count: int) -> Tuple[ValidatorIndex, ...]: - param_hash = (hash_tree_root(indices), seed, index, count) - - if param_hash not in committee_cache: - committee_cache[param_hash] = _compute_committee(indices, seed, index, count) - return committee_cache[param_hash] - - # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Hash] = {} @@ -92,6 +76,22 @@ def hash(x: bytes) -> Hash: return hash_cache[x] +# Monkey patch validator compute committee code +_compute_committee = compute_committee +committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} + + +def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore + seed: Hash, + index: int, + count: int) -> Tuple[ValidatorIndex, ...]: + param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count) + + if param_hash not in committee_cache: + committee_cache[param_hash] = _compute_committee(indices, seed, index, count) + return committee_cache[param_hash] + + # Access to overwrite spec constants based on configuration def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars = globals() diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7cbb9b67b..9a02c16e4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -651,11 +651,13 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Tuple[ValidatorIndex, ...]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]: """ Get active validator indices at ``epoch``. """ - return tuple(ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)) + return List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + i for i, v in enumerate(state.validators) if is_active_validator(v, epoch) + ) ``` ### `increase_balance` @@ -873,7 +875,7 @@ def compute_committee(indices: Tuple[ValidatorIndex, ...], ```python def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: return compute_committee( - indices=get_active_validator_indices(state, epoch), + indices=tuple(get_active_validator_indices(state, epoch)), seed=generate_seed(state, epoch), index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, count=get_epoch_committee_count(state, epoch), diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 67df12fcd..b49a6be1f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -11,7 +11,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING reveal = bls_sign( - message_hash=spec.hash_tree_root(epoch), + message_hash=spec.hash_tree_root(spec.Epoch(epoch)), privkey=privkeys[revealed_index], domain=spec.get_domain( state=state, @@ -20,7 +20,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ), ) mask = bls_sign( - message_hash=spec.hash_tree_root(epoch), + message_hash=spec.hash_tree_root(spec.Epoch(epoch)), privkey=privkeys[masker_index], domain=spec.get_domain( state=state, diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 21583ee92..038b555cf 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -4,7 +4,7 @@ from .hash_function import hash ZERO_BYTES32 = b'\x00' * 32 zerohashes = [ZERO_BYTES32] -for layer in range(1, 32): +for layer in range(1, 100): zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a9c36649b..4b64c9162 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint, ) # SSZ Serialization @@ -143,5 +143,6 @@ def hash_tree_root(obj: SSZValue): def signing_root(obj: Container): # ignore last field - leaves = [hash_tree_root(field) for field in obj[:-1]] + fields = [field for field in obj][:-1] + leaves = [hash_tree_root(f) for f in fields] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 51a790853..381dadf9e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -98,7 +98,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) - elif isinstance(v, bytes): + elif isinstance(v, (bytes, BytesN, Bytes)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) @@ -154,7 +154,8 @@ class Container(Series, metaclass=SSZType): super().__setattr__(name, value) def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_fields().keys()}) + return repr({field: (getattr(self, field) if hasattr(self, field) else 'unset') + for field in self.get_fields().keys()}) def __str__(self): output = [f'{self.__class__.__name__}'] @@ -236,15 +237,24 @@ class ParamsMeta(SSZType): raise TypeError("provided parameters {} mismatch required parameter count {}".format(params, i)) return res - def __instancecheck__(self, obj): - if obj.__class__.__name__ != self.__name__: + def __subclasscheck__(self, subclass): + # check regular class system if we can, solves a lot of the normal cases. + if super().__subclasscheck__(subclass): + return True + # if they are not normal subclasses, they are of the same class. + # then they should have the same name + if subclass.__name__ != self.__name__: return False + # If they do have the same name, they should also have the same params. for name, typ in self.__annotations__.items(): - if hasattr(self, name) and hasattr(obj.__class__, name) \ - and getattr(obj.__class__, name) != getattr(self, name): + if hasattr(self, name) and hasattr(subclass, name) \ + and getattr(subclass, name) != getattr(self, name): return False return True + def __instancecheck__(self, obj): + return self.__subclasscheck__(obj.__class__) + class ElementsType(ParamsMeta): elem_type: SSZType @@ -305,9 +315,6 @@ class Elements(ParamsBase, metaclass=ElementsType): def __iter__(self) -> Iterator[SSZValue]: return iter(self.items) - def __eq__(self, other): - return self.items == other.items - class List(Elements): @@ -366,9 +373,6 @@ class BytesLike(Elements, metaclass=BytesType): cls = self.__class__ return f"{cls.__name__}[{cls.length}]: {self.hex()}" - def hex(self) -> str: - return self.items.hex() - class Bytes(BytesLike): @@ -398,7 +402,7 @@ class BytesN(BytesLike): # Helpers for common BytesN types. -Bytes4 = BytesN[4] -Bytes32 = BytesN[32] -Bytes48 = BytesN[48] -Bytes96 = BytesN[96] +Bytes4: BytesType = BytesN[4] +Bytes32: BytesType = BytesN[32] +Bytes48: BytesType = BytesN[48] +Bytes96: BytesType = BytesN[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 6bb56f4e5..daa923aa7 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,7 +1,8 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256 + uint, uint8, uint16, uint32, uint64, uint128, uint256, + Bytes32, Bytes48 ) @@ -193,3 +194,20 @@ def test_list(): assert False except IndexError: pass + + +def test_bytesn_subclass(): + assert isinstance(BytesN[32](b'\xab' * 32), Bytes32) + assert not isinstance(BytesN[32](b'\xab' * 32), Bytes48) + assert issubclass(BytesN[32](b'\xab' * 32).type(), Bytes32) + assert issubclass(BytesN[32], Bytes32) + + class Hash(Bytes32): + pass + + assert isinstance(Hash(b'\xab' * 32), Bytes32) + assert not isinstance(Hash(b'\xab' * 32), Bytes48) + assert issubclass(Hash(b'\xab' * 32).type(), Bytes32) + assert issubclass(Hash, Bytes32) + + assert not issubclass(Bytes48, Bytes32) From 977856b06fa324440f87cf912b6f250ce60a3982 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:30:42 +0200 Subject: [PATCH 32/68] ssz typing now subclasses list/bytes, much easier to work with than wrapped list/bytes functionality --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 11 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 90 +++++++++++-------- .../eth2spec/utils/ssz/test_ssz_typing.py | 2 + 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 4b64c9162..144201d83 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, uint, ) # SSZ Serialization @@ -46,9 +46,8 @@ def serialize(obj: SSZValue): def encode_series(values: Series): - # bytes and bytesN are already in the right format. - if isinstance(values, (Bytes, BytesN)): - return values.items + if isinstance(values, bytes): # Bytes and BytesN are already like serialized output + return values # Recursively serialize parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] @@ -84,8 +83,8 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, (Bytes, BytesN)): - return values.items + if isinstance(values, bytes): # Bytes and BytesN are already packed + return values return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 381dadf9e..341df880a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -188,10 +188,10 @@ class Container(Series, metaclass=SSZType): class ParamsBase(Series): - _bare = True + _has_params = False def __new__(cls, *args, **kwargs): - if cls._bare: + if not cls._has_params: raise Exception("cannot init bare type without params") return super().__new__(cls, **kwargs) @@ -200,13 +200,13 @@ class ParamsMeta(SSZType): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) - for k, v in attrs.items(): - setattr(out, k, v) + if hasattr(out, "_has_params") and getattr(out, "_has_params"): + for k, v in attrs.items(): + setattr(out, k, v) return out def __getitem__(self, params): o = self.__class__(self.__name__, (self,), self.attr_from_params(params)) - o._bare = False return o def __str__(self): @@ -218,7 +218,7 @@ class ParamsMeta(SSZType): def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. params = p if isinstance(p, tuple) else (p,) - res = {} + res = {'_has_params': True} i = 0 for (name, typ) in self.__annotations__.items(): if hasattr(self.__class__, name): @@ -262,13 +262,17 @@ class ElementsType(ParamsMeta): class Elements(ParamsBase, metaclass=ElementsType): + pass + + +class BaseList(list, Elements): def __init__(self, *args): items = self.extract_args(*args) if not self.value_check(items): raise ValueError(f"Bad input for class {self.__class__}: {items}") - self.items = items + super().__init__(items) @classmethod def value_check(cls, value): @@ -284,39 +288,32 @@ class Elements(ParamsBase, metaclass=ElementsType): def __str__(self): cls = self.__class__ - return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" def __getitem__(self, i) -> SSZValue: - return self.items[i] + if i < 0: + raise IndexError(f"cannot get item in type {self.__class__} at negative index {i}") + if i > len(self): + raise IndexError(f"cannot get item in type {self.__class__}" + f" at out of bounds index {i}") + return super().__getitem__(i) def __setitem__(self, k, v): if k < 0: raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self.items): + if k > len(self): raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") - self.items[k] = coerce_type_maybe(v, self.__class__.elem_type, strict=True) + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): - self.items.append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) - - def pop(self): - if len(self.items) == 0: - raise IndexError("Pop from empty list") - else: - return self.items.pop() - - def __len__(self): - return len(self.items) - - def __repr__(self): - return repr(self.items) + super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def __iter__(self) -> Iterator[SSZValue]: - return iter(self.items) + return super().__iter__() -class List(Elements): +class List(BaseList): @classmethod def default(cls): @@ -327,7 +324,7 @@ class List(Elements): return False -class Vector(Elements): +class Vector(BaseList): @classmethod def value_check(cls, value): @@ -342,27 +339,35 @@ class Vector(Elements): def is_fixed_size(cls): return cls.elem_type.is_fixed_size() + def append(self, v): + raise Exception("cannot modify vector length") + + def pop(self, *args): + raise Exception("cannot modify vector length") + class BytesType(ElementsType): elem_type: SSZType = byte length: int -class BytesLike(Elements, metaclass=BytesType): +class BaseBytes(bytes, Elements, metaclass=BytesType): + + def __new__(cls, *args) -> "BaseBytes": + extracted_val = cls.extract_args(*args) + if not cls.value_check(extracted_val): + raise ValueError(f"Bad input for class {cls}: {extracted_val}") + return super().__new__(cls, extracted_val) @classmethod def extract_args(cls, *args): - x = list(args) - if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, BytesLike)): + x = args + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes)): x = x[0] - if isinstance(x, bytes): + if isinstance(x, bytes): # Includes BytesLike return x - elif isinstance(x, BytesLike): - return x.items - elif isinstance(x, GeneratorType): - return bytes(x) else: - return bytes(x) + return bytes(x) # E.g. GeneratorType put into bytes. @classmethod def value_check(cls, value): @@ -374,7 +379,7 @@ class BytesLike(Elements, metaclass=BytesType): return f"{cls.__name__}[{cls.length}]: {self.hex()}" -class Bytes(BytesLike): +class Bytes(BaseBytes): @classmethod def default(cls): @@ -385,7 +390,14 @@ class Bytes(BytesLike): return False -class BytesN(BytesLike): +class BytesN(BaseBytes): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) @classmethod def default(cls): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index daa923aa7..895a074a9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -211,3 +211,5 @@ def test_bytesn_subclass(): assert issubclass(Hash, Bytes32) assert not issubclass(Bytes48, Bytes32) + + assert len(Bytes32() + Bytes48()) == 80 From 82240d8dbd0f083efb66c8c5b2388eb153d69977 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 18:01:57 +0200 Subject: [PATCH 33/68] fix vector default type --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 341df880a..6595c966f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -333,7 +333,7 @@ class Vector(BaseList): @classmethod def default(cls): - return [cls.elem_type.default() for _ in range(cls.length)] + return cls(cls.elem_type.default() for _ in range(cls.length)) @classmethod def is_fixed_size(cls): From f157745248492114da19b14c45e8831493368853 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:38:42 +0200 Subject: [PATCH 34/68] resolve some remaining list-rework rebase details --- specs/core/0_beacon-chain.md | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9a02c16e4..9c871f5f2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -535,8 +535,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations - previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] - current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -1282,7 +1282,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterab ```python def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: - return [ + return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) ] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6595c966f..bd7fe0950 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Iterator +from typing import Dict, Iterator from types import GeneratorType @@ -40,7 +40,7 @@ class Bit(BasicValue): # can't subclass bool. @classmethod def default(cls): - return cls(False) + return cls(0) def __bool__(self): return self > 0 @@ -103,7 +103,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): elif isinstance(v, GeneratorType): return typ(v) else: - # just return as-is, Value-checkers will take care of it not being coerced. + # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. if strict and not isinstance(v, typ): raise ValueError("Type coercion of {} to {} failed".format(v, typ)) return v @@ -181,10 +181,10 @@ class Container(Series, metaclass=SSZType): @classmethod def is_fixed_size(cls): - return all(t.is_fixed_size() for t in cls.get_field_types()) + return all(t.is_fixed_size() for t in cls.get_fields().values()) def __iter__(self) -> Iterator[SSZValue]: - return iter([getattr(self, field) for field in self.get_fields()]) + return iter([getattr(self, field) for field in self.get_fields().keys()]) class ParamsBase(Series): From 224c98a094a5d537fe05377955d8f30868cb6ba7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:55:17 +0200 Subject: [PATCH 35/68] last() method, no negative index lookups --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../block_processing/test_process_transfer.py | 14 +++++++------- .../test_process_voluntary_exit.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++++ 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index b49a6be1f..681add457 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -4,7 +4,7 @@ from eth2spec.utils.bls import bls_sign def get_valid_early_derived_secret_reveal(spec, state, epoch=None): current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] + revealed_index = spec.get_active_validator_indices(state, current_epoch).last() masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d5b7f7b7f..7b6f71374 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.keys import pubkey_to_privkey def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index acc6a35c5..215cd6fcb 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -9,7 +9,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] + sender_index = spec.get_active_validator_indices(state, current_epoch).last() recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index e9d282b3a..2bed94dc8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -63,7 +63,7 @@ def test_success_withdrawable(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -73,7 +73,7 @@ def test_success_active_above_max_effective(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -94,7 +94,7 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) @@ -115,7 +115,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -128,7 +128,7 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -141,7 +141,7 @@ def test_insufficient_balance(spec, state): @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() balance = state.balances[sender_index] transfer = get_valid_transfer( spec, @@ -161,7 +161,7 @@ def test_no_dust_sender(spec, state): @with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 33cacc4e2..b9f7d6098 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -93,7 +93,7 @@ def test_success_exit_queue(spec, state): continue # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index f8590005e..c86394a0b 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -269,7 +269,7 @@ def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( state, spec.get_current_epoch(state) - )[-1] + ).last() # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -315,7 +315,7 @@ def test_voluntary_exit(spec, state): # overwrite default 0 to test # spec.MAX_TRANSFERS = 1 - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() # amount = get_balance(state, sender_index) # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) @@ -347,7 +347,7 @@ def test_voluntary_exit(spec, state): @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index bd7fe0950..e5f9f66a8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -312,6 +312,10 @@ class BaseList(list, Elements): def __iter__(self) -> Iterator[SSZValue]: return super().__iter__() + def last(self): + # be explict about getting the last item, for the non-python readers, and negative-index safety + return self[len(self)-1] + class List(BaseList): From 8c6d2b42d885f14758f57958f922b66128b2cf8f Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:07:23 +0200 Subject: [PATCH 36/68] update ssz-pyssz decoder for fuzzing --- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 73 +++++++++----------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index a5d3dfd97..a4ebb7e70 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -8,32 +8,31 @@ def translate_typ(typ) -> ssz.BaseSedes: :param typ: The spec type, a class. :return: The Py-SSZ equivalent. """ - if spec_ssz.is_container_type(typ): + if issubclass(typ, spec_ssz.Container): return ssz.Container( - [translate_typ(field_typ) for (field_name, field_typ) in typ.get_fields()]) - elif spec_ssz.is_bytesn_type(typ): + [translate_typ(field_typ) for field_name, field_typ in typ.get_fields().items()]) + elif issubclass(typ, spec_ssz.BytesN): return ssz.ByteVector(typ.length) - elif spec_ssz.is_bytes_type(typ): + elif issubclass(typ, spec_ssz.Bytes): return ssz.ByteList() - elif spec_ssz.is_vector_type(typ): - return ssz.Vector(translate_typ(spec_ssz.read_vector_elem_type(typ)), typ.length) - elif spec_ssz.is_list_type(typ): - return ssz.List(translate_typ(spec_ssz.read_list_elem_type(typ))) - elif spec_ssz.is_bool_type(typ): + elif issubclass(typ, spec_ssz.Vector): + return ssz.Vector(translate_typ(typ.elem_type), typ.length) + elif issubclass(typ, spec_ssz.List): + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bit): return ssz.boolean - elif spec_ssz.is_uint_type(typ): - size = spec_ssz.uint_byte_size(typ) - if size == 1: + elif issubclass(typ, spec_ssz.uint): + if typ.byte_len == 1: return ssz.uint8 - elif size == 2: + elif typ.byte_len == 2: return ssz.uint16 - elif size == 4: + elif typ.byte_len == 4: return ssz.uint32 - elif size == 8: + elif typ.byte_len == 8: return ssz.uint64 - elif size == 16: + elif typ.byte_len == 16: return ssz.uint128 - elif size == 32: + elif typ.byte_len == 32: return ssz.uint256 else: raise TypeError("invalid uint size") @@ -48,37 +47,33 @@ def translate_value(value, typ): :param typ: The type from the spec to translate into :return: the translated value """ - if spec_ssz.is_uint_type(typ): - size = spec_ssz.uint_byte_size(typ) - if size == 1: + if issubclass(typ, spec_ssz.uint): + if typ.byte_len == 1: return spec_ssz.uint8(value) - elif size == 2: + elif typ.byte_len == 2: return spec_ssz.uint16(value) - elif size == 4: + elif typ.byte_len == 4: return spec_ssz.uint32(value) - elif size == 8: - # uint64 is default (TODO this is changing soon) - return value - elif size == 16: + elif typ.byte_len == 8: + return spec_ssz.uint64(value) + elif typ.byte_len == 16: return spec_ssz.uint128(value) - elif size == 32: + elif typ.byte_len == 32: return spec_ssz.uint256(value) else: raise TypeError("invalid uint size") - elif spec_ssz.is_list_type(typ): - elem_typ = spec_ssz.read_elem_type(typ) - return [translate_value(elem, elem_typ) for elem in value] - elif spec_ssz.is_bool_type(typ): + elif issubclass(typ, spec_ssz.List): + return [translate_value(elem, typ.elem_type) for elem in value] + elif issubclass(typ, spec_ssz.Bit): return value - elif spec_ssz.is_vector_type(typ): - elem_typ = spec_ssz.read_elem_type(typ) - return typ(*(translate_value(elem, elem_typ) for elem in value)) - elif spec_ssz.is_bytesn_type(typ): + elif issubclass(typ, spec_ssz.Vector): + return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.BytesN): return typ(value) - elif spec_ssz.is_bytes_type(typ): + elif issubclass(typ, spec_ssz.Bytes): return value - elif spec_ssz.is_container_type(typ): - return typ(**{f_name: translate_value(f_val, f_typ) for (f_name, f_val, f_typ) - in zip(typ.get_field_names(), value, typ.get_field_types())}) + if issubclass(typ, spec_ssz.Container): + return typ(**{f_name: translate_value(f_val, f_typ) for (f_val, (f_name, f_typ)) + in zip(value, typ.get_fields().items())}) else: raise TypeError("Type not supported: {}".format(typ)) From 8bd204827bd5e2d94091ede1e9ff973b011a7678 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:08:34 +0200 Subject: [PATCH 37/68] improve type coercion; coerce between equal-length uint subclasses --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index e5f9f66a8..55acde44b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -94,19 +94,24 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): # shortcut if it's already the type we are looking for if v_typ == typ: return v - elif isinstance(v, int) and not isinstance(v, uint): # do not coerce from one uintX to another uintY - return typ(v) + elif isinstance(v, int): + if isinstance(v, uint): # do not coerce from one uintX to another uintY + if issubclass(typ, uint) and v.type().byte_len == typ.byte_len: + return typ(v) + # revert to default behavior below if-else. (ValueError/bare) + else: + return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) elif isinstance(v, (bytes, BytesN, Bytes)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) - else: - # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. - if strict and not isinstance(v, typ): - raise ValueError("Type coercion of {} to {} failed".format(v, typ)) - return v + + # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. + if strict and not isinstance(v, typ): + raise ValueError("Type coercion of {} to {} failed".format(v, typ)) + return v class Series(SSZValue): From b4ef672f87cb3d3afbd0a7244feabbb3b7158258 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:12:46 +0200 Subject: [PATCH 38/68] deal with deepcopy modifying vector length from 0 to full length during copy --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 55acde44b..39fed9ac8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -349,7 +349,12 @@ class Vector(BaseList): return cls.elem_type.is_fixed_size() def append(self, v): - raise Exception("cannot modify vector length") + # Deep-copy and other utils like to change the internals during work. + # Only complain if we had the right size. + if len(self) == self.__class__.length: + raise Exception("cannot modify vector length") + else: + super().append(v) def pop(self, *args): raise Exception("cannot modify vector length") From 2d6771707925558b54fc5fe6bbc8154198dca2fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:42:55 +0200 Subject: [PATCH 39/68] fix linting issues + make spec builder remove comments in container re-initialization part --- scripts/build_spec.py | 15 ++++++++++++++- specs/core/0_beacon-chain.md | 5 +++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d33ba6642..338093e94 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -106,6 +106,18 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: ''' +def strip_comments(raw: str) -> str: + comment_line_regex = re.compile('^\s+# ') + lines = raw.split('\n') + out = [] + for line in lines: + if not comment_line_regex.match(line): + if ' #' in line: + line = line[:line.index(' #')] + out.append(line) + return '\n'.join(out) + + def objects_to_spec(functions: Dict[str, str], custom_types: Dict[str, str], constants: Dict[str, str], @@ -133,7 +145,8 @@ def objects_to_spec(functions: Dict[str, str], ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) ssz_objects_reinitialization_spec = ( 'def init_SSZ_types() -> None:\n global_vars = globals()\n\n ' - + '\n\n '.join([re.sub(r'(?!\n\n)\n', r'\n ', value[:-1]) for value in ssz_objects.values()]) + + '\n\n '.join([strip_comments(re.sub(r'(?!\n\n)\n', r'\n ', value[:-1])) + for value in ssz_objects.values()]) + '\n\n' + '\n'.join(map(lambda x: ' global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys())) ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9c871f5f2..52d087281 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -533,7 +533,7 @@ class BeaconState(Container): randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Digests of the active registry, for light clients # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] @@ -1525,7 +1525,8 @@ def process_slashings(state: BeaconState) -> None: total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if validator.slashed and current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2: + if validator.slashed and current_epoch == ( + validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2): penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 39fed9ac8..c9516bebf 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -319,7 +319,7 @@ class BaseList(list, Elements): def last(self): # be explict about getting the last item, for the non-python readers, and negative-index safety - return self[len(self)-1] + return self[len(self) - 1] class List(BaseList): From 4dcfee2d2cd8ba978787eda8bf336f2756dc67a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:45:17 +0200 Subject: [PATCH 40/68] remove unused spec-helper from spec builder --- scripts/build_spec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 338093e94..45cb0368f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -61,10 +61,6 @@ from eth2spec.utils.hash_function import hash Deltas = list ''' SUNDRY_FUNCTIONS = ''' -def get_ssz_type_by_name(name: str) -> Container: - return globals()[name] - - # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Hash] = {} From d8f470bb4a81ae5662f28d83daecc27769972ef3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:34:19 +0200 Subject: [PATCH 41/68] enable slicing of SSZ lists/vectors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index c9516bebf..22f76bada 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -295,13 +295,14 @@ class BaseList(list, Elements): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" - def __getitem__(self, i) -> SSZValue: - if i < 0: - raise IndexError(f"cannot get item in type {self.__class__} at negative index {i}") - if i > len(self): - raise IndexError(f"cannot get item in type {self.__class__}" - f" at out of bounds index {i}") - return super().__getitem__(i) + def __getitem__(self, k) -> SSZValue: + if isinstance(k, int): # check if we are just doing a lookup, and not slicing + if k < 0: + raise IndexError(f"cannot get item in type {self.__class__} at negative index {k}") + if k > len(self): + raise IndexError(f"cannot get item in type {self.__class__}" + f" at out of bounds index {k}") + return super().__getitem__(k) def __setitem__(self, k, v): if k < 0: From 6338c5b88051dcefe515e424196bca708b44dd06 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:49:03 +0200 Subject: [PATCH 42/68] fix custody bug, needs review from Carl --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 681add457..bc70c9fa8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -27,7 +27,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): domain_type=spec.DOMAIN_RANDAO, message_epoch=epoch, ), - ) + )[:32] # TODO(Carl): mask is 32 bytes, and signature is 96? Correct to slice the first 32 out? return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, From f27c44b953789b49e533fe36f8be314073c9f844 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:49:34 +0200 Subject: [PATCH 43/68] fix deposit negative index fail --- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 63b94c638..a40c120fb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -183,7 +183,7 @@ def test_bad_merkle_proof(spec, state): deposit = prepare_state_and_deposit(spec, state, validator_index, amount) # mess up merkle branch - deposit.proof[-1] = spec.ZERO_HASH + deposit.proof[5] = spec.ZERO_HASH sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) From c20372409c543b52bf15e71bc1df02b7ccb37434 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:52:16 +0200 Subject: [PATCH 44/68] comment out old deposit test, re-introduced soon maybe, cc Justin --- .../block_processing/test_process_deposit.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index a40c120fb..2f60c6be2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -113,20 +113,21 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): # inconsistent withdrawal credentials, in top-ups, are allowed! yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) - -@with_all_phases -@spec_state_test -def test_wrong_index(spec, state): - validator_index = len(state.validators) - amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(spec, state, validator_index, amount) - - # mess up eth1_deposit_index - deposit.index = state.eth1_deposit_index + 1 - - sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) - - yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) +# Deposit data temporarily has no index field. +# SSZ typing catches that, and raises an exception. Test may be introduced back later +# @with_all_phases +# @spec_state_test +# def test_wrong_index(spec, state): +# validator_index = len(state.validators) +# amount = spec.MAX_EFFECTIVE_BALANCE +# deposit = prepare_state_and_deposit(spec, state, validator_index, amount) +# +# # mess up eth1_deposit_index +# deposit.index = state.eth1_deposit_index + 1 +# +# sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) +# +# yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) @with_all_phases From 3d8466fd6e07dea808fe13b2719917867ffa28b5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 23:02:22 +0200 Subject: [PATCH 45/68] make Bit check not use "is", and remove duplicate line --- specs/core/0_beacon-chain.md | 2 +- specs/core/1_custody-game.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 52d087281..f171bd4df 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -645,7 +645,7 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: """ Check if ``validator`` is slashable. """ - return validator.slashed is False and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) ``` ### `get_active_validator_indices` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 99c85a034..24e9a19e2 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -402,12 +402,11 @@ def process_early_derived_secret_reveal(state: BeaconState, """ revealed_validator = state.validators[reveal.revealed_index] - masker = state.validators[reveal.masker_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS assert reveal.epoch >= get_current_epoch(state) + RANDAO_PENALTY_EPOCHS assert reveal.epoch < get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - assert revealed_validator.slashed is False + assert not revealed_validator.slashed assert reveal.revealed_index not in state.exposed_derived_secrets[derived_secret_location] # Verify signature correctness From 6648b3c49e0cd5535c5a34519d33798cc461712d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 00:23:28 +0200 Subject: [PATCH 46/68] remove old deposits test, there is no deposit index in deposit data anymore --- .../block_processing/test_process_deposit.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 2f60c6be2..8b3d7b413 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -113,22 +113,6 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): # inconsistent withdrawal credentials, in top-ups, are allowed! yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -# Deposit data temporarily has no index field. -# SSZ typing catches that, and raises an exception. Test may be introduced back later -# @with_all_phases -# @spec_state_test -# def test_wrong_index(spec, state): -# validator_index = len(state.validators) -# amount = spec.MAX_EFFECTIVE_BALANCE -# deposit = prepare_state_and_deposit(spec, state, validator_index, amount) -# -# # mess up eth1_deposit_index -# deposit.index = state.eth1_deposit_index + 1 -# -# sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) -# -# yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) - @with_all_phases @spec_state_test @@ -173,9 +157,6 @@ def test_wrong_deposit_for_deposit_count(spec, state): yield from run_deposit_processing(spec, state, deposit_2, index_2, valid=False) -# TODO: test invalid signature - - @with_all_phases @spec_state_test def test_bad_merkle_proof(spec, state): From e99c864ed18ef172adc5795f1777035ef517fc82 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 20 Jun 2019 17:17:12 -0600 Subject: [PATCH 47/68] Deltas = NewType('Deltas', TypingList[Gwei]) --- scripts/build_spec.py | 12 ++++-------- specs/core/0_beacon-chain.md | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 45cb0368f..bd7686ff5 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,8 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Tuple + Any, Callable, Iterable, Dict, Set, Tuple, NewType, + List as TypingList, ) from eth2spec.utils.ssz.ssz_impl import ( @@ -30,12 +31,9 @@ from eth2spec.utils.bls import ( ) from eth2spec.utils.hash_function import hash - - -Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable, + Any, Callable, Dict, Optional, Set, Tuple, Iterable, NewType, List as TypingList ) @@ -56,9 +54,6 @@ from eth2spec.utils.bls import ( ) from eth2spec.utils.hash_function import hash - - -Deltas = list ''' SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache @@ -149,6 +144,7 @@ def objects_to_spec(functions: Dict[str, str], spec = ( imports + '\n\n' + new_type_definitions + + '\n\n' + "Deltas = NewType('Deltas', TypingList[Gwei])" + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f171bd4df..e89f0baed 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1405,8 +1405,8 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = Deltas(0 for _ in range(len(state.validators))) - penalties = Deltas(0 for _ in range(len(state.validators))) + rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) + penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1454,8 +1454,8 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: ```python def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: - rewards = Deltas(0 for _ in range(len(state.validators))) - penalties = Deltas(0 for _ in range(len(state.validators))) + rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) + penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) From b7b2fee6350489ec4cd1c2e118b8bfba21db8055 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 21:12:27 +0200 Subject: [PATCH 48/68] uint add/sub type checking, fixes #1029 --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 ++++- .../eth2spec/utils/ssz/test_ssz_typing.py | 32 +++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 22f76bada..971b2106b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -52,9 +52,15 @@ class uint(BasicValue, metaclass=BasicType): if value < 0: raise ValueError("unsigned types must not be negative") if cls.byte_len and value.bit_length() > (cls.byte_len << 3): - raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) + raise ValueError("value out of bounds for uint{}".format(cls.byte_len * 8)) return super().__new__(cls, value) + def __add__(self, other): + return self.__class__(super().__add__(coerce_type_maybe(other, self.__class__, strict=True))) + + def __sub__(self, other): + return self.__class__(super().__sub__(coerce_type_maybe(other, self.__class__, strict=True))) + @classmethod def default(cls): return cls(0) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 895a074a9..4325501aa 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -6,6 +6,14 @@ from .ssz_typing import ( ) +def expect_value_error(fn, msg): + try: + fn() + raise AssertionError(msg) + except ValueError: + pass + + def test_subclasses(): for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: assert issubclass(u, uint) @@ -55,21 +63,13 @@ def test_basic_value_bounds(): # this should work assert k(v - 1) == v - 1 # but we do not allow overflows - try: - k(v) - assert False - except ValueError: - pass + expect_value_error(lambda: k(v), "no overflows allowed") for k, _ in max.items(): # this should work assert k(0) == 0 # but we do not allow underflows - try: - k(-1) - assert False - except ValueError: - pass + expect_value_error(lambda: k(-1), "no underflows allowed") def test_container(): @@ -213,3 +213,15 @@ def test_bytesn_subclass(): assert not issubclass(Bytes48, Bytes32) assert len(Bytes32() + Bytes48()) == 80 + + +def test_uint_math(): + assert uint8(0) + uint8(uint32(16)) == uint8(16) # allow explict casting to make invalid addition valid + + expect_value_error(lambda: uint8(0) - uint8(1), "no underflows allowed") + expect_value_error(lambda: uint8(1) + uint8(255), "no overflows allowed") + expect_value_error(lambda: uint8(0) + 256, "no overflows allowed") + expect_value_error(lambda: uint8(42) + uint32(123), "no mixed types") + expect_value_error(lambda: uint32(42) + uint8(123), "no mixed types") + + assert type(uint32(1234) + 56) == uint32 From d1fa3acb27802b56627ee71e0e3d292553838146 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 21:27:26 +0200 Subject: [PATCH 49/68] remove unused dependency --- test_libs/pyspec/requirements.txt | 1 - test_libs/pyspec/setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index b96b0a80f..83197af9c 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,6 +2,5 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 -typing_inspect==0.4.0 dataclasses==0.6 ssz==0.1.0a10 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index fe14c498c..d8d54eab7 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -9,7 +9,6 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", - "typing_inspect==0.4.0", "ssz==0.1.0a10", "dataclasses==0.6", ] From c09e45c5fc49e81c5be5c6576642949b18ec0d73 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:43:55 -0600 Subject: [PATCH 50/68] fix rule_4 underflow and split out genesis finality test --- .../pyspec/eth2spec/test/test_finality.py | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5e81f52c8..160a32a4b 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -39,11 +39,13 @@ def next_epoch_with_attestations(spec, state, fill_cur_epoch, fill_prev_epoch): + assert state.slot % spec.SLOTS_PER_EPOCH == 0 + post_state = deepcopy(state) blocks = [] for _ in range(spec.SLOTS_PER_EPOCH): block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch: + if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) @@ -63,11 +65,13 @@ def next_epoch_with_attestations(spec, @with_all_phases @never_bls @spec_state_test -def test_finality_rule_4(spec, state): +def test_finality_no_updates_at_genesis(spec, state): + assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH + yield 'pre', state blocks = [] - for epoch in range(4): + for epoch in range(2): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) blocks += new_blocks @@ -77,9 +81,37 @@ def test_finality_rule_4(spec, state): # justification/finalization skipped at GENESIS_EPOCH + 1 elif epoch == 1: check_finality(spec, state, prev_state, False, False, False) - elif epoch == 2: + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_4(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield 'pre', state + + blocks = [] + for epoch in range(2): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + + # justification/finalization skipped at GENESIS_EPOCH + # if epoch == 0: + # check_finality(spec, state, prev_state, False, False, False) + # justification/finalization skipped at GENESIS_EPOCH + 1 + # elif epoch == 1: + # check_finality(spec, state, prev_state, False, False, False) + if epoch == 0: check_finality(spec, state, prev_state, True, False, False) - elif epoch >= 3: + elif epoch == 1: # rule 4 of finality check_finality(spec, state, prev_state, True, True, True) assert state.finalized_epoch == prev_state.current_justified_epoch From 8f997413445d55a11a00c44ffe4f81a92ca0fe1e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:47:18 -0600 Subject: [PATCH 51/68] remove commented old code --- test_libs/pyspec/eth2spec/test/test_finality.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 160a32a4b..730456286 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -103,12 +103,6 @@ def test_finality_rule_4(spec, state): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) blocks += new_blocks - # justification/finalization skipped at GENESIS_EPOCH - # if epoch == 0: - # check_finality(spec, state, prev_state, False, False, False) - # justification/finalization skipped at GENESIS_EPOCH + 1 - # elif epoch == 1: - # check_finality(spec, state, prev_state, False, False, False) if epoch == 0: check_finality(spec, state, prev_state, True, False, False) elif epoch == 1: From 00aae07d4690417e70ee50b57a34b2382b6642b3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 18:12:42 +0200 Subject: [PATCH 52/68] type annotation clean up --- scripts/build_spec.py | 12 +++--- specs/core/0_beacon-chain.md | 64 +++++++++++++++---------------- specs/core/1_shard-data-chains.md | 18 ++++----- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 943d37877..0b55dfa44 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,8 +12,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Tuple, NewType, - List as TypingList, + Any, Callable, Iterable, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -38,7 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable, NewType, + Any, Callable, Dict, Iterable, Optional, Set, Sequence, Tuple, List as TypingList ) @@ -79,13 +78,13 @@ def hash(x: bytes) -> Hash: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], Sequence[ValidatorIndex]] = {} -def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore +def compute_committee(indices: Sequence[ValidatorIndex], # type: ignore seed: Hash, index: int, - count: int) -> Tuple[ValidatorIndex, ...]: + count: int) -> Sequence[ValidatorIndex]: param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count) if param_hash not in committee_cache: @@ -154,7 +153,6 @@ def objects_to_spec(functions: Dict[str, str], spec = ( imports + '\n\n' + new_type_definitions - + '\n\n' + "Deltas = NewType('Deltas', TypingList[Gwei])" + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf8e69535..f42b4cc13 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -651,13 +651,11 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ Get active validator indices at ``epoch``. """ - return List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( - i for i, v in enumerate(state.validators) if is_active_validator(v, epoch) - ) + return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` ### `increase_balance` @@ -819,7 +817,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: Tuple[Hash, ...], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, proof: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -863,19 +861,19 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V ### `compute_committee` ```python -def compute_committee(indices: Tuple[ValidatorIndex, ...], - seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return tuple(indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)) + return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` ### `get_crosslink_committee` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: return compute_committee( - indices=tuple(get_active_validator_indices(state, epoch)), + indices=get_active_validator_indices(state, epoch), seed=generate_seed(state, epoch), index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, count=get_epoch_committee_count(state, epoch), @@ -887,13 +885,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> T ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Tuple[ValidatorIndex, ...]: + bitfield: bytes) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return tuple(sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])) + return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` ### `int_to_bytes` @@ -1139,7 +1137,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1156,7 +1154,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Process deposits state = BeaconState() for deposit in deposits: @@ -1178,7 +1176,7 @@ def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, eth1_data=eth1_data, @@ -1195,8 +1193,10 @@ def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) + # Populate active_index_roots + genesis_active_index_root = hash_tree_root( + List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE](get_active_validator_indices(state, GENESIS_EPOCH)) + ) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root @@ -1271,17 +1271,17 @@ def process_epoch(state: BeaconState) -> None: ```python def get_total_active_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) ``` ```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) @@ -1289,7 +1289,7 @@ def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterab ``` ```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) @@ -1298,22 +1298,22 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable ```python def get_unslashed_attesting_indices(state: BeaconState, - attestations: Iterable[PendingAttestation]) -> Iterable[ValidatorIndex]: + attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) - return sorted(filter(lambda index: not state.validators[index].slashed, list(output))) + return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` ```python -def get_attesting_balance(state: BeaconState, attestations: Iterable[PendingAttestation]) -> Gwei: +def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, - shard: Shard) -> Tuple[Crosslink, Iterable[ValidatorIndex]]: + shard: Shard) -> Tuple[Crosslink, Set[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1402,11 +1402,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) - penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1453,9 +1453,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: ``` ```python -def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: - rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) - penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) +def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) @@ -1642,7 +1642,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ) # type: Tuple[Tuple[List, Callable], ...] + ) # type: Sequence[Tuple[List, Callable]] for operations, function in all_operations: for operation in operations: function(state, operation) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9c60062f4..9f0de2e6b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -133,7 +133,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, - count: int) -> Tuple[ValidatorIndex, ...]: + count: int) -> Sequence[ValidatorIndex]: """ Return committee for a period. Used to construct persistent committees. """ @@ -159,7 +159,7 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> Tuple[ValidatorIndex, ...]: + slot: Slot) -> Sequence[ValidatorIndex]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ @@ -193,7 +193,7 @@ def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: Slot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee - persistent_committee = get_persistent_committee(state, shard, slot) + persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] @@ -248,7 +248,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: Iterable[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) @@ -287,9 +287,9 @@ Let: * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], +def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], beacon_state: BeaconState, - valid_shard_blocks: Iterable[ShardBlock], + valid_shard_blocks: Sequence[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -336,7 +336,7 @@ def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], assert proposer_index is not None assert bls_verify( pubkey=beacon_state.validators[proposer_index].pubkey, - message_hash=signing_root(block), + message_hash=signing_root(candidate), signature=candidate.signature, domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, slot_to_epoch(candidate.slot)), ) @@ -353,7 +353,7 @@ Let: * `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python -def is_valid_shard_attestation(valid_shard_blocks: Iterable[ShardBlock], +def is_valid_shard_attestation(valid_shard_blocks: Sequence[ShardBlock], beacon_state: BeaconState, candidate: ShardAttestation) -> bool: # Check shard block @@ -383,7 +383,7 @@ Let: ```python def is_valid_beacon_attestation(shard: Shard, - shard_blocks: TypingList[ShardBlock], + shard_blocks: Sequence[ShardBlock], beacon_state: BeaconState, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: From f95e7315b42afe8d879c270a6e354e92ad5840c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 18:34:33 +0200 Subject: [PATCH 53/68] fix get_active_validator_indices typing usage --- specs/core/0_beacon-chain.md | 4 +++- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 3 ++- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../block_processing/test_process_transfer.py | 14 +++++++------- .../test_process_voluntary_exit.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f42b4cc13..1f500bc05 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1553,7 +1553,9 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) + List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) + ) ) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index bc70c9fa8..cd34ee4c8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -4,7 +4,7 @@ from eth2spec.utils.bls import bls_sign def get_valid_early_derived_secret_reveal(spec, state, epoch=None): current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch).last() + revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d1d818908..73738932a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -40,7 +40,8 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(spec.get_active_validator_indices(state, spec.GENESIS_EPOCH)) + genesis_active_index_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index 7b6f71374..d5b7f7b7f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.keys import pubkey_to_privkey def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 215cd6fcb..acc6a35c5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -9,7 +9,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch).last() + sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 2bed94dc8..e9d282b3a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -63,7 +63,7 @@ def test_success_withdrawable(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -73,7 +73,7 @@ def test_success_active_above_max_effective(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -94,7 +94,7 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) @@ -115,7 +115,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -128,7 +128,7 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -141,7 +141,7 @@ def test_insufficient_balance(spec, state): @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] balance = state.balances[sender_index] transfer = get_valid_transfer( spec, @@ -161,7 +161,7 @@ def test_no_dust_sender(spec, state): @with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index b9f7d6098..33cacc4e2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -93,7 +93,7 @@ def test_success_exit_queue(spec, state): continue # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index c86394a0b..f8590005e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -269,7 +269,7 @@ def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( state, spec.get_current_epoch(state) - ).last() + )[-1] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -315,7 +315,7 @@ def test_voluntary_exit(spec, state): # overwrite default 0 to test # spec.MAX_TRANSFERS = 1 - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # amount = get_balance(state, sender_index) # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) @@ -347,7 +347,7 @@ def test_voluntary_exit(spec, state): @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH From dd5ad2e2c56b7bc47da3c47c8fa5ed87d1eaa650 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:48:06 +0200 Subject: [PATCH 54/68] remove unnecessary (and now outdated) type hints, update List encoding for generators --- test_libs/pyspec/eth2spec/debug/encode.py | 4 +-- .../pyspec/eth2spec/test/helpers/genesis.py | 1 + .../eth2spec/test/sanity/test_blocks.py | 28 +++++++++---------- .../pyspec/eth2spec/test/test_finality.py | 11 ++++---- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index d264bd7ff..e45bd1117 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -13,9 +13,9 @@ def encode(value: SSZValue, include_hash_tree_roots=False): elif isinstance(value, Bit): assert value in (True, False) return value - elif isinstance(value, (List, Vector)): + elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] - elif isinstance(value, (Bytes, BytesN)): # both bytes and BytesN + elif isinstance(value, bytes): # both bytes and BytesN return '0x' + value.hex() elif isinstance(value, Container): ret = {} diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 73738932a..680825165 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.keys import pubkeys from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import List def build_mock_validator(spec, i: int, balance: int): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index f8590005e..36ded7bc6 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -28,7 +28,7 @@ def test_empty_block_transition(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.eth1_data_votes) == pre_eth1_votes + 1 @@ -48,7 +48,7 @@ def test_skipped_slots(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -69,7 +69,7 @@ def test_empty_epoch_transition(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -90,7 +90,7 @@ def test_empty_epoch_transition(spec, state): # state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'blocks', [block] # yield 'post', state # assert state.slot == block.slot @@ -120,7 +120,7 @@ def test_proposer_slashing(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state # check if slashed @@ -155,7 +155,7 @@ def test_attester_slashing(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state slashed_validator = state.validators[validator_index] @@ -193,7 +193,7 @@ def test_deposit_in_block(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.validators) == initial_registry_len + 1 @@ -221,7 +221,7 @@ def test_deposit_top_up(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.validators) == initial_registry_len @@ -256,7 +256,7 @@ def test_attestation(spec, state): sign_block(spec, state, epoch_block) state_transition_and_sign_block(spec, state, epoch_block) - yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] + yield 'blocks', [attestation_block, epoch_block] yield 'post', state assert len(state.current_epoch_attestations) == 0 @@ -303,7 +303,7 @@ def test_voluntary_exit(spec, state): sign_block(spec, state, exit_block) state_transition_and_sign_block(spec, state, exit_block) - yield 'blocks', [initiate_exit_block, exit_block], List[spec.BeaconBlock] + yield 'blocks', [initiate_exit_block, exit_block] yield 'post', state assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -334,7 +334,7 @@ def test_voluntary_exit(spec, state): # state_transition_and_sign_block(spec, state, block) - # yield 'blocks', [block], List[spec.BeaconBlock] + # yield 'blocks', [block] # yield 'post', state # sender_balance = get_balance(state, sender_index) @@ -362,7 +362,7 @@ def test_balance_driven_status_transitions(spec, state): sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -379,7 +379,7 @@ def test_historical_batch(spec, state): block = build_empty_block_for_next_slot(spec, state, signed=True) state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -408,7 +408,7 @@ def test_historical_batch(spec, state): # state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'blocks', [block] # yield 'post', state # assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 730456286..cf9e94a71 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,5 +1,4 @@ from copy import deepcopy -from typing import List from eth2spec.test.context import spec_state_test, never_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block @@ -82,7 +81,7 @@ def test_finality_no_updates_at_genesis(spec, state): elif epoch == 1: check_finality(spec, state, prev_state, False, False, False) - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -111,7 +110,7 @@ def test_finality_rule_4(spec, state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -142,7 +141,7 @@ def test_finality_rule_1(spec, state): assert state.finalized_epoch == prev_state.previous_justified_epoch assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -175,7 +174,7 @@ def test_finality_rule_2(spec, state): blocks += new_blocks - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -225,5 +224,5 @@ def test_finality_rule_3(spec, state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state From e873bbd73bc3d05526b3c3f42ea3e1bd88049eb9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:59:15 +0200 Subject: [PATCH 55/68] support list casting --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 14415ac6b..891480a5c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -296,7 +296,7 @@ class BaseList(list, Elements): @classmethod def extract_args(cls, *args): x = list(args) - if len(x) == 1 and isinstance(x[0], GeneratorType): + if len(x) == 1 and isinstance(x[0], (GeneratorType, list, tuple)): x = list(x[0]) x = [coerce_type_maybe(v, cls.elem_type) for v in x] return x diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 4325501aa..bd86a5806 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -161,6 +161,8 @@ def test_list(): assert issubclass(v.type(), Elements) assert isinstance(v.type(), ElementsType) + assert len(typ([i for i in range(10)])) == 10 # cast py list to SSZ list + foo = List[uint32, 128](0 for i in range(128)) foo[0] = 123 foo[1] = 654 From 47034a6c8c40fbe5beb3aba36f264f92863682d2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:59:44 +0200 Subject: [PATCH 56/68] fix imports in helper test file --- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 680825165..d3d67e09e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_SIZE]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root From 0249cf651e2c30c6c343dd6c2b2440f17b59a2cd Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 20:04:17 +0200 Subject: [PATCH 57/68] fix lint, and update encoder to handle the few imported types well --- test_libs/pyspec/eth2spec/debug/encode.py | 5 ++--- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index e45bd1117..9a819a256 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bytes, BytesN, List, Vector, Bit + SSZValue, uint, Container, Bit ) @@ -11,8 +11,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): return str(value) return value elif isinstance(value, Bit): - assert value in (True, False) - return value + return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] elif isinstance(value, bytes): # both bytes and BytesN diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 36ded7bc6..286c0150c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,5 +1,4 @@ from copy import deepcopy -from typing import List from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.bls import bls_sign From 9befe09f82605c6cf103c9c4fbd120ad50977831 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 21:27:56 +0200 Subject: [PATCH 58/68] test merkleize chunks --- .../eth2spec/utils/test_merkle_minimal.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py new file mode 100644 index 000000000..1c72a649b --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -0,0 +1,58 @@ +import pytest +from .merkle_minimal import zerohashes, merkleize_chunks +from .hash_function import hash + + +def h(a: bytes, b: bytes) -> bytes: + return hash(a + b) + + +def e(v: int) -> bytes: + return v.to_bytes(length=32, byteorder='little') + + +def z(i: int) -> bytes: + return zerohashes[i] + + +cases = [ + (0, 0, 1, z(0)), + (0, 1, 1, e(0)), + (1, 0, 2, h(z(0), z(0))), + (1, 1, 2, h(e(0), z(0))), + (1, 2, 2, h(e(0), e(1))), + (2, 0, 4, h(h(z(0), z(0)), z(1))), + (2, 1, 4, h(h(e(0), z(0)), z(1))), + (2, 2, 4, h(h(e(0), e(1)), z(1))), + (2, 3, 4, h(h(e(0), e(1)), h(e(2), z(0)))), + (2, 4, 4, h(h(e(0), e(1)), h(e(2), e(3)))), + (3, 0, 8, h(h(h(z(0), z(0)), z(1)), z(2))), + (3, 1, 8, h(h(h(e(0), z(0)), z(1)), z(2))), + (3, 2, 8, h(h(h(e(0), e(1)), z(1)), z(2))), + (3, 3, 8, h(h(h(e(0), e(1)), h(e(2), z(0))), z(2))), + (3, 4, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), z(2))), + (3, 5, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1)))), + (3, 6, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0))))), + (3, 7, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0))))), + (3, 8, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), + (4, 0, 16, h(h(h(h(z(0), z(0)), z(1)), z(2)), z(3))), + (4, 1, 16, h(h(h(h(e(0), z(0)), z(1)), z(2)), z(3))), + (4, 2, 16, h(h(h(h(e(0), e(1)), z(1)), z(2)), z(3))), + (4, 3, 16, h(h(h(h(e(0), e(1)), h(e(2), z(0))), z(2)), z(3))), + (4, 4, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), z(2)), z(3))), + (4, 5, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1))), z(3))), + (4, 6, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0)))), z(3))), + (4, 7, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0)))), z(3))), + (4, 8, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), z(3))), + (4, 9, 16, + h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), h(h(h(e(8), z(0)), z(1)), z(2)))), +] + + +@pytest.mark.parametrize( + 'depth,count,pow2,value', + cases, +) +def test_merkleize_chunks(depth, count, pow2, value): + chunks = [e(i) for i in range(count)] + assert merkleize_chunks(chunks, pad_to=pow2) == value From da858f1aae3ff5f906e041f025094d0a90ccc0c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 21:49:42 +0200 Subject: [PATCH 59/68] fix int encoding, fix list randomization size limit. --- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/random_value.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 9a819a256..0878a1f94 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -8,8 +8,8 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length if value.type().byte_len > 8: - return str(value) - return value + return str(int(value)) + return int(value) elif isinstance(value, Bit): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index bd40cb832..8e13cd5e1 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -95,6 +95,9 @@ def get_random_ssz_object(rng: Random, elif mode == RandomizationMode.mode_max_count: length = max_list_length + if typ.length < length: # SSZ imposes a hard limit on lists, we can't put in more than that + length = typ.length + return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) From a5a2e29dfe8a424ea53cf0b9203b58463046a24b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 22:15:42 +0200 Subject: [PATCH 60/68] remove unnecessary argument, typing is based on values fully now --- test_generators/ssz_static/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 9d9af8c5e..0dfdebf5d 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -21,8 +21,8 @@ MAX_LIST_LENGTH = 10 @to_dict -def create_test_case_contents(value, typ): - yield "value", encode.encode(value, typ) +def create_test_case_contents(value): + yield "value", encode.encode(value) yield "serialized", '0x' + serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() if hasattr(value, "signature"): @@ -32,7 +32,7 @@ def create_test_case_contents(value, typ): @to_dict def create_test_case(rng: Random, name: str, typ, mode: random_value.RandomizationMode, chaos: bool): value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) - yield name, create_test_case_contents(value, typ) + yield name, create_test_case_contents(value) def get_spec_ssz_types(): From 1408a1ee0daf918a4fdc72facad4e22bf56d9d79 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 23 Jun 2019 00:17:54 +0200 Subject: [PATCH 61/68] undo tuple wrapping --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9f0de2e6b..f84b0dd46 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -180,10 +180,10 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return tuple(sorted(list(set( + return sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - )))) + ))) ``` ### `get_shard_proposer_index` From 82ae1804900e06daf2446c8dfbb1b198d931a5ed Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:38:36 +0200 Subject: [PATCH 62/68] clean up list limit constants --- configs/constant_presets/mainnet.yaml | 4 ++++ configs/constant_presets/minimal.yaml | 4 ++++ specs/core/0_beacon-chain.md | 17 +++++++---------- .../pyspec/eth2spec/test/helpers/genesis.py | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 09ec7bdb7..9f7ca950f 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -77,6 +77,10 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 201ac475f..3e3f7ccb4 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -78,6 +78,10 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 +# 2**24 (= 16,777,216) historical roots +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1f500bc05..7d2459eb3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,7 +224,6 @@ These configurations are updated for releases, but may be out of sync during `de | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | -| `HISTORICAL_ROOTS_LENGTH` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | @@ -238,10 +237,8 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | -| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `VALIDATOR_REGISTRY_SIZE` | `2**40` (= 1,099,511,627,776) | | | +| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | +| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | ### Rewards and penalties @@ -520,14 +517,14 @@ class BeaconState(Container): latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash, HISTORICAL_ROOTS_LENGTH] + historical_roots: List[Hash, HISTORICAL_ROOTS_LIMIT] # Eth1 eth1_data: Eth1Data eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] eth1_deposit_index: uint64 # Registry - validators: List[Validator, VALIDATOR_REGISTRY_SIZE] - balances: List[Gwei, VALIDATOR_REGISTRY_SIZE] + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] @@ -1195,7 +1192,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth # Populate active_index_roots genesis_active_index_root = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE](get_active_validator_indices(state, GENESIS_EPOCH)) + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root @@ -1553,7 +1550,7 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT]( get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d3d67e09e..ce0be19bb 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_SIZE]( + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root From c73417b4cae0335e348f1a4d06968506301aeb64 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:40:47 +0200 Subject: [PATCH 63/68] deserialize-basic detail, make subclass --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 144201d83..e2a264955 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -27,7 +27,7 @@ def deserialize_basic(value, typ: BasicType): return typ(int.from_bytes(value, 'little')) elif issubclass(typ, Bit): assert value in (b'\x00', b'\x01') - return Bit(value == b'\x01') + return typ(value == b'\x01') else: raise Exception(f"Type not supported: {typ}") From 5989e5cd2368a5e7b4913bcfa6ef5025019b0aa7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:56:26 +0200 Subject: [PATCH 64/68] use Bool as base name, make Bit an alias --- scripts/build_spec.py | 6 +++--- specs/core/0_beacon-chain.md | 4 ++-- test_libs/pyspec/eth2spec/debug/decode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- .../pyspec/eth2spec/debug/random_value.py | 8 ++++---- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 ++-- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- .../pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 11 ++++++++--- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 +++++++++++------- 10 files changed, 43 insertions(+), 29 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b55dfa44..08793869f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, uint64, + Bit, Bool, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -53,7 +53,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, uint64, + Bit, Bool, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -179,7 +179,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7d2459eb3..b3e3ba0ea 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -314,7 +314,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bit + slashed: Bool # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -354,7 +354,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit for the custody of crosslink data + custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 743479371..c0b53b0ef 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bit, + SSZType, SSZValue, uint, Container, Bytes, List, Bool, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bit)): + if issubclass(typ, (uint, Bool)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 0878a1f94..02814e441 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bit + SSZValue, uint, Container, Bool ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bit): + elif isinstance(value, Bool): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 8e13cd5e1..c6efb722b 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from random import Random from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bit, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index a4ebb7e70..e533ca5c2 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bit): + elif issubclass(typ, spec_ssz.Bool): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bit): + elif issubclass(typ, spec_ssz.Bool): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index e2a264955..b9c7b6d38 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, ) # SSZ Serialization @@ -13,7 +13,7 @@ BYTES_PER_LENGTH_OFFSET = 4 def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bit): + elif isinstance(value, Bool): if value: return b'\x01' else: @@ -25,7 +25,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bit): + elif issubclass(typ, Bool): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 891480a5c..6079f2866 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bit(BasicValue): # can't subclass bool. +class Bool(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value, *args, **kwargs): @@ -47,6 +47,11 @@ class Bit(BasicValue): # can't subclass bool. return self > 0 +# Alias for Bool +class Bit(Bool): + pass + + class uint(BasicValue, metaclass=BasicType): def __new__(cls, value, *args, **kwargs): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index ae0849098..aa7aee64a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,6 +1,6 @@ from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Container, List, Vector, Bytes, BytesN, + Bit, Bool, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, byte ) @@ -47,11 +47,16 @@ for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): sig_test_data[k] = v test_data = [ - ("bool F", Bit(False), "00"), - ("bool T", Bit(True), "01"), + ("bit F", Bit(False), "00"), + ("bit T", Bit(True), "01"), + ("bool F", Bool(False), "00"), + ("bool T", Bool(True), "01"), ("uint8 00", uint8(0x00), "00"), ("uint8 01", uint8(0x01), "01"), ("uint8 ab", uint8(0xab), "ab"), + ("byte 00", byte(0x00), "00"), + ("byte 01", byte(0x01), "01"), + ("byte ab", byte(0xab), "ab"), ("uint16 0000", uint16(0x0000), "0000"), ("uint16 abcd", uint16(0xabcd), "cdab"), ("uint32 00000000", uint32(0x00000000), "00000000"), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index bd86a5806..2af742360 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,7 +1,7 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256, + Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bit, BasicValue) - assert isinstance(Bit, BasicType) + assert issubclass(Bool, BasicValue) + assert isinstance(Bool, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -38,21 +38,25 @@ def test_subclasses(): def test_basic_instances(): - for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + for u in [uint, uint8, byte, uint16, uint32, uint64, uint128, uint256]: v = u(123) assert isinstance(v, uint) assert isinstance(v, int) assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bit(True), BasicValue) - assert isinstance(Bit(False), BasicValue) + assert isinstance(Bool(True), BasicValue) + assert isinstance(Bool(False), BasicValue) + assert isinstance(Bit(True), Bool) + assert isinstance(Bit(False), Bool) def test_basic_value_bounds(): max = { + Bool: 2 ** 1, Bit: 2 ** 1, uint8: 2 ** (8 * 1), + byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), uint32: 2 ** (8 * 4), uint64: 2 ** (8 * 8), From 9fb58067645da1aa02dcfa995cb61fb86a2dff16 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 00:24:13 +0200 Subject: [PATCH 65/68] be explicit about input for balance sum --- scripts/build_spec.py | 4 ++-- specs/core/0_beacon-chain.md | 6 +++--- specs/core/1_shard-data-chains.md | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 08793869f..237c24384 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Sequence, Tuple, + Any, Callable, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -37,7 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Iterable, Optional, Set, Sequence, Tuple, + Any, Callable, Dict, Optional, Set, Sequence, Tuple, List as TypingList ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d6be89f7..b8cab7eaf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -908,7 +908,7 @@ def bytes_to_int(data: bytes) -> int: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, indices: Iterable[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ @@ -1383,7 +1383,7 @@ def process_crosslinks(state: BeaconState) -> None: for epoch in (get_previous_epoch(state), get_current_epoch(state)): for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - crosslink_committee = get_crosslink_committee(state, epoch, shard) + crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): state.current_crosslinks[shard] = winning_crosslink @@ -1456,7 +1456,7 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[G epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - crosslink_committee = get_crosslink_committee(state, epoch, shard) + crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) attesting_balance = get_total_balance(state, attesting_indices) committee_balance = get_total_balance(state, crosslink_committee) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f84b0dd46..dd48a842a 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -378,7 +378,7 @@ Let: * `shard` be a valid `Shard` * `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot` * `beacon_state` be the canonical `BeaconState` -* `valid_attestations` be the list of valid `Attestation`, recursively defined +* `valid_attestations` be the set of valid `Attestation` objects, recursively defined * `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` ```python @@ -388,7 +388,7 @@ def is_valid_beacon_attestation(shard: Shard, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: # Check if attestation is already determined valid - for _, attestation in enumerate(valid_attestations): + for attestation in valid_attestations: if candidate == attestation: return True From a5b7564c5b6c337a9db8272beac7d9fe61e92f6d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 02:35:55 +0200 Subject: [PATCH 66/68] hash-tree-root tests --- .../eth2spec/utils/ssz/test_ssz_impl.py | 192 ++++++++++++++---- 1 file changed, 155 insertions(+), 37 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index aa7aee64a..82fb4ec68 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,8 +1,10 @@ +from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( Bit, Bool, Container, List, Vector, Bytes, BytesN, - uint8, uint16, uint32, uint64, byte + uint8, uint16, uint32, uint64, uint256, byte ) +from ..hash_function import hash as bytes_hash import pytest @@ -46,37 +48,117 @@ sig_test_data = [0 for i in range(96)] for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): sig_test_data[k] = v + +def chunk(hex: str) -> str: + return (hex + ("00" * 32))[:64] # just pad on the right, to 32 bytes (64 hex chars) + + +def h(a: str, b: str) -> str: + return bytes_hash(bytes.fromhex(a) + bytes.fromhex(b)).hex() + + +# zero hashes, as strings, for +zero_hashes = [chunk("")] +for layer in range(1, 32): + zero_hashes.append(h(zero_hashes[layer - 1], zero_hashes[layer - 1])) + + +def merge(a: str, branch: Iterable[str]) -> str: + """ + Merge (out on left, branch on right) leaf a with branch items, branch is from bottom to top. + """ + out = a + for b in branch: + out = h(out, b) + return out + + test_data = [ - ("bit F", Bit(False), "00"), - ("bit T", Bit(True), "01"), - ("bool F", Bool(False), "00"), - ("bool T", Bool(True), "01"), - ("uint8 00", uint8(0x00), "00"), - ("uint8 01", uint8(0x01), "01"), - ("uint8 ab", uint8(0xab), "ab"), - ("byte 00", byte(0x00), "00"), - ("byte 01", byte(0x01), "01"), - ("byte ab", byte(0xab), "ab"), - ("uint16 0000", uint16(0x0000), "0000"), - ("uint16 abcd", uint16(0xabcd), "cdab"), - ("uint32 00000000", uint32(0x00000000), "00000000"), - ("uint32 01234567", uint32(0x01234567), "67452301"), - ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301"), - ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301"), - ("uint32 01234567", uint32(0x01234567), "67452301"), - ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000"), - ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301"), + ("bit F", Bit(False), "00", chunk("00")), + ("bit T", Bit(True), "01", chunk("01")), + ("bool F", Bool(False), "00", chunk("00")), + ("bool T", Bool(True), "01", chunk("01")), + ("uint8 00", uint8(0x00), "00", chunk("00")), + ("uint8 01", uint8(0x01), "01", chunk("01")), + ("uint8 ab", uint8(0xab), "ab", chunk("ab")), + ("byte 00", byte(0x00), "00", chunk("00")), + ("byte 01", byte(0x01), "01", chunk("01")), + ("byte ab", byte(0xab), "ab", chunk("ab")), + ("uint16 0000", uint16(0x0000), "0000", chunk("0000")), + ("uint16 abcd", uint16(0xabcd), "cdab", chunk("cdab")), + ("uint32 00000000", uint32(0x00000000), "00000000", chunk("00000000")), + ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), + ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301", h(chunk("6745"), chunk("2301"))), + ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301", chunk("67452301")), + ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), + ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000", chunk("0000000000000000")), + ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301", chunk("efcdab8967452301")), ("sig", BytesN[96](*sig_test_data), "0100000000000000000000000000000000000000000000000000000000000000" "0200000000000000000000000000000000000000000000000000000000000000" - "03000000000000000000000000000000000000000000000000000000000000ff"), - ("emptyTestStruct", EmptyTestStruct(), ""), - ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab"), - ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412"), - ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff"), - ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff"), + "03000000000000000000000000000000000000000000000000000000000000ff", + h(h(chunk("01"), chunk("02")), + h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), + ("emptyTestStruct", EmptyTestStruct(), "", chunk("")), + ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab", chunk("ab")), + ("uint16 list", List[uint16, 32](uint16(0xaabb), uint16(0xc0ad), uint16(0xeeff)), "bbaaadc0ffee", + h(h(chunk("bbaaadc0ffee"), chunk("")), chunk("03000000")) # max length: 32 * 2 = 64 bytes = 2 chunks + ), + ("uint32 list", List[uint32, 128](uint32(0xaabb), uint32(0xc0ad), uint32(0xeeff)), "bbaa0000adc00000ffee0000", + # max length: 128 * 4 = 512 bytes = 16 chunks + h(merge(chunk("bbaa0000adc00000ffee0000"), zero_hashes[0:4]), chunk("03000000")) + ), + ("uint256 list", List[uint256, 32](uint256(0xaabb), uint256(0xc0ad), uint256(0xeeff)), + "bbaa000000000000000000000000000000000000000000000000000000000000" + "adc0000000000000000000000000000000000000000000000000000000000000" + "ffee000000000000000000000000000000000000000000000000000000000000", + h(merge(h(h(chunk("bbaa"), chunk("adc0")), h(chunk("ffee"), chunk(""))), zero_hashes[2:5]), chunk("03000000")) + ), + ("uint256 list long", List[uint256, 128](i for i in range(1, 20)), + "".join([i.to_bytes(length=32, byteorder='little').hex() for i in range(1, 20)]), + h(merge( + h( + h( + h( + h(h(chunk("01"), chunk("02")), h(chunk("03"), chunk("04"))), + h(h(chunk("05"), chunk("06")), h(chunk("07"), chunk("08"))), + ), + h( + h(h(chunk("09"), chunk("0a")), h(chunk("0b"), chunk("0c"))), + h(h(chunk("0d"), chunk("0e")), h(chunk("0f"), chunk("10"))), + ) + ), + h( + h( + h(h(chunk("11"), chunk("12")), h(chunk("13"), chunk(""))), + zero_hashes[2] + ), + zero_hashes[3] + ) + ), + zero_hashes[5:7]), chunk("13000000")) # 128 chunks = 7 deep + ), + ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412", + h(h(chunk("ab"), chunk("33221100ddccbbaa")), h(chunk("78563412"), chunk("")))), + ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff", + h(h(chunk("cdab"), h(zero_hashes[6], chunk("00000000"))), h(chunk("ff"), chunk("")))), + ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff", + h(h(chunk("cdab"), h(zero_hashes[6], chunk("00000000"))), h(chunk("ff"), chunk("")))), # log2(1024*2/32)= 6 deep ("varTestStruct some", VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), - "cdab07000000ff010002000300"), + "cdab07000000ff010002000300", + h( + h( + chunk("cdab"), + h( + merge( + chunk("010002000300"), + zero_hashes[0:6] + ), + chunk("03000000") # length mix in + ) + ), + h(chunk("ff"), chunk("")) + )), ("complexTestStruct", ComplexTestStruct( A=0xaabb, @@ -90,8 +172,8 @@ test_data = [ FixedTestStruct(A=0xee, B=0x4444444444444444, C=0x00112233), FixedTestStruct(A=0xff, B=0x5555555555555555, C=0x44556677)), G=Vector[VarTestStruct, 2]( - VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), - VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff)), + VarTestStruct(A=0xdead, B=List[uint16, 1024](1, 2, 3), C=0x11), + VarTestStruct(A=0xbeef, B=List[uint16, 1024](4, 5, 6), C=0x22)), ), "bbaa" "47000000" # offset of B, []uint16 @@ -107,17 +189,53 @@ test_data = [ "666f6f626172" # foobar "cdab07000000ff010002000300" # contents of E "08000000" "15000000" # [start G]: local offsets of [2]varTestStruct - "cdab07000000ff010002000300" - "cdab07000000ff010002000300", - ) + "adde0700000011010002000300" + "efbe0700000022040005000600", + h( + h( + h( # A and B + chunk("bbaa"), + h(merge(chunk("22114433"), zero_hashes[0:3]), chunk("02000000")) # 2*128/32 = 8 chunks + ), + h( # C and D + chunk("ff"), + h(merge(chunk("666f6f626172"), zero_hashes[0:3]), chunk("06000000")) # 256/32 = 8 chunks + ) + ), + h( + h( # E and F + h(h(chunk("cdab"), h(merge(chunk("010002000300"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("ff"), chunk(""))), + h( + h( + h(h(chunk("cc"), chunk("4242424242424242")), h(chunk("37133713"), chunk(""))), + h(h(chunk("dd"), chunk("3333333333333333")), h(chunk("cdabcdab"), chunk(""))), + ), + h( + h(h(chunk("ee"), chunk("4444444444444444")), h(chunk("33221100"), chunk(""))), + h(h(chunk("ff"), chunk("5555555555555555")), h(chunk("77665544"), chunk(""))), + ), + ) + ), + h( # G and padding + h( + h(h(chunk("adde"), h(merge(chunk("010002000300"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("11"), chunk(""))), + h(h(chunk("efbe"), h(merge(chunk("040005000600"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("22"), chunk(""))), + ), + chunk("") + ) + ) + )) ] -@pytest.mark.parametrize("name, value, serialized", test_data) -def test_serialize(name, value, serialized): +@pytest.mark.parametrize("name, value, serialized, _", test_data) +def test_serialize(name, value, serialized, _): assert serialize(value) == bytes.fromhex(serialized) -@pytest.mark.parametrize("name, value, _", test_data) -def test_hash_tree_root(name, value, _): - hash_tree_root(value) +@pytest.mark.parametrize("name, value, _, root", test_data) +def test_hash_tree_root(name, value, _, root): + assert hash_tree_root(value) == bytes.fromhex(root) From 45dbf5a107324e7f5b5c69e2d76950cca888bac3 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 25 Jun 2019 02:41:02 +0200 Subject: [PATCH 67/68] Remove old Deltas reference Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_beacon-chain.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b8cab7eaf..77fdb34e1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -286,8 +286,6 @@ We define the following Python custom types for type hinting and readability: | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | -`Deltas` is a non-SSZ type, a series of changes applied to balances, optimized by clients. - ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. From 054a1579536afdbdb71d6b6da0c778bcbae6a3b5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 02:56:49 +0200 Subject: [PATCH 68/68] get rid of TypingList, add MutableSequence --- scripts/build_spec.py | 3 +-- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 237c24384..d67c0e5c6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -37,8 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Sequence, Tuple, - List as TypingList + Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple, ) from dataclasses import ( diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 24e9a19e2..07f6ec698 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -318,7 +318,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: TypingList[Any], new_element: Any) -> int: +def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int: for i in range(len(list)): if is_empty(list[i]): list[i] = new_element diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index dd48a842a..d1c86eca2 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -252,9 +252,9 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) - def pad_to_power_of_2(values: TypingList[bytes]) -> TypingList[bytes]: + def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: while not is_power_of_two(len(values)): - values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] + values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) return values def hash_tree_root_of_bytes(data: bytes) -> bytes: @@ -264,6 +264,8 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: return data + b'\x00' * (length - len(data)) return hash( + # TODO untested code. + # Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x) hash_tree_root(pad_to_power_of_2([ hash_tree_root_of_bytes( zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY)