From 5ddfe34f0c3363589b402e88abde2c0aa1b05225 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:51:38 +0200 Subject: [PATCH 01/76] Simplified SSZ impl --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 13 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 407 ++++-------------- 2 files changed, 102 insertions(+), 318 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b3c877d48..c88cfed1f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,9 +1,8 @@ -from ..merkle_minimal import merkleize_chunks, hash -from eth2spec.utils.ssz.ssz_typing import ( +from ..merkle_minimal import merkleize_chunks, hash, ZERO_BYTES32 +from .ssz_typing import ( is_uint_type, is_bool_type, is_container_type, is_list_kind, is_vector_kind, - read_vector_elem_type, read_elem_type, - uint_byte_size, + read_elem_type, infer_input_type, get_zero_value, ) @@ -20,7 +19,7 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint_type(typ): - return value.to_bytes(uint_byte_size(typ), 'little') + return value.to_bytes(typ.byte_len, 'little') elif is_bool_type(typ): if value: return b'\x01' @@ -140,6 +139,8 @@ def get_typed_values(obj, typ=None): else: raise Exception("Invalid type") +def item_length(typ): + return 1 if typ == bool else typ.byte_len if is_uint_type(typ) else 32 @infer_input_type def hash_tree_root(obj, typ=None): @@ -150,6 +151,8 @@ def hash_tree_root(obj, typ=None): fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): + full_chunk_length = (item_length(read_elem_type(typ)) * typ.length + 31) // 32 + leaves += [ZERO_BYTES32] * (full_chunk_length - len(obj)) return mix_in_length(merkleize_chunks(leaves), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index f870336e8..dbc3f9523 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -6,74 +6,38 @@ from typing_inspect import get_origin # SSZ integers # ----------------------------- - class uint(int): byte_len = 0 def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") + if value.byte_len and value.bit_length() > value.byte_len: + raise ValueError("value out of bounds for uint{}".format(value.byte_len)) return super().__new__(cls, value) class uint8(uint): byte_len = 1 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 8: - raise ValueError("value out of bounds for uint8") - return super().__new__(cls, value) - - # Alias for uint8 byte = NewType('byte', uint8) - class uint16(uint): byte_len = 2 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 16: - raise ValueError("value out of bounds for uint16") - return super().__new__(cls, value) - - class uint32(uint): byte_len = 4 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 32: - raise ValueError("value out of bounds for uint16") - return super().__new__(cls, value) - - class uint64(uint): byte_len = 8 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 64: - raise ValueError("value out of bounds for uint64") - return super().__new__(cls, value) - - class uint128(uint): byte_len = 16 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 128: - raise ValueError("value out of bounds for uint128") - return super().__new__(cls, value) - - class uint256(uint): byte_len = 32 - def __new__(cls, value, *args, **kwargs): - if value.bit_length() > 256: - raise ValueError("value out of bounds for uint256") - return super().__new__(cls, value) - - def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. @@ -84,21 +48,6 @@ def is_uint_type(typ): return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) - -def uint_byte_size(typ): - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ - - if isinstance(typ, type): - if issubclass(typ, uint): - return typ.byte_len - elif issubclass(typ, int): - # Default to uint64 - return 8 - else: - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) - - # SSZ Container base class # ----------------------------- @@ -166,45 +115,19 @@ class Container(object): return list(cls.__annotations__.values()) -# SSZ vector -# ----------------------------- - - -def _is_vector_instance_of(a, b): - # Other must not be a BytesN - if issubclass(b, bytes): - return False - elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector (b) is not an instance of Vector[X, Y] (a) - return False - elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): - # Vector[X, Y] (b) is an instance of Vector (a) - return True +def get_zero_value(typ): + if typ == int: + return 0 else: - # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) - return a.elem_type == b.elem_type and a.length == b.length + return typ.default() - -def _is_equal_vector_type(a, b): - # Other must not be a BytesN - if issubclass(b, bytes): - return False - elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector == Vector - return True - else: - # Vector != Vector[X, Y] - return False - elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): - # Vector[X, Y] != Vector - return False +def type_check(typ, value): + if typ == int or typ == uint64: + return isinstance(value, int) else: - # Vector[X, Y] == Vector[X, Y] - return a.elem_type == b.elem_type and a.length == b.length + return typ.value_check(value) - -class VectorMeta(type): +class AbstractListMeta(type): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) if 'elem_type' in attrs and 'length' in attrs: @@ -214,239 +137,115 @@ class VectorMeta(type): def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: - raise Exception("Vector must be instantiated with two args: elem type and length") - o = self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) - o._name = 'Vector' + raise Exception("List must be instantiated with two args: elem type and length") + o = self.__class__(self.__name__, (self,), {'elem_type': params[0], 'length': params[1]}) + o._name = 'AbstractList' return o - def __subclasscheck__(self, sub): - return _is_vector_instance_of(self, sub) + def __instancecheck__(self, obj): + if obj.__class__.__name__ != self.__name__: + return False + if hasattr(self, 'elem_type') and obj.__class__.elem_type != self.elem_type: + return False + if hasattr(self, 'length') and obj.__class__.length != self.length: + return False + return True - def __instancecheck__(self, other): - return _is_vector_instance_of(self, other.__class__) +class ValueCheckError(Exception): + pass - def __eq__(self, other): - return _is_equal_vector_type(self, other) +class AbstractList(metaclass=AbstractListMeta): + def __init__(self, *args): + items = self.extract_args(args) + + if not self.value_check(items): + raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) + self.items = items + + def value_check(self, value): + for v in value: + if not type_check(self.__class__.elem_type, v): + return False + return True - def __ne__(self, other): - return not _is_equal_vector_type(self, other) + def extract_args(self, args): + return list(args) if len(args) > 0 else self.default() - def __hash__(self): - return hash(self.__class__) + def default(self): + raise Exception("Not implemented") + def __getitem__(self, i): + return self.items[i] -class Vector(metaclass=VectorMeta): - - def __init__(self, *args: Iterable): - cls = self.__class__ - if not hasattr(cls, 'elem_type'): - raise TypeError("Type Vector without elem_type data cannot be instantiated") - elif not hasattr(cls, 'length'): - raise TypeError("Type Vector without length data cannot be instantiated") - - if len(args) != cls.length: - if len(args) == 0: - args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] - else: - raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) - - self.items = list(args) - - # cannot check non-type objects, or parametrized types - if isinstance(cls.elem_type, type) and not hasattr(cls.elem_type, '__args__'): - for i, item in enumerate(self.items): - if not issubclass(cls.elem_type, type(item)): - raise TypeError("Typed vector cannot hold differently typed value" - " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) - - def serialize(self): - from .ssz_impl import serialize - return serialize(self, self.__class__) - - def hash_tree_root(self): - from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) - - def __repr__(self): - return repr({'length': self.__class__.length, 'items': self.items}) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) + def __setitem__(self, k, v): + self.items[k] = v def __len__(self): return len(self.items) - def __eq__(self, other): - return self.hash_tree_root() == other.hash_tree_root() + def __repr__(self): + return repr(self.items) - -# SSZ BytesN -# ----------------------------- - - -def _is_bytes_n_instance_of(a, b): - # Other has to be a Bytes derivative class to be a BytesN - if not issubclass(b, bytes): - return False - elif not hasattr(b, 'length'): - # BytesN (b) is not an instance of BytesN[X] (a) - return False - elif not hasattr(a, 'length'): - # BytesN[X] (b) is an instance of BytesN (a) - return True - else: - # BytesN[X] (a) is an instance of BytesN[X] (b) - return a.length == b.length - - -def _is_equal_bytes_n_type(a, b): - # Other has to be a Bytes derivative class to be a BytesN - if not issubclass(b, bytes): - return False - elif not hasattr(a, 'length'): - if not hasattr(b, 'length'): - # BytesN == BytesN - return True - else: - # BytesN != BytesN[X] - return False - elif not hasattr(b, 'length'): - # BytesN[X] != BytesN - return False - else: - # BytesN[X] == BytesN[X] - return a.length == b.length - - -class BytesNMeta(type): - def __new__(cls, class_name, parents, attrs): - out = type.__new__(cls, class_name, parents, attrs) - if 'length' in attrs: - setattr(out, 'length', attrs['length']) - out._name = 'BytesN' - out.elem_type = byte - return out - - def __getitem__(self, n): - return self.__class__(self.__name__, (BytesN,), {'length': n}) - - def __subclasscheck__(self, sub): - return _is_bytes_n_instance_of(self, sub) - - def __instancecheck__(self, other): - return _is_bytes_n_instance_of(self, other.__class__) + def __iter__(self): + return iter(self.items) def __eq__(self, other): - return _is_equal_bytes_n_type(self, other) + return self.items == other.items - def __ne__(self, other): - return not _is_equal_bytes_n_type(self, other) +class List(AbstractList, metaclass=AbstractListMeta): + def value_check(self, value): + return len(value) <= self.__class__.length and super().value_check(value) - def __hash__(self): - return hash(self.__class__) - - -def parse_bytes(val): - if val is None: - return None - elif isinstance(val, str): - # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) - return None - elif isinstance(val, bytes): - return val - elif isinstance(val, int): - return bytes([val]) - elif isinstance(val, (list, GeneratorType)): - return bytes(val) - else: - return None - - -class BytesN(bytes, metaclass=BytesNMeta): - def __new__(cls, *args): - if not hasattr(cls, 'length'): - return - bytesval = None - if len(args) == 1: - val: Union[bytes, int, str] = args[0] - bytesval = parse_bytes(val) - elif len(args) > 1: - # TODO: each int is 1 byte, check size, create bytesval - bytesval = bytes(args) - - if bytesval is None: - if cls.length == 0: - bytesval = b'' - else: - bytesval = b'\x00' * cls.length - if len(bytesval) != cls.length: - raise TypeError("BytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) - return super().__new__(cls, bytesval) - - def serialize(self): - from .ssz_impl import serialize - return serialize(self, self.__class__) - - def hash_tree_root(self): - from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) - - -class Bytes4(BytesN): - length = 4 - - -class Bytes32(BytesN): - length = 32 - - -class Bytes48(BytesN): - length = 48 - - -class Bytes96(BytesN): - length = 96 - - -# SSZ Defaults -# ----------------------------- -def get_zero_value(typ): - if is_uint_type(typ): - return uint64(0) - elif is_list_type(typ): + def default(self): return [] - elif is_bool_type(typ): - return False - elif is_vector_type(typ): - return typ() - elif is_bytesn_type(typ): - return typ() - elif is_bytes_type(typ): + +class Vector(AbstractList, metaclass=AbstractListMeta): + def value_check(self, value): + return len(value) == self.__class__.length and super().value_check(value) + + def default(self): + return [get_zero_value(self.__class__.elem_type) for _ in range(self.__class__.length)] + +class BytesMeta(AbstractListMeta): + def __getitem__(self, params): + if not isinstance(params, int): + raise Exception("Bytes must be instantiated with one arg: length") + o = self.__class__(self.__name__, (self,), {'length': params}) + o._name = 'Bytes' + return o + +def single_item_extractor(cls, args): + assert len(args) < 2 + return args[0] if len(args) > 0 else cls.default() + +class Bytes(AbstractList, metaclass=BytesMeta): + def value_check(self, value): + return len(value) <= self.__class__.length and isinstance(value, bytes) + + extract_args = single_item_extractor + + def default(self): return b'' - elif is_container_type(typ): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - else: - raise Exception("Type not supported: {}".format(typ)) + +class BytesN(AbstractList, metaclass=BytesMeta): + def value_check(self, value): + return len(value) == self.__class__.length and isinstance(value, bytes) + + extract_args = single_item_extractor + + def default(self): + return b'\x00' * self.__class__.length # Type helpers # ----------------------------- - def infer_type(obj): if is_uint_type(obj.__class__): return obj.__class__ elif isinstance(obj, int): return uint64 - elif isinstance(obj, list): - return List[infer_type(obj[0])] - elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): + elif isinstance(obj, (List, Vector, Container, bool, BytesN, Bytes)): return obj.__class__ else: raise Exception("Unknown type for {}".format(obj)) @@ -476,15 +275,14 @@ def is_list_type(typ): """ Check if the given type is a list. """ - return get_origin(typ) is List or get_origin(typ) is list + return isinstance(typ, type) and issubclass(typ, List) def is_bytes_type(typ): """ Check if the given type is a ``bytes``. """ - # Do not accept subclasses of bytes here, to avoid confusion with BytesN - return typ == bytes + return isinstance(typ, type) and issubclass(typ, Bytes) def is_bytesn_type(typ): @@ -526,22 +324,5 @@ T = TypeVar('T') L = TypeVar('L') -def read_list_elem_type(list_typ: Type[List[T]]) -> T: - if list_typ.__args__ is None or len(list_typ.__args__) != 1: - raise TypeError("Supplied list-type is invalid, no element type found.") - return list_typ.__args__[0] - - -def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: - return vector_typ.elem_type - - def read_elem_type(typ): - if typ == bytes or (isinstance(typ, type) and issubclass(typ, bytes)): # bytes or bytesN - return byte - elif is_list_type(typ): - return read_list_elem_type(typ) - elif is_vector_type(typ): - return read_vector_elem_type(typ) - else: - raise TypeError("Unexpected type: {}".format(typ)) + return typ.elem_type From 7c4232455c305b58143d9f62dfd29ebbaa2eb5e2 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Fri, 14 Jun 2019 14:07:25 -0400 Subject: [PATCH 02/76] Added get_container_type to get_zero_value --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index dbc3f9523..de54bbf05 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -118,6 +118,8 @@ class Container(object): def get_zero_value(typ): if typ == int: return 0 + elif is_container_type(typ): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: return typ.default() From 8919f628cbb9f17472716cb8cd850b963a7f37c7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 14 Jun 2019 18:32:30 -0400 Subject: [PATCH 03/76] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: Diederik Loerakker --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index c88cfed1f..b08a3d4e2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,4 +1,5 @@ -from ..merkle_minimal import merkleize_chunks, hash, ZERO_BYTES32 +from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 +from .hash_function import hash from .ssz_typing import ( is_uint_type, is_bool_type, is_container_type, is_list_kind, is_vector_kind, From d1ecfd510ec2738dd267994a1e2b305176bb4eda Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:53:32 +0200 Subject: [PATCH 04/76] typing improvements --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 297 ++++++++---------- 1 file changed, 128 insertions(+), 169 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index de54bbf05..9aafb5294 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,52 +1,56 @@ from types import GeneratorType -from typing import List, Iterable, TypeVar, Type, NewType -from typing import Union -from typing_inspect import get_origin + + +class DefaultingTypeMeta(type): + def default(cls): + raise Exception("Not implemented") # SSZ integers # ----------------------------- -class uint(int): + +class uint(int, metaclass=DefaultingTypeMeta): byte_len = 0 def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") - if value.byte_len and value.bit_length() > value.byte_len: - raise ValueError("value out of bounds for uint{}".format(value.byte_len)) + if cls.byte_len and (value.bit_length() >> 3) > cls.byte_len: + raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) return super().__new__(cls, value) + @classmethod + def default(cls): + return cls(0) + class uint8(uint): byte_len = 1 + # Alias for uint8 byte = NewType('byte', uint8) + class uint16(uint): byte_len = 2 + class uint32(uint): byte_len = 4 + class uint64(uint): byte_len = 8 + class uint128(uint): byte_len = 16 + class uint256(uint): byte_len = 32 -def is_uint_type(typ): - # All integers are uint in the scope of the spec here. - # Since we default to uint64. Bounds can be checked elsewhere. - # However, some are wrapped in a NewType - if hasattr(typ, '__supertype__'): - # get the type that the NewType is wrapping - typ = typ.__supertype__ - - return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) # SSZ Container base class # ----------------------------- @@ -59,7 +63,7 @@ class Container(object): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, get_zero_value(t)) + setattr(self, f, t.default()) else: setattr(self, f, kwargs[f]) @@ -83,9 +87,9 @@ class Container(object): return repr({field: getattr(self, field) for field in self.get_field_names()}) def __str__(self): - output = [] + output = [f'{self.__class__.__name__}'] for field in self.get_field_names(): - output.append(f'{field}: {getattr(self, field)}') + output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) def __eq__(self, other): @@ -114,67 +118,94 @@ class Container(object): # values of annotations are the types corresponding to the fields, not instance values. return list(cls.__annotations__.values()) + @classmethod + def default(cls): + return cls(**{f: t.default() for f, t in cls.get_fields()}) -def get_zero_value(typ): - if typ == int: - return 0 - elif is_container_type(typ): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) - else: - return typ.default() -def type_check(typ, value): - if typ == int or typ == uint64: - return isinstance(value, int) - else: - return typ.value_check(value) +class ParamsBase: + _bare = True + + def __new__(cls, *args, **kwargs): + if cls._bare: + raise Exception("cannot init bare type without params") + return super().__new__(cls, **kwargs) + + +class ParamsMeta(DefaultingTypeMeta): -class AbstractListMeta(type): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) - if 'elem_type' in attrs and 'length' in attrs: - setattr(out, 'elem_type', attrs['elem_type']) - setattr(out, 'length', attrs['length']) + for k, v in attrs.items(): + setattr(out, k, v) return out def __getitem__(self, params): - if not isinstance(params, tuple) or len(params) != 2: - raise Exception("List must be instantiated with two args: elem type and length") - o = self.__class__(self.__name__, (self,), {'elem_type': params[0], 'length': params[1]}) - o._name = 'AbstractList' + o = self.__class__(self.__name__, (self,), self.attr_from_params(params)) + o._bare = False return o + def attr_from_params(self, p): + # single key params are valid too. Wrap them in a tuple. + params = p if isinstance(p, tuple) else (p,) + res = {} + i = 0 + for (name, typ) in self.__annotations__.items(): + param = params[i] + if hasattr(self.__class__, name): + res[name] = getattr(self.__class__, name) + else: + if not isinstance(param, typ): + raise TypeError( + "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) + res[name] = param + i += 1 + if len(params) != i: + raise TypeError("provided parameters {} mismatch required parameter count {}".format(params, i)) + return res + def __instancecheck__(self, obj): if obj.__class__.__name__ != self.__name__: return False - if hasattr(self, 'elem_type') and obj.__class__.elem_type != self.elem_type: - return False - if hasattr(self, 'length') and obj.__class__.length != self.length: - return False + for name, typ in self.__annotations__: + if hasattr(self, name) and hasattr(obj.__class__, name) \ + and getattr(obj.__class__, name) != getattr(self, name): + return False return True + class ValueCheckError(Exception): pass -class AbstractList(metaclass=AbstractListMeta): + +class AbstractListMeta(ParamsMeta): + elem_type: DefaultingTypeMeta + length: int + + +class AbstractList(ParamsBase, metaclass=AbstractListMeta): + def __init__(self, *args): - items = self.extract_args(args) - + items = self.extract_args(*args) + if not self.value_check(items): raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) self.items = items - - def value_check(self, value): - for v in value: - if not type_check(self.__class__.elem_type, v): - return False - return True - def extract_args(self, args): - return list(args) if len(args) > 0 else self.default() + @classmethod + def value_check(cls, value): + return all(isinstance(v, cls.elem_type) for v in value) - def default(self): - raise Exception("Not implemented") + @classmethod + def extract_args(cls, *args): + x = list(args) + if len(x) == 1 and isinstance(x[0], GeneratorType): + x = list(x[0]) + return x if len(x) > 0 else cls.default() + + def __str__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" def __getitem__(self, i): return self.items[i] @@ -194,137 +225,65 @@ class AbstractList(metaclass=AbstractListMeta): def __eq__(self, other): return self.items == other.items -class List(AbstractList, metaclass=AbstractListMeta): + +class List(AbstractList): def value_check(self, value): return len(value) <= self.__class__.length and super().value_check(value) - def default(self): - return [] + @classmethod + def default(cls): + return cls() + class Vector(AbstractList, metaclass=AbstractListMeta): def value_check(self, value): return len(value) == self.__class__.length and super().value_check(value) - def default(self): - return [get_zero_value(self.__class__.elem_type) for _ in range(self.__class__.length)] + @classmethod + def default(cls): + return [cls.elem_type.default() for _ in range(cls.length)] + class BytesMeta(AbstractListMeta): - def __getitem__(self, params): - if not isinstance(params, int): - raise Exception("Bytes must be instantiated with one arg: length") - o = self.__class__(self.__name__, (self,), {'length': params}) - o._name = 'Bytes' - return o + elem_type: DefaultingTypeMeta = byte + length: int -def single_item_extractor(cls, args): - assert len(args) < 2 - return args[0] if len(args) > 0 else cls.default() -class Bytes(AbstractList, metaclass=BytesMeta): +class BytesLike(AbstractList, metaclass=BytesMeta): + + @classmethod + def extract_args(cls, args): + if isinstance(args, bytes): + return args + elif isinstance(args, BytesLike): + return args.items + elif isinstance(args, GeneratorType): + return bytes(args) + else: + return bytes(args) + + @classmethod + def value_check(cls, value): + return len(value) == cls.length and isinstance(value, bytes) + + def __str__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.length}]: {self.items.hex()}" + + +class Bytes(BytesLike): + def value_check(self, value): return len(value) <= self.__class__.length and isinstance(value, bytes) - extract_args = single_item_extractor - - def default(self): + @classmethod + def default(cls): return b'' -class BytesN(AbstractList, metaclass=BytesMeta): - def value_check(self, value): - return len(value) == self.__class__.length and isinstance(value, bytes) - extract_args = single_item_extractor +class BytesN(BytesLike): - def default(self): - return b'\x00' * self.__class__.length + @classmethod + def default(cls): + return b'\x00' * cls.length - -# Type helpers -# ----------------------------- - -def infer_type(obj): - if is_uint_type(obj.__class__): - return obj.__class__ - elif isinstance(obj, int): - return uint64 - elif isinstance(obj, (List, Vector, Container, bool, BytesN, Bytes)): - return obj.__class__ - else: - raise Exception("Unknown type for {}".format(obj)) - - -def infer_input_type(fn): - """ - Decorator to run infer_type on the obj if typ argument is None - """ - def infer_helper(obj, typ=None, **kwargs): - if typ is None: - typ = infer_type(obj) - return fn(obj, typ=typ, **kwargs) - return infer_helper - - -def is_bool_type(typ): - """ - Check if the given type is a bool. - """ - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ - return isinstance(typ, type) and issubclass(typ, bool) - - -def is_list_type(typ): - """ - Check if the given type is a list. - """ - return isinstance(typ, type) and issubclass(typ, List) - - -def is_bytes_type(typ): - """ - Check if the given type is a ``bytes``. - """ - return isinstance(typ, type) and issubclass(typ, Bytes) - - -def is_bytesn_type(typ): - """ - Check if the given type is a BytesN. - """ - return isinstance(typ, type) and issubclass(typ, BytesN) - - -def is_list_kind(typ): - """ - Check if the given type is a kind of list. Can be bytes. - """ - return is_list_type(typ) or is_bytes_type(typ) - - -def is_vector_type(typ): - """ - Check if the given type is a vector. - """ - return isinstance(typ, type) and issubclass(typ, Vector) - - -def is_vector_kind(typ): - """ - Check if the given type is a kind of vector. Can be BytesN. - """ - return is_vector_type(typ) or is_bytesn_type(typ) - - -def is_container_type(typ): - """ - Check if the given type is a container. - """ - return isinstance(typ, type) and issubclass(typ, Container) - - -T = TypeVar('T') -L = TypeVar('L') - - -def read_elem_type(typ): - return typ.elem_type From b6cf809d9b686f090dc4e19a68fbf311cd247831 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:54:59 +0200 Subject: [PATCH 05/76] more improvements, and implement new space-efficient merkleization with padding support --- .../pyspec/eth2spec/utils/merkle_minimal.py | 36 +++++++--- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 66 +++++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 32 +++++---- 3 files changed, 86 insertions(+), 48 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index c508f0df2..ebfb4faf6 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -44,11 +44,31 @@ def next_power_of_two(v: int) -> int: return 1 << (v - 1).bit_length() -def merkleize_chunks(chunks): - tree = chunks[::] - margin = next_power_of_two(len(chunks)) - len(chunks) - tree.extend([ZERO_BYTES32] * margin) - tree = [ZERO_BYTES32] * len(tree) + tree - for i in range(len(tree) // 2 - 1, 0, -1): - tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) - return tree[1] +def merkleize_chunks(chunks, pad_to: int = None): + count = len(chunks) + depth = max(count - 1, 0).bit_length() + max_depth = max(depth, (pad_to - 1).bit_length()) + tmp = [None for _ in range(max_depth + 1)] + + def merge(h, i): + j = 0 + while True: + if i & (1 << j) == 0: + if i == count and j < depth: + h = hash(h + zerohashes[j]) + else: + break + else: + h = hash(tmp[j] + h) + j += 1 + tmp[j] = h + + for i in range(count): + merge(chunks[i], i) + + merge(zerohashes[0], count) + + for j in range(depth, max_depth): + tmp[j + 1] = hash(tmp[j] + zerohashes[j]) + + return tmp[max_depth] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b08a3d4e2..1a556bc7d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,11 +1,7 @@ from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 -from .hash_function import hash +from ..hash_function import hash from .ssz_typing import ( - is_uint_type, is_bool_type, is_container_type, - is_list_kind, is_vector_kind, - read_elem_type, - infer_input_type, - get_zero_value, + get_zero_value, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -15,13 +11,13 @@ BYTES_PER_LENGTH_OFFSET = 4 def is_basic_type(typ): - return is_uint_type(typ) or is_bool_type(typ) + return issubclass(typ, (bool, uint)) def serialize_basic(value, typ): - if is_uint_type(typ): + if issubclass(typ, uint): return value.to_bytes(typ.byte_len, 'little') - elif is_bool_type(typ): + elif issubclass(typ, bool): if value: return b'\x01' else: @@ -31,22 +27,34 @@ def serialize_basic(value, typ): def deserialize_basic(value, typ): - if is_uint_type(typ): + if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif is_bool_type(typ): + elif issubclass(typ, bool): assert value in (b'\x00', b'\x01') return True if value == b'\x01' else False else: raise Exception("Type not supported: {}".format(typ)) +def is_list_kind(typ): + return issubclass(typ, (List, Bytes)) + + +def is_vector_kind(typ): + return issubclass(typ, (Vector, BytesN)) + + +def is_container_type(typ): + return issubclass(typ, Container) + + def is_fixed_size(typ): if is_basic_type(typ): return True elif is_list_kind(typ): return False elif is_vector_kind(typ): - return is_fixed_size(read_vector_elem_type(typ)) + return is_fixed_size(typ.elem_type) elif is_container_type(typ): return all(is_fixed_size(t) for t in typ.get_field_types()) else: @@ -57,12 +65,11 @@ def is_empty(obj): return get_zero_value(type(obj)) == obj -@infer_input_type -def serialize(obj, typ=None): +def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [read_elem_type(typ)] * len(obj)) + return encode_series(obj, [typ.elem_type] * len(obj)) elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: @@ -126,40 +133,41 @@ def mix_in_length(root, length): def is_bottom_layer_kind(typ): return ( is_basic_type(typ) or - (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(read_elem_type(typ)) + (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(typ.elem_type) ) -@infer_input_type -def get_typed_values(obj, typ=None): +def get_typed_values(obj, typ): if is_container_type(typ): return obj.get_typed_values() elif is_list_kind(typ) or is_vector_kind(typ): - elem_type = read_elem_type(typ) - return list(zip(obj, [elem_type] * len(obj))) + return list(zip(obj, [typ.elem_type] * len(obj))) else: raise Exception("Invalid type") -def item_length(typ): - return 1 if typ == bool else typ.byte_len if is_uint_type(typ) else 32 -@infer_input_type -def hash_tree_root(obj, typ=None): +def item_length(typ): + if typ == bool: + return 1 + elif issubclass(typ, uint): + return typ.byte_len + else: + return 32 + + +def hash_tree_root(obj, typ): if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, typ.elem_type) leaves = chunkify(data) else: fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): - full_chunk_length = (item_length(read_elem_type(typ)) * typ.length + 31) // 32 - leaves += [ZERO_BYTES32] * (full_chunk_length - len(obj)) - return mix_in_length(merkleize_chunks(leaves), len(obj)) + return mix_in_length(merkleize_chunks(leaves, pad_to=typ.length), len(obj)) else: return merkleize_chunks(leaves) -@infer_input_type def signing_root(obj, typ): assert is_container_type(typ) # ignore last field diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 9aafb5294..30f71f87d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,10 +1,26 @@ +from typing import NewType, Union from types import GeneratorType +class ValueCheckError(Exception): + pass + + class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") +# Every type is subclassed and has a default() method, except bool. +TypeWithDefault = Union[DefaultingTypeMeta, bool] + + +def get_zero_value(typ: TypeWithDefault): + if issubclass(typ, bool): + return False + else: + return typ.default() + + # SSZ integers # ----------------------------- @@ -63,7 +79,7 @@ class Container(object): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, t.default()) + setattr(self, f, get_zero_value(t)) else: setattr(self, f, kwargs[f]) @@ -120,7 +136,7 @@ class Container(object): @classmethod def default(cls): - return cls(**{f: t.default() for f, t in cls.get_fields()}) + return cls(**{f: get_zero_value(t) for f, t in cls.get_fields()}) class ParamsBase: @@ -174,12 +190,8 @@ class ParamsMeta(DefaultingTypeMeta): return True -class ValueCheckError(Exception): - pass - - class AbstractListMeta(ParamsMeta): - elem_type: DefaultingTypeMeta + elem_type: TypeWithDefault length: int @@ -227,8 +239,6 @@ class AbstractList(ParamsBase, metaclass=AbstractListMeta): class List(AbstractList): - def value_check(self, value): - return len(value) <= self.__class__.length and super().value_check(value) @classmethod def default(cls): @@ -241,11 +251,11 @@ class Vector(AbstractList, metaclass=AbstractListMeta): @classmethod def default(cls): - return [cls.elem_type.default() for _ in range(cls.length)] + return [get_zero_value(cls.elem_type) for _ in range(cls.length)] class BytesMeta(AbstractListMeta): - elem_type: DefaultingTypeMeta = byte + elem_type: TypeWithDefault = byte length: int From cd5f59eb74cf8c398033ccd7e862d89c4d6ceffa Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:55:53 +0200 Subject: [PATCH 06/76] fix bytes value check, fix default-type checking --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 30f71f87d..6f578796d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -171,7 +171,10 @@ class ParamsMeta(DefaultingTypeMeta): if hasattr(self.__class__, name): res[name] = getattr(self.__class__, name) else: - if not isinstance(param, typ): + if typ == TypeWithDefault: + if not (isinstance(param, bool) or isinstance(param, DefaultingTypeMeta)): + raise TypeError("expected param {} as {} to have a type default".format(param, name, typ)) + elif not isinstance(param, typ): raise TypeError( "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) res[name] = param @@ -246,8 +249,10 @@ class List(AbstractList): class Vector(AbstractList, metaclass=AbstractListMeta): - def value_check(self, value): - return len(value) == self.__class__.length and super().value_check(value) + + @classmethod + def value_check(cls, value): + return len(value) == cls.length and super().value_check(value) @classmethod def default(cls): @@ -274,7 +279,7 @@ class BytesLike(AbstractList, metaclass=BytesMeta): @classmethod def value_check(cls, value): - return len(value) == cls.length and isinstance(value, bytes) + return isinstance(value, bytes) def __str__(self): cls = self.__class__ @@ -283,9 +288,6 @@ class BytesLike(AbstractList, metaclass=BytesMeta): class Bytes(BytesLike): - def value_check(self, value): - return len(value) <= self.__class__.length and isinstance(value, bytes) - @classmethod def default(cls): return b'' @@ -297,3 +299,7 @@ class BytesN(BytesLike): def default(cls): return b'\x00' * cls.length + @classmethod + def value_check(cls, value): + return len(value) == cls.length and super().value_check(value) + From 54a1fa9abec52a747ea47bf73c679a7b2f2bef11 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 15 Jun 2019 20:57:22 +0200 Subject: [PATCH 07/76] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: vbuterin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1a556bc7d..ed8dbf1e0 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -153,7 +153,13 @@ def item_length(typ): return typ.byte_len else: return 32 - +def chunk_count(typ): + if is_basic_type(typ): + return 1 + elif is_list_kind(typ) or is_vector_kind(typ): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + else: + return len(typ.get_fields()) def hash_tree_root(obj, typ): if is_bottom_layer_kind(typ): From 3a9b1fb72c97e4512a935fb3c99c084ac411cae1 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Sat, 15 Jun 2019 20:57:30 +0200 Subject: [PATCH 08/76] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: vbuterin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index ed8dbf1e0..bcdef3988 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -169,7 +169,7 @@ def hash_tree_root(obj, typ): fields = get_typed_values(obj, typ=typ) leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] if is_list_kind(typ): - return mix_in_length(merkleize_chunks(leaves, pad_to=typ.length), len(obj)) + return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(typ)), len(obj)) else: return merkleize_chunks(leaves) From 82e7392b17f7d63f58a2745d00555bb8e479cf18 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 21:13:01 +0200 Subject: [PATCH 09/76] default method for container is recognized now --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6f578796d..7cf44f6b3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -73,7 +73,7 @@ class uint256(uint): # Note: importing ssz functionality locally, to avoid import loop -class Container(object): +class Container(object, metaclass=DefaultingTypeMeta): def __init__(self, **kwargs): cls = self.__class__ From 108410d8626a2db20bb7e44422180f98d41162c6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 15 Jun 2019 22:12:59 +0200 Subject: [PATCH 10/76] Change byte to explict class instead of newtype --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7cf44f6b3..40901ad97 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -45,7 +45,8 @@ class uint8(uint): # Alias for uint8 -byte = NewType('byte', uint8) +class byte(uint8): + pass class uint16(uint): From 4ebdceaf129e40cd27c2b95ffbd36af0d72e9db6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 19:57:50 +0200 Subject: [PATCH 11/76] highly experimental typing --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 142 +++++++----------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 141 +++++++++++------ 2 files changed, 146 insertions(+), 137 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index bcdef3988..679574891 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ -from ..merkle_minimal import merkleize_chunks, ZERO_BYTES32 +from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - get_zero_value, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -10,79 +10,48 @@ from .ssz_typing import ( BYTES_PER_LENGTH_OFFSET = 4 -def is_basic_type(typ): - return issubclass(typ, (bool, uint)) - - -def serialize_basic(value, typ): - if issubclass(typ, uint): - return value.to_bytes(typ.byte_len, 'little') - elif issubclass(typ, bool): +def serialize_basic(value: SSZValue): + if isinstance(value, uint): + return value.to_bytes(value.type().byte_len, 'little') + elif isinstance(value, Bit): if value: return b'\x01' else: return b'\x00' else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {type(value)}") -def deserialize_basic(value, typ): +def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, bool): + elif issubclass(typ, Bit): assert value in (b'\x00', b'\x01') - return True if value == b'\x01' else False + return Bit(value == b'\x01') else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {typ}") -def is_list_kind(typ): - return issubclass(typ, (List, Bytes)) +def is_empty(obj: SSZValue): + return type(obj).default() == obj -def is_vector_kind(typ): - return issubclass(typ, (Vector, BytesN)) - - -def is_container_type(typ): - return issubclass(typ, Container) - - -def is_fixed_size(typ): - if is_basic_type(typ): - return True - elif is_list_kind(typ): - return False - elif is_vector_kind(typ): - return is_fixed_size(typ.elem_type) - elif is_container_type(typ): - return all(is_fixed_size(t) for t in typ.get_field_types()) +def serialize(obj: SSZValue): + if isinstance(obj, BasicValue): + return serialize_basic(obj) + elif isinstance(obj, Series): + return encode_series(obj) else: - raise Exception("Type not supported: {}".format(typ)) + raise Exception(f"Type not supported: {type(obj)}") -def is_empty(obj): - return get_zero_value(type(obj)) == obj - - -def serialize(obj, typ): - if is_basic_type(typ): - return serialize_basic(obj, typ) - elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [typ.elem_type] * len(obj)) - elif is_container_type(typ): - return encode_series(obj.get_field_values(), typ.get_field_types()) - else: - raise Exception("Type not supported: {}".format(typ)) - - -def encode_series(values, types): +def encode_series(values: Series): # bytes and bytesN are already in the right format. if isinstance(values, bytes): return values # Recursively serialize - parts = [(is_fixed_size(types[i]), serialize(values[i], typ=types[i])) for i in range(len(values))] + parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] # Compute and check lengths fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET @@ -114,10 +83,10 @@ def encode_series(values, types): # ----------------------------- -def pack(values, subtype): +def pack(values: Series): if isinstance(values, bytes): return values - return b''.join([serialize_basic(value, subtype) for value in values]) + return b''.join([serialize_basic(value) for value in values]) def chunkify(bytez): @@ -130,52 +99,49 @@ def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) -def is_bottom_layer_kind(typ): +def is_bottom_layer_kind(typ: SSZType): return ( - is_basic_type(typ) or - (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(typ.elem_type) + issubclass(typ, BasicType) or + (issubclass(typ, Elements) and issubclass(typ.elem_type, BasicType)) ) -def get_typed_values(obj, typ): - if is_container_type(typ): - return obj.get_typed_values() - elif is_list_kind(typ) or is_vector_kind(typ): - return list(zip(obj, [typ.elem_type] * len(obj))) - else: - raise Exception("Invalid type") - - -def item_length(typ): - if typ == bool: - return 1 - elif issubclass(typ, uint): +def item_length(typ: SSZType) -> int: + if issubclass(typ, BasicType): return typ.byte_len else: return 32 -def chunk_count(typ): - if is_basic_type(typ): - return 1 - elif is_list_kind(typ) or is_vector_kind(typ): - return (typ.length * item_length(typ.elem_type) + 31) // 32 - else: - return len(typ.get_fields()) -def hash_tree_root(obj, typ): - if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, typ.elem_type) - leaves = chunkify(data) + +def chunk_count(typ: SSZType) -> int: + if issubclass(typ, BasicType): + return 1 + elif issubclass(typ, Elements): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + elif issubclass(typ, Container): + return len(typ.get_fields()) else: - fields = get_typed_values(obj, typ=typ) - leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] - if is_list_kind(typ): - return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(typ)), len(obj)) + raise Exception(f"Type not supported: {typ}") + + +def hash_tree_root(obj: SSZValue): + if isinstance(obj, Series): + if is_bottom_layer_kind(obj.type()): + leaves = chunkify(pack(obj)) + else: + leaves = [hash_tree_root(value) for value in obj] + elif isinstance(obj, BasicValue): + leaves = chunkify(serialize_basic(obj)) + else: + raise Exception(f"Type not supported: {obj.type()}") + + if isinstance(obj, (List, Bytes)): + return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) -def signing_root(obj, typ): - assert is_container_type(typ) +def signing_root(obj: Container): # ignore last field - leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] + leaves = [hash_tree_root(field) for field in obj[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 40901ad97..b79789f27 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import NewType, Union +from typing import Tuple, Dict, Iterator from types import GeneratorType @@ -10,24 +10,42 @@ class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") -# Every type is subclassed and has a default() method, except bool. -TypeWithDefault = Union[DefaultingTypeMeta, bool] + +class SSZType(DefaultingTypeMeta): + + def is_fixed_size(cls): + raise Exception("Not implemented") -def get_zero_value(typ: TypeWithDefault): - if issubclass(typ, bool): - return False - else: - return typ.default() +class SSZValue(object, metaclass=SSZType): + + def type(self): + return self.__class__ -# SSZ integers -# ----------------------------- - - -class uint(int, metaclass=DefaultingTypeMeta): +class BasicType(SSZType): byte_len = 0 + def is_fixed_size(cls): + return True + + +class BasicValue(int, SSZValue, metaclass=BasicType): + pass + + +class Bit(BasicValue): # can't subclass bool. + + @classmethod + def default(cls): + return cls(False) + + def __bool__(self): + return self > 0 + + +class uint(BasicValue, metaclass=BasicType): + def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") @@ -69,36 +87,39 @@ class uint256(uint): byte_len = 32 -# SSZ Container base class -# ----------------------------- +class Series(SSZValue): + + def __iter__(self) -> Iterator[SSZValue]: + raise Exception("Not implemented") + # Note: importing ssz functionality locally, to avoid import loop -class Container(object, metaclass=DefaultingTypeMeta): +class Container(Series, metaclass=SSZType): def __init__(self, **kwargs): cls = self.__class__ for f, t in cls.get_fields(): if f not in kwargs: - setattr(self, f, get_zero_value(t)) + setattr(self, f, t.default()) else: setattr(self, f, kwargs[f]) def serialize(self): from .ssz_impl import serialize - return serialize(self, self.__class__) + return serialize(self) def hash_tree_root(self): from .ssz_impl import hash_tree_root - return hash_tree_root(self, self.__class__) + return hash_tree_root(self) def signing_root(self): from .ssz_impl import signing_root - return signing_root(self, self.__class__) + return signing_root(self) - def get_field_values(self): + def get_field_values(self) -> Tuple[SSZValue, ...]: cls = self.__class__ - return [getattr(self, field) for field in cls.get_field_names()] + return tuple(getattr(self, field) for field in cls.get_field_names()) def __repr__(self): return repr({field: getattr(self, field) for field in self.get_field_names()}) @@ -116,31 +137,38 @@ class Container(object, metaclass=DefaultingTypeMeta): return hash(self.hash_tree_root()) @classmethod - def get_fields_dict(cls): + def get_fields_dict(cls) -> Dict[str, SSZType]: return dict(cls.__annotations__) @classmethod - def get_fields(cls): - return list(dict(cls.__annotations__).items()) + def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + return tuple((f, SSZType(t)) for f, t in dict(cls.__annotations__).items()) def get_typed_values(self): - return list(zip(self.get_field_values(), self.get_field_types())) + return tuple(zip(self.get_field_values(), self.get_field_types())) @classmethod - def get_field_names(cls): - return list(cls.__annotations__.keys()) + def get_field_names(cls) -> Tuple[str]: + return tuple(cls.__annotations__.keys()) @classmethod - def get_field_types(cls): + def get_field_types(cls) -> Tuple[SSZType, ...]: # values of annotations are the types corresponding to the fields, not instance values. - return list(cls.__annotations__.values()) + return tuple(cls.__annotations__.values()) @classmethod def default(cls): - return cls(**{f: get_zero_value(t) for f, t in cls.get_fields()}) + return cls(**{f: t.default() for f, t in cls.get_fields()}) + + @classmethod + def is_fixed_size(cls): + return all(t.is_fixed_size() for t in cls.get_field_types()) + + def __iter__(self) -> Iterator[SSZValue]: + return iter(self.get_field_values()) -class ParamsBase: +class ParamsBase(Series): _bare = True def __new__(cls, *args, **kwargs): @@ -149,7 +177,7 @@ class ParamsBase: return super().__new__(cls, **kwargs) -class ParamsMeta(DefaultingTypeMeta): +class ParamsMeta(SSZType): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) @@ -168,14 +196,14 @@ class ParamsMeta(DefaultingTypeMeta): res = {} i = 0 for (name, typ) in self.__annotations__.items(): - param = params[i] if hasattr(self.__class__, name): res[name] = getattr(self.__class__, name) else: - if typ == TypeWithDefault: - if not (isinstance(param, bool) or isinstance(param, DefaultingTypeMeta)): - raise TypeError("expected param {} as {} to have a type default".format(param, name, typ)) - elif not isinstance(param, typ): + if i >= len(params): + i += 1 + continue + param = params[i] + if not isinstance(param, typ): raise TypeError( "cannot create parametrized class with param {} as {} of type {}".format(param, name, typ)) res[name] = param @@ -194,12 +222,12 @@ class ParamsMeta(DefaultingTypeMeta): return True -class AbstractListMeta(ParamsMeta): - elem_type: TypeWithDefault +class Elements(ParamsMeta): + elem_type: SSZType length: int -class AbstractList(ParamsBase, metaclass=AbstractListMeta): +class ElementsBase(ParamsBase, metaclass=Elements): def __init__(self, *args): items = self.extract_args(*args) @@ -223,7 +251,7 @@ class AbstractList(ParamsBase, metaclass=AbstractListMeta): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" - def __getitem__(self, i): + def __getitem__(self, i) -> SSZValue: return self.items[i] def __setitem__(self, k, v): @@ -235,21 +263,25 @@ class AbstractList(ParamsBase, metaclass=AbstractListMeta): def __repr__(self): return repr(self.items) - def __iter__(self): + def __iter__(self) -> Iterator[SSZValue]: return iter(self.items) def __eq__(self, other): return self.items == other.items -class List(AbstractList): +class List(ElementsBase): @classmethod def default(cls): return cls() + @classmethod + def is_fixed_size(cls): + return False -class Vector(AbstractList, metaclass=AbstractListMeta): + +class Vector(ElementsBase): @classmethod def value_check(cls, value): @@ -257,15 +289,19 @@ class Vector(AbstractList, metaclass=AbstractListMeta): @classmethod def default(cls): - return [get_zero_value(cls.elem_type) for _ in range(cls.length)] + return [cls.elem_type.default() for _ in range(cls.length)] + + @classmethod + def is_fixed_size(cls): + return cls.elem_type.is_fixed_size() -class BytesMeta(AbstractListMeta): - elem_type: TypeWithDefault = byte +class BytesMeta(Elements): + elem_type: SSZType = byte length: int -class BytesLike(AbstractList, metaclass=BytesMeta): +class BytesLike(ElementsBase, metaclass=BytesMeta): @classmethod def extract_args(cls, args): @@ -293,6 +329,10 @@ class Bytes(BytesLike): def default(cls): return b'' + @classmethod + def is_fixed_size(cls): + return False + class BytesN(BytesLike): @@ -304,3 +344,6 @@ class BytesN(BytesLike): def value_check(cls, value): return len(value) == cls.length and super().value_check(value) + @classmethod + def is_fixed_size(cls): + return True From 97025c51ac399140445ee103f20ab488c2f3ca2f Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:03:11 +0200 Subject: [PATCH 12/76] start updating virtual sizes of lists --- specs/core/0_beacon-chain.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1dcdbdef9..ddc4c8f61 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -367,7 +367,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: bytes # Bit set for every attesting participant within a committee + aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] # Bit set for every attesting participant within a committee data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -434,9 +434,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: bytes + aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] data: AttestationData - custody_bitfield: bytes + custody_bitfield: Bytes[MAX_COMMITTEE_SIZE] signature: BLSSignature ``` @@ -480,12 +480,12 @@ class BeaconBlockBody(Container): eth1_data: Eth1Data # Eth1 data vote graffiti: Bytes32 # Arbitrary data # Operations - proposer_slashings: List[ProposerSlashing] - attester_slashings: List[AttesterSlashing] - attestations: List[Attestation] - deposits: List[Deposit] - voluntary_exits: List[VoluntaryExit] - transfers: List[Transfer] + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] + transfers: List[Transfer, MAX_TRANSFERS] ``` #### `BeaconBlock` From 08e6f32f3897889a62dcdf0887e4ba42403aa3a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 16 Jun 2019 03:21:58 +0200 Subject: [PATCH 13/76] typing improvements, type testing --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 21 ++- .../eth2spec/utils/ssz/test_ssz_typing.py | 131 ++++++++++++++++++ 2 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index b79789f27..5cab68aa8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -36,6 +36,11 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bit(BasicValue): # can't subclass bool. + def __new__(cls, value, *args, **kwargs): + if value < 0 or value > 1: + raise ValueError(f"value {value} out of bounds for bit") + return super().__new__(cls, value) + @classmethod def default(cls): return cls(False) @@ -49,7 +54,7 @@ class uint(BasicValue, metaclass=BasicType): def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") - if cls.byte_len and (value.bit_length() >> 3) > cls.byte_len: + if cls.byte_len and value.bit_length() > (cls.byte_len << 3): raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) return super().__new__(cls, value) @@ -142,7 +147,7 @@ class Container(Series, metaclass=SSZType): @classmethod def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: - return tuple((f, SSZType(t)) for f, t in dict(cls.__annotations__).items()) + return tuple((f, t) for f, t in cls.__annotations__.items()) def get_typed_values(self): return tuple(zip(self.get_field_values(), self.get_field_types())) @@ -190,6 +195,12 @@ class ParamsMeta(SSZType): o._bare = False return o + def __str__(self): + return f"{self.__name__}~{self.__class__.__name__}" + + def __repr__(self): + return self, self.__class__ + def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. params = p if isinstance(p, tuple) else (p,) @@ -215,7 +226,7 @@ class ParamsMeta(SSZType): def __instancecheck__(self, obj): if obj.__class__.__name__ != self.__name__: return False - for name, typ in self.__annotations__: + for name, typ in self.__annotations__.items(): if hasattr(self, name) and hasattr(obj.__class__, name) \ and getattr(obj.__class__, name) != getattr(self, name): return False @@ -233,7 +244,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): items = self.extract_args(*args) if not self.value_check(items): - raise ValueCheckError("Bad input for class {}: {}".format(self.__class__, items)) + raise ValueCheckError(f"Bad input for class {self.__class__}: {items}") self.items = items @classmethod @@ -245,7 +256,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): x = list(args) if len(x) == 1 and isinstance(x[0], GeneratorType): x = list(x[0]) - return x if len(x) > 0 else cls.default() + return x def __str__(self): cls = self.__class__ diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py new file mode 100644 index 000000000..a0705f8d3 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -0,0 +1,131 @@ +from .ssz_typing import ( + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, + uint, uint8, uint16, uint32, uint64, uint128, uint256 +) + + +def test_subclasses(): + for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + assert issubclass(u, uint) + assert issubclass(u, int) + assert issubclass(u, BasicValue) + assert issubclass(u, SSZValue) + assert isinstance(u, SSZType) + assert isinstance(u, BasicType) + assert issubclass(Bit, BasicValue) + assert isinstance(Bit, BasicType) + + for c in [Container, List, Vector, Bytes, BytesN]: + assert issubclass(c, Series) + assert issubclass(c, SSZValue) + assert isinstance(c, SSZType) + assert not issubclass(c, BasicValue) + assert not isinstance(c, BasicType) + + for c in [List, Vector, Bytes, BytesN]: + assert isinstance(c, Elements) + + +def test_basic_instances(): + for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + v = u(123) + assert isinstance(v, uint) + assert isinstance(v, int) + assert isinstance(v, BasicValue) + assert isinstance(v, SSZValue) + + assert isinstance(Bit(True), BasicValue) + assert isinstance(Bit(False), BasicValue) + + +def test_basic_value_bounds(): + max = { + Bit: 2**1, + uint8: 2**(8 * 1), + uint16: 2**(8 * 2), + uint32: 2**(8 * 4), + uint64: 2**(8 * 8), + uint128: 2**(8 * 16), + uint256: 2**(8 * 32), + } + for k, v in max.items(): + # this should work + assert k(v - 1) == v - 1 + # but we do not allow overflows + try: + k(v) + assert False + except ValueError: + pass + + for k, _ in max.items(): + # this should work + assert k(0) == 0 + # but we do not allow underflows + try: + k(-1) + assert False + except ValueError: + pass + + +def test_container(): + class Foo(Container): + a: uint8 + b: uint32 + + assert issubclass(Foo, Container) + assert issubclass(Foo, SSZValue) + assert issubclass(Foo, Series) + + assert Foo.is_fixed_size() + x = Foo(a=uint8(123), b=uint32(45)) + assert x.a == 123 + assert x.b == 45 + assert isinstance(x.a, uint8) + assert isinstance(x.b, uint32) + assert x.type().is_fixed_size() + + class Bar(Container): + a: uint8 + b: List[uint8, 1024] + + assert not Bar.is_fixed_size() + + y = Bar(a=uint8(123), b=List[uint8, 1024](uint8(1), uint8(2))) + assert y.a == 123 + assert len(y.b) == 2 + assert isinstance(y.a, uint8) + assert isinstance(y.b, List[uint8, 1024]) + assert not y.type().is_fixed_size() + assert y.b[0] == 1 + v: List = y.b + assert v.type().elem_type == uint8 + assert v.type().length == 1024 + + +def test_list(): + typ = List[uint64, 128] + assert issubclass(typ, List) + assert issubclass(typ, SSZValue) + assert issubclass(typ, Series) + assert isinstance(typ, Elements) + + assert not typ.is_fixed_size() + + assert len(typ()) == 0 # empty + assert len(typ(uint64(0))) == 1 # single arg + assert len(typ(uint64(i) for i in range(10))) == 10 # generator + assert len(typ(uint64(0), uint64(1), uint64(2))) == 3 # args + + v = typ(uint64(0)) + v[0] = uint64(123) + assert v[0] == 123 + assert isinstance(v[0], uint64) + + assert isinstance(v, List) + assert isinstance(v, List[uint64, 128]) + assert isinstance(v, typ) + assert isinstance(v, SSZValue) + assert isinstance(v, Series) + assert isinstance(v.type(), Elements) From 8bd2e878ef308c58da54f35bd8dd67c13756048c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 17 Jun 2019 01:39:39 +0200 Subject: [PATCH 14/76] bugfixes and typing improvements --- .../pyspec/eth2spec/utils/merkle_minimal.py | 10 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 18 +-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 69 ++++++---- .../eth2spec/utils/ssz/test_ssz_impl.py | 118 ++++++++++++++++++ .../eth2spec/utils/ssz/test_ssz_typing.py | 11 +- 5 files changed, 187 insertions(+), 39 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index ebfb4faf6..21583ee92 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -44,7 +44,7 @@ def next_power_of_two(v: int) -> int: return 1 << (v - 1).bit_length() -def merkleize_chunks(chunks, pad_to: int = None): +def merkleize_chunks(chunks, pad_to: int = 1): count = len(chunks) depth = max(count - 1, 0).bit_length() max_depth = max(depth, (pad_to - 1).bit_length()) @@ -55,7 +55,7 @@ def merkleize_chunks(chunks, pad_to: int = None): while True: if i & (1 << j) == 0: if i == count and j < depth: - h = hash(h + zerohashes[j]) + h = hash(h + zerohashes[j]) # keep going if we are complementing the void to the next power of 2 else: break else: @@ -63,11 +63,15 @@ def merkleize_chunks(chunks, pad_to: int = None): j += 1 tmp[j] = h + # merge in leaf by leaf. for i in range(count): merge(chunks[i], i) - merge(zerohashes[0], count) + # complement with 0 if empty, or if not the right power of 2 + if 1 << depth != count: + merge(zerohashes[0], count) + # the next power of two may be smaller than the ultimate virtual size, complement with zero-hashes at each depth. for j in range(depth, max_depth): tmp[j + 1] = hash(tmp[j] + zerohashes[j]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 679574891..1b59b276b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint ) # SSZ Serialization @@ -47,8 +47,8 @@ def serialize(obj: SSZValue): def encode_series(values: Series): # bytes and bytesN are already in the right format. - if isinstance(values, bytes): - return values + if isinstance(values, (Bytes, BytesN)): + return values.items # Recursively serialize parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] @@ -84,8 +84,8 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, bytes): - return values + if isinstance(values, (Bytes, BytesN)): + return values.items return b''.join([serialize_basic(value) for value in values]) @@ -101,8 +101,8 @@ def mix_in_length(root, length): def is_bottom_layer_kind(typ: SSZType): return ( - issubclass(typ, BasicType) or - (issubclass(typ, Elements) and issubclass(typ.elem_type, BasicType)) + isinstance(typ, BasicType) or + (issubclass(typ, Elements) and isinstance(typ.elem_type, BasicType)) ) @@ -114,7 +114,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: - if issubclass(typ, BasicType): + if isinstance(typ, BasicType): return 1 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 @@ -133,7 +133,7 @@ def hash_tree_root(obj: SSZValue): elif isinstance(obj, BasicValue): leaves = chunkify(serialize_basic(obj)) else: - raise Exception(f"Type not supported: {obj.type()}") + raise Exception(f"Type not supported: {type(obj)}") if isinstance(obj, (List, Bytes)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5cab68aa8..7662971c4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -92,6 +92,22 @@ class uint256(uint): byte_len = 32 +def coerce_type_maybe(v, typ: SSZType): + v_typ = type(v) + # shortcut if it's already the type we are looking for + if v_typ == typ: + return v + elif isinstance(v, int): + return typ(v) + elif isinstance(v, (list, tuple)): + return typ(*v) + elif isinstance(v, GeneratorType): + return typ(v) + else: + # just return as-is, Value-checkers will take care of it not being coerced. + return v + + class Series(SSZValue): def __iter__(self) -> Iterator[SSZValue]: @@ -108,7 +124,11 @@ class Container(Series, metaclass=SSZType): if f not in kwargs: setattr(self, f, t.default()) else: - setattr(self, f, kwargs[f]) + value = coerce_type_maybe(kwargs[f], t) + if not isinstance(value, t): + raise ValueCheckError(f"Bad input for class {self.__class__}:" + f" field: {f} type: {t} value: {value} value type: {type(value)}") + setattr(self, f, value) def serialize(self): from .ssz_impl import serialize @@ -141,23 +161,22 @@ class Container(Series, metaclass=SSZType): def __hash__(self): return hash(self.hash_tree_root()) - @classmethod - def get_fields_dict(cls) -> Dict[str, SSZType]: - return dict(cls.__annotations__) - @classmethod def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () return tuple((f, t) for f, t in cls.__annotations__.items()) - def get_typed_values(self): - return tuple(zip(self.get_field_values(), self.get_field_types())) - @classmethod - def get_field_names(cls) -> Tuple[str]: + def get_field_names(cls) -> Tuple[str, ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () return tuple(cls.__annotations__.keys()) @classmethod def get_field_types(cls) -> Tuple[SSZType, ...]: + if not hasattr(cls, '__annotations__'): # no container fields + return () # values of annotations are the types corresponding to the fields, not instance values. return tuple(cls.__annotations__.values()) @@ -233,12 +252,12 @@ class ParamsMeta(SSZType): return True -class Elements(ParamsMeta): +class ElementsType(ParamsMeta): elem_type: SSZType length: int -class ElementsBase(ParamsBase, metaclass=Elements): +class Elements(ParamsBase, metaclass=ElementsType): def __init__(self, *args): items = self.extract_args(*args) @@ -256,6 +275,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): x = list(args) if len(x) == 1 and isinstance(x[0], GeneratorType): x = list(x[0]) + x = [coerce_type_maybe(v, cls.elem_type) for v in x] return x def __str__(self): @@ -281,7 +301,7 @@ class ElementsBase(ParamsBase, metaclass=Elements): return self.items == other.items -class List(ElementsBase): +class List(Elements): @classmethod def default(cls): @@ -292,7 +312,7 @@ class List(ElementsBase): return False -class Vector(ElementsBase): +class Vector(Elements): @classmethod def value_check(cls, value): @@ -307,23 +327,26 @@ class Vector(ElementsBase): return cls.elem_type.is_fixed_size() -class BytesMeta(Elements): +class BytesType(ElementsType): elem_type: SSZType = byte length: int -class BytesLike(ElementsBase, metaclass=BytesMeta): +class BytesLike(Elements, metaclass=BytesType): @classmethod - def extract_args(cls, args): - if isinstance(args, bytes): - return args - elif isinstance(args, BytesLike): - return args.items - elif isinstance(args, GeneratorType): - return bytes(args) + def extract_args(cls, *args): + x = list(args) + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, BytesLike)): + x = x[0] + if isinstance(x, bytes): + return x + elif isinstance(x, BytesLike): + return x.items + elif isinstance(x, GeneratorType): + return bytes(x) else: - return bytes(args) + return bytes(x) @classmethod def value_check(cls, value): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py new file mode 100644 index 000000000..8dd04a86d --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -0,0 +1,118 @@ +from .ssz_impl import serialize, serialize_basic, encode_series, signing_root, hash_tree_root +from .ssz_typing import ( + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Bit, Container, List, Vector, Bytes, BytesN, + uint, uint8, uint16, uint32, uint64, uint128, uint256, byte +) + +import pytest + + +class EmptyTestStruct(Container): + pass + + +class SingleFieldTestStruct(Container): + A: byte + + +class SmallTestStruct(Container): + A: uint16 + B: uint16 + + +class FixedTestStruct(Container): + A: uint8 + B: uint64 + C: uint32 + + +class VarTestStruct(Container): + A: uint16 + B: List[uint16, 1024] + C: uint8 + + +class ComplexTestStruct(Container): + A: uint16 + B: List[uint16, 128] + C: uint8 + D: Bytes[256] + E: VarTestStruct + F: Vector[FixedTestStruct, 4] + G: Vector[VarTestStruct, 2] + + +sig_test_data = [0 for i in range(96)] +for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): + sig_test_data[k] = v + +test_data = [ + ("bool F", Bit(False), "00"), + ("bool T", Bit(True), "01"), + ("uint8 00", uint8(0x00), "00"), + ("uint8 01", uint8(0x01), "01"), + ("uint8 ab", uint8(0xab), "ab"), + ("uint16 0000", uint16(0x0000), "0000"), + ("uint16 abcd", uint16(0xabcd), "cdab"), + ("uint32 00000000", uint32(0x00000000), "00000000"), + ("uint32 01234567", uint32(0x01234567), "67452301"), + ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301"), + ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301"), + ("uint32 01234567", uint32(0x01234567), "67452301"), + ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000"), + ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301"), + ("sig", BytesN[96](*sig_test_data), + "0100000000000000000000000000000000000000000000000000000000000000" + "0200000000000000000000000000000000000000000000000000000000000000" + "03000000000000000000000000000000000000000000000000000000000000ff"), + ("emptyTestStruct", EmptyTestStruct(), ""), + ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab"), + ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412"), + ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff"), + ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff"), + ("varTestStruct some", VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + "cdab07000000ff010002000300"), + ("complexTestStruct", + ComplexTestStruct( + A=0xaabb, + B=List[uint16, 128](0x1122, 0x3344), + C=0xff, + D=Bytes[256](b"foobar"), + E=VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + F=Vector[FixedTestStruct, 4]( + FixedTestStruct(A=0xcc, B=0x4242424242424242, C=0x13371337), + FixedTestStruct(A=0xdd, B=0x3333333333333333, C=0xabcdabcd), + FixedTestStruct(A=0xee, B=0x4444444444444444, C=0x00112233), + FixedTestStruct(A=0xff, B=0x5555555555555555, C=0x44556677)), + G=Vector[VarTestStruct, 2]( + VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), + VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff)), + ), + "bbaa" + "47000000" # offset of B, []uint16 + "ff" + "4b000000" # offset of foobar + "51000000" # offset of E + "cc424242424242424237133713" + "dd3333333333333333cdabcdab" + "ee444444444444444433221100" + "ff555555555555555577665544" + "5e000000" # pointer to G + "22114433" # contents of B + "666f6f626172" # foobar + "cdab07000000ff010002000300" # contents of E + "08000000" "15000000" # [start G]: local offsets of [2]varTestStruct + "cdab07000000ff010002000300" + "cdab07000000ff010002000300", + ) +] + + +@pytest.mark.parametrize("name, value, serialized", test_data) +def test_serialize(name, value, serialized): + assert serialize(value) == bytes.fromhex(serialized) + + +@pytest.mark.parametrize("name, value, _", test_data) +def test_hash_tree_root(name, value, _): + hash_tree_root(value) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index a0705f8d3..e59d29b91 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,5 +1,5 @@ from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Vector, Bytes, BytesN, + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint, uint8, uint16, uint32, uint64, uint128, uint256 ) @@ -23,7 +23,8 @@ def test_subclasses(): assert not isinstance(c, BasicType) for c in [List, Vector, Bytes, BytesN]: - assert isinstance(c, Elements) + assert issubclass(c, Elements) + assert isinstance(c, ElementsType) def test_basic_instances(): @@ -109,7 +110,8 @@ def test_list(): assert issubclass(typ, List) assert issubclass(typ, SSZValue) assert issubclass(typ, Series) - assert isinstance(typ, Elements) + assert issubclass(typ, Elements) + assert isinstance(typ, ElementsType) assert not typ.is_fixed_size() @@ -128,4 +130,5 @@ def test_list(): assert isinstance(v, typ) assert isinstance(v, SSZValue) assert isinstance(v, Series) - assert isinstance(v.type(), Elements) + assert issubclass(v.type(), Elements) + assert isinstance(v.type(), ElementsType) From 0a43003b426b581de664ff27bc0e5e766a960b29 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 17 Jun 2019 02:11:50 +0200 Subject: [PATCH 15/76] minor test improvements --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index e59d29b91..99416e333 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -93,8 +93,9 @@ def test_container(): assert not Bar.is_fixed_size() - y = Bar(a=uint8(123), b=List[uint8, 1024](uint8(1), uint8(2))) + y = Bar(a=123, b=List[uint8, 1024](uint8(1), uint8(2))) assert y.a == 123 + assert isinstance(y.a, uint8) assert len(y.b) == 2 assert isinstance(y.a, uint8) assert isinstance(y.b, List[uint8, 1024]) @@ -119,6 +120,8 @@ def test_list(): assert len(typ(uint64(0))) == 1 # single arg assert len(typ(uint64(i) for i in range(10))) == 10 # generator assert len(typ(uint64(0), uint64(1), uint64(2))) == 3 # args + assert isinstance(typ(1, 2, 3, 4, 5)[4], uint64) # coercion + assert isinstance(typ(i for i in range(10))[9], uint64) # coercion in generator v = typ(uint64(0)) v[0] = uint64(123) From b89183ae69faf82a1eec798928da7c358d759621 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:05:34 +0200 Subject: [PATCH 16/76] Update spec for new SSZ with list max length --- specs/core/0_beacon-chain.md | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ddc4c8f61..44c20ec84 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,6 +224,7 @@ These configurations are updated for releases, but may be out of sync during `de | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | +| `HISTORICAL_ROOTS_LENGTH` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | @@ -237,6 +238,12 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | +| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | +| `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | + +* Assuming the maximum 16 deposits per slot, the validator registry will last for at least 13,065 years (but likely much much longer) ### Rewards and penalties @@ -357,8 +364,8 @@ class AttestationDataAndCustodyBit(Container): ```python class IndexedAttestation(Container): - custody_bit_0_indices: List[ValidatorIndex] # Indices with custody bit equal to 0 - custody_bit_1_indices: List[ValidatorIndex] # Indices with custody bit equal to 1 + custody_bit_0_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 0 + custody_bit_1_indices: List[ValidatorIndex, MAX_INDICES_PER_ATTESTATION] # Indices with custody bit equal to 1 data: AttestationData signature: BLSSignature ``` @@ -513,14 +520,14 @@ class BeaconState(Container): latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash] + historical_roots: List[Hash, HISTORICAL_ROOTS_LENGTH] # Eth1 eth1_data: Eth1Data - eth1_data_votes: List[Eth1Data] + eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] eth1_deposit_index: uint64 # Registry - validators: List[Validator] - balances: List[Gwei] + validators: List[Validator, VALIDATOR_REGISTRY_SIZE] + balances: List[Gwei, VALIDATOR_REGISTRY_SIZE] # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] @@ -528,8 +535,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations - previous_epoch_attestations: List[PendingAttestation] - current_epoch_attestations: List[PendingAttestation] + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -984,8 +991,6 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 - # Verify max number of indices - assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted @@ -1624,6 +1629,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) +<<<<<<< HEAD all_operations = [ (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), @@ -1634,6 +1640,17 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ] # type: List[Tuple[List[Container], int, Callable]] for operations, max_operations, function in all_operations: assert len(operations) <= max_operations +======= + + for operations, max_operations, function in ( + (body.proposer_slashings, process_proposer_slashing), + (body.attester_slashings, process_attester_slashing), + (body.attestations, process_attestation), + (body.deposits, process_deposit), + (body.voluntary_exits, process_voluntary_exit), + (body.transfers, process_transfer), + ): +>>>>>>> f6a2345f... Update spec for new SSZ with list max length for operation in operations: function(state, operation) ``` From 4c2adcc5e643279c462ba3d25ce474ff4bb741f2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 17 Jun 2019 19:07:20 -0400 Subject: [PATCH 17/76] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 44c20ec84..8f2078956 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -243,8 +243,6 @@ These configurations are updated for releases, but may be out of sync during `de | `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | -* Assuming the maximum 16 deposits per slot, the validator registry will last for at least 13,065 years (but likely much much longer) - ### Rewards and penalties | Name | Value | @@ -991,6 +989,8 @@ def validate_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify no index has custody bit equal to 1 [to be removed in phase 1] assert len(bit_1_indices) == 0 + # Verify max number of indices + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION # Verify index sets are disjoint assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 # Verify indices are sorted From 73ba419d640920a1427c357e487625ba70ae87b4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:04:10 +0200 Subject: [PATCH 18/76] check virtual lengths, fix imports --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7662971c4..efcb8f207 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -268,7 +268,7 @@ class Elements(ParamsBase, metaclass=ElementsType): @classmethod def value_check(cls, value): - return all(isinstance(v, cls.elem_type) for v in value) + return all(isinstance(v, cls.elem_type) for v in value) and len(value) <= cls.length @classmethod def extract_args(cls, *args): @@ -316,6 +316,7 @@ class Vector(Elements): @classmethod def value_check(cls, value): + # check length limit strictly return len(value) == cls.length and super().value_check(value) @classmethod @@ -350,7 +351,8 @@ class BytesLike(Elements, metaclass=BytesType): @classmethod def value_check(cls, value): - return isinstance(value, bytes) + # check type and virtual length limit + return isinstance(value, bytes) and len(value) <= cls.length def __str__(self): cls = self.__class__ @@ -376,6 +378,7 @@ class BytesN(BytesLike): @classmethod def value_check(cls, value): + # check length limit strictly return len(value) == cls.length and super().value_check(value) @classmethod From 8c6ddd5233d51bc8937dc6a7e84d8b95f381fc80 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:18:00 +0200 Subject: [PATCH 19/76] container field coercion --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 12 +++++++++++- .../pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index efcb8f207..942095622 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Dict, Iterator +from typing import Tuple, Iterator from types import GeneratorType @@ -142,6 +142,16 @@ class Container(Series, metaclass=SSZType): from .ssz_impl import signing_root return signing_root(self) + def __setattr__(self, name, value): + if name not in self.__class__.__annotations__: + raise AttributeError("Cannot change non-existing SSZ-container attribute") + field_typ = self.__class__.__annotations__[name] + value = coerce_type_maybe(value, field_typ) + if not isinstance(value, field_typ): + raise ValueCheckError(f"Cannot set field of {self.__class__}:" + f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") + super().__setattr__(name, value) + def get_field_values(self) -> Tuple[SSZValue, ...]: cls = self.__class__ return tuple(getattr(self, field) for field in cls.get_field_names()) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 99416e333..291c4b955 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -105,6 +105,20 @@ def test_container(): assert v.type().elem_type == uint8 assert v.type().length == 1024 + y.a = 42 + y.a = uint16(255) + try: + y.a = uint16(256) + assert False + except ValueError: + pass + + try: + y.not_here = 5 + assert False + except AttributeError: + pass + def test_list(): typ = List[uint64, 128] From 4aefc078e9e6cd9a11757681c4cb55abc2b908dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 18 Jun 2019 02:54:08 +0200 Subject: [PATCH 20/76] list-rework type fixes --- specs/core/0_beacon-chain.md | 23 +++------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++++---- .../eth2spec/utils/ssz/test_ssz_typing.py | 43 ++++++++++++++++++- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f2078956..a505fb451 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -241,7 +241,7 @@ These configurations are updated for releases, but may be out of sync during `de | `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `VALIDATOR_REGISTRY_SIZE` | `2**40 (= 1,099,511,627,776)` | | | +| `VALIDATOR_REGISTRY_SIZE` | `2**40` (= 1,099,511,627,776) | | | ### Rewards and penalties @@ -372,7 +372,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] # Bit set for every attesting participant within a committee + aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -439,9 +439,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_COMMITTEE_SIZE] + aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] data: AttestationData - custody_bitfield: Bytes[MAX_COMMITTEE_SIZE] + custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] signature: BLSSignature ``` @@ -1629,20 +1629,8 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) -<<<<<<< HEAD - all_operations = [ - (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), - (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), - (body.attestations, MAX_ATTESTATIONS, process_attestation), - (body.deposits, MAX_DEPOSITS, process_deposit), - (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), - (body.transfers, MAX_TRANSFERS, process_transfer), - ] # type: List[Tuple[List[Container], int, Callable]] - for operations, max_operations, function in all_operations: - assert len(operations) <= max_operations -======= - for operations, max_operations, function in ( + for operations, function in ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), @@ -1650,7 +1638,6 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), ): ->>>>>>> f6a2345f... Update spec for new SSZ with list max length for operation in operations: function(state, operation) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 942095622..5aadfae32 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -2,10 +2,6 @@ from typing import Tuple, Iterator from types import GeneratorType -class ValueCheckError(Exception): - pass - - class DefaultingTypeMeta(type): def default(cls): raise Exception("Not implemented") @@ -97,7 +93,7 @@ def coerce_type_maybe(v, typ: SSZType): # shortcut if it's already the type we are looking for if v_typ == typ: return v - elif isinstance(v, int): + elif isinstance(v, int) and not isinstance(v, uint): # do not coerce from one uintX to another uintY return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) @@ -126,7 +122,7 @@ class Container(Series, metaclass=SSZType): else: value = coerce_type_maybe(kwargs[f], t) if not isinstance(value, t): - raise ValueCheckError(f"Bad input for class {self.__class__}:" + raise ValueError(f"Bad input for class {self.__class__}:" f" field: {f} type: {t} value: {value} value type: {type(value)}") setattr(self, f, value) @@ -148,7 +144,7 @@ class Container(Series, metaclass=SSZType): field_typ = self.__class__.__annotations__[name] value = coerce_type_maybe(value, field_typ) if not isinstance(value, field_typ): - raise ValueCheckError(f"Cannot set field of {self.__class__}:" + raise ValueError(f"Cannot set field of {self.__class__}:" f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) @@ -273,7 +269,7 @@ class Elements(ParamsBase, metaclass=ElementsType): items = self.extract_args(*args) if not self.value_check(items): - raise ValueCheckError(f"Bad input for class {self.__class__}: {items}") + raise ValueError(f"Bad input for class {self.__class__}: {items}") self.items = items @classmethod @@ -296,6 +292,16 @@ class Elements(ParamsBase, metaclass=ElementsType): return self.items[i] def __setitem__(self, k, v): + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self.items): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") + typ = self.__class__.elem_type + v = coerce_type_maybe(v, typ) + if not isinstance(v, typ): + raise ValueError(f"Cannot set item in type {self.__class__}," + f" mismatched element type: {v} of {type(v)}, expected {typ}") self.items[k] = v def __len__(self): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 291c4b955..0f4a06c5f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -106,9 +106,14 @@ def test_container(): assert v.type().length == 1024 y.a = 42 - y.a = uint16(255) try: - y.a = uint16(256) + y.a = 256 # out of bounds + assert False + except ValueError: + pass + + try: + y.a = uint16(255) # within bounds, wrong type assert False except ValueError: pass @@ -149,3 +154,37 @@ def test_list(): assert isinstance(v, Series) assert issubclass(v.type(), Elements) assert isinstance(v.type(), ElementsType) + + foo = List[uint32, 128](0 for i in range(128)) + foo[0] = 123 + foo[1] = 654 + foo[127] = 222 + assert sum(foo) == 999 + try: + foo[3] = 2**32 # out of bounds + except ValueError: + pass + + try: + foo[3] = uint64(2**32 - 1) # within bounds, wrong type + assert False + except ValueError: + pass + + try: + foo[128] = 100 + assert False + except IndexError: + pass + + try: + foo[-1] = 100 # valid in normal python lists + assert False + except IndexError: + pass + + try: + foo[128] = 100 # out of bounds + assert False + except IndexError: + pass From 439e4d485938b07a9cc52b5e7745d052245569b4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:12:17 +0200 Subject: [PATCH 21/76] Build spec --- scripts/build_spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 3a1b22efd..61613376f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,8 +26,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, - Bytes4, Bytes32, Bytes48, Bytes96, + Bit, Container, List, Vector, Bytes, BytesN, uint64 ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -179,8 +178,9 @@ def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str """ items = list(objects.items()) for key, value in items: - dependencies = re.findall(r'(: [A-Z][\w[]*)', value) - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|Bytes\d+|bytes', '', x), dependencies) + dependencies = re.findall(r'(: [A-Z][\w\[]*)', value) + dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants + dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|BytesN\[.+\\]|Bytes\[.+\\]|Bytes\d*|bytes\d*', '', x), dependencies) for dep in dependencies: if dep in custom_types or len(dep) == 0: continue From c9747b634f1cca8994af1b97207769b222b4165e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:13:55 +0200 Subject: [PATCH 22/76] improve build spec, get clean dependencies list --- scripts/build_spec.py | 48 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 61613376f..ca648a754 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,12 +12,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, - Callable, - Dict, - List, - Set, - Tuple, + Any, Callable, Iterable, Dict, Set, Tuple ) from eth2spec.utils.ssz.ssz_impl import ( @@ -33,18 +28,14 @@ from eth2spec.utils.bls import ( bls_verify, bls_verify_multiple, ) -# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation. from eth2spec.utils.hash_function import hash + + +Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, + Any, Callable, Dict, Optional, Set, Tuple, Iterable ) from eth2spec.utils.ssz.ssz_impl import ( @@ -54,8 +45,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - # unused: uint8, uint16, uint32, uint128, uint256, - uint64, Container, Vector, + Bit, Container, List, Vector, Bytes, BytesN, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -65,8 +55,10 @@ from eth2spec.utils.bls import ( ) from eth2spec.utils.hash_function import hash + + +Deltas = list ''' -BYTE_TYPES = [4, 32, 48, 96] SUNDRY_FUNCTIONS = ''' def get_ssz_type_by_name(name: str) -> Container: return globals()[name] @@ -172,18 +164,32 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st return old_constants +ignored_dependencies = [ + 'Bit', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', + 'bytes' # to be removed after updating spec doc +] + + def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None: """ Determines which SSZ Object is depenedent on which other and orders them appropriately """ items = list(objects.items()) for key, value in items: - dependencies = re.findall(r'(: [A-Z][\w\[]*)', value) + dependencies = [] + for line in value.split('\n'): + if not re.match(r'\s+\w+: .+', line): + continue # skip whitespace etc. + line = line[line.index(':') + 1:] # strip of field name + if '#' in line: + line = line[:line.index('#')] # strip of comment + dependencies.extend(re.findall(r'(\w+)', line)) # catch all legible words, potential dependencies dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies) # filter out constants - dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|BytesN\[.+\\]|Bytes\[.+\\]|Bytes\d*|bytes\d*', '', x), dependencies) + dependencies = filter(lambda x: x not in ignored_dependencies, dependencies) + dependencies = filter(lambda x: x not in custom_types, dependencies) for dep in dependencies: - if dep in custom_types or len(dep) == 0: - continue key_list = list(objects.keys()) for item in [dep, key] + key_list[key_list.index(dep)+1:]: objects[item] = objects.pop(item) From 8344d50ae5ceaa20998eda8f8f063dc1f53e7f03 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:15:48 +0200 Subject: [PATCH 23/76] update beacon chain doc, use new types, avoid List --- specs/core/0_beacon-chain.md | 64 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a505fb451..7cbb9b67b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -291,6 +291,8 @@ We define the following Python custom types for type hinting and readability: | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | +`Deltas` is a non-SSZ type, a series of changes applied to balances, optimized by clients. + ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. @@ -315,7 +317,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: bool + slashed: Bit # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -355,7 +357,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: bool # Challengeable bit for the custody of crosslink data + custody_bit: Bit # Challengeable bit for the custody of crosslink data ``` #### `IndexedAttestation` @@ -649,11 +651,11 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Tuple[ValidatorIndex, ...]: """ Get active validator indices at ``epoch``. """ - return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] + return tuple(ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)) ``` ### `increase_balance` @@ -815,7 +817,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: List[Hash], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, proof: Tuple[Hash, ...], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -859,16 +861,17 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V ### `compute_committee` ```python -def compute_committee(indices: List[ValidatorIndex], seed: Hash, index: int, count: int) -> List[ValidatorIndex]: +def compute_committee(indices: Tuple[ValidatorIndex, ...], + seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] + return tuple(indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)) ``` ### `get_crosslink_committee` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> List[ValidatorIndex]: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: return compute_committee( indices=get_active_validator_indices(state, epoch), seed=generate_seed(state, epoch), @@ -882,13 +885,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> L ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: + bitfield: bytes) -> Tuple[ValidatorIndex, ...]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return tuple(sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])) ``` ### `int_to_bytes` @@ -908,7 +911,7 @@ def bytes_to_int(data: bytes) -> int: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, indices: Iterable[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ @@ -1134,7 +1137,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1151,7 +1154,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: # Process deposits state = BeaconState() for deposit in deposits: @@ -1173,10 +1176,10 @@ def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, - eth1_data=genesis_eth1_data, + eth1_data=eth1_data, latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), ) @@ -1270,21 +1273,21 @@ def get_total_active_balance(state: BeaconState) -> Gwei: ``` ```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: - return [ +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: + return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) ] ``` ```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) @@ -1293,7 +1296,7 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[Pen ```python def get_unslashed_attesting_indices(state: BeaconState, - attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + attestations: Iterable[PendingAttestation]) -> Iterable[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) @@ -1301,14 +1304,14 @@ def get_unslashed_attesting_indices(state: BeaconState, ``` ```python -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: +def get_attesting_balance(state: BeaconState, attestations: Iterable[PendingAttestation]) -> Gwei: return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, - shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: + shard: Shard) -> Tuple[Crosslink, Iterable[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1397,11 +1400,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: +def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = [Gwei(0) for _ in range(len(state.validators))] - penalties = [Gwei(0) for _ in range(len(state.validators))] + rewards = Deltas(0 for _ in range(len(state.validators))) + penalties = Deltas(0 for _ in range(len(state.validators))) eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1448,9 +1451,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: ``` ```python -def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - rewards = [Gwei(0) for index in range(len(state.validators))] - penalties = [Gwei(0) for index in range(len(state.validators))] +def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: + rewards = Deltas(0 for _ in range(len(state.validators))) + penalties = Deltas(0 for _ in range(len(state.validators))) epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) @@ -1630,14 +1633,15 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) - for operations, function in ( + all_operations = ( (body.proposer_slashings, process_proposer_slashing), (body.attester_slashings, process_attester_slashing), (body.attestations, process_attestation), (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ): + ) # type: Tuple[Tuple[List, Callable], ...] + for operations, function in all_operations: for operation in operations: function(state, operation) ``` From 4b4bf87e476769d56227bdd3dc2486a6886947fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 00:55:01 +0200 Subject: [PATCH 24/76] update shard doc, use new types, avoid List --- specs/core/1_shard-data-chains.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b83cd54f7..79e2478fe 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -154,7 +154,7 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> List[ValidatorIndex]: + slot: Slot) -> Tuple[ValidatorIndex, ...]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ @@ -175,10 +175,10 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return sorted(list(set( + return tuple(sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - ))) + )))) ``` ### `get_shard_proposer_index` @@ -284,7 +284,7 @@ Let: ```python def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, - valid_shard_blocks: List[ShardBlock], + valid_shard_blocks: Iterable[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -348,7 +348,7 @@ Let: * `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python -def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], +def is_valid_shard_attestation(valid_shard_blocks: Iterable[ShardBlock], beacon_state: BeaconState, candidate: ShardAttestation) -> bool: # Check shard block @@ -380,7 +380,7 @@ Let: def is_valid_beacon_attestation(shard: Shard, shard_blocks: List[ShardBlock], beacon_state: BeaconState, - valid_attestations: List[Attestation], + valid_attestations: Set[Attestation], candidate: Attestation) -> bool: # Check if attestation is already determined valid for _, attestation in enumerate(valid_attestations): From 5be0c57aade56b95c0d8b8b2d54d437cdc14ddb4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 01:03:04 +0200 Subject: [PATCH 25/76] fix linting + mypy --- scripts/build_spec.py | 6 +++--- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 13 +++++++++--- .../eth2spec/utils/ssz/test_ssz_impl.py | 6 +++--- .../eth2spec/utils/ssz/test_ssz_typing.py | 21 ++++++++++--------- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index ca648a754..20e90ee33 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -20,8 +20,8 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - # unused: uint8, uint16, uint32, uint128, uint256, - Bit, Container, List, Vector, Bytes, BytesN, uint64 + Bit, Container, List, Vector, Bytes, uint64, + Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -45,7 +45,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, BytesN, uint64, + Bit, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1b59b276b..fd17e29f9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint ) # SSZ Serialization diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5aadfae32..082e3ed30 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -123,7 +123,7 @@ class Container(Series, metaclass=SSZType): value = coerce_type_maybe(kwargs[f], t) if not isinstance(value, t): raise ValueError(f"Bad input for class {self.__class__}:" - f" field: {f} type: {t} value: {value} value type: {type(value)}") + f" field: {f} type: {t} value: {value} value type: {type(value)}") setattr(self, f, value) def serialize(self): @@ -145,7 +145,7 @@ class Container(Series, metaclass=SSZType): value = coerce_type_maybe(value, field_typ) if not isinstance(value, field_typ): raise ValueError(f"Cannot set field of {self.__class__}:" - f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") + f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) def get_field_values(self) -> Tuple[SSZValue, ...]: @@ -301,7 +301,7 @@ class Elements(ParamsBase, metaclass=ElementsType): v = coerce_type_maybe(v, typ) if not isinstance(v, typ): raise ValueError(f"Cannot set item in type {self.__class__}," - f" mismatched element type: {v} of {type(v)}, expected {typ}") + f" mismatched element type: {v} of {type(v)}, expected {typ}") self.items[k] = v def __len__(self): @@ -400,3 +400,10 @@ class BytesN(BytesLike): @classmethod def is_fixed_size(cls): return True + + +# Helpers for common BytesN types. +Bytes4 = BytesN[4] +Bytes32 = BytesN[32] +Bytes48 = BytesN[48] +Bytes96 = BytesN[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 8dd04a86d..ae0849098 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ -from .ssz_impl import serialize, serialize_basic, encode_series, signing_root, hash_tree_root +from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256, byte + Bit, Container, List, Vector, Bytes, BytesN, + uint8, uint16, uint32, uint64, byte ) import pytest diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 0f4a06c5f..f604b6468 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,5 +1,6 @@ from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, + SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, + Elements, Bit, Container, List, Vector, Bytes, BytesN, uint, uint8, uint16, uint32, uint64, uint128, uint256 ) @@ -41,13 +42,13 @@ def test_basic_instances(): def test_basic_value_bounds(): max = { - Bit: 2**1, - uint8: 2**(8 * 1), - uint16: 2**(8 * 2), - uint32: 2**(8 * 4), - uint64: 2**(8 * 8), - uint128: 2**(8 * 16), - uint256: 2**(8 * 32), + Bit: 2 ** 1, + uint8: 2 ** (8 * 1), + uint16: 2 ** (8 * 2), + uint32: 2 ** (8 * 4), + uint64: 2 ** (8 * 8), + uint128: 2 ** (8 * 16), + uint256: 2 ** (8 * 32), } for k, v in max.items(): # this should work @@ -161,12 +162,12 @@ def test_list(): foo[127] = 222 assert sum(foo) == 999 try: - foo[3] = 2**32 # out of bounds + foo[3] = 2 ** 32 # out of bounds except ValueError: pass try: - foo[3] = uint64(2**32 - 1) # within bounds, wrong type + foo[3] = uint64(2 ** 32 - 1) # within bounds, wrong type assert False except ValueError: pass From 6f46c1d837646bd6906845253c13f2e6ee1d2d72 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 01:12:51 +0200 Subject: [PATCH 26/76] fix typing in spec builder monkey patch --- scripts/build_spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 20e90ee33..7d42c218e 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -66,13 +66,13 @@ def get_ssz_type_by_name(name: str) -> Container: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], List[ValidatorIndex]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} -def compute_committee(indices: List[ValidatorIndex], # type: ignore +def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore seed: Hash, index: int, - count: int) -> List[ValidatorIndex]: + count: int) -> Tuple[ValidatorIndex, ...]: param_hash = (hash_tree_root(indices), seed, index, count) if param_hash not in committee_cache: From 6b82e3faa56707b0c1fd11cd82e2f8eb36037370 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:20:07 +0200 Subject: [PATCH 27/76] Modifications from Vitalik, to enable SSZ Partials --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 54 ++++++++----------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index fd17e29f9..a9c36649b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -107,7 +107,7 @@ def is_bottom_layer_kind(typ: SSZType): def item_length(typ: SSZType) -> int: - if issubclass(typ, BasicType): + if issubclass(typ, BasicValue): return typ.byte_len else: return 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 082e3ed30..981f30d9b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,6 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bit(BasicValue): # can't subclass bool. + byte_len = 1 def __new__(cls, value, *args, **kwargs): if value < 0 or value > 1: @@ -88,7 +89,7 @@ class uint256(uint): byte_len = 32 -def coerce_type_maybe(v, typ: SSZType): +def coerce_type_maybe(v, typ: SSZType, strict: bool = False): v_typ = type(v) # shortcut if it's already the type we are looking for if v_typ == typ: @@ -97,10 +98,14 @@ def coerce_type_maybe(v, typ: SSZType): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) + elif isinstance(v, bytes): + return typ(v) elif isinstance(v, GeneratorType): return typ(v) else: # just return as-is, Value-checkers will take care of it not being coerced. + if strict and not isinstance(v, typ): + raise ValueError("Type coercion of {} to {} failed".format(v, typ)) return v @@ -116,7 +121,7 @@ class Container(Series, metaclass=SSZType): def __init__(self, **kwargs): cls = self.__class__ - for f, t in cls.get_fields(): + for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, t.default()) else: @@ -148,16 +153,12 @@ class Container(Series, metaclass=SSZType): f" field: {name} type: {field_typ} value: {value} value type: {type(value)}") super().__setattr__(name, value) - def get_field_values(self) -> Tuple[SSZValue, ...]: - cls = self.__class__ - return tuple(getattr(self, field) for field in cls.get_field_names()) - def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_field_names()}) + return repr({field: getattr(self, field) for field in self.get_fields()}) def __str__(self): output = [f'{self.__class__.__name__}'] - for field in self.get_field_names(): + for field in self.get_fields(): output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) @@ -168,23 +169,10 @@ class Container(Series, metaclass=SSZType): return hash(self.hash_tree_root()) @classmethod - def get_fields(cls) -> Tuple[Tuple[str, SSZType], ...]: + def get_fields(cls) -> Dict[str, SSZType]: if not hasattr(cls, '__annotations__'): # no container fields - return () - return tuple((f, t) for f, t in cls.__annotations__.items()) - - @classmethod - def get_field_names(cls) -> Tuple[str, ...]: - if not hasattr(cls, '__annotations__'): # no container fields - return () - return tuple(cls.__annotations__.keys()) - - @classmethod - def get_field_types(cls) -> Tuple[SSZType, ...]: - if not hasattr(cls, '__annotations__'): # no container fields - return () - # values of annotations are the types corresponding to the fields, not instance values. - return tuple(cls.__annotations__.values()) + return {} + return dict(cls.__annotations__) @classmethod def default(cls): @@ -195,7 +183,7 @@ class Container(Series, metaclass=SSZType): return all(t.is_fixed_size() for t in cls.get_field_types()) def __iter__(self) -> Iterator[SSZValue]: - return iter(self.get_field_values()) + return iter([getattr(self, field) for field in self.get_fields()]) class ParamsBase(Series): @@ -297,12 +285,16 @@ class Elements(ParamsBase, metaclass=ElementsType): if k > len(self.items): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") - typ = self.__class__.elem_type - v = coerce_type_maybe(v, typ) - if not isinstance(v, typ): - raise ValueError(f"Cannot set item in type {self.__class__}," - f" mismatched element type: {v} of {type(v)}, expected {typ}") - self.items[k] = v + self.items[k] = coerce_type_maybe(v, self.__class__.elem_type, strict=True) + + def append(self, v): + self.items.append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + + def pop(self): + if len(self.items) == 0: + raise IndexError("Pop from empty list") + else: + return self.items.pop() def __len__(self): return len(self.items) From 5048b9e87a1f260c4c307100e489407adcade2ac Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:14:13 +0200 Subject: [PATCH 28/76] temporary fix for phase-1 spec typing --- scripts/build_spec.py | 3 ++- specs/core/1_custody-game.md | 34 +++++++++++++++++++------------ specs/core/1_shard-data-chains.md | 24 ++++++++++++++-------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7d42c218e..612edbd00 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -35,7 +35,8 @@ from eth2spec.utils.hash_function import hash Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable + Any, Callable, Dict, Optional, Set, Tuple, Iterable, + List as TypingList ) from eth2spec.utils.ssz.ssz_impl import ( diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 3fe132c07..99c85a034 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -113,6 +113,13 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | - | - | | `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | + +### TODO PLACEHOLDER + +| Name | Value | +| - | - | +| `PLACEHOLDER` | `2**32` | + ## Data structures ### Custody objects @@ -134,7 +141,7 @@ class CustodyBitChallenge(Container): attestation: Attestation challenger_index: ValidatorIndex responder_key: BLSSignature - chunk_bits: bytes + chunk_bits: Bytes[PLACEHOLDER] signature: BLSSignature ``` @@ -171,9 +178,9 @@ class CustodyBitChallengeRecord(Container): class CustodyResponse(Container): challenge_index: uint64 chunk_index: uint64 - chunk: Vector[bytes, BYTES_PER_CUSTODY_CHUNK] - data_branch: List[Bytes32] - chunk_bits_branch: List[Bytes32] + chunk: Vector[Bytes[PLACEHOLDER], BYTES_PER_CUSTODY_CHUNK] + data_branch: List[Bytes32, PLACEHOLDER] + chunk_bits_branch: List[Bytes32, PLACEHOLDER] chunk_bits_leaf: Bytes32 ``` @@ -226,24 +233,25 @@ class Validator(Container): ```python class BeaconState(Container): - custody_chunk_challenge_records: List[CustodyChunkChallengeRecord] - custody_bit_challenge_records: List[CustodyBitChallengeRecord] + custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, PLACEHOLDER] + custody_bit_challenge_records: List[CustodyBitChallengeRecord, PLACEHOLDER] custody_challenge_index: uint64 # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - exposed_derived_secrets: Vector[List[ValidatorIndex], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + exposed_derived_secrets: Vector[List[ValidatorIndex, PLACEHOLDER], + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] ``` #### `BeaconBlockBody` ```python class BeaconBlockBody(Container): - custody_chunk_challenges: List[CustodyChunkChallenge] - custody_bit_challenges: List[CustodyBitChallenge] - custody_responses: List[CustodyResponse] - custody_key_reveals: List[CustodyKeyReveal] - early_derived_secret_reveals: List[EarlyDerivedSecretReveal] + custody_chunk_challenges: List[CustodyChunkChallenge, PLACEHOLDER] + custody_bit_challenges: List[CustodyBitChallenge, PLACEHOLDER] + custody_responses: List[CustodyResponse, PLACEHOLDER] + custody_key_reveals: List[CustodyKeyReveal, PLACEHOLDER] + early_derived_secret_reveals: List[EarlyDerivedSecretReveal, PLACEHOLDER] ``` ## Helpers @@ -310,7 +318,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: List[Any], new_element: Any) -> int: +def replace_empty_or_append(list: TypingList[Any], new_element: Any) -> int: for i in range(len(list)): if is_empty(list[i]): list[i] = new_element diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 79e2478fe..bdf6550ff 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -70,13 +70,19 @@ This document describes the shard data layer and the shard fork choice rule in P | `DOMAIN_SHARD_PROPOSER` | `128` | | `DOMAIN_SHARD_ATTESTER` | `129` | +### TODO PLACEHOLDER + +| Name | Value | +| - | - | +| `PLACEHOLDER` | `2**32` | + ## Data structures ### `ShardBlockBody` ```python class ShardBlockBody(Container): - data: Vector[bytes, BYTES_PER_SHARD_BLOCK_BODY] + data: Vector[Bytes[PLACEHOLDER], BYTES_PER_SHARD_BLOCK_BODY] ``` ### `ShardAttestation` @@ -87,7 +93,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: bytes + aggregation_bitfield: Bytes[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -101,7 +107,7 @@ class ShardBlock(Container): parent_root: Bytes32 data: ShardBlockBody state_root: Bytes32 - attestations: List[ShardAttestation] + attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -115,7 +121,7 @@ class ShardBlockHeader(Container): parent_root: Bytes32 body_root: Bytes32 state_root: Bytes32 - attestations: List[ShardAttestation] + attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -128,7 +134,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, - count: int) -> List[ValidatorIndex]: + count: int) -> Tuple[ValidatorIndex, ...]: """ Return committee for a period. Used to construct persistent committees. """ @@ -243,11 +249,11 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Iterable[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) - def pad_to_power_of_2(values: List[bytes]) -> List[bytes]: + def pad_to_power_of_2(values: TypingList[bytes]) -> TypingList[bytes]: while not is_power_of_two(len(values)): values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] return values @@ -282,7 +288,7 @@ Let: * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: List[BeaconBlock], +def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: Iterable[ShardBlock], candidate: ShardBlock) -> bool: @@ -378,7 +384,7 @@ Let: ```python def is_valid_beacon_attestation(shard: Shard, - shard_blocks: List[ShardBlock], + shard_blocks: TypingList[ShardBlock], beacon_state: BeaconState, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: From a33c67894ebd9593fea94d38fd6914a3725ffe04 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:37:22 +0200 Subject: [PATCH 29/76] update ssz testing/debug utils --- test_libs/pyspec/eth2spec/debug/decode.py | 44 +++---- test_libs/pyspec/eth2spec/debug/encode.py | 34 +++--- .../pyspec/eth2spec/debug/random_value.py | 112 ++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 5 +- 4 files changed, 84 insertions(+), 111 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 5ce116025..743479371 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,39 +1,29 @@ +from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, - is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, - read_vector_elem_type, read_list_elem_type, + SSZType, SSZValue, uint, Container, Bytes, List, Bit, Vector, BytesN ) -def decode(data, typ): - if is_uint_type(typ): - return data - elif is_bool_type(typ): - assert data in (True, False) - return data - elif is_list_type(typ): - elem_typ = read_list_elem_type(typ) - return [decode(element, elem_typ) for element in data] - elif is_vector_type(typ): - elem_typ = read_vector_elem_type(typ) - return Vector(decode(element, elem_typ) for element in data) - elif is_bytes_type(typ): - return bytes.fromhex(data[2:]) - elif is_bytesn_type(typ): - return BytesN(bytes.fromhex(data[2:])) - elif is_container_type(typ): +def decode(data: Any, typ: SSZType) -> SSZValue: + if issubclass(typ, (uint, Bit)): + return typ(data) + elif issubclass(typ, (List, Vector)): + return typ(decode(element, typ.elem_type) for element in data) + elif issubclass(typ, (Bytes, BytesN)): + return typ(bytes.fromhex(data[2:])) + elif issubclass(typ, Container): temp = {} - for field, subtype in typ.get_fields(): - temp[field] = decode(data[field], subtype) - if field + "_hash_tree_root" in data: - assert(data[field + "_hash_tree_root"][2:] == - hash_tree_root(temp[field], subtype).hex()) + for field_name, field_type in typ.get_fields().items(): + temp[field_name] = decode(data[field_name], field_type) + if field_name + "_hash_tree_root" in data: + assert (data[field_name + "_hash_tree_root"][2:] == + hash_tree_root(temp[field_name]).hex()) ret = typ(**temp) if "hash_tree_root" in data: - assert(data["hash_tree_root"][2:] == - hash_tree_root(ret, typ).hex()) + assert (data["hash_tree_root"][2:] == + hash_tree_root(ret).hex()) return ret else: raise Exception(f"Type not recognized: data={data}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 61dd87928..d264bd7ff 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,36 +1,30 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, is_vector_type, is_container_type, - read_elem_type, - uint + SSZValue, uint, Container, Bytes, BytesN, List, Vector, Bit ) -def encode(value, typ, include_hash_tree_roots=False): - if is_uint_type(typ): - if hasattr(typ, '__supertype__'): - typ = typ.__supertype__ +def encode(value: SSZValue, include_hash_tree_roots=False): + if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length - if issubclass(typ, uint) and typ.byte_len > 8: + if value.type().byte_len > 8: return str(value) return value - elif is_bool_type(typ): + elif isinstance(value, Bit): assert value in (True, False) return value - elif is_list_type(typ) or is_vector_type(typ): - elem_typ = read_elem_type(typ) - return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif isinstance(typ, type) and issubclass(typ, bytes): # both bytes and BytesN + elif isinstance(value, (List, Vector)): + return [encode(element, include_hash_tree_roots) for element in value] + elif isinstance(value, (Bytes, BytesN)): # both bytes and BytesN return '0x' + value.hex() - elif is_container_type(typ): + elif isinstance(value, Container): ret = {} - for field, subtype in typ.get_fields(): - field_value = getattr(value, field) - ret[field] = encode(field_value, subtype, include_hash_tree_roots) + for field_value, field_name in zip(value, value.get_fields().keys()): + ret[field_name] = encode(field_value, include_hash_tree_roots) if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(field_value, subtype).hex() + ret[field_name + "_hash_tree_root"] = '0x' + hash_tree_root(field_value).hex() if include_hash_tree_roots: - ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() + ret["hash_tree_root"] = '0x' + hash_tree_root(value).hex() return ret else: - raise Exception(f"Type not recognized: value={value}, typ={typ}") + raise Exception(f"Type not recognized: value={value}, typ={value.type()}") diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 3edcc8808..bd40cb832 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -1,18 +1,13 @@ from random import Random -from typing import Any from enum import Enum -from eth2spec.utils.ssz.ssz_impl import is_basic_type - from eth2spec.utils.ssz.ssz_typing import ( - is_uint_type, is_bool_type, is_list_type, - is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, - read_vector_elem_type, read_list_elem_type, - uint_byte_size + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bit, + Vector, BytesN ) # in bytes -UINT_SIZES = (1, 2, 4, 8, 16, 32) +UINT_BYTE_SIZES = (1, 2, 4, 8, 16, 32) random_mode_names = ("random", "zero", "max", "nil", "one", "lengthy") @@ -39,11 +34,11 @@ class RandomizationMode(Enum): def get_random_ssz_object(rng: Random, - typ: Any, + typ: SSZType, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, - chaos: bool) -> Any: + chaos: bool) -> SSZValue: """ Create an object for a given type, filled with random data. :param rng: The random number generator to use. @@ -56,33 +51,31 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if is_bytes_type(typ): + if issubclass(typ, Bytes): # Bytes array if mode == RandomizationMode.mode_nil_count: - return b'' + return typ(b'') elif mode == RandomizationMode.mode_max_count: - return get_random_bytes_list(rng, max_bytes_length) + return typ(get_random_bytes_list(rng, max_bytes_length)) elif mode == RandomizationMode.mode_one_count: - return get_random_bytes_list(rng, 1) + return typ(get_random_bytes_list(rng, 1)) elif mode == RandomizationMode.mode_zero: - return b'\x00' + return typ(b'\x00') elif mode == RandomizationMode.mode_max: - return b'\xff' + return typ(b'\xff') else: - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) - elif is_bytesn_type(typ): - # BytesN - length = typ.length + return typ(get_random_bytes_list(rng, rng.randint(0, max_bytes_length))) + elif issubclass(typ, BytesN): # Sanity, don't generate absurdly big random values # If a client is aiming to performance-test, they should create a benchmark suite. - assert length <= max_bytes_length + assert typ.length <= max_bytes_length if mode == RandomizationMode.mode_zero: - return b'\x00' * length + return typ(b'\x00' * typ.length) elif mode == RandomizationMode.mode_max: - return b'\xff' * length + return typ(b'\xff' * typ.length) else: - return get_random_bytes_list(rng, length) - elif is_basic_type(typ): + return typ(get_random_bytes_list(rng, typ.length)) + elif issubclass(typ, BasicValue): # Basic types if mode == RandomizationMode.mode_zero: return get_min_basic_value(typ) @@ -90,32 +83,28 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif is_vector_type(typ): - # Vector - elem_typ = read_vector_elem_type(typ) - return [ - get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + elif issubclass(typ, Vector): + return typ( + get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) - ] - elif is_list_type(typ): - # List - elem_typ = read_list_elem_type(typ) - length = rng.randint(0, max_list_length) + ) + elif issubclass(typ, List): + length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 elif mode == RandomizationMode.mode_max_count: length = max_list_length - return [ - get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + return typ( + get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) - ] - elif is_container_type(typ): + ) + elif issubclass(typ, Container): # Container return typ(**{ - field: - get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) - for field, subtype in typ.get_fields() + field_name: + get_random_ssz_object(rng, field_type, max_bytes_length, max_list_length, mode, chaos) + for field_name, field_type in typ.get_fields().items() }) else: raise Exception(f"Type not recognized: typ={typ}") @@ -125,34 +114,31 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: return bytes(rng.getrandbits(8) for _ in range(length)) -def get_random_basic_value(rng: Random, typ) -> Any: - if is_bool_type(typ): - return rng.choice((True, False)) - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return rng.randint(0, 256**size - 1) +def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(rng.choice((True, False))) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(rng.randint(0, 256 ** typ.byte_len - 1)) else: raise ValueError(f"Not a basic type: typ={typ}") -def get_min_basic_value(typ) -> Any: - if is_bool_type(typ): - return False - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return 0 +def get_min_basic_value(typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(False) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(0) else: raise ValueError(f"Not a basic type: typ={typ}") -def get_max_basic_value(typ) -> Any: - if is_bool_type(typ): - return True - elif is_uint_type(typ): - size = uint_byte_size(typ) - assert size in UINT_SIZES - return 256**size - 1 +def get_max_basic_value(typ: BasicType) -> BasicValue: + if issubclass(typ, Bit): + return typ(True) + elif issubclass(typ, uint): + assert typ.byte_len in UINT_BYTE_SIZES + return typ(256 ** typ.byte_len - 1) else: raise ValueError(f"Not a basic type: typ={typ}") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 981f30d9b..ea4d85e82 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,7 +364,10 @@ class BytesLike(Elements, metaclass=BytesType): def __str__(self): cls = self.__class__ - return f"{cls.__name__}[{cls.length}]: {self.items.hex()}" + return f"{cls.__name__}[{cls.length}]: {self.hex()}" + + def hex(self) -> str: + return self.items.hex() class Bytes(BytesLike): From 7cdec746b47bacff0e2d4352cb4838dd6d28182b Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 19 Jun 2019 02:41:43 +0200 Subject: [PATCH 30/76] fix field iteration crash in ssz typing --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea4d85e82..51a790853 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -154,11 +154,11 @@ class Container(Series, metaclass=SSZType): super().__setattr__(name, value) def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_fields()}) + return repr({field: getattr(self, field) for field in self.get_fields().keys()}) def __str__(self): output = [f'{self.__class__.__name__}'] - for field in self.get_fields(): + for field in self.get_fields().keys(): output.append(f' {field}: {getattr(self, field)}') return "\n".join(output) @@ -176,7 +176,7 @@ class Container(Series, metaclass=SSZType): @classmethod def default(cls): - return cls(**{f: t.default() for f, t in cls.get_fields()}) + return cls(**{f: t.default() for f, t in cls.get_fields().items()}) @classmethod def is_fixed_size(cls): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index f604b6468..6bb56f4e5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -76,6 +76,10 @@ def test_container(): a: uint8 b: uint32 + empty = Foo() + assert empty.a == uint8(0) + assert empty.b == uint32(0) + assert issubclass(Foo, Container) assert issubclass(Foo, SSZValue) assert issubclass(Foo, Series) From 4e747fb8879540012655e534e9d8fd214e47be87 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:25:22 +0200 Subject: [PATCH 31/76] fixes for class based ssz typing --- scripts/build_spec.py | 32 ++++++++--------- specs/core/0_beacon-chain.md | 8 +++-- .../pyspec/eth2spec/test/helpers/custody.py | 4 +-- .../pyspec/eth2spec/utils/merkle_minimal.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 +-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 36 ++++++++++--------- .../eth2spec/utils/ssz/test_ssz_typing.py | 20 ++++++++++- 7 files changed, 66 insertions(+), 41 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 612edbd00..d33ba6642 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -65,22 +65,6 @@ def get_ssz_type_by_name(name: str) -> Container: return globals()[name] -# Monkey patch validator compute committee code -_compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} - - -def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore - seed: Hash, - index: int, - count: int) -> Tuple[ValidatorIndex, ...]: - param_hash = (hash_tree_root(indices), seed, index, count) - - if param_hash not in committee_cache: - committee_cache[param_hash] = _compute_committee(indices, seed, index, count) - return committee_cache[param_hash] - - # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Hash] = {} @@ -92,6 +76,22 @@ def hash(x: bytes) -> Hash: return hash_cache[x] +# Monkey patch validator compute committee code +_compute_committee = compute_committee +committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} + + +def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore + seed: Hash, + index: int, + count: int) -> Tuple[ValidatorIndex, ...]: + param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count) + + if param_hash not in committee_cache: + committee_cache[param_hash] = _compute_committee(indices, seed, index, count) + return committee_cache[param_hash] + + # Access to overwrite spec constants based on configuration def apply_constants_preset(preset: Dict[str, Any]) -> None: global_vars = globals() diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7cbb9b67b..9a02c16e4 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -651,11 +651,13 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Tuple[ValidatorIndex, ...]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]: """ Get active validator indices at ``epoch``. """ - return tuple(ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)) + return List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + i for i, v in enumerate(state.validators) if is_active_validator(v, epoch) + ) ``` ### `increase_balance` @@ -873,7 +875,7 @@ def compute_committee(indices: Tuple[ValidatorIndex, ...], ```python def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: return compute_committee( - indices=get_active_validator_indices(state, epoch), + indices=tuple(get_active_validator_indices(state, epoch)), seed=generate_seed(state, epoch), index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, count=get_epoch_committee_count(state, epoch), diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 67df12fcd..b49a6be1f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -11,7 +11,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING reveal = bls_sign( - message_hash=spec.hash_tree_root(epoch), + message_hash=spec.hash_tree_root(spec.Epoch(epoch)), privkey=privkeys[revealed_index], domain=spec.get_domain( state=state, @@ -20,7 +20,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ), ) mask = bls_sign( - message_hash=spec.hash_tree_root(epoch), + message_hash=spec.hash_tree_root(spec.Epoch(epoch)), privkey=privkeys[masker_index], domain=spec.get_domain( state=state, diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 21583ee92..038b555cf 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -4,7 +4,7 @@ from .hash_function import hash ZERO_BYTES32 = b'\x00' * 32 zerohashes = [ZERO_BYTES32] -for layer in range(1, 32): +for layer in range(1, 100): zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a9c36649b..4b64c9162 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint, ) # SSZ Serialization @@ -143,5 +143,6 @@ def hash_tree_root(obj: SSZValue): def signing_root(obj: Container): # ignore last field - leaves = [hash_tree_root(field) for field in obj[:-1]] + fields = [field for field in obj][:-1] + leaves = [hash_tree_root(f) for f in fields] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 51a790853..381dadf9e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -98,7 +98,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) - elif isinstance(v, bytes): + elif isinstance(v, (bytes, BytesN, Bytes)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) @@ -154,7 +154,8 @@ class Container(Series, metaclass=SSZType): super().__setattr__(name, value) def __repr__(self): - return repr({field: getattr(self, field) for field in self.get_fields().keys()}) + return repr({field: (getattr(self, field) if hasattr(self, field) else 'unset') + for field in self.get_fields().keys()}) def __str__(self): output = [f'{self.__class__.__name__}'] @@ -236,15 +237,24 @@ class ParamsMeta(SSZType): raise TypeError("provided parameters {} mismatch required parameter count {}".format(params, i)) return res - def __instancecheck__(self, obj): - if obj.__class__.__name__ != self.__name__: + def __subclasscheck__(self, subclass): + # check regular class system if we can, solves a lot of the normal cases. + if super().__subclasscheck__(subclass): + return True + # if they are not normal subclasses, they are of the same class. + # then they should have the same name + if subclass.__name__ != self.__name__: return False + # If they do have the same name, they should also have the same params. for name, typ in self.__annotations__.items(): - if hasattr(self, name) and hasattr(obj.__class__, name) \ - and getattr(obj.__class__, name) != getattr(self, name): + if hasattr(self, name) and hasattr(subclass, name) \ + and getattr(subclass, name) != getattr(self, name): return False return True + def __instancecheck__(self, obj): + return self.__subclasscheck__(obj.__class__) + class ElementsType(ParamsMeta): elem_type: SSZType @@ -305,9 +315,6 @@ class Elements(ParamsBase, metaclass=ElementsType): def __iter__(self) -> Iterator[SSZValue]: return iter(self.items) - def __eq__(self, other): - return self.items == other.items - class List(Elements): @@ -366,9 +373,6 @@ class BytesLike(Elements, metaclass=BytesType): cls = self.__class__ return f"{cls.__name__}[{cls.length}]: {self.hex()}" - def hex(self) -> str: - return self.items.hex() - class Bytes(BytesLike): @@ -398,7 +402,7 @@ class BytesN(BytesLike): # Helpers for common BytesN types. -Bytes4 = BytesN[4] -Bytes32 = BytesN[32] -Bytes48 = BytesN[48] -Bytes96 = BytesN[96] +Bytes4: BytesType = BytesN[4] +Bytes32: BytesType = BytesN[32] +Bytes48: BytesType = BytesN[48] +Bytes96: BytesType = BytesN[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 6bb56f4e5..daa923aa7 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,7 +1,8 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, Elements, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256 + uint, uint8, uint16, uint32, uint64, uint128, uint256, + Bytes32, Bytes48 ) @@ -193,3 +194,20 @@ def test_list(): assert False except IndexError: pass + + +def test_bytesn_subclass(): + assert isinstance(BytesN[32](b'\xab' * 32), Bytes32) + assert not isinstance(BytesN[32](b'\xab' * 32), Bytes48) + assert issubclass(BytesN[32](b'\xab' * 32).type(), Bytes32) + assert issubclass(BytesN[32], Bytes32) + + class Hash(Bytes32): + pass + + assert isinstance(Hash(b'\xab' * 32), Bytes32) + assert not isinstance(Hash(b'\xab' * 32), Bytes48) + assert issubclass(Hash(b'\xab' * 32).type(), Bytes32) + assert issubclass(Hash, Bytes32) + + assert not issubclass(Bytes48, Bytes32) From 977856b06fa324440f87cf912b6f250ce60a3982 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:30:42 +0200 Subject: [PATCH 32/76] ssz typing now subclasses list/bytes, much easier to work with than wrapped list/bytes functionality --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 11 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 90 +++++++++++-------- .../eth2spec/utils/ssz/test_ssz_typing.py | 2 + 3 files changed, 58 insertions(+), 45 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 4b64c9162..144201d83 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, BytesN, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, uint, ) # SSZ Serialization @@ -46,9 +46,8 @@ def serialize(obj: SSZValue): def encode_series(values: Series): - # bytes and bytesN are already in the right format. - if isinstance(values, (Bytes, BytesN)): - return values.items + if isinstance(values, bytes): # Bytes and BytesN are already like serialized output + return values # Recursively serialize parts = [(v.type().is_fixed_size(), serialize(v)) for v in values] @@ -84,8 +83,8 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, (Bytes, BytesN)): - return values.items + if isinstance(values, bytes): # Bytes and BytesN are already packed + return values return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 381dadf9e..341df880a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -188,10 +188,10 @@ class Container(Series, metaclass=SSZType): class ParamsBase(Series): - _bare = True + _has_params = False def __new__(cls, *args, **kwargs): - if cls._bare: + if not cls._has_params: raise Exception("cannot init bare type without params") return super().__new__(cls, **kwargs) @@ -200,13 +200,13 @@ class ParamsMeta(SSZType): def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) - for k, v in attrs.items(): - setattr(out, k, v) + if hasattr(out, "_has_params") and getattr(out, "_has_params"): + for k, v in attrs.items(): + setattr(out, k, v) return out def __getitem__(self, params): o = self.__class__(self.__name__, (self,), self.attr_from_params(params)) - o._bare = False return o def __str__(self): @@ -218,7 +218,7 @@ class ParamsMeta(SSZType): def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. params = p if isinstance(p, tuple) else (p,) - res = {} + res = {'_has_params': True} i = 0 for (name, typ) in self.__annotations__.items(): if hasattr(self.__class__, name): @@ -262,13 +262,17 @@ class ElementsType(ParamsMeta): class Elements(ParamsBase, metaclass=ElementsType): + pass + + +class BaseList(list, Elements): def __init__(self, *args): items = self.extract_args(*args) if not self.value_check(items): raise ValueError(f"Bad input for class {self.__class__}: {items}") - self.items = items + super().__init__(items) @classmethod def value_check(cls, value): @@ -284,39 +288,32 @@ class Elements(ParamsBase, metaclass=ElementsType): def __str__(self): cls = self.__class__ - return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self.items)})" + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" def __getitem__(self, i) -> SSZValue: - return self.items[i] + if i < 0: + raise IndexError(f"cannot get item in type {self.__class__} at negative index {i}") + if i > len(self): + raise IndexError(f"cannot get item in type {self.__class__}" + f" at out of bounds index {i}") + return super().__getitem__(i) def __setitem__(self, k, v): if k < 0: raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self.items): + if k > len(self): raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self.items)})") - self.items[k] = coerce_type_maybe(v, self.__class__.elem_type, strict=True) + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): - self.items.append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) - - def pop(self): - if len(self.items) == 0: - raise IndexError("Pop from empty list") - else: - return self.items.pop() - - def __len__(self): - return len(self.items) - - def __repr__(self): - return repr(self.items) + super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def __iter__(self) -> Iterator[SSZValue]: - return iter(self.items) + return super().__iter__() -class List(Elements): +class List(BaseList): @classmethod def default(cls): @@ -327,7 +324,7 @@ class List(Elements): return False -class Vector(Elements): +class Vector(BaseList): @classmethod def value_check(cls, value): @@ -342,27 +339,35 @@ class Vector(Elements): def is_fixed_size(cls): return cls.elem_type.is_fixed_size() + def append(self, v): + raise Exception("cannot modify vector length") + + def pop(self, *args): + raise Exception("cannot modify vector length") + class BytesType(ElementsType): elem_type: SSZType = byte length: int -class BytesLike(Elements, metaclass=BytesType): +class BaseBytes(bytes, Elements, metaclass=BytesType): + + def __new__(cls, *args) -> "BaseBytes": + extracted_val = cls.extract_args(*args) + if not cls.value_check(extracted_val): + raise ValueError(f"Bad input for class {cls}: {extracted_val}") + return super().__new__(cls, extracted_val) @classmethod def extract_args(cls, *args): - x = list(args) - if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, BytesLike)): + x = args + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes)): x = x[0] - if isinstance(x, bytes): + if isinstance(x, bytes): # Includes BytesLike return x - elif isinstance(x, BytesLike): - return x.items - elif isinstance(x, GeneratorType): - return bytes(x) else: - return bytes(x) + return bytes(x) # E.g. GeneratorType put into bytes. @classmethod def value_check(cls, value): @@ -374,7 +379,7 @@ class BytesLike(Elements, metaclass=BytesType): return f"{cls.__name__}[{cls.length}]: {self.hex()}" -class Bytes(BytesLike): +class Bytes(BaseBytes): @classmethod def default(cls): @@ -385,7 +390,14 @@ class Bytes(BytesLike): return False -class BytesN(BytesLike): +class BytesN(BaseBytes): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) @classmethod def default(cls): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index daa923aa7..895a074a9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -211,3 +211,5 @@ def test_bytesn_subclass(): assert issubclass(Hash, Bytes32) assert not issubclass(Bytes48, Bytes32) + + assert len(Bytes32() + Bytes48()) == 80 From 82240d8dbd0f083efb66c8c5b2388eb153d69977 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 18:01:57 +0200 Subject: [PATCH 33/76] fix vector default type --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 341df880a..6595c966f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -333,7 +333,7 @@ class Vector(BaseList): @classmethod def default(cls): - return [cls.elem_type.default() for _ in range(cls.length)] + return cls(cls.elem_type.default() for _ in range(cls.length)) @classmethod def is_fixed_size(cls): From f157745248492114da19b14c45e8831493368853 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:38:42 +0200 Subject: [PATCH 34/76] resolve some remaining list-rework rebase details --- specs/core/0_beacon-chain.md | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9a02c16e4..9c871f5f2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -535,8 +535,8 @@ class BeaconState(Container): # Slashings slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators # Attestations - previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] - current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * EPOCH_LENGTH] + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # Crosslinks previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -1282,7 +1282,7 @@ def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterab ```python def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: - return [ + return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) ] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6595c966f..bd7fe0950 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Iterator +from typing import Dict, Iterator from types import GeneratorType @@ -40,7 +40,7 @@ class Bit(BasicValue): # can't subclass bool. @classmethod def default(cls): - return cls(False) + return cls(0) def __bool__(self): return self > 0 @@ -103,7 +103,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): elif isinstance(v, GeneratorType): return typ(v) else: - # just return as-is, Value-checkers will take care of it not being coerced. + # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. if strict and not isinstance(v, typ): raise ValueError("Type coercion of {} to {} failed".format(v, typ)) return v @@ -181,10 +181,10 @@ class Container(Series, metaclass=SSZType): @classmethod def is_fixed_size(cls): - return all(t.is_fixed_size() for t in cls.get_field_types()) + return all(t.is_fixed_size() for t in cls.get_fields().values()) def __iter__(self) -> Iterator[SSZValue]: - return iter([getattr(self, field) for field in self.get_fields()]) + return iter([getattr(self, field) for field in self.get_fields().keys()]) class ParamsBase(Series): From 224c98a094a5d537fe05377955d8f30868cb6ba7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 20:55:17 +0200 Subject: [PATCH 35/76] last() method, no negative index lookups --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../block_processing/test_process_transfer.py | 14 +++++++------- .../test_process_voluntary_exit.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++++ 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index b49a6be1f..681add457 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -4,7 +4,7 @@ from eth2spec.utils.bls import bls_sign def get_valid_early_derived_secret_reveal(spec, state, epoch=None): current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] + revealed_index = spec.get_active_validator_indices(state, current_epoch).last() masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index d5b7f7b7f..7b6f71374 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.keys import pubkey_to_privkey def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index acc6a35c5..215cd6fcb 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -9,7 +9,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] + sender_index = spec.get_active_validator_indices(state, current_epoch).last() recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index e9d282b3a..2bed94dc8 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -63,7 +63,7 @@ def test_success_withdrawable(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -73,7 +73,7 @@ def test_success_active_above_max_effective(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -94,7 +94,7 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) @@ -115,7 +115,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -128,7 +128,7 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -141,7 +141,7 @@ def test_insufficient_balance(spec, state): @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() balance = state.balances[sender_index] transfer = get_valid_transfer( spec, @@ -161,7 +161,7 @@ def test_no_dust_sender(spec, state): @with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 33cacc4e2..b9f7d6098 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -93,7 +93,7 @@ def test_success_exit_queue(spec, state): continue # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index f8590005e..c86394a0b 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -269,7 +269,7 @@ def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( state, spec.get_current_epoch(state) - )[-1] + ).last() # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -315,7 +315,7 @@ def test_voluntary_exit(spec, state): # overwrite default 0 to test # spec.MAX_TRANSFERS = 1 - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() # amount = get_balance(state, sender_index) # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) @@ -347,7 +347,7 @@ def test_voluntary_exit(spec, state): @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + validator_index = spec.get_active_validator_indices(state, current_epoch).last() assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index bd7fe0950..e5f9f66a8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -312,6 +312,10 @@ class BaseList(list, Elements): def __iter__(self) -> Iterator[SSZValue]: return super().__iter__() + def last(self): + # be explict about getting the last item, for the non-python readers, and negative-index safety + return self[len(self)-1] + class List(BaseList): From 8c6d2b42d885f14758f57958f922b66128b2cf8f Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:07:23 +0200 Subject: [PATCH 36/76] update ssz-pyssz decoder for fuzzing --- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 73 +++++++++----------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index a5d3dfd97..a4ebb7e70 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -8,32 +8,31 @@ def translate_typ(typ) -> ssz.BaseSedes: :param typ: The spec type, a class. :return: The Py-SSZ equivalent. """ - if spec_ssz.is_container_type(typ): + if issubclass(typ, spec_ssz.Container): return ssz.Container( - [translate_typ(field_typ) for (field_name, field_typ) in typ.get_fields()]) - elif spec_ssz.is_bytesn_type(typ): + [translate_typ(field_typ) for field_name, field_typ in typ.get_fields().items()]) + elif issubclass(typ, spec_ssz.BytesN): return ssz.ByteVector(typ.length) - elif spec_ssz.is_bytes_type(typ): + elif issubclass(typ, spec_ssz.Bytes): return ssz.ByteList() - elif spec_ssz.is_vector_type(typ): - return ssz.Vector(translate_typ(spec_ssz.read_vector_elem_type(typ)), typ.length) - elif spec_ssz.is_list_type(typ): - return ssz.List(translate_typ(spec_ssz.read_list_elem_type(typ))) - elif spec_ssz.is_bool_type(typ): + elif issubclass(typ, spec_ssz.Vector): + return ssz.Vector(translate_typ(typ.elem_type), typ.length) + elif issubclass(typ, spec_ssz.List): + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bit): return ssz.boolean - elif spec_ssz.is_uint_type(typ): - size = spec_ssz.uint_byte_size(typ) - if size == 1: + elif issubclass(typ, spec_ssz.uint): + if typ.byte_len == 1: return ssz.uint8 - elif size == 2: + elif typ.byte_len == 2: return ssz.uint16 - elif size == 4: + elif typ.byte_len == 4: return ssz.uint32 - elif size == 8: + elif typ.byte_len == 8: return ssz.uint64 - elif size == 16: + elif typ.byte_len == 16: return ssz.uint128 - elif size == 32: + elif typ.byte_len == 32: return ssz.uint256 else: raise TypeError("invalid uint size") @@ -48,37 +47,33 @@ def translate_value(value, typ): :param typ: The type from the spec to translate into :return: the translated value """ - if spec_ssz.is_uint_type(typ): - size = spec_ssz.uint_byte_size(typ) - if size == 1: + if issubclass(typ, spec_ssz.uint): + if typ.byte_len == 1: return spec_ssz.uint8(value) - elif size == 2: + elif typ.byte_len == 2: return spec_ssz.uint16(value) - elif size == 4: + elif typ.byte_len == 4: return spec_ssz.uint32(value) - elif size == 8: - # uint64 is default (TODO this is changing soon) - return value - elif size == 16: + elif typ.byte_len == 8: + return spec_ssz.uint64(value) + elif typ.byte_len == 16: return spec_ssz.uint128(value) - elif size == 32: + elif typ.byte_len == 32: return spec_ssz.uint256(value) else: raise TypeError("invalid uint size") - elif spec_ssz.is_list_type(typ): - elem_typ = spec_ssz.read_elem_type(typ) - return [translate_value(elem, elem_typ) for elem in value] - elif spec_ssz.is_bool_type(typ): + elif issubclass(typ, spec_ssz.List): + return [translate_value(elem, typ.elem_type) for elem in value] + elif issubclass(typ, spec_ssz.Bit): return value - elif spec_ssz.is_vector_type(typ): - elem_typ = spec_ssz.read_elem_type(typ) - return typ(*(translate_value(elem, elem_typ) for elem in value)) - elif spec_ssz.is_bytesn_type(typ): + elif issubclass(typ, spec_ssz.Vector): + return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.BytesN): return typ(value) - elif spec_ssz.is_bytes_type(typ): + elif issubclass(typ, spec_ssz.Bytes): return value - elif spec_ssz.is_container_type(typ): - return typ(**{f_name: translate_value(f_val, f_typ) for (f_name, f_val, f_typ) - in zip(typ.get_field_names(), value, typ.get_field_types())}) + if issubclass(typ, spec_ssz.Container): + return typ(**{f_name: translate_value(f_val, f_typ) for (f_val, (f_name, f_typ)) + in zip(value, typ.get_fields().items())}) else: raise TypeError("Type not supported: {}".format(typ)) From 8bd204827bd5e2d94091ede1e9ff973b011a7678 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:08:34 +0200 Subject: [PATCH 37/76] improve type coercion; coerce between equal-length uint subclasses --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index e5f9f66a8..55acde44b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -94,19 +94,24 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): # shortcut if it's already the type we are looking for if v_typ == typ: return v - elif isinstance(v, int) and not isinstance(v, uint): # do not coerce from one uintX to another uintY - return typ(v) + elif isinstance(v, int): + if isinstance(v, uint): # do not coerce from one uintX to another uintY + if issubclass(typ, uint) and v.type().byte_len == typ.byte_len: + return typ(v) + # revert to default behavior below if-else. (ValueError/bare) + else: + return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) elif isinstance(v, (bytes, BytesN, Bytes)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) - else: - # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. - if strict and not isinstance(v, typ): - raise ValueError("Type coercion of {} to {} failed".format(v, typ)) - return v + + # just return as-is, Value-checkers will take care of it not being coerced, if we are not strict. + if strict and not isinstance(v, typ): + raise ValueError("Type coercion of {} to {} failed".format(v, typ)) + return v class Series(SSZValue): From b4ef672f87cb3d3afbd0a7244feabbb3b7158258 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:12:46 +0200 Subject: [PATCH 38/76] deal with deepcopy modifying vector length from 0 to full length during copy --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 55acde44b..39fed9ac8 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -349,7 +349,12 @@ class Vector(BaseList): return cls.elem_type.is_fixed_size() def append(self, v): - raise Exception("cannot modify vector length") + # Deep-copy and other utils like to change the internals during work. + # Only complain if we had the right size. + if len(self) == self.__class__.length: + raise Exception("cannot modify vector length") + else: + super().append(v) def pop(self, *args): raise Exception("cannot modify vector length") From 2d6771707925558b54fc5fe6bbc8154198dca2fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:42:55 +0200 Subject: [PATCH 39/76] fix linting issues + make spec builder remove comments in container re-initialization part --- scripts/build_spec.py | 15 ++++++++++++++- specs/core/0_beacon-chain.md | 5 +++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d33ba6642..338093e94 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -106,6 +106,18 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None: ''' +def strip_comments(raw: str) -> str: + comment_line_regex = re.compile('^\s+# ') + lines = raw.split('\n') + out = [] + for line in lines: + if not comment_line_regex.match(line): + if ' #' in line: + line = line[:line.index(' #')] + out.append(line) + return '\n'.join(out) + + def objects_to_spec(functions: Dict[str, str], custom_types: Dict[str, str], constants: Dict[str, str], @@ -133,7 +145,8 @@ def objects_to_spec(functions: Dict[str, str], ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) ssz_objects_reinitialization_spec = ( 'def init_SSZ_types() -> None:\n global_vars = globals()\n\n ' - + '\n\n '.join([re.sub(r'(?!\n\n)\n', r'\n ', value[:-1]) for value in ssz_objects.values()]) + + '\n\n '.join([strip_comments(re.sub(r'(?!\n\n)\n', r'\n ', value[:-1])) + for value in ssz_objects.values()]) + '\n\n' + '\n'.join(map(lambda x: ' global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys())) ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9c871f5f2..52d087281 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -533,7 +533,7 @@ class BeaconState(Container): randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] active_index_roots: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] # Digests of the active registry, for light clients # Slashings - slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the effective balances of slashed validators + slashed_balances: Vector[Gwei, EPOCHS_PER_SLASHED_BALANCES_VECTOR] # Sums of the slashed effective balances # Attestations previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] @@ -1525,7 +1525,8 @@ def process_slashings(state: BeaconState) -> None: total_penalties = total_at_end - total_at_start for index, validator in enumerate(state.validators): - if validator.slashed and current_epoch == validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2: + if validator.slashed and current_epoch == ( + validator.withdrawable_epoch - EPOCHS_PER_SLASHED_BALANCES_VECTOR // 2): penalty = max( validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 39fed9ac8..c9516bebf 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -319,7 +319,7 @@ class BaseList(list, Elements): def last(self): # be explict about getting the last item, for the non-python readers, and negative-index safety - return self[len(self)-1] + return self[len(self) - 1] class List(BaseList): From 4dcfee2d2cd8ba978787eda8bf336f2756dc67a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 21:45:17 +0200 Subject: [PATCH 40/76] remove unused spec-helper from spec builder --- scripts/build_spec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 338093e94..45cb0368f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -61,10 +61,6 @@ from eth2spec.utils.hash_function import hash Deltas = list ''' SUNDRY_FUNCTIONS = ''' -def get_ssz_type_by_name(name: str) -> Container: - return globals()[name] - - # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Hash] = {} From d8f470bb4a81ae5662f28d83daecc27769972ef3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:34:19 +0200 Subject: [PATCH 41/76] enable slicing of SSZ lists/vectors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index c9516bebf..22f76bada 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -295,13 +295,14 @@ class BaseList(list, Elements): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" - def __getitem__(self, i) -> SSZValue: - if i < 0: - raise IndexError(f"cannot get item in type {self.__class__} at negative index {i}") - if i > len(self): - raise IndexError(f"cannot get item in type {self.__class__}" - f" at out of bounds index {i}") - return super().__getitem__(i) + def __getitem__(self, k) -> SSZValue: + if isinstance(k, int): # check if we are just doing a lookup, and not slicing + if k < 0: + raise IndexError(f"cannot get item in type {self.__class__} at negative index {k}") + if k > len(self): + raise IndexError(f"cannot get item in type {self.__class__}" + f" at out of bounds index {k}") + return super().__getitem__(k) def __setitem__(self, k, v): if k < 0: From 6338c5b88051dcefe515e424196bca708b44dd06 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:49:03 +0200 Subject: [PATCH 42/76] fix custody bug, needs review from Carl --- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 681add457..bc70c9fa8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -27,7 +27,7 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): domain_type=spec.DOMAIN_RANDAO, message_epoch=epoch, ), - ) + )[:32] # TODO(Carl): mask is 32 bytes, and signature is 96? Correct to slice the first 32 out? return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, From f27c44b953789b49e533fe36f8be314073c9f844 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:49:34 +0200 Subject: [PATCH 43/76] fix deposit negative index fail --- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 63b94c638..a40c120fb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -183,7 +183,7 @@ def test_bad_merkle_proof(spec, state): deposit = prepare_state_and_deposit(spec, state, validator_index, amount) # mess up merkle branch - deposit.proof[-1] = spec.ZERO_HASH + deposit.proof[5] = spec.ZERO_HASH sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) From c20372409c543b52bf15e71bc1df02b7ccb37434 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 22:52:16 +0200 Subject: [PATCH 44/76] comment out old deposit test, re-introduced soon maybe, cc Justin --- .../block_processing/test_process_deposit.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index a40c120fb..2f60c6be2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -113,20 +113,21 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): # inconsistent withdrawal credentials, in top-ups, are allowed! yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) - -@with_all_phases -@spec_state_test -def test_wrong_index(spec, state): - validator_index = len(state.validators) - amount = spec.MAX_EFFECTIVE_BALANCE - deposit = prepare_state_and_deposit(spec, state, validator_index, amount) - - # mess up eth1_deposit_index - deposit.index = state.eth1_deposit_index + 1 - - sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) - - yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) +# Deposit data temporarily has no index field. +# SSZ typing catches that, and raises an exception. Test may be introduced back later +# @with_all_phases +# @spec_state_test +# def test_wrong_index(spec, state): +# validator_index = len(state.validators) +# amount = spec.MAX_EFFECTIVE_BALANCE +# deposit = prepare_state_and_deposit(spec, state, validator_index, amount) +# +# # mess up eth1_deposit_index +# deposit.index = state.eth1_deposit_index + 1 +# +# sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) +# +# yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) @with_all_phases From 3d8466fd6e07dea808fe13b2719917867ffa28b5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 20 Jun 2019 23:02:22 +0200 Subject: [PATCH 45/76] make Bit check not use "is", and remove duplicate line --- specs/core/0_beacon-chain.md | 2 +- specs/core/1_custody-game.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 52d087281..f171bd4df 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -645,7 +645,7 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: """ Check if ``validator`` is slashable. """ - return validator.slashed is False and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) ``` ### `get_active_validator_indices` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 99c85a034..24e9a19e2 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -402,12 +402,11 @@ def process_early_derived_secret_reveal(state: BeaconState, """ revealed_validator = state.validators[reveal.revealed_index] - masker = state.validators[reveal.masker_index] derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS assert reveal.epoch >= get_current_epoch(state) + RANDAO_PENALTY_EPOCHS assert reveal.epoch < get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS - assert revealed_validator.slashed is False + assert not revealed_validator.slashed assert reveal.revealed_index not in state.exposed_derived_secrets[derived_secret_location] # Verify signature correctness From 6648b3c49e0cd5535c5a34519d33798cc461712d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 00:23:28 +0200 Subject: [PATCH 46/76] remove old deposits test, there is no deposit index in deposit data anymore --- .../block_processing/test_process_deposit.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 2f60c6be2..8b3d7b413 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -113,22 +113,6 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): # inconsistent withdrawal credentials, in top-ups, are allowed! yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -# Deposit data temporarily has no index field. -# SSZ typing catches that, and raises an exception. Test may be introduced back later -# @with_all_phases -# @spec_state_test -# def test_wrong_index(spec, state): -# validator_index = len(state.validators) -# amount = spec.MAX_EFFECTIVE_BALANCE -# deposit = prepare_state_and_deposit(spec, state, validator_index, amount) -# -# # mess up eth1_deposit_index -# deposit.index = state.eth1_deposit_index + 1 -# -# sign_deposit_data(spec, state, deposit.data, privkeys[validator_index]) -# -# yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) - @with_all_phases @spec_state_test @@ -173,9 +157,6 @@ def test_wrong_deposit_for_deposit_count(spec, state): yield from run_deposit_processing(spec, state, deposit_2, index_2, valid=False) -# TODO: test invalid signature - - @with_all_phases @spec_state_test def test_bad_merkle_proof(spec, state): From e99c864ed18ef172adc5795f1777035ef517fc82 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 20 Jun 2019 17:17:12 -0600 Subject: [PATCH 47/76] Deltas = NewType('Deltas', TypingList[Gwei]) --- scripts/build_spec.py | 12 ++++-------- specs/core/0_beacon-chain.md | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 45cb0368f..bd7686ff5 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,8 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Tuple + Any, Callable, Iterable, Dict, Set, Tuple, NewType, + List as TypingList, ) from eth2spec.utils.ssz.ssz_impl import ( @@ -30,12 +31,9 @@ from eth2spec.utils.bls import ( ) from eth2spec.utils.hash_function import hash - - -Deltas = list ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable, + Any, Callable, Dict, Optional, Set, Tuple, Iterable, NewType, List as TypingList ) @@ -56,9 +54,6 @@ from eth2spec.utils.bls import ( ) from eth2spec.utils.hash_function import hash - - -Deltas = list ''' SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache @@ -149,6 +144,7 @@ def objects_to_spec(functions: Dict[str, str], spec = ( imports + '\n\n' + new_type_definitions + + '\n\n' + "Deltas = NewType('Deltas', TypingList[Gwei])" + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f171bd4df..e89f0baed 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1405,8 +1405,8 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = Deltas(0 for _ in range(len(state.validators))) - penalties = Deltas(0 for _ in range(len(state.validators))) + rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) + penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1454,8 +1454,8 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: ```python def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: - rewards = Deltas(0 for _ in range(len(state.validators))) - penalties = Deltas(0 for _ in range(len(state.validators))) + rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) + penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) From b7b2fee6350489ec4cd1c2e118b8bfba21db8055 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 21:12:27 +0200 Subject: [PATCH 48/76] uint add/sub type checking, fixes #1029 --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 ++++- .../eth2spec/utils/ssz/test_ssz_typing.py | 32 +++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 22f76bada..971b2106b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -52,9 +52,15 @@ class uint(BasicValue, metaclass=BasicType): if value < 0: raise ValueError("unsigned types must not be negative") if cls.byte_len and value.bit_length() > (cls.byte_len << 3): - raise ValueError("value out of bounds for uint{}".format(cls.byte_len)) + raise ValueError("value out of bounds for uint{}".format(cls.byte_len * 8)) return super().__new__(cls, value) + def __add__(self, other): + return self.__class__(super().__add__(coerce_type_maybe(other, self.__class__, strict=True))) + + def __sub__(self, other): + return self.__class__(super().__sub__(coerce_type_maybe(other, self.__class__, strict=True))) + @classmethod def default(cls): return cls(0) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 895a074a9..4325501aa 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -6,6 +6,14 @@ from .ssz_typing import ( ) +def expect_value_error(fn, msg): + try: + fn() + raise AssertionError(msg) + except ValueError: + pass + + def test_subclasses(): for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: assert issubclass(u, uint) @@ -55,21 +63,13 @@ def test_basic_value_bounds(): # this should work assert k(v - 1) == v - 1 # but we do not allow overflows - try: - k(v) - assert False - except ValueError: - pass + expect_value_error(lambda: k(v), "no overflows allowed") for k, _ in max.items(): # this should work assert k(0) == 0 # but we do not allow underflows - try: - k(-1) - assert False - except ValueError: - pass + expect_value_error(lambda: k(-1), "no underflows allowed") def test_container(): @@ -213,3 +213,15 @@ def test_bytesn_subclass(): assert not issubclass(Bytes48, Bytes32) assert len(Bytes32() + Bytes48()) == 80 + + +def test_uint_math(): + assert uint8(0) + uint8(uint32(16)) == uint8(16) # allow explict casting to make invalid addition valid + + expect_value_error(lambda: uint8(0) - uint8(1), "no underflows allowed") + expect_value_error(lambda: uint8(1) + uint8(255), "no overflows allowed") + expect_value_error(lambda: uint8(0) + 256, "no overflows allowed") + expect_value_error(lambda: uint8(42) + uint32(123), "no mixed types") + expect_value_error(lambda: uint32(42) + uint8(123), "no mixed types") + + assert type(uint32(1234) + 56) == uint32 From d1fa3acb27802b56627ee71e0e3d292553838146 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 21 Jun 2019 21:27:26 +0200 Subject: [PATCH 49/76] remove unused dependency --- test_libs/pyspec/requirements.txt | 1 - test_libs/pyspec/setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index b96b0a80f..83197af9c 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,6 +2,5 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 -typing_inspect==0.4.0 dataclasses==0.6 ssz==0.1.0a10 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index fe14c498c..d8d54eab7 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -9,7 +9,6 @@ setup( "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", - "typing_inspect==0.4.0", "ssz==0.1.0a10", "dataclasses==0.6", ] From c09e45c5fc49e81c5be5c6576642949b18ec0d73 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:43:55 -0600 Subject: [PATCH 50/76] fix rule_4 underflow and split out genesis finality test --- .../pyspec/eth2spec/test/test_finality.py | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 5e81f52c8..160a32a4b 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -39,11 +39,13 @@ def next_epoch_with_attestations(spec, state, fill_cur_epoch, fill_prev_epoch): + assert state.slot % spec.SLOTS_PER_EPOCH == 0 + post_state = deepcopy(state) blocks = [] for _ in range(spec.SLOTS_PER_EPOCH): block = build_empty_block_for_next_slot(spec, post_state) - if fill_cur_epoch: + if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest) @@ -63,11 +65,13 @@ def next_epoch_with_attestations(spec, @with_all_phases @never_bls @spec_state_test -def test_finality_rule_4(spec, state): +def test_finality_no_updates_at_genesis(spec, state): + assert spec.get_current_epoch(state) == spec.GENESIS_EPOCH + yield 'pre', state blocks = [] - for epoch in range(4): + for epoch in range(2): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) blocks += new_blocks @@ -77,9 +81,37 @@ def test_finality_rule_4(spec, state): # justification/finalization skipped at GENESIS_EPOCH + 1 elif epoch == 1: check_finality(spec, state, prev_state, False, False, False) - elif epoch == 2: + + yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'post', state + + +@with_all_phases +@never_bls +@spec_state_test +def test_finality_rule_4(spec, state): + # get past first two epochs that finality does not run on + next_epoch(spec, state) + apply_empty_block(spec, state) + next_epoch(spec, state) + apply_empty_block(spec, state) + + yield 'pre', state + + blocks = [] + for epoch in range(2): + prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) + blocks += new_blocks + + # justification/finalization skipped at GENESIS_EPOCH + # if epoch == 0: + # check_finality(spec, state, prev_state, False, False, False) + # justification/finalization skipped at GENESIS_EPOCH + 1 + # elif epoch == 1: + # check_finality(spec, state, prev_state, False, False, False) + if epoch == 0: check_finality(spec, state, prev_state, True, False, False) - elif epoch >= 3: + elif epoch == 1: # rule 4 of finality check_finality(spec, state, prev_state, True, True, True) assert state.finalized_epoch == prev_state.current_justified_epoch From 8f997413445d55a11a00c44ffe4f81a92ca0fe1e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 21 Jun 2019 14:47:18 -0600 Subject: [PATCH 51/76] remove commented old code --- test_libs/pyspec/eth2spec/test/test_finality.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 160a32a4b..730456286 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -103,12 +103,6 @@ def test_finality_rule_4(spec, state): prev_state, new_blocks, state = next_epoch_with_attestations(spec, state, True, False) blocks += new_blocks - # justification/finalization skipped at GENESIS_EPOCH - # if epoch == 0: - # check_finality(spec, state, prev_state, False, False, False) - # justification/finalization skipped at GENESIS_EPOCH + 1 - # elif epoch == 1: - # check_finality(spec, state, prev_state, False, False, False) if epoch == 0: check_finality(spec, state, prev_state, True, False, False) elif epoch == 1: From 00aae07d4690417e70ee50b57a34b2382b6642b3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 18:12:42 +0200 Subject: [PATCH 52/76] type annotation clean up --- scripts/build_spec.py | 12 +++--- specs/core/0_beacon-chain.md | 64 +++++++++++++++---------------- specs/core/1_shard-data-chains.md | 18 ++++----- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 943d37877..0b55dfa44 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,8 +12,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Tuple, NewType, - List as TypingList, + Any, Callable, Iterable, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -38,7 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, Iterable, NewType, + Any, Callable, Dict, Iterable, Optional, Set, Sequence, Tuple, List as TypingList ) @@ -79,13 +78,13 @@ def hash(x: bytes) -> Hash: # Monkey patch validator compute committee code _compute_committee = compute_committee -committee_cache: Dict[Tuple[Hash, Hash, int, int], Tuple[ValidatorIndex, ...]] = {} +committee_cache: Dict[Tuple[Hash, Hash, int, int], Sequence[ValidatorIndex]] = {} -def compute_committee(indices: Tuple[ValidatorIndex, ...], # type: ignore +def compute_committee(indices: Sequence[ValidatorIndex], # type: ignore seed: Hash, index: int, - count: int) -> Tuple[ValidatorIndex, ...]: + count: int) -> Sequence[ValidatorIndex]: param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count) if param_hash not in committee_cache: @@ -154,7 +153,6 @@ def objects_to_spec(functions: Dict[str, str], spec = ( imports + '\n\n' + new_type_definitions - + '\n\n' + "Deltas = NewType('Deltas', TypingList[Gwei])" + '\n\n' + constants_spec + '\n\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cf8e69535..f42b4cc13 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -651,13 +651,11 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: ### `get_active_validator_indices` ```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]: +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ Get active validator indices at ``epoch``. """ - return List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( - i for i, v in enumerate(state.validators) if is_active_validator(v, epoch) - ) + return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` ### `increase_balance` @@ -819,7 +817,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: ### `verify_merkle_branch` ```python -def verify_merkle_branch(leaf: Hash, proof: Tuple[Hash, ...], depth: int, index: int, root: Hash) -> bool: +def verify_merkle_branch(leaf: Hash, proof: Sequence[Hash], depth: int, index: int, root: Hash) -> bool: """ Verify that the given ``leaf`` is on the merkle branch ``proof`` starting with the given ``root``. @@ -863,19 +861,19 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Hash) -> V ### `compute_committee` ```python -def compute_committee(indices: Tuple[ValidatorIndex, ...], - seed: Hash, index: int, count: int) -> Tuple[ValidatorIndex, ...]: +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Hash, index: int, count: int) -> Sequence[ValidatorIndex]: start = (len(indices) * index) // count end = (len(indices) * (index + 1)) // count - return tuple(indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)) + return [indices[get_shuffled_index(ValidatorIndex(i), len(indices), seed)] for i in range(start, end)] ``` ### `get_crosslink_committee` ```python -def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[ValidatorIndex, ...]: +def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: return compute_committee( - indices=tuple(get_active_validator_indices(state, epoch)), + indices=get_active_validator_indices(state, epoch), seed=generate_seed(state, epoch), index=(shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT, count=get_epoch_committee_count(state, epoch), @@ -887,13 +885,13 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> T ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Tuple[ValidatorIndex, ...]: + bitfield: bytes) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) - return tuple(sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1])) + return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` ### `int_to_bytes` @@ -1139,7 +1137,7 @@ def slash_validator(state: BeaconState, ### Genesis trigger -Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool` where: +Before genesis has been triggered and whenever the deposit contract emits a `Deposit` log, call the function `is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool` where: * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log @@ -1156,7 +1154,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: *Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: ```python -def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: +def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Process deposits state = BeaconState() for deposit in deposits: @@ -1178,7 +1176,7 @@ def is_genesis_trigger(deposits: Iterable[Deposit], timestamp: uint64) -> bool: Let `genesis_state = get_genesis_beacon_state(genesis_deposits, genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState( genesis_time=genesis_time, eth1_data=eth1_data, @@ -1195,8 +1193,10 @@ def get_genesis_beacon_state(deposits: Iterable[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) + # Populate active_index_roots + genesis_active_index_root = hash_tree_root( + List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE](get_active_validator_indices(state, GENESIS_EPOCH)) + ) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root @@ -1271,17 +1271,17 @@ def process_epoch(state: BeaconState) -> None: ```python def get_total_active_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) ``` ```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: assert epoch in (get_current_epoch(state), get_previous_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` ```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.target_root == get_block_root(state, epoch) @@ -1289,7 +1289,7 @@ def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Iterab ``` ```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable[PendingAttestation]: +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) @@ -1298,22 +1298,22 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Iterable ```python def get_unslashed_attesting_indices(state: BeaconState, - attestations: Iterable[PendingAttestation]) -> Iterable[ValidatorIndex]: + attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) - return sorted(filter(lambda index: not state.validators[index].slashed, list(output))) + return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` ```python -def get_attesting_balance(state: BeaconState, attestations: Iterable[PendingAttestation]) -> Gwei: +def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, - shard: Shard) -> Tuple[Crosslink, Iterable[ValidatorIndex]]: + shard: Shard) -> Tuple[Crosslink, Set[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1402,11 +1402,11 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: ``` ```python -def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: previous_epoch = get_previous_epoch(state) total_balance = get_total_active_balance(state) - rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) - penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] eligible_validator_indices = [ ValidatorIndex(index) for index, v in enumerate(state.validators) if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) @@ -1453,9 +1453,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: ``` ```python -def get_crosslink_deltas(state: BeaconState) -> Tuple[Deltas, Deltas]: - rewards = Deltas([Gwei(0) for _ in range(len(state.validators))]) - penalties = Deltas([Gwei(0) for _ in range(len(state.validators))]) +def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + rewards = [Gwei(0) for _ in range(len(state.validators))] + penalties = [Gwei(0) for _ in range(len(state.validators))] epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) @@ -1642,7 +1642,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - ) # type: Tuple[Tuple[List, Callable], ...] + ) # type: Sequence[Tuple[List, Callable]] for operations, function in all_operations: for operation in operations: function(state, operation) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9c60062f4..9f0de2e6b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -133,7 +133,7 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, - count: int) -> Tuple[ValidatorIndex, ...]: + count: int) -> Sequence[ValidatorIndex]: """ Return committee for a period. Used to construct persistent committees. """ @@ -159,7 +159,7 @@ def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex ```python def get_persistent_committee(state: BeaconState, shard: Shard, - slot: Slot) -> Tuple[ValidatorIndex, ...]: + slot: Slot) -> Sequence[ValidatorIndex]: """ Return the persistent committee for the given ``shard`` at the given ``slot``. """ @@ -193,7 +193,7 @@ def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: Slot) -> Optional[ValidatorIndex]: # Randomly shift persistent committee - persistent_committee = get_persistent_committee(state, shard, slot) + persistent_committee = list(get_persistent_committee(state, shard, slot)) seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] @@ -248,7 +248,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: Iterable[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) @@ -287,9 +287,9 @@ Let: * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python -def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], +def is_valid_shard_block(beacon_blocks: Sequence[BeaconBlock], beacon_state: BeaconState, - valid_shard_blocks: Iterable[ShardBlock], + valid_shard_blocks: Sequence[ShardBlock], candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -336,7 +336,7 @@ def is_valid_shard_block(beacon_blocks: TypingList[BeaconBlock], assert proposer_index is not None assert bls_verify( pubkey=beacon_state.validators[proposer_index].pubkey, - message_hash=signing_root(block), + message_hash=signing_root(candidate), signature=candidate.signature, domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, slot_to_epoch(candidate.slot)), ) @@ -353,7 +353,7 @@ Let: * `candidate` be a candidate `ShardAttestation` for which validity is to be determined by running `is_valid_shard_attestation` ```python -def is_valid_shard_attestation(valid_shard_blocks: Iterable[ShardBlock], +def is_valid_shard_attestation(valid_shard_blocks: Sequence[ShardBlock], beacon_state: BeaconState, candidate: ShardAttestation) -> bool: # Check shard block @@ -383,7 +383,7 @@ Let: ```python def is_valid_beacon_attestation(shard: Shard, - shard_blocks: TypingList[ShardBlock], + shard_blocks: Sequence[ShardBlock], beacon_state: BeaconState, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: From f95e7315b42afe8d879c270a6e354e92ad5840c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 18:34:33 +0200 Subject: [PATCH 53/76] fix get_active_validator_indices typing usage --- specs/core/0_beacon-chain.md | 4 +++- test_libs/pyspec/eth2spec/test/helpers/custody.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 3 ++- .../eth2spec/test/helpers/proposer_slashings.py | 2 +- .../pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../block_processing/test_process_transfer.py | 14 +++++++------- .../test_process_voluntary_exit.py | 2 +- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f42b4cc13..1f500bc05 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1553,7 +1553,9 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) + List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) + ) ) # Set total slashed balances state.slashed_balances[next_epoch % EPOCHS_PER_SLASHED_BALANCES_VECTOR] = ( diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index bc70c9fa8..cd34ee4c8 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -4,7 +4,7 @@ from eth2spec.utils.bls import bls_sign def get_valid_early_derived_secret_reveal(spec, state, epoch=None): current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch).last() + revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d1d818908..73738932a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -40,7 +40,8 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(spec.get_active_validator_indices(state, spec.GENESIS_EPOCH)) + genesis_active_index_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py index 7b6f71374..d5b7f7b7f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py +++ b/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py @@ -6,7 +6,7 @@ from eth2spec.test.helpers.keys import pubkey_to_privkey def get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=False): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] slot = state.slot diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 215cd6fcb..acc6a35c5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -9,7 +9,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f slot = state.slot current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch).last() + sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 2bed94dc8..e9d282b3a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -63,7 +63,7 @@ def test_success_withdrawable(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -73,7 +73,7 @@ def test_success_active_above_max_effective(spec, state): @with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -94,7 +94,7 @@ def test_invalid_signature(spec, state): @with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=amount, fee=0, signed=True) @@ -115,7 +115,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -128,7 +128,7 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test def test_insufficient_balance(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -141,7 +141,7 @@ def test_insufficient_balance(spec, state): @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] balance = state.balances[sender_index] transfer = get_valid_transfer( spec, @@ -161,7 +161,7 @@ def test_no_dust_sender(spec, state): @with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + 1 transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) state.balances[transfer.recipient] = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index b9f7d6098..33cacc4e2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -93,7 +93,7 @@ def test_success_exit_queue(spec, state): continue # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validators[validator_index].pubkey] voluntary_exit = build_voluntary_exit( spec, diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index c86394a0b..f8590005e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -269,7 +269,7 @@ def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( state, spec.get_current_epoch(state) - ).last() + )[-1] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -315,7 +315,7 @@ def test_voluntary_exit(spec, state): # overwrite default 0 to test # spec.MAX_TRANSFERS = 1 - # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).last() + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] # amount = get_balance(state, sender_index) # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) @@ -347,7 +347,7 @@ def test_voluntary_exit(spec, state): @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch).last() + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] assert state.validators[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH From dd5ad2e2c56b7bc47da3c47c8fa5ed87d1eaa650 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:48:06 +0200 Subject: [PATCH 54/76] remove unnecessary (and now outdated) type hints, update List encoding for generators --- test_libs/pyspec/eth2spec/debug/encode.py | 4 +-- .../pyspec/eth2spec/test/helpers/genesis.py | 1 + .../eth2spec/test/sanity/test_blocks.py | 28 +++++++++---------- .../pyspec/eth2spec/test/test_finality.py | 11 ++++---- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index d264bd7ff..e45bd1117 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -13,9 +13,9 @@ def encode(value: SSZValue, include_hash_tree_roots=False): elif isinstance(value, Bit): assert value in (True, False) return value - elif isinstance(value, (List, Vector)): + elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] - elif isinstance(value, (Bytes, BytesN)): # both bytes and BytesN + elif isinstance(value, bytes): # both bytes and BytesN return '0x' + value.hex() elif isinstance(value, Container): ret = {} diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 73738932a..680825165 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.keys import pubkeys from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import List def build_mock_validator(spec, i: int, balance: int): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index f8590005e..36ded7bc6 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -28,7 +28,7 @@ def test_empty_block_transition(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.eth1_data_votes) == pre_eth1_votes + 1 @@ -48,7 +48,7 @@ def test_skipped_slots(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -69,7 +69,7 @@ def test_empty_epoch_transition(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -90,7 +90,7 @@ def test_empty_epoch_transition(spec, state): # state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'blocks', [block] # yield 'post', state # assert state.slot == block.slot @@ -120,7 +120,7 @@ def test_proposer_slashing(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state # check if slashed @@ -155,7 +155,7 @@ def test_attester_slashing(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state slashed_validator = state.validators[validator_index] @@ -193,7 +193,7 @@ def test_deposit_in_block(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.validators) == initial_registry_len + 1 @@ -221,7 +221,7 @@ def test_deposit_top_up(spec, state): state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert len(state.validators) == initial_registry_len @@ -256,7 +256,7 @@ def test_attestation(spec, state): sign_block(spec, state, epoch_block) state_transition_and_sign_block(spec, state, epoch_block) - yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] + yield 'blocks', [attestation_block, epoch_block] yield 'post', state assert len(state.current_epoch_attestations) == 0 @@ -303,7 +303,7 @@ def test_voluntary_exit(spec, state): sign_block(spec, state, exit_block) state_transition_and_sign_block(spec, state, exit_block) - yield 'blocks', [initiate_exit_block, exit_block], List[spec.BeaconBlock] + yield 'blocks', [initiate_exit_block, exit_block] yield 'post', state assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -334,7 +334,7 @@ def test_voluntary_exit(spec, state): # state_transition_and_sign_block(spec, state, block) - # yield 'blocks', [block], List[spec.BeaconBlock] + # yield 'blocks', [block] # yield 'post', state # sender_balance = get_balance(state, sender_index) @@ -362,7 +362,7 @@ def test_balance_driven_status_transitions(spec, state): sign_block(spec, state, block) state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.validators[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -379,7 +379,7 @@ def test_historical_batch(spec, state): block = build_empty_block_for_next_slot(spec, state, signed=True) state_transition_and_sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + yield 'blocks', [block] yield 'post', state assert state.slot == block.slot @@ -408,7 +408,7 @@ def test_historical_batch(spec, state): # state_transition_and_sign_block(spec, state, block) -# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'blocks', [block] # yield 'post', state # assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 730456286..cf9e94a71 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,5 +1,4 @@ from copy import deepcopy -from typing import List from eth2spec.test.context import spec_state_test, never_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block @@ -82,7 +81,7 @@ def test_finality_no_updates_at_genesis(spec, state): elif epoch == 1: check_finality(spec, state, prev_state, False, False, False) - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -111,7 +110,7 @@ def test_finality_rule_4(spec, state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -142,7 +141,7 @@ def test_finality_rule_1(spec, state): assert state.finalized_epoch == prev_state.previous_justified_epoch assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -175,7 +174,7 @@ def test_finality_rule_2(spec, state): blocks += new_blocks - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state @@ -225,5 +224,5 @@ def test_finality_rule_3(spec, state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, List[spec.BeaconBlock] + yield 'blocks', blocks yield 'post', state From e873bbd73bc3d05526b3c3f42ea3e1bd88049eb9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:59:15 +0200 Subject: [PATCH 55/76] support list casting --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 14415ac6b..891480a5c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -296,7 +296,7 @@ class BaseList(list, Elements): @classmethod def extract_args(cls, *args): x = list(args) - if len(x) == 1 and isinstance(x[0], GeneratorType): + if len(x) == 1 and isinstance(x[0], (GeneratorType, list, tuple)): x = list(x[0]) x = [coerce_type_maybe(v, cls.elem_type) for v in x] return x diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 4325501aa..bd86a5806 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -161,6 +161,8 @@ def test_list(): assert issubclass(v.type(), Elements) assert isinstance(v.type(), ElementsType) + assert len(typ([i for i in range(10)])) == 10 # cast py list to SSZ list + foo = List[uint32, 128](0 for i in range(128)) foo[0] = 123 foo[1] = 654 From 47034a6c8c40fbe5beb3aba36f264f92863682d2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 19:59:44 +0200 Subject: [PATCH 56/76] fix imports in helper test file --- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 680825165..d3d67e09e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_SIZE]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root From 0249cf651e2c30c6c343dd6c2b2440f17b59a2cd Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 20:04:17 +0200 Subject: [PATCH 57/76] fix lint, and update encoder to handle the few imported types well --- test_libs/pyspec/eth2spec/debug/encode.py | 5 ++--- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index e45bd1117..9a819a256 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bytes, BytesN, List, Vector, Bit + SSZValue, uint, Container, Bit ) @@ -11,8 +11,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): return str(value) return value elif isinstance(value, Bit): - assert value in (True, False) - return value + return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] elif isinstance(value, bytes): # both bytes and BytesN diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 36ded7bc6..286c0150c 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,5 +1,4 @@ from copy import deepcopy -from typing import List from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.bls import bls_sign From 9befe09f82605c6cf103c9c4fbd120ad50977831 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 21:27:56 +0200 Subject: [PATCH 58/76] test merkleize chunks --- .../eth2spec/utils/test_merkle_minimal.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py new file mode 100644 index 000000000..1c72a649b --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -0,0 +1,58 @@ +import pytest +from .merkle_minimal import zerohashes, merkleize_chunks +from .hash_function import hash + + +def h(a: bytes, b: bytes) -> bytes: + return hash(a + b) + + +def e(v: int) -> bytes: + return v.to_bytes(length=32, byteorder='little') + + +def z(i: int) -> bytes: + return zerohashes[i] + + +cases = [ + (0, 0, 1, z(0)), + (0, 1, 1, e(0)), + (1, 0, 2, h(z(0), z(0))), + (1, 1, 2, h(e(0), z(0))), + (1, 2, 2, h(e(0), e(1))), + (2, 0, 4, h(h(z(0), z(0)), z(1))), + (2, 1, 4, h(h(e(0), z(0)), z(1))), + (2, 2, 4, h(h(e(0), e(1)), z(1))), + (2, 3, 4, h(h(e(0), e(1)), h(e(2), z(0)))), + (2, 4, 4, h(h(e(0), e(1)), h(e(2), e(3)))), + (3, 0, 8, h(h(h(z(0), z(0)), z(1)), z(2))), + (3, 1, 8, h(h(h(e(0), z(0)), z(1)), z(2))), + (3, 2, 8, h(h(h(e(0), e(1)), z(1)), z(2))), + (3, 3, 8, h(h(h(e(0), e(1)), h(e(2), z(0))), z(2))), + (3, 4, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), z(2))), + (3, 5, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1)))), + (3, 6, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0))))), + (3, 7, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0))))), + (3, 8, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), + (4, 0, 16, h(h(h(h(z(0), z(0)), z(1)), z(2)), z(3))), + (4, 1, 16, h(h(h(h(e(0), z(0)), z(1)), z(2)), z(3))), + (4, 2, 16, h(h(h(h(e(0), e(1)), z(1)), z(2)), z(3))), + (4, 3, 16, h(h(h(h(e(0), e(1)), h(e(2), z(0))), z(2)), z(3))), + (4, 4, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), z(2)), z(3))), + (4, 5, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1))), z(3))), + (4, 6, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0)))), z(3))), + (4, 7, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0)))), z(3))), + (4, 8, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), z(3))), + (4, 9, 16, + h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), h(h(h(e(8), z(0)), z(1)), z(2)))), +] + + +@pytest.mark.parametrize( + 'depth,count,pow2,value', + cases, +) +def test_merkleize_chunks(depth, count, pow2, value): + chunks = [e(i) for i in range(count)] + assert merkleize_chunks(chunks, pad_to=pow2) == value From da858f1aae3ff5f906e041f025094d0a90ccc0c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 21:49:42 +0200 Subject: [PATCH 59/76] fix int encoding, fix list randomization size limit. --- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/random_value.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 9a819a256..0878a1f94 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -8,8 +8,8 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if isinstance(value, uint): # Larger uints are boxed and the class declares their byte length if value.type().byte_len > 8: - return str(value) - return value + return str(int(value)) + return int(value) elif isinstance(value, Bit): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index bd40cb832..8e13cd5e1 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -95,6 +95,9 @@ def get_random_ssz_object(rng: Random, elif mode == RandomizationMode.mode_max_count: length = max_list_length + if typ.length < length: # SSZ imposes a hard limit on lists, we can't put in more than that + length = typ.length + return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) From a5a2e29dfe8a424ea53cf0b9203b58463046a24b Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 22 Jun 2019 22:15:42 +0200 Subject: [PATCH 60/76] remove unnecessary argument, typing is based on values fully now --- test_generators/ssz_static/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 9d9af8c5e..0dfdebf5d 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -21,8 +21,8 @@ MAX_LIST_LENGTH = 10 @to_dict -def create_test_case_contents(value, typ): - yield "value", encode.encode(value, typ) +def create_test_case_contents(value): + yield "value", encode.encode(value) yield "serialized", '0x' + serialize(value).hex() yield "root", '0x' + hash_tree_root(value).hex() if hasattr(value, "signature"): @@ -32,7 +32,7 @@ def create_test_case_contents(value, typ): @to_dict def create_test_case(rng: Random, name: str, typ, mode: random_value.RandomizationMode, chaos: bool): value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) - yield name, create_test_case_contents(value, typ) + yield name, create_test_case_contents(value) def get_spec_ssz_types(): From 1408a1ee0daf918a4fdc72facad4e22bf56d9d79 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 23 Jun 2019 00:17:54 +0200 Subject: [PATCH 61/76] undo tuple wrapping --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9f0de2e6b..f84b0dd46 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -180,10 +180,10 @@ def get_persistent_committee(state: BeaconState, # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated - return tuple(sorted(list(set( + return sorted(list(set( [i for i in earlier_committee if epoch % PERSISTENT_COMMITTEE_PERIOD < get_switchover_epoch(state, epoch, i)] + [i for i in later_committee if epoch % PERSISTENT_COMMITTEE_PERIOD >= get_switchover_epoch(state, epoch, i)] - )))) + ))) ``` ### `get_shard_proposer_index` From 82ae1804900e06daf2446c8dfbb1b198d931a5ed Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:38:36 +0200 Subject: [PATCH 62/76] clean up list limit constants --- configs/constant_presets/mainnet.yaml | 4 ++++ configs/constant_presets/minimal.yaml | 4 ++++ specs/core/0_beacon-chain.md | 17 +++++++---------- .../pyspec/eth2spec/test/helpers/genesis.py | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 09ec7bdb7..9f7ca950f 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -77,6 +77,10 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 EPOCHS_PER_HISTORICAL_VECTOR: 65536 # 2**13 (= 8,192) epochs ~36 days EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 201ac475f..3e3f7ccb4 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -78,6 +78,10 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 EPOCHS_PER_HISTORICAL_VECTOR: 64 # [customized] smaller state EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64 +# 2**24 (= 16,777,216) historical roots +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 # Reward and penalty quotients diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1f500bc05..7d2459eb3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -224,7 +224,6 @@ These configurations are updated for releases, but may be out of sync during `de | `ACTIVATION_EXIT_DELAY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | -| `HISTORICAL_ROOTS_LENGTH` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | @@ -238,10 +237,8 @@ These configurations are updated for releases, but may be out of sync during `de | - | - | :-: | :-: | | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHED_BALANCES_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | -| `RANDAO_MIXES_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -| `VALIDATOR_REGISTRY_SIZE` | `2**40` (= 1,099,511,627,776) | | | +| `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | +| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | | ### Rewards and penalties @@ -520,14 +517,14 @@ class BeaconState(Container): latest_block_header: BeaconBlockHeader block_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Hash, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Hash, HISTORICAL_ROOTS_LENGTH] + historical_roots: List[Hash, HISTORICAL_ROOTS_LIMIT] # Eth1 eth1_data: Eth1Data eth1_data_votes: List[Eth1Data, SLOTS_PER_ETH1_VOTING_PERIOD] eth1_deposit_index: uint64 # Registry - validators: List[Validator, VALIDATOR_REGISTRY_SIZE] - balances: List[Gwei, VALIDATOR_REGISTRY_SIZE] + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] # Shuffling start_shard: Shard randao_mixes: Vector[Hash, EPOCHS_PER_HISTORICAL_VECTOR] @@ -1195,7 +1192,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth # Populate active_index_roots genesis_active_index_root = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE](get_active_validator_indices(state, GENESIS_EPOCH)) + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) for index in range(EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root @@ -1553,7 +1550,7 @@ def process_final_updates(state: BeaconState) -> None: # Set active index root index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % EPOCHS_PER_HISTORICAL_VECTOR state.active_index_roots[index_root_position] = hash_tree_root( - List[ValidatorIndex, VALIDATOR_REGISTRY_SIZE]( + List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT]( get_active_validator_indices(state, Epoch(next_epoch + ACTIVATION_EXIT_DELAY)) ) ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index d3d67e09e..ce0be19bb 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -41,7 +41,7 @@ def create_genesis_state(spec, num_validators): validator.activation_eligibility_epoch = spec.GENESIS_EPOCH validator.activation_epoch = spec.GENESIS_EPOCH - genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_SIZE]( + genesis_active_index_root = hash_tree_root(List[spec.ValidatorIndex, spec.VALIDATOR_REGISTRY_LIMIT]( spec.get_active_validator_indices(state, spec.GENESIS_EPOCH))) for index in range(spec.EPOCHS_PER_HISTORICAL_VECTOR): state.active_index_roots[index] = genesis_active_index_root From c73417b4cae0335e348f1a4d06968506301aeb64 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:40:47 +0200 Subject: [PATCH 63/76] deserialize-basic detail, make subclass --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 144201d83..e2a264955 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -27,7 +27,7 @@ def deserialize_basic(value, typ: BasicType): return typ(int.from_bytes(value, 'little')) elif issubclass(typ, Bit): assert value in (b'\x00', b'\x01') - return Bit(value == b'\x01') + return typ(value == b'\x01') else: raise Exception(f"Type not supported: {typ}") From 5989e5cd2368a5e7b4913bcfa6ef5025019b0aa7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 24 Jun 2019 23:56:26 +0200 Subject: [PATCH 64/76] use Bool as base name, make Bit an alias --- scripts/build_spec.py | 6 +++--- specs/core/0_beacon-chain.md | 4 ++-- test_libs/pyspec/eth2spec/debug/decode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/encode.py | 4 ++-- .../pyspec/eth2spec/debug/random_value.py | 8 ++++---- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 ++-- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +++--- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- .../pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 11 ++++++++--- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 +++++++++++------- 10 files changed, 43 insertions(+), 29 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b55dfa44..08793869f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, uint64, + Bit, Bool, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -53,7 +53,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Container, List, Vector, Bytes, uint64, + Bit, Bool, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -179,7 +179,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7d2459eb3..b3e3ba0ea 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -314,7 +314,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bit + slashed: Bool # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -354,7 +354,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit for the custody of crosslink data + custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 743479371..c0b53b0ef 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bit, + SSZType, SSZValue, uint, Container, Bytes, List, Bool, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bit)): + if issubclass(typ, (uint, Bool)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 0878a1f94..02814e441 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bit + SSZValue, uint, Container, Bool ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bit): + elif isinstance(value, Bool): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 8e13cd5e1..c6efb722b 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from random import Random from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bit, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bit): + if issubclass(typ, Bool): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index a4ebb7e70..e533ca5c2 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bit): + elif issubclass(typ, spec_ssz.Bool): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bit): + elif issubclass(typ, spec_ssz.Bool): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index e2a264955..b9c7b6d38 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bit, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, ) # SSZ Serialization @@ -13,7 +13,7 @@ BYTES_PER_LENGTH_OFFSET = 4 def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bit): + elif isinstance(value, Bool): if value: return b'\x01' else: @@ -25,7 +25,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bit): + elif issubclass(typ, Bool): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 891480a5c..6079f2866 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bit(BasicValue): # can't subclass bool. +class Bool(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value, *args, **kwargs): @@ -47,6 +47,11 @@ class Bit(BasicValue): # can't subclass bool. return self > 0 +# Alias for Bool +class Bit(Bool): + pass + + class uint(BasicValue, metaclass=BasicType): def __new__(cls, value, *args, **kwargs): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index ae0849098..aa7aee64a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,6 +1,6 @@ from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Container, List, Vector, Bytes, BytesN, + Bit, Bool, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, byte ) @@ -47,11 +47,16 @@ for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): sig_test_data[k] = v test_data = [ - ("bool F", Bit(False), "00"), - ("bool T", Bit(True), "01"), + ("bit F", Bit(False), "00"), + ("bit T", Bit(True), "01"), + ("bool F", Bool(False), "00"), + ("bool T", Bool(True), "01"), ("uint8 00", uint8(0x00), "00"), ("uint8 01", uint8(0x01), "01"), ("uint8 ab", uint8(0xab), "ab"), + ("byte 00", byte(0x00), "00"), + ("byte 01", byte(0x01), "01"), + ("byte ab", byte(0xab), "ab"), ("uint16 0000", uint16(0x0000), "0000"), ("uint16 abcd", uint16(0xabcd), "cdab"), ("uint32 00000000", uint32(0x00000000), "00000000"), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index bd86a5806..2af742360 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,7 +1,7 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Container, List, Vector, Bytes, BytesN, - uint, uint8, uint16, uint32, uint64, uint128, uint256, + Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bit, BasicValue) - assert isinstance(Bit, BasicType) + assert issubclass(Bool, BasicValue) + assert isinstance(Bool, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -38,21 +38,25 @@ def test_subclasses(): def test_basic_instances(): - for u in [uint, uint8, uint16, uint32, uint64, uint128, uint256]: + for u in [uint, uint8, byte, uint16, uint32, uint64, uint128, uint256]: v = u(123) assert isinstance(v, uint) assert isinstance(v, int) assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bit(True), BasicValue) - assert isinstance(Bit(False), BasicValue) + assert isinstance(Bool(True), BasicValue) + assert isinstance(Bool(False), BasicValue) + assert isinstance(Bit(True), Bool) + assert isinstance(Bit(False), Bool) def test_basic_value_bounds(): max = { + Bool: 2 ** 1, Bit: 2 ** 1, uint8: 2 ** (8 * 1), + byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), uint32: 2 ** (8 * 4), uint64: 2 ** (8 * 8), From 9fb58067645da1aa02dcfa995cb61fb86a2dff16 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 00:24:13 +0200 Subject: [PATCH 65/76] be explicit about input for balance sum --- scripts/build_spec.py | 4 ++-- specs/core/0_beacon-chain.md | 6 +++--- specs/core/1_shard-data-chains.md | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 08793869f..237c24384 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -12,7 +12,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Callable, Iterable, Dict, Set, Sequence, Tuple, + Any, Callable, Dict, Set, Sequence, Tuple, ) from dataclasses import ( @@ -37,7 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Iterable, Optional, Set, Sequence, Tuple, + Any, Callable, Dict, Optional, Set, Sequence, Tuple, List as TypingList ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d6be89f7..b8cab7eaf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -908,7 +908,7 @@ def bytes_to_int(data: bytes) -> int: ### `get_total_balance` ```python -def get_total_balance(state: BeaconState, indices: Iterable[ValidatorIndex]) -> Gwei: +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ @@ -1383,7 +1383,7 @@ def process_crosslinks(state: BeaconState) -> None: for epoch in (get_previous_epoch(state), get_current_epoch(state)): for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - crosslink_committee = get_crosslink_committee(state, epoch, shard) + crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) if 3 * get_total_balance(state, attesting_indices) >= 2 * get_total_balance(state, crosslink_committee): state.current_crosslinks[shard] = winning_crosslink @@ -1456,7 +1456,7 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[G epoch = get_previous_epoch(state) for offset in range(get_epoch_committee_count(state, epoch)): shard = Shard((get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT) - crosslink_committee = get_crosslink_committee(state, epoch, shard) + crosslink_committee = set(get_crosslink_committee(state, epoch, shard)) winning_crosslink, attesting_indices = get_winning_crosslink_and_attesting_indices(state, epoch, shard) attesting_balance = get_total_balance(state, attesting_indices) committee_balance = get_total_balance(state, crosslink_committee) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f84b0dd46..dd48a842a 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -378,7 +378,7 @@ Let: * `shard` be a valid `Shard` * `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot` * `beacon_state` be the canonical `BeaconState` -* `valid_attestations` be the list of valid `Attestation`, recursively defined +* `valid_attestations` be the set of valid `Attestation` objects, recursively defined * `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` ```python @@ -388,7 +388,7 @@ def is_valid_beacon_attestation(shard: Shard, valid_attestations: Set[Attestation], candidate: Attestation) -> bool: # Check if attestation is already determined valid - for _, attestation in enumerate(valid_attestations): + for attestation in valid_attestations: if candidate == attestation: return True From a5b7564c5b6c337a9db8272beac7d9fe61e92f6d Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 02:35:55 +0200 Subject: [PATCH 66/76] hash-tree-root tests --- .../eth2spec/utils/ssz/test_ssz_impl.py | 192 ++++++++++++++---- 1 file changed, 155 insertions(+), 37 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index aa7aee64a..82fb4ec68 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,8 +1,10 @@ +from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( Bit, Bool, Container, List, Vector, Bytes, BytesN, - uint8, uint16, uint32, uint64, byte + uint8, uint16, uint32, uint64, uint256, byte ) +from ..hash_function import hash as bytes_hash import pytest @@ -46,37 +48,117 @@ sig_test_data = [0 for i in range(96)] for k, v in {0: 1, 32: 2, 64: 3, 95: 0xff}.items(): sig_test_data[k] = v + +def chunk(hex: str) -> str: + return (hex + ("00" * 32))[:64] # just pad on the right, to 32 bytes (64 hex chars) + + +def h(a: str, b: str) -> str: + return bytes_hash(bytes.fromhex(a) + bytes.fromhex(b)).hex() + + +# zero hashes, as strings, for +zero_hashes = [chunk("")] +for layer in range(1, 32): + zero_hashes.append(h(zero_hashes[layer - 1], zero_hashes[layer - 1])) + + +def merge(a: str, branch: Iterable[str]) -> str: + """ + Merge (out on left, branch on right) leaf a with branch items, branch is from bottom to top. + """ + out = a + for b in branch: + out = h(out, b) + return out + + test_data = [ - ("bit F", Bit(False), "00"), - ("bit T", Bit(True), "01"), - ("bool F", Bool(False), "00"), - ("bool T", Bool(True), "01"), - ("uint8 00", uint8(0x00), "00"), - ("uint8 01", uint8(0x01), "01"), - ("uint8 ab", uint8(0xab), "ab"), - ("byte 00", byte(0x00), "00"), - ("byte 01", byte(0x01), "01"), - ("byte ab", byte(0xab), "ab"), - ("uint16 0000", uint16(0x0000), "0000"), - ("uint16 abcd", uint16(0xabcd), "cdab"), - ("uint32 00000000", uint32(0x00000000), "00000000"), - ("uint32 01234567", uint32(0x01234567), "67452301"), - ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301"), - ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301"), - ("uint32 01234567", uint32(0x01234567), "67452301"), - ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000"), - ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301"), + ("bit F", Bit(False), "00", chunk("00")), + ("bit T", Bit(True), "01", chunk("01")), + ("bool F", Bool(False), "00", chunk("00")), + ("bool T", Bool(True), "01", chunk("01")), + ("uint8 00", uint8(0x00), "00", chunk("00")), + ("uint8 01", uint8(0x01), "01", chunk("01")), + ("uint8 ab", uint8(0xab), "ab", chunk("ab")), + ("byte 00", byte(0x00), "00", chunk("00")), + ("byte 01", byte(0x01), "01", chunk("01")), + ("byte ab", byte(0xab), "ab", chunk("ab")), + ("uint16 0000", uint16(0x0000), "0000", chunk("0000")), + ("uint16 abcd", uint16(0xabcd), "cdab", chunk("cdab")), + ("uint32 00000000", uint32(0x00000000), "00000000", chunk("00000000")), + ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), + ("small (4567, 0123)", SmallTestStruct(A=0x4567, B=0x0123), "67452301", h(chunk("6745"), chunk("2301"))), + ("small [4567, 0123]::2", Vector[uint16, 2](uint16(0x4567), uint16(0x0123)), "67452301", chunk("67452301")), + ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), + ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000", chunk("0000000000000000")), + ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301", chunk("efcdab8967452301")), ("sig", BytesN[96](*sig_test_data), "0100000000000000000000000000000000000000000000000000000000000000" "0200000000000000000000000000000000000000000000000000000000000000" - "03000000000000000000000000000000000000000000000000000000000000ff"), - ("emptyTestStruct", EmptyTestStruct(), ""), - ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab"), - ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412"), - ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff"), - ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff"), + "03000000000000000000000000000000000000000000000000000000000000ff", + h(h(chunk("01"), chunk("02")), + h("03000000000000000000000000000000000000000000000000000000000000ff", chunk("")))), + ("emptyTestStruct", EmptyTestStruct(), "", chunk("")), + ("singleFieldTestStruct", SingleFieldTestStruct(A=0xab), "ab", chunk("ab")), + ("uint16 list", List[uint16, 32](uint16(0xaabb), uint16(0xc0ad), uint16(0xeeff)), "bbaaadc0ffee", + h(h(chunk("bbaaadc0ffee"), chunk("")), chunk("03000000")) # max length: 32 * 2 = 64 bytes = 2 chunks + ), + ("uint32 list", List[uint32, 128](uint32(0xaabb), uint32(0xc0ad), uint32(0xeeff)), "bbaa0000adc00000ffee0000", + # max length: 128 * 4 = 512 bytes = 16 chunks + h(merge(chunk("bbaa0000adc00000ffee0000"), zero_hashes[0:4]), chunk("03000000")) + ), + ("uint256 list", List[uint256, 32](uint256(0xaabb), uint256(0xc0ad), uint256(0xeeff)), + "bbaa000000000000000000000000000000000000000000000000000000000000" + "adc0000000000000000000000000000000000000000000000000000000000000" + "ffee000000000000000000000000000000000000000000000000000000000000", + h(merge(h(h(chunk("bbaa"), chunk("adc0")), h(chunk("ffee"), chunk(""))), zero_hashes[2:5]), chunk("03000000")) + ), + ("uint256 list long", List[uint256, 128](i for i in range(1, 20)), + "".join([i.to_bytes(length=32, byteorder='little').hex() for i in range(1, 20)]), + h(merge( + h( + h( + h( + h(h(chunk("01"), chunk("02")), h(chunk("03"), chunk("04"))), + h(h(chunk("05"), chunk("06")), h(chunk("07"), chunk("08"))), + ), + h( + h(h(chunk("09"), chunk("0a")), h(chunk("0b"), chunk("0c"))), + h(h(chunk("0d"), chunk("0e")), h(chunk("0f"), chunk("10"))), + ) + ), + h( + h( + h(h(chunk("11"), chunk("12")), h(chunk("13"), chunk(""))), + zero_hashes[2] + ), + zero_hashes[3] + ) + ), + zero_hashes[5:7]), chunk("13000000")) # 128 chunks = 7 deep + ), + ("fixedTestStruct", FixedTestStruct(A=0xab, B=0xaabbccdd00112233, C=0x12345678), "ab33221100ddccbbaa78563412", + h(h(chunk("ab"), chunk("33221100ddccbbaa")), h(chunk("78563412"), chunk("")))), + ("varTestStruct nil", VarTestStruct(A=0xabcd, C=0xff), "cdab07000000ff", + h(h(chunk("cdab"), h(zero_hashes[6], chunk("00000000"))), h(chunk("ff"), chunk("")))), + ("varTestStruct empty", VarTestStruct(A=0xabcd, B=List[uint16, 1024](), C=0xff), "cdab07000000ff", + h(h(chunk("cdab"), h(zero_hashes[6], chunk("00000000"))), h(chunk("ff"), chunk("")))), # log2(1024*2/32)= 6 deep ("varTestStruct some", VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), - "cdab07000000ff010002000300"), + "cdab07000000ff010002000300", + h( + h( + chunk("cdab"), + h( + merge( + chunk("010002000300"), + zero_hashes[0:6] + ), + chunk("03000000") # length mix in + ) + ), + h(chunk("ff"), chunk("")) + )), ("complexTestStruct", ComplexTestStruct( A=0xaabb, @@ -90,8 +172,8 @@ test_data = [ FixedTestStruct(A=0xee, B=0x4444444444444444, C=0x00112233), FixedTestStruct(A=0xff, B=0x5555555555555555, C=0x44556677)), G=Vector[VarTestStruct, 2]( - VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), - VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff)), + VarTestStruct(A=0xdead, B=List[uint16, 1024](1, 2, 3), C=0x11), + VarTestStruct(A=0xbeef, B=List[uint16, 1024](4, 5, 6), C=0x22)), ), "bbaa" "47000000" # offset of B, []uint16 @@ -107,17 +189,53 @@ test_data = [ "666f6f626172" # foobar "cdab07000000ff010002000300" # contents of E "08000000" "15000000" # [start G]: local offsets of [2]varTestStruct - "cdab07000000ff010002000300" - "cdab07000000ff010002000300", - ) + "adde0700000011010002000300" + "efbe0700000022040005000600", + h( + h( + h( # A and B + chunk("bbaa"), + h(merge(chunk("22114433"), zero_hashes[0:3]), chunk("02000000")) # 2*128/32 = 8 chunks + ), + h( # C and D + chunk("ff"), + h(merge(chunk("666f6f626172"), zero_hashes[0:3]), chunk("06000000")) # 256/32 = 8 chunks + ) + ), + h( + h( # E and F + h(h(chunk("cdab"), h(merge(chunk("010002000300"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("ff"), chunk(""))), + h( + h( + h(h(chunk("cc"), chunk("4242424242424242")), h(chunk("37133713"), chunk(""))), + h(h(chunk("dd"), chunk("3333333333333333")), h(chunk("cdabcdab"), chunk(""))), + ), + h( + h(h(chunk("ee"), chunk("4444444444444444")), h(chunk("33221100"), chunk(""))), + h(h(chunk("ff"), chunk("5555555555555555")), h(chunk("77665544"), chunk(""))), + ), + ) + ), + h( # G and padding + h( + h(h(chunk("adde"), h(merge(chunk("010002000300"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("11"), chunk(""))), + h(h(chunk("efbe"), h(merge(chunk("040005000600"), zero_hashes[0:6]), chunk("03000000"))), + h(chunk("22"), chunk(""))), + ), + chunk("") + ) + ) + )) ] -@pytest.mark.parametrize("name, value, serialized", test_data) -def test_serialize(name, value, serialized): +@pytest.mark.parametrize("name, value, serialized, _", test_data) +def test_serialize(name, value, serialized, _): assert serialize(value) == bytes.fromhex(serialized) -@pytest.mark.parametrize("name, value, _", test_data) -def test_hash_tree_root(name, value, _): - hash_tree_root(value) +@pytest.mark.parametrize("name, value, _, root", test_data) +def test_hash_tree_root(name, value, _, root): + assert hash_tree_root(value) == bytes.fromhex(root) From 45dbf5a107324e7f5b5c69e2d76950cca888bac3 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 25 Jun 2019 02:41:02 +0200 Subject: [PATCH 67/76] Remove old Deltas reference Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_beacon-chain.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b8cab7eaf..77fdb34e1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -286,8 +286,6 @@ We define the following Python custom types for type hinting and readability: | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | -`Deltas` is a non-SSZ type, a series of changes applied to balances, optimized by clients. - ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. From 054a1579536afdbdb71d6b6da0c778bcbae6a3b5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 02:56:49 +0200 Subject: [PATCH 68/76] get rid of TypingList, add MutableSequence --- scripts/build_spec.py | 3 +-- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 237c24384..d67c0e5c6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -37,8 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Callable, Dict, Optional, Set, Sequence, Tuple, - List as TypingList + Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple, ) from dataclasses import ( diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 24e9a19e2..07f6ec698 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -318,7 +318,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ### `replace_empty_or_append` ```python -def replace_empty_or_append(list: TypingList[Any], new_element: Any) -> int: +def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int: for i in range(len(list)): if is_empty(list[i]): list[i] = new_element diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index dd48a842a..d1c86eca2 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -252,9 +252,9 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) - def pad_to_power_of_2(values: TypingList[bytes]) -> TypingList[bytes]: + def pad_to_power_of_2(values: MutableSequence[bytes]) -> Sequence[bytes]: while not is_power_of_two(len(values)): - values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] + values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) return values def hash_tree_root_of_bytes(data: bytes) -> bytes: @@ -264,6 +264,8 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: return data + b'\x00' * (length - len(data)) return hash( + # TODO untested code. + # Need to either pass a typed list to hash-tree-root, or merkleize_chunks(values, pad_to=2**x) hash_tree_root(pad_to_power_of_2([ hash_tree_root_of_bytes( zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY) From 37b1872634097f4d2b22242e0645f3175d3bcf2b Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 25 Jun 2019 08:30:59 -0500 Subject: [PATCH 69/76] Remove duplicate custom type definitions (#1214) Seems to be an accidental duplication of the type defs --- specs/core/0_beacon-chain.md | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 103c436b5..6898bf459 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -21,7 +21,6 @@ - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - [Signature domains](#signature-domains) - - [Custom types](#custom-types-1) - [Containers](#containers) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) @@ -270,20 +269,6 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_VOLUNTARY_EXIT` | `4` | | `DOMAIN_TRANSFER` | `5` | -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `Slot` | `uint64` | a slot number | -| `Epoch` | `uint64` | an epoch number | -| `Shard` | `uint64` | a shard number | -| `ValidatorIndex` | `uint64` | a validator registry index | -| `Gwei` | `uint64` | an amount in Gwei | -| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | -| `BLSSignature` | `Bytes96` | a BLS12-381 signature | - ## Containers The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. From 20602bc92ba98a8b32a7b1ae81cfeb68018f55d5 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Tue, 25 Jun 2019 08:32:56 -0500 Subject: [PATCH 70/76] phase 0 doc standardization b4 spec freeze (#1212) --- scripts/README.md | 12 +++---- specs/core/0_beacon-chain.md | 28 +++++++-------- specs/core/0_deposit-contract.md | 2 +- specs/core/0_fork-choice.md | 6 ++-- specs/simple-serialize.md | 8 ++--- specs/validator/0_beacon-chain-validator.md | 36 +++++++++---------- .../validator/0_beacon-node-validator-api.md | 19 +++++----- 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 25b46decf..9d5849053 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,18 +1,18 @@ # Building pyspecs from specs.md -The benefit of the particular spec design is that the given markdown files can be converted to a `spec.py` file for the purposes of testing and linting. The result of this is that bugs are discovered and patched more quickly. +The benefit of the particular spec design is that the given Markdown files can be converted to a `spec.py` file for the purposes of testing and linting. As a result, bugs are discovered and patched more quickly. -Specs can be built from either a single markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. +Specs can be built from either a single Markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. ## Usage -For usage of the spec builder run `python3 -m build_spec --help`. +For usage of the spec builder, run `python3 -m build_spec --help`. ## `@Labels` and inserts The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows: -* In the old file, a label is added as a python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: +* In the old file, a label is added as a Python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: ```python def foo(x): @@ -21,7 +21,7 @@ def foo(x): return x ``` -* In spec1, the new code could then be inserted by having a code-block that looked as follows: +* In spec1, the new code can then be inserted by having a code-block that looks as follows: ```python #begin insert @YourLabelHere @@ -29,4 +29,4 @@ def foo(x): #end insert @YourLabelHere ``` -**Note** that the code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point. +*Note*: The code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point. diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6898bf459..b3b2b2166 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -140,7 +140,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. * **Committee**—a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". * **Proposer**—the [validator](#dfn-validator) that creates a beacon chain block. * **Attester**—a [validator](#dfn-validator) that is part of a committee that needs to sign off on a beacon chain block while simultaneously creating a link (crosslink) to a recent shard block on a particular shard chain. -* **Beacon chain**—the central PoS chain that is the base of the sharding system. +* **Beacon chain**—the central proof-of-stake chain that is the base of the sharding system. * **Shard chain**—one of the chains on which user transactions take place and account data is stored. * **Block root**—a 32-byte Merkle root of a beacon chain block or shard chain block. Previously called "block hash". * **Crosslink**—a set of signatures from a committee attesting to a block in a shard chain that can be included into the beacon chain. Crosslinks are the main means by which the beacon chain "learns about" the updated state of shard chains. @@ -179,7 +179,7 @@ The following values are (non-configurable) constants used throughout the specif ## Configuration -*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. These configurations are updated for releases and may be out of sync during `dev` changes. +*Note*: The default mainnet configuration values are included here for spec-design purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the [`configs/constant_presets`](../../configs/constant_presets) directory. These configurations are updated for releases and may be out of sync during `dev` changes. ### Misc @@ -192,7 +192,7 @@ The following values are (non-configurable) constants used throughout the specif | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -* For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +* For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values @@ -226,7 +226,7 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | | `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | -* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` +* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH`. ### State list lengths @@ -245,7 +245,7 @@ The following values are (non-configurable) constants used throughout the specif | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. +* The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (about 18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -548,11 +548,11 @@ The `hash` function is SHA256. ### `hash_tree_root` -`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root utilizing a hash tree structure. `hash_tree_root` is defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). +`def hash_tree_root(object: SSZSerializable) -> Hash` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SimpleSerialize spec](../simple-serialize.md#merkleization). ### `signing_root` -`def signing_root(object: Container) -> Hash` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +`def signing_root(object: Container) -> Hash` is a function for computing signing messages, as defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers). ### `bls_domain` @@ -1046,15 +1046,15 @@ def get_churn_limit(state: BeaconState) -> int: ### `bls_verify` -`bls_verify` is a function for verifying a BLS signature, defined in the [BLS Signature spec](../bls_signature.md#bls_verify). +`bls_verify` is a function for verifying a BLS signature, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify). ### `bls_verify_multiple` -`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). +`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). ### `bls_aggregate_pubkeys` -`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). +`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). ### Routines for updating validator status @@ -1119,7 +1119,7 @@ Before genesis has been triggered and whenever the deposit contract emits a `Dep * `deposits` is the list of all deposits, ordered chronologically, up to and including the deposit triggering the latest `Deposit` log * `timestamp` is the Unix timestamp in the Ethereum 1.0 block that emitted the latest `Deposit` log -When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: +When `is_genesis_trigger(deposits, timestamp) is True` for the first time, let: * `genesis_deposits = deposits` * `genesis_time = timestamp - timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY` where `SECONDS_PER_DAY = 86400` @@ -1128,7 +1128,7 @@ When `is_genesis_trigger(deposits, timestamp) is True` for the first time let: * `genesis_eth1_data.deposit_root` is the deposit root for the last deposit in `deposits` * `genesis_eth1_data.deposit_count = len(genesis_deposits)` -*Note*: The function `is_genesis_trigger` has yet to be agreed by the community, and can be updated as necessary. We define the following testing placeholder: +*Note*: The function `is_genesis_trigger` has yet to be agreed upon by the community, and can be updated as necessary. We define the following testing placeholder: ```python def is_genesis_trigger(deposits: List[Deposit], timestamp: uint64) -> bool: @@ -1227,7 +1227,7 @@ def process_slot(state: BeaconState) -> None: ### Epoch processing -*Note*: the `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. +*Note*: The `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. ```python def process_epoch(state: BeaconState) -> None: @@ -1735,7 +1735,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the deposit signature (proof of possession). # Invalid signatures are allowed by the deposit contract, # and hence included on-chain, but must not be processed. - # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` + # Note: Deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify( pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) ): diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 338ece487..d06dbaea1 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -61,4 +61,4 @@ Every Ethereum 1.0 deposit emits a `Deposit` log for consumption by the beacon c The deposit contract source code, written in Vyper, is available [here](https://github.com/ethereum/eth2.0-specs/blob/dev/deposit_contract/contracts/validator_registration.v.py). -*Note*: To save on gas the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. +*Note*: To save on gas, the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 25b33ab30..f896caa3b 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,7 +8,7 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Constants](#constants) + - [Configuration](#configuration) - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) @@ -39,7 +39,7 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0 ## Fork choice -The head block root associated with a `store` is defined as `get_head(store)`. At genesis let `store = get_genesis_store(genesis_state)` and update `store` by running: +The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_genesis_store(genesis_state)` and update `store` by running: * `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time * `on_block(block)` whenever a block `block` is received @@ -49,7 +49,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. -3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](../validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. 5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 2adff2388..3318fe45b 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -70,17 +70,17 @@ The default value of a type upon initialization is recursively defined using `0` #### `is_empty` -An SSZ object is called empty (and thus `is_empty(object)` returns true) if it is equal to the default value for that type. +An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it is equal to the default value for that type. ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e., with type index zero). +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e. with type index zero). ## Serialization We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. -> *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. +*Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. ### `"uintN"` @@ -164,7 +164,7 @@ Let `value` be a self-signed container object. The convention is that the signat | Python | Ethereum 2.0 | Ethereum Foundation | [https://github.com/ethereum/py-ssz](https://github.com/ethereum/py-ssz) | | Rust | Lighthouse | Sigma Prime | [https://github.com/sigp/lighthouse/tree/master/eth2/utils/ssz](https://github.com/sigp/lighthouse/tree/master/eth2/utils/ssz) | | Nim | Nimbus | Status | [https://github.com/status-im/nim-beacon-chain/blob/master/beacon_chain/ssz.nim](https://github.com/status-im/nim-beacon-chain/blob/master/beacon_chain/ssz.nim) | -| Rust | Shasper | ParityTech | [https://github.com/paritytech/shasper/tree/master/util/ssz](https://github.com/paritytech/shasper/tree/master/util/ssz) | +| Rust | Shasper | ParityTech | [https://github.com/paritytech/shasper/tree/master/utils/ssz](https://github.com/paritytech/shasper/tree/master/util/ssz) | | TypeScript | Lodestar | ChainSafe Systems | [https://github.com/ChainSafe/ssz-js](https://github.com/ChainSafe/ssz-js) | | Java | Cava | ConsenSys | [https://www.github.com/ConsenSys/cava/tree/master/ssz](https://www.github.com/ConsenSys/cava/tree/master/ssz) | | Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/go-ssz](https://github.com/prysmaticlabs/go-ssz) | diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ecbc5af27..aa8350b66 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -57,7 +57,7 @@ This document represents the expected behavior of an "honest validator" with respect to Phase 0 of the Ethereum 2.0 protocol. This document does not distinguish between a "node" (i.e. the functionality of following and reading the beacon chain) and a "validator client" (i.e. the functionality of actively participating in consensus). The separation of concerns between these (potentially) two pieces of software is left as a design decision that is out of scope. -A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof of work networks in which a miner provides collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. +A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof-of-work networks in which miners provide collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. ## Prerequisites @@ -91,7 +91,7 @@ The validator constructs their `withdrawal_credentials` via the following: ### Submit deposit -In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. +In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 proof-of-work chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. To submit a deposit: @@ -113,7 +113,7 @@ Once a validator has been processed and added to the beacon state's `validators` ### Activation -In normal operation, the validator is quickly activated at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes). +In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes). The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: @@ -124,7 +124,7 @@ is_active = is_active_validator(validator, get_current_epoch(state)) Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. -*Note*: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. +*Note*: There is a maximum validator churn per finalized epoch, so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. ## Validator assignments @@ -183,7 +183,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). -There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). +There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~3 weeks). #### Block header @@ -255,15 +255,15 @@ block_signature = bls_sign( ##### Proposer slashings -Up to `MAX_PROPOSER_SLASHINGS` [`ProposerSlashing`](../core/0_beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](../core/0_beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. +Up to `MAX_PROPOSER_SLASHINGS`, [`ProposerSlashing`](../core/0_beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](../core/0_beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. ##### Attester slashings -Up to `MAX_ATTESTER_SLASHINGS` [`AttesterSlashing`](../core/0_beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [Attester slashings processing](../core/0_beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. +Up to `MAX_ATTESTER_SLASHINGS`, [`AttesterSlashing`](../core/0_beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [attester slashings processing](../core/0_beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. ##### Attestations -Up to `MAX_ATTESTATIONS` aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](../core/0_beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. +Up to `MAX_ATTESTATIONS`, aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](../core/0_beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. ##### Deposits @@ -273,17 +273,17 @@ The `proof` for each deposit must be constructed against the deposit root contai ##### Voluntary exits -Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](../core/0_beacon-chain.md#voluntary-exits). +Up to `MAX_VOLUNTARY_EXITS`, [`VoluntaryExit`](../core/0_beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](../core/0_beacon-chain.md#voluntary-exits). ### Attestations -A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `shard`, and assigned `slot` for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. +A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `shard`, and assigned `slot` for which the validator performs this role during an epoch are defined by `get_committee_assignment(state, epoch, validator_index)`. -A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. +A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned―that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. #### Attestation data -First the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. +First, the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. * Let `head_block` be the result of running the fork choice during the assigned slot. * Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. @@ -316,7 +316,7 @@ Construct `attestation_data.crosslink` via the following. #### Construct attestation -Next the validator creates `attestation`, an [`Attestation`](../core/0_beacon-chain.md#attestation) object. +Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-chain.md#attestation) object. ##### Data @@ -362,9 +362,9 @@ signed_attestation_data = bls_sign( ## How to avoid slashing -"Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed -- [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. +"Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed: [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. -*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102 and vice versa. +*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102, and vice versa. ### Proposer slashing @@ -376,14 +376,14 @@ Specifically, when signing a `BeaconBlock`, a validator should perform the follo 1. Save a record to hard disk that a beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`. 2. Generate and broadcast the block. -If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. +If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. ### Attester slashing To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](../core/0_beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](../core/0_beacon-chain.md#is_slashable_attestation_data). Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an attestation has been signed for source -- `attestation_data.source_epoch` -- and target -- `slot_to_epoch(attestation_data.slot)`. +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source_epoch`) and target (i.e. `slot_to_epoch(attestation_data.slot)`). 2. Generate and broadcast attestation. -If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. +If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 2a5fe7fcd..20379eef0 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -1,28 +1,27 @@ # Ethereum 2.0 Phase 0 -- Beacon Node API for Validator -__NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. +**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. ## Outline -This document outlines a minimal application programming interface (API) which is exposed by a beacon node for use by a validator client implementation which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. +This document outlines a minimal application programming interface (API) which is exposed by a beacon node for use by a validator client implementation which aims to facilitate [Phase 0](../../README.md#phase-0) of Ethereum 2.0. -The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. The only supported return data type is currently JSON. +The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. Currently, the only supported return data type is JSON. -### Background -The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. +## Background +The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum 2.0 network. Conceptually, it does not maintain keypairs that participate with the beacon chain. -The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties include the production of beacon blocks and signing of attestations. +The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks, called "duties", on the beacon chain. These duties include the production of beacon blocks and signing of attestations. Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. -The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator client implementations, independently from beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client which utilises the same API. +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator client implementations, independently from beacon node development. For example, the validator client from [Lighthouse](https://github.com/sigp/lighthouse) could communicate with a running instance of the beacon node from [Prysm](https://github.com/prysmaticlabs/prysm), or a staking pool might create a decentrally managed validator client which utilizes the same API. -This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) +This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012). ## Specification The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. -For convenience, this specification has been uploaded to [SwaggerHub](https://swagger.io/tools/swaggerhub/) at the following URL: -[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator) +For convenience, this specification has been uploaded to SwaggerHub [here](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator). From 4986311d5bb1e716e9dd6eea59864d44732b97ec Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 25 Jun 2019 10:24:14 -0600 Subject: [PATCH 71/76] Update 0_beacon-chain.md fix typo w/ refactor --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 99425c6c2..56a6fd06a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1142,7 +1142,7 @@ def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Count active validators at genesis active_validator_count = 0 - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 From d90d56c610e8b58dc697c2dc92f9c1608c974520 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 18:42:34 +0200 Subject: [PATCH 72/76] Change uint aliases to just be subclasses, do not override init with no-op --- scripts/build_spec.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index d67c0e5c6..99c5cd69d 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -130,11 +130,7 @@ def objects_to_spec(functions: Dict[str, str], new_type_definitions = ( '\n\n'.join( [ - f"class {key}({value}):\n" - f" def __init__(self, _x: {value}) -> None:\n" - f" ...\n" - if value.startswith("uint") - else f"class {key}({value}):\n pass\n" + f"class {key}({value}):\n pass\n" for key, value in custom_types.items() ] ) From 3b5c7f243a4ec6555ea9f762411c8541f7c20e3a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 19:32:49 +0200 Subject: [PATCH 73/76] type hint uint input --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6079f2866..58e66ca68 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -34,7 +34,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): class Bool(BasicValue): # can't subclass bool. byte_len = 1 - def __new__(cls, value, *args, **kwargs): + def __new__(cls, value: int): # int value, but can be any subclass of int (bool, Bit, Bool, etc...) if value < 0 or value > 1: raise ValueError(f"value {value} out of bounds for bit") return super().__new__(cls, value) @@ -54,7 +54,7 @@ class Bit(Bool): class uint(BasicValue, metaclass=BasicType): - def __new__(cls, value, *args, **kwargs): + def __new__(cls, value: int): if value < 0: raise ValueError("unsigned types must not be negative") if cls.byte_len and value.bit_length() > (cls.byte_len << 3): From 2c6f4f2597b3a6e821b03007de329058871f1d3f Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 25 Jun 2019 19:33:24 +0200 Subject: [PATCH 74/76] update validator_registry -> validators missed case --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 99425c6c2..56a6fd06a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1142,7 +1142,7 @@ def is_genesis_trigger(deposits: Sequence[Deposit], timestamp: uint64) -> bool: # Count active validators at genesis active_validator_count = 0 - for validator in state.validator_registry: + for validator in state.validators: if validator.effective_balance == MAX_EFFECTIVE_BALANCE: active_validator_count += 1 From d587c4fe619458b9e036ab44ecb8318cd91a0de6 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Wed, 26 Jun 2019 21:21:17 +0200 Subject: [PATCH 75/76] Critical fix: introduce back total-value check (#1220) This was dropped in a376b6607fe5e6406371f44254960e891ee5ee8d, as improvement in dust checking. Now that dust-checking is done, we still need to check if the sender has the minimum value, as decrease balance just clips to 0. See be86f966f87958856584b3f20c095abf910a3d0c for older dust-creation problem work around, which was dropped in the above. The bug enabled you to transfer your full balance to someone else, and pay the same amount in fee, possibly to a puppet proposer to collect back funds. Effectively enabling printing of money. Silly bug, good to fix and introduce tests for. --- specs/core/0_beacon-chain.md | 4 +- .../block_processing/test_process_transfer.py | 138 +++++++++++++++++- 2 files changed, 138 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a..264c6d23b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1795,8 +1795,8 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: """ Process ``Transfer`` operation. """ - # Verify the amount and fee are not individually too big (for anti-overflow purposes) - assert state.balances[transfer.sender] >= max(transfer.amount, transfer.fee) + # Verify the balance the covers amount and fee (with overflow protection) + assert state.balances[transfer.sender] >= max(transfer.amount + transfer.fee, transfer.amount, transfer.fee) # A transfer is valid in only one slot assert state.slot == transfer.slot # Sender must satisfy at least one of the following conditions in the parenthesis: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index e9d282b3a..89246cc51 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -114,7 +114,7 @@ def test_incorrect_slot(spec, state): @with_all_phases @spec_state_test -def test_insufficient_balance_for_fee(spec, state): +def test_insufficient_balance_for_fee_result_dust(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=0, fee=1, signed=True) @@ -127,7 +127,20 @@ def test_insufficient_balance_for_fee(spec, state): @with_all_phases @spec_state_test -def test_insufficient_balance(spec, state): +def test_insufficient_balance_for_fee_result_full(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=0, fee=state.balances[sender_index] + 1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_amount_result_dust(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=0, signed=True) @@ -138,6 +151,127 @@ def test_insufficient_balance(spec, state): yield from run_transfer_processing(spec, state, transfer, False) +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_amount_result_full(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=state.balances[sender_index] + 1, fee=0, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_result_dust(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee without dust, and amount without dust, but not both. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, fee=1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_result_full(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT + 1, + fee=spec.MIN_DEPOSIT_AMOUNT + 1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_big_amount(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to create a dust balance (off by 1) with combination of fee and amount. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=spec.MIN_DEPOSIT_AMOUNT + 1, fee=1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_for_combined_big_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to create a dust balance (off by 1) with combination of fee and amount. + state.balances[sender_index] = spec.MIN_DEPOSIT_AMOUNT * 2 + 1 + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=1, fee=spec.MIN_DEPOSIT_AMOUNT + 1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_off_by_1_fee(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to print money by using the full balance as amount, plus 1 for fee. + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=state.balances[sender_index], fee=1, signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_off_by_1_amount(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to print money by using the full balance as fee, plus 1 for amount. + transfer = get_valid_transfer(spec, state, sender_index=sender_index, amount=1, + fee=state.balances[sender_index], signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + +@with_all_phases +@spec_state_test +def test_insufficient_balance_duplicate_as_fee_and_amount(spec, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # Enough to pay fee fully without dust left, and amount fully without dust left, but not both. + # Try to print money by using the full balance, twice. + transfer = get_valid_transfer(spec, state, sender_index=sender_index, + amount=state.balances[sender_index], + fee=state.balances[sender_index], signed=True) + + # un-activate so validator can transfer + state.validators[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + + yield from run_transfer_processing(spec, state, transfer, False) + + @with_all_phases @spec_state_test def test_no_dust_sender(spec, state): From dbb697dadd3b027979baaffaafc62d5932b895a0 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 26 Jun 2019 19:40:11 -0400 Subject: [PATCH 76/76] Small update to typing in BLS spec file [uint384] -> Tuple[uint384, uint384] --- specs/bls_signature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index 3fe1bcc0e..b901b9345 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -71,7 +71,7 @@ We require: G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -def hash_to_G2(message_hash: Bytes32, domain: uint64) -> [uint384]: +def hash_to_G2(message_hash: Bytes32, domain: uint64) -> Tuple[uint384, uint384]: # Initial candidate x coordinate x_re = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x01'), 'big') x_im = int.from_bytes(hash(message_hash + bytes8(domain) + b'\x02'), 'big')