From 02f6ba36f048d4b79b556df69eed612f660d8e42 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 09:51:06 +0100 Subject: [PATCH 01/28] Add Bitvector and Bitlist Bool, Bit -> boolean, bit Fix simple-serialize.md --- scripts/build_spec.py | 6 +- specs/core/0_beacon-chain.md | 4 +- specs/simple-serialize.md | 87 ++++++++++++++----- test_libs/pyspec/eth2spec/debug/decode.py | 4 +- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 8 +- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 19 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 22 ++++- .../eth2spec/utils/ssz/test_ssz_impl.py | 10 +-- .../eth2spec/utils/ssz/test_ssz_typing.py | 18 ++-- 11 files changed, 127 insertions(+), 59 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 99c5cd69d..0b1512111 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -52,7 +52,7 @@ from eth2spec.utils.ssz.ssz_impl import ( is_empty, ) from eth2spec.utils.ssz.ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, Bytes, uint64, Bytes4, Bytes32, Bytes48, Bytes96, ) from eth2spec.utils.bls import ( @@ -174,7 +174,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 56a6fd06a..7b9aeee4b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -297,7 +297,7 @@ class Validator(Container): pubkey: BLSPubkey withdrawal_credentials: Hash # Commitment to pubkey for withdrawals and transfers effective_balance: Gwei # Balance at stake - slashed: Bool + slashed: boolean # Status epochs activation_eligibility_epoch: Epoch # When criteria for activation were met activation_epoch: Epoch @@ -337,7 +337,7 @@ class AttestationData(Container): ```python class AttestationDataAndCustodyBit(Container): data: AttestationData - custody_bit: Bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data + custody_bit: bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of crosslink data ``` #### `IndexedAttestation` diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3318fe45b..53c5649ed 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -15,9 +15,9 @@ - [Default values](#default-values) - [Illegal types](#illegal-types) - [Serialization](#serialization) - - [`"uintN"`](#uintn) - - [`"bool"`](#bool) - - [`"null`](#null) + - [`uintN`](#uintn) + - [`boolean`](#boolean) + - [`null`](#null) - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) @@ -37,36 +37,45 @@ ## Typing ### Basic types -* `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) -* `"bool"`: `True` or `False` +* `uintN`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) +* `boolean`: `True` or `False` ### Composite types * **container**: ordered heterogeneous collection of values - * key-pair curly bracket notation `{}`, e.g. `{"foo": "uint64", "bar": "bool"}` + * python dataclass notation with key-type pairs, e.g. +```python +class ContainerExample(Container): + foo: uint64 + bar: boolean +``` * **vector**: ordered fixed-length homogeneous collection of values - * angle bracket notation `[type, N]`, e.g. `["uint64", N]` -* **list**: ordered variable-length homogeneous collection of values - * angle bracket notation `[type]`, e.g. `["uint64"]` + * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` +* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` + * notation `List[type, N]`, e.g. `List[uint64, N]` * **union**: union type containing one of the given subtypes - * round bracket notation `(type_1, type_2, ...)`, e.g. `("null", "uint64")` + * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` +* **Bitvector**: a fixed-length list of `boolean` values + * notation `Bitvector[N]` +* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` + * notation `Bitlist[N]` ### Variable-size and fixed-size -We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". +We recursively define "variable-size" types to be lists, unions, `Bitlist` and all types that contain a variable-size type. All other types are said to be "fixed-size". ### Aliases For convenience we alias: -* `"byte"` to `"uint8"` (this is a basic type) -* `"bytes"` to `["byte"]` (this is *not* a basic type) -* `"bytesN"` to `["byte", N]` (this is *not* a basic type) -* `"null"`: `{}`, i.e. the empty container +* `bit` to `boolean` +* `byte` to `uint8` (this is a basic type) +* `BytesN` to `Vector[byte, N]` (this is *not* a basic type) +* `null`: `{}`, i.e. the empty container ### Default values -The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. Unions default to the first type in the union (with type index zero), which is `"null"` if present in the union. +The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union. #### `is_empty` @@ -74,34 +83,50 @@ An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e. with type index zero). +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `null` type is only legal as the first type in a union subtype (i.e. with type index zero). ## Serialization -We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. +We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `bytes`. *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. -### `"uintN"` +### `uintN` ```python assert N in [8, 16, 32, 64, 128, 256] return value.to_bytes(N // 8, "little") ``` -### `"bool"` +### `boolean` ```python assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### `"null"` +### `null` ```python return b"" ``` +### `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((N + 7) // 8, "little") +``` + +### `Bitlist[N]` + +Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. + +```python +as_integer = (1 << len(value)) + sum([value[i] << i for i in range(len(value))]) +return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") +``` + ### Vectors, containers, lists, unions ```python @@ -142,17 +167,33 @@ We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. +* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type +### Merkleization of `Bitvector[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return merkleize(as_integer.to_bytes((N + 7) // 8, "little")) +``` + +### `Bitlist[N]` + +```python +as_integer = sum([value[i] << i for i in range(len(value))]) +return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) +``` + + ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c0b53b0ef..c0b977ab3 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,13 +1,13 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, uint, Container, Bytes, List, boolean, Vector, BytesN ) def decode(data: Any, typ: SSZType) -> SSZValue: - if issubclass(typ, (uint, Bool)): + if issubclass(typ, (uint, boolean)): return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 02814e441..670f580b2 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZValue, uint, Container, Bool + SSZValue, uint, Container, boolean ) @@ -10,7 +10,7 @@ def encode(value: SSZValue, include_hash_tree_roots=False): if value.type().byte_len > 8: return str(int(value)) return int(value) - elif isinstance(value, Bool): + elif isinstance(value, boolean): return value == 1 elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index c6efb722b..cdcba343a 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,7 +2,7 @@ from random import Random from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, Bool, + SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, Vector, BytesN ) @@ -118,7 +118,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(rng.choice((True, False))) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -128,7 +128,7 @@ def get_random_basic_value(rng: Random, typ: BasicType) -> BasicValue: def get_min_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(False) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES @@ -138,7 +138,7 @@ def get_min_basic_value(typ: BasicType) -> BasicValue: def get_max_basic_value(typ: BasicType) -> BasicValue: - if issubclass(typ, Bool): + if issubclass(typ, boolean): return typ(True) elif issubclass(typ, uint): assert typ.byte_len in UINT_BYTE_SIZES diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index e533ca5c2..130956235 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -19,7 +19,7 @@ def translate_typ(typ) -> ssz.BaseSedes: return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): return ssz.List(translate_typ(typ.elem_type)) - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): if typ.byte_len == 1: @@ -64,7 +64,7 @@ def translate_value(value, typ): raise TypeError("invalid uint size") elif issubclass(typ, spec_ssz.List): return [translate_value(elem, typ.elem_type) for elem in value] - elif issubclass(typ, spec_ssz.Bool): + elif issubclass(typ, spec_ssz.boolean): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index b9c7b6d38..a7f6f9da1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,8 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bool, Container, List, Bytes, uint, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + Bitlist, Bitvector, uint, ) # SSZ Serialization @@ -13,7 +14,7 @@ BYTES_PER_LENGTH_OFFSET = 4 def serialize_basic(value: SSZValue): if isinstance(value, uint): return value.to_bytes(value.type().byte_len, 'little') - elif isinstance(value, Bool): + elif isinstance(value, boolean): if value: return b'\x01' else: @@ -39,6 +40,12 @@ def is_empty(obj: SSZValue): def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) + elif isinstance(obj, Bitvector): + as_integer = sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((len(obj) + 7) // 8, "little") + elif isinstance(obj, Bitlist): + as_integer = (1 << len(obj)) + sum([obj[i] << i for i in range(len(obj))]) + return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") elif isinstance(obj, Series): return encode_series(obj) else: @@ -85,6 +92,12 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values + elif isinstance(values, Bitvector): + as_integer = sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") + elif isinstance(values, Bitlist): + as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) @@ -134,7 +147,7 @@ def hash_tree_root(obj: SSZValue): else: raise Exception(f"Type not supported: {type(obj)}") - if isinstance(obj, (List, Bytes)): + if isinstance(obj, (List, Bytes, Bitlist)): return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 58e66ca68..ea07359b2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -31,7 +31,7 @@ class BasicValue(int, SSZValue, metaclass=BasicType): pass -class Bool(BasicValue): # can't subclass bool. +class boolean(BasicValue): # can't subclass bool. byte_len = 1 def __new__(cls, value: int): # int value, but can be any subclass of int (bool, Bit, Bool, etc...) @@ -48,7 +48,7 @@ class Bool(BasicValue): # can't subclass bool. # Alias for Bool -class Bit(Bool): +class bit(boolean): pass @@ -233,7 +233,7 @@ class ParamsMeta(SSZType): return f"{self.__name__}~{self.__class__.__name__}" def __repr__(self): - return self, self.__class__ + return f"{self.__name__}~{self.__class__.__name__}" def attr_from_params(self, p): # single key params are valid too. Wrap them in a tuple. @@ -280,11 +280,12 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int +class BitElementsType(ElementsType): + elem_type = boolean class Elements(ParamsBase, metaclass=ElementsType): pass - class BaseList(list, Elements): def __init__(self, *args): @@ -310,6 +311,10 @@ class BaseList(list, Elements): cls = self.__class__ return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __repr__(self): + cls = self.__class__ + return f"{cls.__name__}[{cls.elem_type.__name__}, {cls.length}]({', '.join(str(v) for v in self)})" + def __getitem__(self, k) -> SSZValue: if isinstance(k, int): # check if we are just doing a lookup, and not slicing if k < 0: @@ -337,6 +342,15 @@ class BaseList(list, Elements): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] +class BaseBitfield(BaseList, metaclass=BitElementsType): + elem_type = bool + +class Bitlist(BaseBitfield): + pass + +class Bitvector(BaseBitfield): + pass + class List(BaseList): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 82fb4ec68..33badcf4a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - Bit, Bool, Container, List, Vector, Bytes, BytesN, + bit, boolean, Container, List, Vector, Bytes, BytesN, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -74,10 +74,10 @@ def merge(a: str, branch: Iterable[str]) -> str: test_data = [ - ("bit F", Bit(False), "00", chunk("00")), - ("bit T", Bit(True), "01", chunk("01")), - ("bool F", Bool(False), "00", chunk("00")), - ("bool T", Bool(True), "01", chunk("01")), + ("bit F", bit(False), "00", chunk("00")), + ("bit T", bit(True), "01", chunk("01")), + ("boolean F", boolean(False), "00", chunk("00")), + ("boolean T", boolean(True), "01", chunk("01")), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index 2af742360..f746a29c9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,6 +1,6 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, Bit, Bool, Container, List, Vector, Bytes, BytesN, + Elements, bit, boolean, Container, List, Vector, Bytes, BytesN, byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -22,8 +22,8 @@ def test_subclasses(): assert issubclass(u, SSZValue) assert isinstance(u, SSZType) assert isinstance(u, BasicType) - assert issubclass(Bool, BasicValue) - assert isinstance(Bool, BasicType) + assert issubclass(boolean, BasicValue) + assert isinstance(boolean, BasicType) for c in [Container, List, Vector, Bytes, BytesN]: assert issubclass(c, Series) @@ -45,16 +45,16 @@ def test_basic_instances(): assert isinstance(v, BasicValue) assert isinstance(v, SSZValue) - assert isinstance(Bool(True), BasicValue) - assert isinstance(Bool(False), BasicValue) - assert isinstance(Bit(True), Bool) - assert isinstance(Bit(False), Bool) + assert isinstance(boolean(True), BasicValue) + assert isinstance(boolean(False), BasicValue) + assert isinstance(bit(True), boolean) + assert isinstance(bit(False), boolean) def test_basic_value_bounds(): max = { - Bool: 2 ** 1, - Bit: 2 ** 1, + boolean: 2 ** 1, + bit: 2 ** 1, uint8: 2 ** (8 * 1), byte: 2 ** (8 * 1), uint16: 2 ** (8 * 2), From 23c743570e5b42119f7e7fdb9ffcf8584e98c1d1 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:26:45 +0100 Subject: [PATCH 02/28] Add some tests and fix pack --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a7f6f9da1..d6689892d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -96,7 +96,7 @@ def pack(values: Series): as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") elif isinstance(values, Bitlist): - as_integer = (1 << len(values)) + sum([values[i] << i for i in range(len(values))]) + as_integer = sum([values[i] << i for i in range(len(values))]) return as_integer.to_bytes((values.length + 7) // 8, "little") return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 33badcf4a..1522ce200 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -2,6 +2,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, BytesN, + Bitlist, Bitvector, uint8, uint16, uint32, uint64, uint256, byte ) from ..hash_function import hash as bytes_hash @@ -78,6 +79,10 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 494984f7d3129d8d3c4798d285824b59dc7f68b2 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 10:42:14 +0100 Subject: [PATCH 03/28] Fix linting errors --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 ++++++ test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d6689892d..7298fb3ca 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -26,7 +26,7 @@ def serialize_basic(value: SSZValue): def deserialize_basic(value, typ: BasicType): if issubclass(typ, uint): return typ(int.from_bytes(value, 'little')) - elif issubclass(typ, Bool): + elif issubclass(typ, boolean): assert value in (b'\x00', b'\x01') return typ(value == b'\x01') else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ea07359b2..047abd4fe 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -280,12 +280,15 @@ class ElementsType(ParamsMeta): elem_type: SSZType length: int + class BitElementsType(ElementsType): elem_type = boolean + class Elements(ParamsBase, metaclass=ElementsType): pass + class BaseList(list, Elements): def __init__(self, *args): @@ -342,12 +345,15 @@ class BaseList(list, Elements): # be explict about getting the last item, for the non-python readers, and negative-index safety return self[len(self) - 1] + class BaseBitfield(BaseList, metaclass=BitElementsType): elem_type = bool + class Bitlist(BaseBitfield): pass + class Bitvector(BaseBitfield): pass diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 1522ce200..63f0c835d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,10 +79,12 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), - ("bitvector TFTFFFTTFT", Bitvector[10](1,0,1,0,0,0,1,1,0,1), "c502", chunk("c502")), - ("bitlist TFTFFFTTFT", Bitlist[16](1,0,1,0,0,0,1,1,0,1), "c506", h(chunk("c502"), chunk("0A"))), - ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c2", chunk("c5c2")), - ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), + ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), + ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c2", chunk("c5c2")), + ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), + "c5c201", h(chunk("c5c2"), chunk("10"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From d641e941519efc8cacd5e2ddabcaaf1722aa0797 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 27 Jun 2019 11:30:23 +0100 Subject: [PATCH 04/28] Cleanups --- specs/simple-serialize.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 53c5649ed..97b1d560c 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -43,7 +43,7 @@ ### Composite types * **container**: ordered heterogeneous collection of values - * python dataclass notation with key-type pairs, e.g. + * python dataclass notation with key-type pairs, e.g. ```python class ContainerExample(Container): foo: uint64 @@ -53,12 +53,12 @@ class ContainerExample(Container): * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` * **list**: ordered variable-length homogeneous collection of values, with maximum length `N` * notation `List[type, N]`, e.g. `List[uint64, N]` +* **bitvector**: ordered fixed-length collection of `boolean` values + * notation `Bitvector[N]` +* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` + * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` -* **Bitvector**: a fixed-length list of `boolean` values - * notation `Bitvector[N]` -* **Bitlist**: a variable-length list of `boolean` values with maximum length `N` - * notation `Bitlist[N]` ### Variable-size and fixed-size @@ -193,7 +193,6 @@ as_integer = sum([value[i] << i for i in range(len(value))]) return mix_in_length(merkleize(as_integer.to_bytes((N + 7) // 8, "little")), len(value)) ``` - ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. From 67c50cb197824a015ff9946fb6fd5c684e9e9639 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 11:45:35 +0100 Subject: [PATCH 05/28] Changed attestation and custody bitfields --- scripts/build_spec.py | 6 +-- specs/core/0_beacon-chain.md | 41 +++---------------- .../pyspec/eth2spec/debug/random_value.py | 6 +-- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 11 +++++ .../eth2spec/test/helpers/attestations.py | 9 ++-- .../pyspec/eth2spec/test/helpers/bitfields.py | 11 ----- .../test_process_attestation.py | 8 +++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 8 +++- 8 files changed, 38 insertions(+), 62 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/helpers/bitfields.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0b1512111..0a41fe5c0 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -26,7 +26,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -53,7 +53,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( bit, boolean, Container, List, Vector, Bytes, uint64, - Bytes4, Bytes32, Bytes48, Bytes96, + Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, @@ -175,7 +175,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' - 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', + 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', 'bytes' # to be removed after updating spec doc ] diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 7b9aeee4b..13ab4cdae 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -80,8 +80,6 @@ - [`bytes_to_int`](#bytes_to_int) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) - - [`get_bitfield_bit`](#get_bitfield_bit) - - [`verify_bitfield`](#verify_bitfield) - [`convert_to_indexed`](#convert_to_indexed) - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) @@ -354,7 +352,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -421,9 +419,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bytes[MAX_INDICES_PER_ATTESTATION // 8] + custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -865,13 +863,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, attestation_data: AttestationData, - bitfield: bytes) -> Sequence[ValidatorIndex]: + bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) - assert verify_bitfield(bitfield, len(committee)) - return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) + return sorted([index for i, index in enumerate(committee) if bitfield[i]]) ``` ### `int_to_bytes` @@ -912,34 +909,6 @@ def get_domain(state: BeaconState, return bls_domain(domain_type, fork_version) ``` -### `get_bitfield_bit` - -```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 -``` - -### `verify_bitfield` - -```python -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True -``` - ### `convert_to_indexed` ```python diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index cdcba343a..95a3ae970 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -3,7 +3,7 @@ from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, - Vector, BytesN + Vector, BytesN, Bitlist, Bitvector ) # in bytes @@ -83,12 +83,12 @@ def get_random_ssz_object(rng: Random, return get_max_basic_value(typ) else: return get_random_basic_value(rng, typ) - elif issubclass(typ, Vector): + elif issubclass(typ, Vector) or issubclass(typ, Bitvector): return typ( get_random_ssz_object(rng, typ.elem_type, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) ) - elif issubclass(typ, List): + elif issubclass(typ, List) or issubclass(typ, Bitlist): length = rng.randint(0, min(typ.length, max_list_length)) if mode == RandomizationMode.mode_one_count: length = 1 diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index 130956235..ccca17385 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -18,7 +18,14 @@ def translate_typ(typ) -> ssz.BaseSedes: elif issubclass(typ, spec_ssz.Vector): return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.List): + # TODO: Make py-ssz List support the new fixed length list return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitlist): + # TODO: Once Bitlist implemented in py-ssz, use appropriate type + return ssz.List(translate_typ(typ.elem_type)) + elif issubclass(typ, spec_ssz.Bitvector): + # TODO: Once Bitvector implemented in py-ssz, use appropriate type + return ssz.Vector(translate_typ(typ.elem_type), typ.length) elif issubclass(typ, spec_ssz.boolean): return ssz.boolean elif issubclass(typ, spec_ssz.uint): @@ -68,6 +75,10 @@ def translate_value(value, typ): return value elif issubclass(typ, spec_ssz.Vector): return typ(*(translate_value(elem, typ.elem_type) for elem in value)) + elif issubclass(typ, spec_ssz.Bitlist): + return typ(value) + elif issubclass(typ, spec_ssz.Bitvector): + return typ(value) elif issubclass(typ, spec_ssz.BytesN): return typ(value) elif issubclass(typ, spec_ssz.Bytes): diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 4c8b5c7eb..2e637d42f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -1,10 +1,10 @@ from typing import List -from eth2spec.test.helpers.bitfields import set_bitfield_bit from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import Bitlist def build_attestation_data(spec, state, slot, shard): @@ -69,9 +69,8 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\x00' * bitfield_length - custody_bitfield = b'\x00' * bitfield_length + aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( aggregation_bitfield=aggregation_bitfield, data=attestation_data, @@ -138,7 +137,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) + attestation.aggregation_bitfield[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py deleted file mode 100644 index 50e5b6cba..000000000 --- a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py +++ /dev/null @@ -1,11 +0,0 @@ -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 2b34ab405..ee76ab23d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -10,6 +10,7 @@ from eth2spec.test.helpers.state import ( next_slot, ) from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.utils.ssz.ssz_typing import Bitlist def run_attestation_processing(spec, state, attestation, valid=True): @@ -281,7 +282,10 @@ def test_inconsistent_bitfields(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + b'\x00' + custody_bitfield = deepcopy(attestation.aggregation_bitfield) + custody_bitfield.append(False) + + attestation.custody_bitfield = custody_bitfield sign_attestation(spec, state, attestation) @@ -307,7 +311,7 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 047abd4fe..d0aef1b44 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -351,11 +351,15 @@ class BaseBitfield(BaseList, metaclass=BitElementsType): class Bitlist(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return False class Bitvector(BaseBitfield): - pass + @classmethod + def is_fixed_size(cls): + return True class List(BaseList): From becb7a032ab4ef1cbc2fe0e3d12ae381250f6175 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:14:36 +0100 Subject: [PATCH 06/28] justification_bitfield -> Bitvector[4] --- specs/core/0_beacon-chain.md | 16 ++++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 13ab4cdae..c1f1a7bcf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -523,7 +523,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: uint64 # Bit set for every recent justified epoch + justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,38 +1291,38 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 + state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 1) + state.justification_bitfield[1] = True current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield |= (1 << 0) + state.justification_bitfield[0] = True # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: + if all(bitfield[1:4]) and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: + if all(bitfield[1:3]) and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: + if all(bitfield[0:3]) and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: + if all(bitfield[0:2]) and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d0aef1b44..6ce2b1538 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -355,12 +355,19 @@ class Bitlist(BaseBitfield): def is_fixed_size(cls): return False + @classmethod + def default(cls): + return cls() + class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + + @classmethod + def default(cls): + return cls(0 for _ in range(cls.length)) class List(BaseList): From 80c680e6147ffc3971856802b4e54d2b776c6862 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 12:41:22 +0100 Subject: [PATCH 07/28] Phase 1 to Bitvector/Bitlist --- specs/core/1_custody-game.md | 12 +++++++++++- specs/core/1_shard-data-chains.md | 5 ++--- specs/light_client/sync_protocol.md | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 07f6ec698..17f599fcd 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,6 +272,16 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` +### `get_bitfield_bit` + +```python +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 +``` + ### `get_custody_chunk_bit` ```python @@ -566,7 +576,7 @@ def process_bit_challenge(state: BeaconState, chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) + custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d1c86eca2..b2a5ea9ea 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bytes[PLACEHOLDER] + aggregation_bitfield: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -230,10 +230,9 @@ def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data persistent_committee = get_persistent_committee(state, data.shard, data.slot) - assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: + if attestation.aggregation_bitfield[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 045bf5608..a1b5777cf 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': 'bytes', + 'shard_bitfield': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if get_bitfield_bit(proof.shard_bitfield, i) is True]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if get_bitfield_bit(proof.shard_bitfield, index) is True + if proof.shard_bitfield[index] ]) assert bls_verify( pubkey=group_public_key, From f57387cc83f16e2ec6b19ee544964254786fa7fa Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 13:09:48 +0100 Subject: [PATCH 08/28] Justification bitvector length to constant --- specs/core/0_beacon-chain.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c1f1a7bcf..4d3fe9773 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -189,6 +189,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | +| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -523,7 +524,7 @@ class BeaconState(Container): previous_justified_root: Hash # Previous epoch snapshot current_justified_epoch: Epoch current_justified_root: Hash - justification_bitfield: Bitvector[4] # Bit set for every recent justified epoch + justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch # Finality finalized_epoch: Epoch finalized_root: Hash @@ -1291,21 +1292,21 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0] + state.justification_bitfield[0:3])) + state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[1] = True + state.justification_bitfield[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) - state.justification_bitfield[0] = True + state.justification_bitfield[0] = 0b1 # Process finalizations bitfield = state.justification_bitfield From a5154da1ff76e8b768e64987471820f958c391d1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 27 Jun 2019 15:40:40 +0200 Subject: [PATCH 09/28] suggestion to implement bitfield like --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 7298fb3ca..f0ee944bd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,6 +128,8 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 + elif issubclass(typ, Bitfield): + return (typ.length + 7) // 8 // 32 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538..53ab42743 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -281,10 +281,6 @@ class ElementsType(ParamsMeta): length: int -class BitElementsType(ElementsType): - elem_type = boolean - - class Elements(ParamsBase, metaclass=ElementsType): pass @@ -346,11 +342,16 @@ class BaseList(list, Elements): return self[len(self) - 1] -class BaseBitfield(BaseList, metaclass=BitElementsType): - elem_type = bool +class BitElementsType(ElementsType): + elem_type: SSZType = boolean + length: int -class Bitlist(BaseBitfield): +class Bitfield(BaseList, metaclass=BitElementsType): + pass + + +class Bitlist(Bitfield): @classmethod def is_fixed_size(cls): return False @@ -360,15 +361,29 @@ class Bitlist(BaseBitfield): return cls() -class Bitvector(BaseBitfield): +class Bitvector(Bitfield): + + @classmethod + def extract_args(cls, *args): + if len(args) == 0: + return cls.default() + else: + return super().extract_args(*args) + + @classmethod + def value_check(cls, value): + # check length limit strictly + return len(value) == cls.length and super().value_check(value) + @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From b574a581094b8a00b360de573bee6a949b44650a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 14:45:17 +0100 Subject: [PATCH 10/28] Remove not working py-ssz decoder tests --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index 26ee6e913..b362d503b 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,9 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - for typ in [spec.BeaconBlock, spec.BeaconState, spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: + # TODO: Once has Bitfields and Bitvectors, add back + # spec.BeaconState and spec.BeaconBlock + for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value original = random_value.get_random_ssz_object(rng, typ, 100, 10, mode=random_value.RandomizationMode.mode_random, From 8ed638bb84ae3dd88e6658f7bfd5eeda874b97f9 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 15:21:04 +0100 Subject: [PATCH 11/28] Linter fixes --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/phase_0/block_processing/test_process_attestation.py | 3 ++- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index b362d503b..c707c840a 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitfields and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index ee76ab23d..d17d93e6d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -311,7 +311,8 @@ def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bitfield))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6ce2b1538..85f5e0bdb 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -364,11 +364,12 @@ class Bitvector(BaseBitfield): @classmethod def is_fixed_size(cls): return True - + @classmethod def default(cls): return cls(0 for _ in range(cls.length)) + class List(BaseList): @classmethod From afd86f71de09e556a7a4cc5d82a38b8d514e0e93 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:31:33 +0100 Subject: [PATCH 12/28] Fixes in ssz impl --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index f0ee944bd..53075845b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -129,7 +129,7 @@ def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 elif issubclass(typ, Bitfield): - return (typ.length + 7) // 8 // 32 + return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 elif issubclass(typ, Container): From 93ce1688629f45f92f7a261a958e99351299606a Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 16:47:48 +0100 Subject: [PATCH 13/28] More linting fixes --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 3 ++- specs/core/1_custody-game.md | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0a41fe5c0..1f5fe1ee6 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,7 +25,7 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - bit, boolean, Container, List, Vector, Bytes, uint64, + bit, boolean, Container, List, Vector, uint64, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d3fe9773..f596da520 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,7 +1292,8 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[4](*([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( + *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 17f599fcd..9ad00516e 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -574,7 +574,6 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) - assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) From 7adf07ea5f1fca3696cc52cfa6e7127aa7e80883 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 27 Jun 2019 22:58:44 +0100 Subject: [PATCH 14/28] A few more tests for Bitvector/Bitlist --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 63f0c835d..88ccc838c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -85,6 +85,16 @@ test_data = [ "c5c2", chunk("c5c2")), ("bitlist TFTFFFTTFTFFFFTT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), "c5c201", h(chunk("c5c2"), chunk("10"))), + ("long bitvector", Bitvector[512](1 for i in range(512)), + "ff" * 64, h("ff" * 32, "ff" * 32)), + ("long bitlist", Bitlist[512](1), + "03", h(h(chunk("01"), chunk("")), chunk("01"))), + ("long bitlist", Bitlist[512](1 for i in range(512)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), chunk("0002"))), + ("odd bitvector", Bitvector[513](1 for i in range(513)), + "ff" * 64 + "01", h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk("")))), + ("odd bitlist", Bitlist[513](1 for i in range(513)), + "ff" * 64 + "03", h(h(h("ff" * 32, "ff" * 32), h(chunk("01"), chunk(""))), chunk("0102"))), ("uint8 00", uint8(0x00), "00", chunk("00")), ("uint8 01", uint8(0x01), "01", chunk("01")), ("uint8 ab", uint8(0xab), "ab", chunk("ab")), From 237b41df5b41a37972c66c51d5acd551134189b5 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:18:54 +0100 Subject: [PATCH 15/28] Slice notation for justification_bitfield shift --- specs/core/0_beacon-chain.md | 5 +++-- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f596da520..0ea24a255 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1292,8 +1292,9 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root - state.justification_bitfield = Bitvector[JUSTIFICATION_BITVECTOR_LENGTH]( - *([0b0] + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1])) + state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ + state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] + state.justification_bitfield[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 53ab42743..ba773b443 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -324,12 +324,18 @@ class BaseList(list, Elements): return super().__getitem__(k) def __setitem__(self, k, v): - if k < 0: - raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") - if k > len(self): - raise IndexError(f"cannot set item in type {self.__class__}" - f" at out of bounds index {k} (to {v}, bound: {len(self)})") - super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) + if type(k) == slice: + if k.start < 0 or k.stop > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds slice {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) + else: + if k < 0: + raise IndexError(f"cannot set item in type {self.__class__} at negative index {k} (to {v})") + if k > len(self): + raise IndexError(f"cannot set item in type {self.__class__}" + f" at out of bounds index {k} (to {v}, bound: {len(self)})") + super().__setitem__(k, coerce_type_maybe(v, self.__class__.elem_type, strict=True)) def append(self, v): super().append(coerce_type_maybe(v, self.__class__.elem_type, strict=True)) From 2677d233a830b07a7f6f60392a1a54b888d3ac60 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 00:31:37 +0100 Subject: [PATCH 16/28] Some more (shorter) Bitvector and Bitlist tests --- test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 88ccc838c..637d9c5c4 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -79,6 +79,12 @@ test_data = [ ("bit T", bit(True), "01", chunk("01")), ("boolean F", boolean(False), "00", chunk("00")), ("boolean T", boolean(True), "01", chunk("01")), + ("bitvector TTFTFTFF", Bitvector[8](1, 1, 0, 1, 0, 1, 0, 0), "2b", chunk("2b")), + ("bitlist TTFTFTFF", Bitlist[8](1, 1, 0, 1, 0, 1, 0, 0), "2b01", h(chunk("2b"), chunk("08"))), + ("bitvector FTFT", Bitvector[4](0, 1, 0, 1), "0a", chunk("0a")), + ("bitlist FTFT", Bitlist[4](0, 1, 0, 1), "1a", h(chunk("0a"), chunk("04"))), + ("bitvector FTF", Bitvector[3](0, 1, 0), "02", chunk("02")), + ("bitlist FTF", Bitlist[3](0, 1, 0), "0a", h(chunk("02"), chunk("03"))), ("bitvector TFTFFFTTFT", Bitvector[10](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c502", chunk("c502")), ("bitlist TFTFFFTTFT", Bitlist[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1), "c506", h(chunk("c502"), chunk("0A"))), ("bitvector TFTFFFTTFTFFFFTT", Bitvector[16](1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1), From 196ac4202595a716a7acba91e7a437d80cc1a4ad Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:23:22 +0100 Subject: [PATCH 17/28] Cleanup naming --- specs/core/0_beacon-chain.md | 49 +++++++++---------- specs/core/1_custody-game.md | 26 +++++----- specs/core/1_shard-data-chains.md | 4 +- specs/light_client/sync_protocol.md | 8 +-- specs/validator/0_beacon-chain-validator.md | 20 ++++---- specs/validator/beacon_node_oapi.yaml | 8 +-- .../pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/fork_choice/test_on_attestation.py | 2 +- .../eth2spec/test/helpers/attestations.py | 12 ++--- .../test_process_attestation.py | 18 +++---- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +-- 12 files changed, 79 insertions(+), 80 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9ce7976b5..d41173424 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -190,7 +190,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `SHUFFLE_ROUND_COUNT` | `90` | -| `JUSTIFICATION_BITVECTOR_LENGTH` | `4` | +| `JUSTIFICATION_BITS_LENGTH` | `4` | * For the safety of crosslinks, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -360,7 +360,7 @@ class IndexedAttestation(Container): ```python class PendingAttestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex @@ -427,9 +427,9 @@ class AttesterSlashing(Container): ```python class Attestation(Container): - aggregation_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + aggregation_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] data: AttestationData - custody_bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION] + custody_bits: Bitlist[MAX_INDICES_PER_ATTESTATION] signature: BLSSignature ``` @@ -527,7 +527,7 @@ class BeaconState(Container): previous_crosslinks: Vector[Crosslink, SHARD_COUNT] # Previous epoch snapshot current_crosslinks: Vector[Crosslink, SHARD_COUNT] # Finality - justification_bitfield: Bitvector[JUSTIFICATION_BITVECTOR_LENGTH] # Bit set for every recent justified epoch + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint @@ -867,12 +867,12 @@ def get_crosslink_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> S ```python def get_attesting_indices(state: BeaconState, data: AttestationData, - bitfield: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: + bits: Bitlist[MAX_INDICES_PER_ATTESTATION]) -> Sequence[ValidatorIndex]: """ - Return the sorted attesting indices corresponding to ``data`` and ``bitfield``. + Return the sorted attesting indices corresponding to ``data`` and ``bits``. """ committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard) - return sorted([index for i, index in enumerate(committee) if bitfield[i]]) + return sorted([index for i, index in enumerate(committee) if bits[i]]) ``` ### `int_to_bytes` @@ -920,8 +920,8 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA """ Convert ``attestation`` to (almost) indexed-verifiable form. """ - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) - custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bitfield) + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) assert set(custody_bit_1_indices).issubset(attesting_indices) custody_bit_0_indices = [index for index in attesting_indices if index not in custody_bit_1_indices] @@ -1146,7 +1146,7 @@ def get_genesis_beacon_state(deposits: Sequence[Deposit], genesis_time: int, eth validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH - # Populate active_index_roots + # Populate active_index_roots genesis_active_index_root = hash_tree_root( List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, GENESIS_EPOCH)) ) @@ -1254,7 +1254,7 @@ def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: output = set() # type: Set[ValidatorIndex] for a in attestations: - output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, list(output))) ``` @@ -1294,36 +1294,35 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bitfield[1:JUSTIFICATION_BITVECTOR_LENGTH] = \ - state.justification_bitfield[0:JUSTIFICATION_BITVECTOR_LENGTH - 1] - state.justification_bitfield[0] = 0b0 + state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 previous_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, previous_epoch) ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) - state.justification_bitfield[1] = 0b1 + state.justification_bits[1] = 0b1 current_epoch_matching_target_balance = get_attesting_balance( state, get_matching_target_attestations(state, current_epoch) ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) - state.justification_bitfield[0] = 0b1 + state.justification_bits[0] = 0b1 # Process finalizations - bitfield = state.justification_bitfield + bits = state.justification_bits # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if all(bitfield[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if all(bitfield[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_previous_justified_checkpoint # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if all(bitfield[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if all(bitfield[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` @@ -1379,7 +1378,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence index = ValidatorIndex(index) attestation = min([ a for a in matching_source_attestations - if index in get_attesting_indices(state, a.data, a.aggregation_bitfield) + if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward @@ -1661,7 +1660,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: pending_attestation = PendingAttestation( data=data, - aggregation_bitfield=attestation.aggregation_bitfield, + aggregation_bits=attestation.aggregation_bits, inclusion_delay=state.slot - attestation_slot, proposer_index=get_beacon_proposer_index(state), ) @@ -1680,7 +1679,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] - + # Check signature validate_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9ad00516e..883476c58 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -272,14 +272,14 @@ def get_custody_chunk_count(crosslink: Crosslink) -> int: return crosslink_length * chunks_per_epoch ``` -### `get_bitfield_bit` +### `get_bit` ```python -def get_bitfield_bit(bitfield: bytes, i: int) -> int: +def get_bit(serialization: bytes, i: int) -> int: """ - Extract the bit in ``bitfield`` at position ``i``. + Extract the bit in ``serialization`` at position ``i``. """ - return (bitfield[i // 8] >> (i % 8)) % 2 + return (serialization[i // 8] >> (i % 8)) % 2 ``` ### `get_custody_chunk_bit` @@ -287,17 +287,17 @@ def get_bitfield_bit(bitfield: bytes, i: int) -> int: ```python def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol - return bool(get_bitfield_bit(hash(key + chunk), 0)) + return bool(get_bit(hash(key + chunk), 0)) ``` ### `get_chunk_bits_root` ```python -def get_chunk_bits_root(chunk_bitfield: bytes) -> Bytes32: +def get_chunk_bits_root(chunk_bits: bytes) -> Bytes32: aggregated_bits = bytearray([0] * 32) - for i in range(0, len(chunk_bitfield), 32): + for i in range(0, len(chunk_bits), 32): for j in range(32): - aggregated_bits[j] ^= chunk_bitfield[i + j] + aggregated_bits[j] ^= chunk_bits[i + j] return hash(aggregated_bits) ``` @@ -489,7 +489,7 @@ def process_chunk_challenge(state: BeaconState, responder = state.validators[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) assert challenge.responder_index in attesters # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: @@ -546,7 +546,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period(state, challenge.responder_index)) # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert challenge.responder_index in attesters # A validator can be the challenger for at most one challenge at a time @@ -575,8 +575,8 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = attestation.custody_bitfield[attesters.index(challenge.responder_index)] - assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) + custody_bit = attestation.custody_bits[attesters.index(challenge.responder_index)] + assert custody_bit != get_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( challenge_index=state.custody_challenge_index, @@ -670,7 +670,7 @@ def process_bit_challenge_response(state: BeaconState, ) # Verify the chunk bit does not match the challenge chunk bit assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) + != get_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index b2a5ea9ea..432820b7c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -92,7 +92,7 @@ class ShardAttestation(Container): slot: Slot shard: Shard shard_block_root: Bytes32 - aggregation_bitfield: Bitlist[PLACEHOLDER] + aggregation_bits: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -232,7 +232,7 @@ def verify_shard_attestation_signature(state: BeaconState, persistent_committee = get_persistent_committee(state, data.shard, data.slot) pubkeys = [] for i, index in enumerate(persistent_committee): - if attestation.aggregation_bitfield[i]: + if attestation.aggregation_bits[i]: validator = state.validators[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index a1b5777cf..3f5a37289 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -168,7 +168,7 @@ If a client wants to update its `finalized_header` it asks the network for a `Bl { 'header': BeaconBlockHeader, 'shard_aggregate_signature': BLSSignature, - 'shard_bitfield': Bitlist[PLACEHOLDER], + 'shard_bits': Bitlist[PLACEHOLDER], 'shard_parent_block': ShardBlock, } ``` @@ -180,13 +180,13 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val assert proof.shard_parent_block.beacon_chain_root == hash_tree_root(proof.header) committee = compute_committee(proof.header, validator_memory) # Verify that we have >=50% support - support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bitfield[i]]) + support_balance = sum([v.effective_balance for i, v in enumerate(committee) if proof.shard_bits[i]]) total_balance = sum([v.effective_balance for i, v in enumerate(committee)]) assert support_balance * 2 > total_balance # Verify shard attestations group_public_key = bls_aggregate_pubkeys([ v.pubkey for v, index in enumerate(committee) - if proof.shard_bitfield[index] + if proof.shard_bits[index] ]) assert bls_verify( pubkey=group_public_key, @@ -196,4 +196,4 @@ def verify_block_validity_proof(proof: BlockValidityProof, validator_memory: Val ) ``` -The size of this proof is only 200 (header) + 96 (signature) + 16 (bitfield) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. +The size of this proof is only 200 (header) + 96 (signature) + 16 (bits) + 352 (shard block) = 664 bytes. It can be reduced further by replacing `ShardBlock` with `MerklePartial(lambda x: x.beacon_chain_root, ShardBlock)`, which would cut off ~220 bytes. diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index aa8350b66..813b8bcfe 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -44,8 +44,8 @@ - [Crosslink vote](#crosslink-vote) - [Construct attestation](#construct-attestation) - [Data](#data) - - [Aggregation bitfield](#aggregation-bitfield) - - [Custody bitfield](#custody-bitfield) + - [Aggregation bits](#aggregation-bits) + - [Custody bits](#custody-bits) - [Aggregate signature](#aggregate-signature) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) @@ -322,19 +322,19 @@ Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-c Set `attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object defined in the previous section, [attestation data](#attestation-data). -##### Aggregation bitfield +##### Aggregation bits -* Let `aggregation_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. * Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bitfield[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bitfield = aggregation_bitfield`. +* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. +* Set `attestation.aggregation_bits = aggregation_bits`. -*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield)` should return a list of length equal to 1, containing `validator_index`. +*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. -##### Custody bitfield +##### Custody bits -* Let `custody_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bitfield = custody_bitfield`. +* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. +* Set `attestation.custody_bits = custody_bits`. *Note*: This is a stub for Phase 0. diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 74be21fac..4da8f7933 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -415,16 +415,16 @@ components: type: object description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." properties: - aggregation_bitfield: + aggregation_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Attester aggregation bitfield." - custody_bitfield: + description: "Attester aggregation bits." + custody_bits: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Custody bitfield." + description: "Custody bits." signature: type: string format: byte diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index c707c840a..ea1f1d47f 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - # TODO: Once has Bitfields and Bitvectors, add back + # TODO: Once has Bitlists and Bitvectors, add back # spec.BeaconState and spec.BeaconBlock for typ in [spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: # create a random pyspec value diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 1a67e01d5..5adb022a6 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -118,5 +118,5 @@ def test_on_attestation_invalid_attestation(spec, state): attestation = get_valid_attestation(spec, state, slot=block.slot) # make attestation invalid - attestation.custody_bitfield[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] + attestation.custody_bits[0:8] = [0, 0, 0, 0, 1, 1, 1, 1] run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index a184a5f5e..6fdf1538b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -67,12 +67,12 @@ def get_valid_attestation(spec, state, slot=None, signed=False): ) committee_size = len(crosslink_committee) - aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) - custody_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) + custody_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION](*([0] * committee_size)) attestation = spec.Attestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=attestation_data, - custody_bitfield=custody_bitfield, + custody_bits=custody_bits, ) fill_aggregate_attestation(spec, state, attestation) if signed: @@ -105,7 +105,7 @@ def sign_attestation(spec, state, attestation): participants = spec.get_attesting_indices( state, attestation.data, - attestation.aggregation_bitfield, + attestation.aggregation_bits, ) attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) @@ -135,7 +135,7 @@ def fill_aggregate_attestation(spec, state, attestation): attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield[i] = True + attestation.aggregation_bits[i] = True def add_attestation_to_state(spec, state, attestation, slot): diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 1ea7c92cf..916724001 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -275,14 +275,14 @@ def test_bad_crosslink_end_epoch(spec, state): @with_all_phases @spec_state_test -def test_inconsistent_bitfields(spec, state): +def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bitfield = deepcopy(attestation.aggregation_bitfield) - custody_bitfield.append(False) + custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits.append(False) - attestation.custody_bitfield = custody_bitfield + attestation.custody_bits = custody_bits sign_attestation(spec, state, attestation) @@ -291,11 +291,11 @@ def test_inconsistent_bitfields(spec, state): @with_phases(['phase0']) @spec_state_test -def test_non_empty_custody_bitfield(spec, state): +def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) + attestation.custody_bits = deepcopy(attestation.aggregation_bits) sign_attestation(spec, state, attestation) @@ -304,12 +304,12 @@ def test_non_empty_custody_bitfield(spec, state): @with_all_phases @spec_state_test -def test_empty_aggregation_bitfield(spec, state): +def test_empty_aggregation_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.aggregation_bitfield = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( - *([0b0] * len(attestation.aggregation_bitfield))) + attestation.aggregation_bits = Bitlist[spec.MAX_INDICES_PER_ATTESTATION]( + *([0b0] * len(attestation.aggregation_bits))) sign_attestation(spec, state, attestation) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 53075845b..d5855a755 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bitfield, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, Bitlist, Bitvector, uint, ) @@ -128,7 +128,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: if isinstance(typ, BasicType): return 1 - elif issubclass(typ, Bitfield): + elif issubclass(typ, Bits): return (typ.length + 255) // 256 elif issubclass(typ, Elements): return (typ.length * item_length(typ.elem_type) + 31) // 32 diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ba773b443..91d3ff4f2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -353,11 +353,11 @@ class BitElementsType(ElementsType): length: int -class Bitfield(BaseList, metaclass=BitElementsType): +class Bits(BaseList, metaclass=BitElementsType): pass -class Bitlist(Bitfield): +class Bitlist(Bits): @classmethod def is_fixed_size(cls): return False @@ -367,7 +367,7 @@ class Bitlist(Bitfield): return cls() -class Bitvector(Bitfield): +class Bitvector(Bits): @classmethod def extract_args(cls, *args): From 6f9d37485d624cb96188714db495cdb07ace9aa6 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:34:01 +0100 Subject: [PATCH 18/28] Cleanups --- specs/core/0_beacon-chain.md | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d41173424..9de024909 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,20 +1294,14 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:JUSTIFICATION_BITS_LENGTH] = state.justification_bits[0:JUSTIFICATION_BITS_LENGTH - 1] - state.justification_bits[0] = 0b0 - previous_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, previous_epoch) - ) - if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, - root=get_block_root(state, previous_epoch)) + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 - current_epoch_matching_target_balance = get_attesting_balance( - state, get_matching_target_attestations(state, current_epoch) - ) - if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch + if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: + state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From e36593b155d3fdffdb8ebe451236d5dfb54e22c9 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 28 Jun 2019 12:35:50 +0100 Subject: [PATCH 19/28] Add comment --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9de024909..96b8981fe 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) From 05842f83718644184ce997277e7817bada07d568 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 28 Jun 2019 15:26:02 +0100 Subject: [PATCH 20/28] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 96b8981fe..b9c3f5f0c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,14 +1294,15 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits = [0b0] + state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] # Bitshift + state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(previous_epoch, get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(current_epoch, get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From 5ff13dd81a95d7852c2afa65dd3851d5642c17dc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:07:36 +0200 Subject: [PATCH 21/28] be explicit about limiting for HTR and chunk padding --- specs/simple-serialize.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 97b1d560c..3fba59fba 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -49,13 +49,13 @@ class ContainerExample(Container): foo: uint64 bar: boolean ``` -* **vector**: ordered fixed-length homogeneous collection of values +* **vector**: ordered fixed-length homogeneous collection, with `N` values * notation `Vector[type, N]`, e.g. `Vector[uint64, N]` -* **list**: ordered variable-length homogeneous collection of values, with maximum length `N` +* **list**: ordered variable-length homogeneous collection, limited to `N` values * notation `List[type, N]`, e.g. `List[uint64, N]` -* **bitvector**: ordered fixed-length collection of `boolean` values +* **bitvector**: ordered fixed-length collection of `boolean` values, with `N` bits * notation `Bitvector[N]` -* **bitlist**: ordered variable-length collection of `boolean` values, with maximum length `N` +* **bitlist**: ordered variable-length collection of `boolean` values, limited to `N` bits * notation `Bitlist[N]` * **union**: union type containing one of the given subtypes * notation `Union[type_1, type_2, ...]`, e.g. `union[null, uint64]` @@ -161,22 +161,31 @@ return serialized_type_index + serialized_bytes Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). +Note that deserialization requires hardening against invalid inputs. A non-exhaustive list: +- Offsets: out of order, out of range, mismatching minimum element size +- Scope: Extra unused bytes, not aligned with element size. +- More elements than a list limit allows. Part of enforcing consensus. + ## Merkleization We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. -* `pad`: given a list `l` and a length `N`, adds `N-len(l)` empty objects to the end of the list (the type of the empty object is implicit in the list type) +* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + There merkleization depends on the input length: + - `0` chunks: A chunk filled with zeroes. + - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - `1+` chunks: pad to the next power of 2, merkleize as binary tree. + - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(pad(value, N))), len(value))` if `value` is a list of basic objects +* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in pad(value, N)]), len(value))` if `value` is a list of composite objects +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]` From 128bbbc665c8eabe10640c019ccb0765c5be8f11 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:27:59 +0200 Subject: [PATCH 22/28] fix slicing, and support partial slice bounds --- specs/core/0_beacon-chain.md | 2 +- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b9c3f5f0c..887ae6802 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1294,7 +1294,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] == state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 91d3ff4f2..d87a22399 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -325,7 +325,7 @@ class BaseList(list, Elements): def __setitem__(self, k, v): if type(k) == slice: - if k.start < 0 or k.stop > len(self): + if (k.start is not None and k.start < 0) or (k.stop is not None and k.stop > len(self)): raise IndexError(f"cannot set item in type {self.__class__}" f" at out of bounds slice {k} (to {v}, bound: {len(self)})") super().__setitem__(k, [coerce_type_maybe(x, self.__class__.elem_type) for x in v]) From 25db39755047ce6ae28332d90080c502ce899423 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 17:34:31 +0200 Subject: [PATCH 23/28] fix line length lint problem in checkpoint --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 887ae6802..ba5b591d1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1298,11 +1298,13 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, root=get_block_root(state, previous_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) state.justification_bits[1] = 0b1 matching_target_attestations = get_matching_target_attestations(state, current_epoch) # Current epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, root=get_block_root(state, current_epoch)) + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) state.justification_bits[0] = 0b1 # Process finalizations From fa84c496592d722b544c775065b9258cd22dbd73 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:23:34 +0100 Subject: [PATCH 24/28] Update specs/core/0_beacon-chain.md Co-Authored-By: Danny Ryan --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index cb50e2f86..6f28c0e3d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1295,7 +1295,7 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process justifications state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[1:] = state.justification_bits[:-1] state.justification_bits[0] = 0b0 matching_target_attestations = get_matching_target_attestations(state, previous_epoch) # Previous epoch if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2: From 6a2d2c84a8f3fa3ea846d17dff5d10ebfb3508e6 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Fri, 28 Jun 2019 20:49:57 +0100 Subject: [PATCH 25/28] Bitlist for attestation doc --- specs/validator/0_beacon-chain-validator.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ad01476e9..7d5630c7b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -331,17 +331,13 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregation bits -* Let `aggregation_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Let `index_into_committee` be the index into the validator's `committee` at which `validator_index` is located. -* Set `aggregation_bits[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. -* Set `attestation.aggregation_bits = aggregation_bits`. +* Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bits -* Let `custody_bits` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. -* Set `attestation.custody_bits = custody_bits`. +* Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. *Note*: This is a stub for Phase 0. From 4dcb47e393ca68ab9e32af8ba913eaf829d6cb69 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:06 +0100 Subject: [PATCH 26/28] Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 916724001..b114c85e9 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -279,7 +279,7 @@ def test_inconsistent_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - custody_bits = deepcopy(attestation.aggregation_bits) + custody_bits = attestation.aggregation_bits[:] custody_bits.append(False) attestation.custody_bits = custody_bits From be04eb2673563b6250cc5637777badc76913ec5f Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 28 Jun 2019 20:52:16 +0100 Subject: [PATCH 27/28] Change copy style, and remove deepcopy import Update test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py Co-Authored-By: Danny Ryan --- .../test/phase_0/block_processing/test_process_attestation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index b114c85e9..41207fdf4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,5 +1,3 @@ -from copy import deepcopy - from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, @@ -295,7 +293,7 @@ def test_non_empty_custody_bits(spec, state): attestation = get_valid_attestation(spec, state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.custody_bits = deepcopy(attestation.aggregation_bits) + attestation.custody_bits = attestation.aggregation_bits[:] sign_attestation(spec, state, attestation) From 4f31207b7f2cd4ba2bf21976b6e8bc6644e24202 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 Jun 2019 22:45:20 +0200 Subject: [PATCH 28/28] reword merkleize with limit / length --- specs/simple-serialize.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 3fba59fba..7076b6410 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -171,21 +171,21 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize(data, pad_to)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. - There merkleization depends on the input length: - - `0` chunks: A chunk filled with zeroes. - - `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. - - `1+` chunks: pad to the next power of 2, merkleize as binary tree. - - with `pad_to` set: pad the `data` with zeroed chunks to this power of two (virtually for memory efficiency). +* `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` +* `merkleize(data, pad_for)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. + The merkleization depends on the effective input, which can be padded: if `pad_for=L`, then pad the `data` with zeroed chunks to `next_pow_of_two(L)` (virtually for memory efficiency). + Then, merkleize the chunks (empty input is padded to 1 zero chunk): + - If `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. + - If `> 1` chunks: pad to `next_pow_of_two(len(chunks))`, merkleize as binary tree. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects -* `mix_in_length(merkleize(pack(value), pad_to=N), len(value))` if `value` is a list of basic objects. `N` is the amount of chunks needed for the list limit (*this depends on element-type size*). +* `mix_in_length(merkleize(pack(value), pad_for=(N * elem_size / BYTES_PER_CHUNK)), len(value))` if `value` is a list of basic objects. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container -* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_to=N), len(value))` if `value` is a list of composite objects. `N` is the list limit (*1 chunk per element*). +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_for=N), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ### Merkleization of `Bitvector[N]`