From 5dd001591262139719a959221fc2b2396668fbca Mon Sep 17 00:00:00 2001 From: Nathaniel Jensen Date: Wed, 30 Oct 2019 10:29:42 +1100 Subject: [PATCH 01/48] Update link to current bls standard draft. --- specs/bls_signature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index 652279cd7..30003699c 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -1,6 +1,6 @@ # BLS signature verification -**Notice**: This document is a placeholder to facilitate the emergence of cross-client testnets. Substantive changes are postponed until [BLS standardisation](https://github.com/pairingwg/bls_standard) is finalized. +**Notice**: This document is a placeholder to facilitate the emergence of cross-client testnets. Substantive changes are postponed until [BLS standardisation](https://github.com/cfrg/draft-irtf-cfrg-bls-signature) is finalized. **Warning**: The constructions in this document should not be considered secure. In particular, the `hash_to_G2` function is known to be unsecure. From 2159df2d1228a7b31074167c1f04b749258baf1c Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 31 Oct 2019 21:31:08 -0700 Subject: [PATCH 02/48] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9f0f21435..39d650f11 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -204,8 +204,8 @@ The following values are (non-configurable) constants used throughout the specif | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | | `MAX_SEED_LOOKAHEAD` | `2**2` (= 4) | epochs | 25.6 minutes | -| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~1.7 hours | -| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | +| `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~3.4 hours | +| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | | `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | From 2c322748709aa938e5940ff341b377867a0fb832 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 15 Nov 2019 14:50:58 +0100 Subject: [PATCH 03/48] rename BytesN to ByteVector, rename Bytes to ByteList --- scripts/build_spec.py | 6 +++--- specs/core/1_custody-game.md | 2 +- specs/light_client/merkle_proofs.md | 4 ++-- test_libs/pyspec/eth2spec/debug/decode.py | 6 +++--- test_libs/pyspec/eth2spec/debug/encode.py | 2 +- .../pyspec/eth2spec/debug/random_value.py | 10 +++++----- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 8 ++++---- .../pyspec/eth2spec/test/helpers/custody.py | 8 ++++---- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 8 ++++---- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 20 +++++++++---------- .../eth2spec/utils/ssz/test_ssz_impl.py | 8 ++++---- .../eth2spec/utils/ssz/test_ssz_typing.py | 14 ++++++------- 12 files changed, 48 insertions(+), 48 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index e05907014..0dfdede22 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -56,7 +56,7 @@ from eth2spec.utils.ssz.ssz_impl import ( ) from eth2spec.utils.ssz.ssz_typing import ( BasicValue, Elements, BaseBytes, BaseList, SSZType, - Container, List, Vector, Bytes, BytesN, Bitlist, Bitvector, Bits, + Container, List, Vector, ByteList, ByteVector, Bitlist, Bitvector, Bits, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, uint64, bit, boolean, byte, ) @@ -211,10 +211,10 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st ignored_dependencies = [ - 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN' + 'bit', 'boolean', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'ByteList', 'ByteVector' 'Bytes1', 'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96', 'Bitlist', 'Bitvector', 'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256', - 'bytes', 'byte', 'BytesN' # to be removed after updating spec doc + 'bytes', 'byte', 'ByteVector' # to be removed after updating spec doc ] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 67e12a08c..3ee082846 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -195,7 +195,7 @@ class CustodyBitChallengeRecord(Container): class CustodyResponse(Container): challenge_index: uint64 chunk_index: uint64 - chunk: BytesN[BYTES_PER_CUSTODY_CHUNK] + chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] data_branch: List[Hash, CUSTODY_DATA_DEPTH] chunk_bits_branch: List[Hash, CUSTODY_CHUNK_BIT_DEPTH] chunk_bits_leaf: Bitvector[256] diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 906e7d241..182aac9d5 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -162,11 +162,11 @@ def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableNam assert not issubclass(typ, BasicValue) # If we descend to a basic type, the path cannot continue further if p == '__len__': typ = uint64 - assert issubclass(typ, (List, Bytes)) + assert issubclass(typ, (List, ByteList)) root = GeneralizedIndex(root * 2 + 1) else: pos, _, _ = get_item_position(typ, p) - base_index = (GeneralizedIndex(2) if issubclass(typ, (List, Bytes)) else GeneralizedIndex(1)) + base_index = (GeneralizedIndex(2) if issubclass(typ, (List, ByteList)) else GeneralizedIndex(1)) root = GeneralizedIndex(root * base_index * get_next_power_of_two(chunk_count(typ)) + pos) typ = get_elem_type(typ, p) return root diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c0b977ab3..da2682276 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,8 +1,8 @@ from typing import Any from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, uint, Container, Bytes, List, boolean, - Vector, BytesN + SSZType, SSZValue, uint, Container, ByteList, List, boolean, + Vector, ByteVector ) @@ -11,7 +11,7 @@ def decode(data: Any, typ: SSZType) -> SSZValue: return typ(data) elif issubclass(typ, (List, Vector)): return typ(decode(element, typ.elem_type) for element in data) - elif issubclass(typ, (Bytes, BytesN)): + elif issubclass(typ, (ByteList, ByteVector)): return typ(bytes.fromhex(data[2:])) elif issubclass(typ, Container): temp = {} diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index d59f15640..4857f7114 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -17,7 +17,7 @@ def encode(value, include_hash_tree_roots=False): return '0x' + serialize(value).hex() elif isinstance(value, list): # normal python lists, ssz-List, Vector return [encode(element, include_hash_tree_roots) for element in value] - elif isinstance(value, bytes): # both bytes and BytesN + elif isinstance(value, bytes): # both bytes and ByteVector return '0x' + value.hex() elif isinstance(value, Container): ret = {} diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index ce1522c32..8055312d7 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,8 +2,8 @@ from random import Random from enum import Enum from eth2spec.utils.ssz.ssz_typing import ( - SSZType, SSZValue, BasicValue, BasicType, uint, Container, Bytes, List, boolean, - Vector, BytesN, Bitlist, Bitvector + SSZType, SSZValue, BasicValue, BasicType, uint, Container, ByteList, List, boolean, + Vector, ByteVector, Bitlist, Bitvector ) # in bytes @@ -51,8 +51,8 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if issubclass(typ, Bytes): - # Bytes array + if issubclass(typ, ByteList): + # ByteList array if mode == RandomizationMode.mode_nil_count: return typ(b'') elif mode == RandomizationMode.mode_max_count: @@ -65,7 +65,7 @@ def get_random_ssz_object(rng: Random, return typ(b'\xff' * min(1, typ.length)) else: return typ(get_random_bytes_list(rng, rng.randint(0, min(max_bytes_length, typ.length)))) - elif issubclass(typ, BytesN): + elif issubclass(typ, ByteVector): # Sanity, don't generate absurdly big random values # If a client is aiming to performance-test, they should create a benchmark suite. assert typ.length <= max_bytes_length diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index 272ed0c44..007e96845 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -11,9 +11,9 @@ def translate_typ(typ) -> ssz.BaseSedes: if issubclass(typ, spec_ssz.Container): return ssz.Container( [translate_typ(field_typ) for field_name, field_typ in typ.get_fields().items()]) - elif issubclass(typ, spec_ssz.BytesN): + elif issubclass(typ, spec_ssz.ByteVector): return ssz.ByteVector(typ.length) - elif issubclass(typ, spec_ssz.Bytes): + elif issubclass(typ, spec_ssz.ByteList): return ssz.ByteList() elif issubclass(typ, spec_ssz.Vector): return ssz.Vector(translate_typ(typ.elem_type), typ.length) @@ -76,9 +76,9 @@ def translate_value(value, typ): return typ(value) elif issubclass(typ, spec_ssz.Bitvector): return typ(value) - elif issubclass(typ, spec_ssz.BytesN): + elif issubclass(typ, spec_ssz.ByteVector): return typ(value) - elif issubclass(typ, spec_ssz.Bytes): + elif issubclass(typ, spec_ssz.ByteList): return value if issubclass(typ, spec_ssz.Container): return typ(**{f_name: translate_value(f_val, f_typ) for (f_val, (f_name, f_typ)) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 205a335f4..f6ca8ecd9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures from eth2spec.utils.hash_function import hash -from eth2spec.utils.ssz.ssz_typing import Bitlist, BytesN, Bitvector +from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector from eth2spec.utils.ssz.ssz_impl import chunkify, pack, hash_tree_root from eth2spec.utils.merkle_minimal import get_merkle_tree, get_merkle_proof @@ -136,9 +136,9 @@ def get_valid_custody_response(spec, state, bit_challenge, custody_data, challen chunk_index -= 1 chunk_bit = spec.get_custody_chunk_bit(bit_challenge.responder_key, chunks[chunk_index]) - chunks_hash_tree_roots = [hash_tree_root(BytesN[spec.BYTES_PER_CUSTODY_CHUNK](chunk)) for chunk in chunks] + chunks_hash_tree_roots = [hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunk)) for chunk in chunks] chunks_hash_tree_roots += [ - hash_tree_root(BytesN[spec.BYTES_PER_CUSTODY_CHUNK](b"\0" * spec.BYTES_PER_CUSTODY_CHUNK)) + hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](b"\0" * spec.BYTES_PER_CUSTODY_CHUNK)) for i in range(2 ** spec.ceillog2(len(chunks)) - len(chunks))] data_tree = get_merkle_tree(chunks_hash_tree_roots) @@ -158,7 +158,7 @@ def get_valid_custody_response(spec, state, bit_challenge, custody_data, challen return spec.CustodyResponse( challenge_index=challenge_index, chunk_index=chunk_index, - chunk=BytesN[spec.BYTES_PER_CUSTODY_CHUNK](chunks[chunk_index]), + chunk=ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunks[chunk_index]), data_branch=data_branch, chunk_bits_branch=bitlist_chunk_branch, chunk_bits_leaf=chunk_bits_leaf, diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 748386733..ea6d82bbd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,7 +1,7 @@ from ..merkle_minimal import merkleize_chunks from ..hash_function import hash from .ssz_typing import ( - SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, Bytes, + SSZValue, SSZType, BasicValue, BasicType, Series, Elements, Bits, boolean, Container, List, ByteList, Bitlist, Bitvector, uint, ) @@ -56,7 +56,7 @@ def serialize(obj: SSZValue): def encode_series(values: Series): - if isinstance(values, bytes): # Bytes and BytesN are already like serialized output + if isinstance(values, bytes): # ByteList and ByteVector are already like serialized output return values # Recursively serialize @@ -93,7 +93,7 @@ def encode_series(values: Series): def pack(values: Series): - if isinstance(values, bytes): # Bytes and BytesN are already packed + if isinstance(values, bytes): # ByteList and ByteVector are already packed return values elif isinstance(values, Bits): # packs the bits in bytes, left-aligned. @@ -151,7 +151,7 @@ def hash_tree_root(obj: SSZValue): else: raise Exception(f"Type not supported: {type(obj)}") - if isinstance(obj, (List, Bytes, Bitlist)): + if isinstance(obj, (List, ByteList, Bitlist)): return mix_in_length(merkleize_chunks(leaves, limit=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6bc8f8326..afa388364 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -115,7 +115,7 @@ def coerce_type_maybe(v, typ: SSZType, strict: bool = False): return typ(v) elif isinstance(v, (list, tuple)): return typ(*v) - elif isinstance(v, (bytes, BytesN, Bytes)): + elif isinstance(v, (bytes, ByteVector, ByteList)): return typ(v) elif isinstance(v, GeneratorType): return typ(v) @@ -472,7 +472,7 @@ class BaseBytes(bytes, Elements, metaclass=BytesType): return f"{cls.__name__}[{cls.length}]: {self.hex()}" -class Bytes(BaseBytes): +class ByteList(BaseBytes): @classmethod def default(cls): @@ -483,7 +483,7 @@ class Bytes(BaseBytes): return False -class BytesN(BaseBytes): +class ByteVector(BaseBytes): @classmethod def extract_args(cls, *args): @@ -506,10 +506,10 @@ class BytesN(BaseBytes): return True -# Helpers for common BytesN types -Bytes1: BytesType = BytesN[1] -Bytes4: BytesType = BytesN[4] -Bytes8: BytesType = BytesN[8] -Bytes32: BytesType = BytesN[32] -Bytes48: BytesType = BytesN[48] -Bytes96: BytesType = BytesN[96] +# Helpers for common ByteVector types +Bytes1: BytesType = ByteVector[1] +Bytes4: BytesType = ByteVector[4] +Bytes8: BytesType = ByteVector[8] +Bytes32: BytesType = ByteVector[32] +Bytes48: BytesType = ByteVector[48] +Bytes96: BytesType = ByteVector[96] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py index 637d9c5c4..6166d14ee 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py @@ -1,7 +1,7 @@ from typing import Iterable from .ssz_impl import serialize, hash_tree_root from .ssz_typing import ( - bit, boolean, Container, List, Vector, Bytes, BytesN, + bit, boolean, Container, List, Vector, ByteList, ByteVector, Bitlist, Bitvector, uint8, uint16, uint32, uint64, uint256, byte ) @@ -39,7 +39,7 @@ class ComplexTestStruct(Container): A: uint16 B: List[uint16, 128] C: uint8 - D: Bytes[256] + D: ByteList[256] E: VarTestStruct F: Vector[FixedTestStruct, 4] G: Vector[VarTestStruct, 2] @@ -116,7 +116,7 @@ test_data = [ ("uint32 01234567", uint32(0x01234567), "67452301", chunk("67452301")), ("uint64 0000000000000000", uint64(0x00000000), "0000000000000000", chunk("0000000000000000")), ("uint64 0123456789abcdef", uint64(0x0123456789abcdef), "efcdab8967452301", chunk("efcdab8967452301")), - ("sig", BytesN[96](*sig_test_data), + ("sig", ByteVector[96](*sig_test_data), "0100000000000000000000000000000000000000000000000000000000000000" "0200000000000000000000000000000000000000000000000000000000000000" "03000000000000000000000000000000000000000000000000000000000000ff", @@ -187,7 +187,7 @@ test_data = [ A=0xaabb, B=List[uint16, 128](0x1122, 0x3344), C=0xff, - D=Bytes[256](b"foobar"), + D=ByteList[256](b"foobar"), E=VarTestStruct(A=0xabcd, B=List[uint16, 1024](1, 2, 3), C=0xff), F=Vector[FixedTestStruct, 4]( FixedTestStruct(A=0xcc, B=0x4242424242424242, C=0x13371337), diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py index d5a53c5fa..e6a69f2e2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py @@ -1,6 +1,6 @@ from .ssz_typing import ( SSZValue, SSZType, BasicValue, BasicType, Series, ElementsType, - Elements, bit, boolean, Container, List, Vector, Bytes, BytesN, + Elements, bit, boolean, Container, List, Vector, ByteList, ByteVector, byte, uint, uint8, uint16, uint32, uint64, uint128, uint256, Bytes32, Bytes48 ) @@ -25,14 +25,14 @@ def test_subclasses(): assert issubclass(boolean, BasicValue) assert isinstance(boolean, BasicType) - for c in [Container, List, Vector, Bytes, BytesN]: + for c in [Container, List, Vector, ByteList, ByteVector]: assert issubclass(c, Series) assert issubclass(c, SSZValue) assert isinstance(c, SSZType) assert not issubclass(c, BasicValue) assert not isinstance(c, BasicType) - for c in [List, Vector, Bytes, BytesN]: + for c in [List, Vector, ByteList, ByteVector]: assert issubclass(c, Elements) assert isinstance(c, ElementsType) @@ -203,10 +203,10 @@ def test_list(): def test_bytesn_subclass(): - assert isinstance(BytesN[32](b'\xab' * 32), Bytes32) - assert not isinstance(BytesN[32](b'\xab' * 32), Bytes48) - assert issubclass(BytesN[32](b'\xab' * 32).type(), Bytes32) - assert issubclass(BytesN[32], Bytes32) + assert isinstance(ByteVector[32](b'\xab' * 32), Bytes32) + assert not isinstance(ByteVector[32](b'\xab' * 32), Bytes48) + assert issubclass(ByteVector[32](b'\xab' * 32).type(), Bytes32) + assert issubclass(ByteVector[32], Bytes32) class Hash(Bytes32): pass From b15669b7a51dc460deeaef3d563de89746f486fa Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Fri, 15 Nov 2019 22:27:04 +0100 Subject: [PATCH 04/48] Backport master (v0.9.1) to dev (#1482) * p2p-interface: clarify that signing_root is used for block requests * hash cleanups * one more hash tree root gone for blocks - block hashes are always signing roots! * use simple serialize data types consistently * Describe which finalized root/epoch to use * remove custody_bits from attestation * remove AttestationDataAndCustodyBit * Specify inclusive range for genesis deposits * add initial fork choice bounce prevention and tests * PR feedback * further test bounce attack * wipe queued justified after epoch transition * remove extra var * minor fmt * only allow attestatiosn to be considered from current and previous epoch * use best_justified_checkpoint instead of queued_justified_checkpoints * use helper for slots since epoch start * be explicit about use of genesis epoch for previous epoch in fork choice on_block * pr feedback * add note aboutgenesis attestations * cleanup get_eth1_vote * make eth1_follow_distance clearer * Update the expected proposer period Since `SECONDS_PER_SLOT` is now `12` * minor fix to comment in mainnet config * Update 0_beacon-chain.md --- README.md | 1 - configs/mainnet.yaml | 7 +- configs/minimal.yaml | 6 + scripts/build_spec.py | 3 +- specs/core/0_beacon-chain.md | 58 ++-------- specs/core/0_fork-choice.md | 72 +++++++++++- specs/networking/p2p-interface.md | 26 ++--- specs/validator/0_beacon-chain-validator.md | 41 +++---- .../pyspec/eth2spec/fuzzing/test_decoder.py | 2 +- .../test/fork_choice/test_on_attestation.py | 65 +++++++++-- .../test/fork_choice/test_on_block.py | 71 ++++++++++++ .../eth2spec/test/fork_choice/test_on_tick.py | 105 ++++++++++++++++++ .../eth2spec/test/helpers/attestations.py | 13 +-- .../test_process_attestation.py | 60 +--------- .../test_process_attester_slashing.py | 59 +++------- .../test_process_rewards_and_penalties.py | 2 +- .../eth2spec/test/sanity/test_blocks.py | 3 +- 17 files changed, 374 insertions(+), 220 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/fork_choice/test_on_tick.py diff --git a/README.md b/README.md index a6f23db9d..fa103394d 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,6 @@ The following are the broad design goals for Ethereum 2.0: ## For spec contributors - Documentation on the different components used during spec writing can be found here: * [YAML Test Generators](test_generators/README.md) * [Executable Python Spec, with Py-tests](test_libs/pyspec/README.md) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index af446d575..a9459fb64 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -23,6 +23,11 @@ MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 65536 MIN_GENESIS_TIME: 1578009600 +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + # Deposit contract # --------------------------------------------------------------- @@ -53,7 +58,7 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # --------------------------------------------------------------- # 12 seconds SECONDS_PER_SLOT: 12 -# 2**0 (= 1) slots 6 seconds +# 2**0 (= 1) slots 12 seconds MIN_ATTESTATION_INCLUSION_DELAY: 1 # 2**5 (= 32) slots 6.4 minutes SLOTS_PER_EPOCH: 32 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 53599e83a..fbc961ab1 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -21,6 +21,12 @@ MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64 # Jan 3, 2020 MIN_GENESIS_TIME: 1578009600 +# +# +# Fork Choice +# --------------------------------------------------------------- +# 2**1 (= 1) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 2 # Deposit contract diff --git a/scripts/build_spec.py b/scripts/build_spec.py index e05907014..9ed9d3ae4 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -24,14 +24,13 @@ from eth2spec.utils.ssz.ssz_impl import ( signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - bit, boolean, Container, List, Vector, uint64, + boolean, Container, List, Vector, uint64, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( bls_aggregate_signatures, bls_aggregate_pubkeys, bls_verify, - bls_verify_multiple, bls_sign, ) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 39d650f11..4a952a2c6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -26,7 +26,6 @@ - [`Checkpoint`](#checkpoint) - [`Validator`](#validator) - [`AttestationData`](#attestationdata) - - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) - [`Eth1Data`](#eth1data) @@ -55,7 +54,6 @@ - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) - [`bls_verify`](#bls_verify) - - [`bls_verify_multiple`](#bls_verify_multiple) - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) - [Predicates](#predicates) - [`is_active_validator`](#is_active_validator) @@ -308,20 +306,11 @@ class AttestationData(Container): target: Checkpoint ``` -#### `AttestationDataAndCustodyBit` - -```python -class AttestationDataAndCustodyBit(Container): - data: AttestationData - custody_bit: bit # Challengeable bit (SSZ-bool, 1 byte) for the custody of shard data -``` - #### `IndexedAttestation` ```python class IndexedAttestation(Container): - custody_bit_0_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] # Indices with custody bit equal to 0 - custody_bit_1_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] # Indices with custody bit equal to 1 + attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData signature: BLSSignature ``` @@ -399,7 +388,6 @@ class AttesterSlashing(Container): class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData - custody_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] signature: BLSSignature ``` @@ -553,10 +541,6 @@ def bytes_to_int(data: bytes) -> uint64: `bls_verify` is a function for verifying a BLS signature, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify). -#### `bls_verify_multiple` - -`bls_verify_multiple` is a function for verifying a BLS signature constructed from multiple messages, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify_multiple). - #### `bls_aggregate_pubkeys` `bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). @@ -605,31 +589,18 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe """ Check if ``indexed_attestation`` has valid indices and signature. """ - bit_0_indices = indexed_attestation.custody_bit_0_indices - bit_1_indices = indexed_attestation.custody_bit_1_indices + indices = indexed_attestation.attesting_indices - # Verify no index has custody bit equal to 1 [to be removed in phase 1] - if not len(bit_1_indices) == 0: # [to be removed in phase 1] - return False # [to be removed in phase 1] # Verify max number of indices - if not len(bit_0_indices) + len(bit_1_indices) <= MAX_VALIDATORS_PER_COMMITTEE: - return False - # Verify index sets are disjoint - if not len(set(bit_0_indices).intersection(bit_1_indices)) == 0: + if not len(indices) <= MAX_VALIDATORS_PER_COMMITTEE: return False # Verify indices are sorted - if not (bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices)): + if not indices == sorted(indices): return False # Verify aggregate signature - if not bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_0_indices]), - bls_aggregate_pubkeys([state.validators[i].pubkey for i in bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), - ], + if not bls_verify( + pubkey=bls_aggregate_pubkeys([state.validators[i].pubkey for i in indices]), + message_hash=hash_tree_root(indexed_attestation.data), signature=indexed_attestation.signature, domain=get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch), ): @@ -922,13 +893,9 @@ def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> Ind Return the indexed attestation corresponding to ``attestation``. """ attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - custody_bit_1_indices = get_attesting_indices(state, attestation.data, attestation.custody_bits) - assert custody_bit_1_indices.issubset(attesting_indices) - custody_bit_0_indices = attesting_indices.difference(custody_bit_1_indices) return IndexedAttestation( - custody_bit_0_indices=sorted(custody_bit_0_indices), - custody_bit_1_indices=sorted(custody_bit_1_indices), + attesting_indices=sorted(attesting_indices), data=attestation.data, signature=attestation.signature, ) @@ -1026,7 +993,7 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b - `eth1_block_hash` is the hash of the Ethereum 1.0 block - `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` -- `deposits` is the sequence of all deposits, ordered chronologically, up to the block with hash `eth1_block_hash` +- `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Hash, @@ -1460,9 +1427,8 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla assert is_valid_indexed_attestation(state, attestation_2) slashed_any = False - attesting_indices_1 = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices - attesting_indices_2 = attestation_2.custody_bit_0_indices + attestation_2.custody_bit_1_indices - for index in sorted(set(attesting_indices_1).intersection(attesting_indices_2)): + indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices) + for index in sorted(indices): if is_slashable_validator(state.validators[index], get_current_epoch(state)): slash_validator(state, index) slashed_any = True @@ -1479,7 +1445,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH committee = get_beacon_committee(state, data.slot, data.index) - assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len(committee) + assert len(attestation.aggregation_bits) == len(committee) pending_attestation = PendingAttestation( data=data, diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index b909ce732..051c6ef8d 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -43,6 +43,12 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. 5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). +### Configuration + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `SAFE_SLOTS_TO_UPDATE_JUSTIFIED` | `2**3` (= 8) | slots | 96 seconds | + ### Helpers #### `LatestMessage` @@ -60,8 +66,10 @@ class LatestMessage(object): @dataclass class Store(object): time: uint64 + genesis_time: uint64 justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint blocks: Dict[Hash, BeaconBlock] = field(default_factory=dict) block_states: Dict[Hash, BeaconState] = field(default_factory=dict) checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) @@ -78,14 +86,30 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root) return Store( time=genesis_state.genesis_time, + genesis_time=genesis_state.genesis_time, justified_checkpoint=justified_checkpoint, finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, blocks={root: genesis_block}, block_states={root: genesis_state.copy()}, checkpoint_states={justified_checkpoint: genesis_state.copy()}, ) ``` +#### `get_current_slot` + +```python +def get_current_slot(store: Store) -> Slot: + return Slot((store.time - store.genesis_time) // SECONDS_PER_SLOT) +``` + +#### `compute_slots_since_epoch_start` + +```python +def compute_slots_since_epoch_start(slot: Slot) -> int: + return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) +``` + #### `get_ancestor` ```python @@ -130,13 +154,50 @@ def get_head(store: Store) -> Hash: head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) ``` +#### `should_update_justified_checkpoint` + +```python +def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: + """ + To address the bouncing attack, only update conflicting justified + checkpoints in the fork choice if in the early slots of the epoch. + Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. + + See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. + """ + if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: + return True + + new_justified_block = store.blocks[new_justified_checkpoint.root] + if new_justified_block.slot <= compute_start_slot_at_epoch(store.justified_checkpoint.epoch): + return False + if not ( + get_ancestor(store, new_justified_checkpoint.root, store.blocks[store.justified_checkpoint.root].slot) == + store.justified_checkpoint.root + ): + return False + + return True +``` + ### Handlers #### `on_tick` ```python def on_tick(store: Store, time: uint64) -> None: + previous_slot = get_current_slot(store) + + # update store time store.time = time + + current_slot = get_current_slot(store) + # Not a new epoch, return + if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): + return + # Update store.justified_checkpoint if a better checkpoint is known + if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = store.best_justified_checkpoint ``` #### `on_block` @@ -164,7 +225,9 @@ def on_block(store: Store, block: BeaconBlock) -> None: # Update justified checkpoint if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = state.current_justified_checkpoint + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: @@ -177,6 +240,11 @@ def on_block(store: Store, block: BeaconBlock) -> None: def on_attestation(store: Store, attestation: Attestation) -> None: target = attestation.data.target + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + assert target.epoch in [current_epoch, previous_epoch] # Cannot calculate the current shuffling if have not seen the target assert target.root in store.blocks @@ -199,7 +267,7 @@ def on_attestation(store: Store, attestation: Attestation) -> None: assert is_valid_indexed_attestation(target_state, indexed_attestation) # Update latest messages - for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices: + for i in indexed_attestation.attesting_indices: if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root) ``` diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 5bccd25a9..30ca43fd3 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -320,7 +320,7 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: -- `ssz`: the contents are [SSZ-encoded](../simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `HashTreeRoots`'s. +- `ssz`: the contents are [SSZ-encoded](../simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Bytes32`'s. - `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. #### SSZ-encoding strategy (with or without Snappy) @@ -344,20 +344,20 @@ constituents individually as `response_chunk`s. For example, the Request, Response Content: ``` ( - head_fork_version: bytes4 - finalized_root: bytes32 + head_fork_version: Bytes4 + finalized_root: Bytes32 finalized_epoch: uint64 - head_root: bytes32 + head_root: Bytes32 head_slot: uint64 ) ``` -The fields are: +The fields are, as seen by the client at the time of sending the message: - `head_fork_version`: The beacon_state `Fork` version. -- `finalized_root`: The latest finalized root the node knows about. -- `finalized_epoch`: The latest finalized epoch the node knows about. -- `head_root`: The block hash tree root corresponding to the head of the chain as seen by the sending node. -- `head_slot`: The slot corresponding to the `head_root`. +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block. +- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. +- `head_root`: The signing root of the current head block. +- `head_slot`: The slot of the block corresponding to the `head_root`. The dialing client MUST send a `Status` request upon connection. @@ -403,7 +403,7 @@ The response MUST consist of a single `response_chunk`. Request Content: ``` ( - head_block_root: HashTreeRoot + head_block_root: Bytes32 start_slot: uint64 count: uint64 step: uint64 @@ -417,7 +417,7 @@ Response Content: ) ``` -Requests count beacon blocks from the peer starting from `start_slot` on the chain defined by `head_block_root`. The response MUST contain no more than count blocks. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. +Requests count beacon blocks from the peer starting from `start_slot` on the chain defined by `head_block_root` (= `signing_root(BeaconBlock)`). The response MUST contain no more than count blocks. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. A step value of 1 returns all blocks on the range `[start_slot, start_slot + count)`. The request MUST be encoded as an SSZ-container. @@ -441,7 +441,7 @@ Request Content: ``` ( - []HashTreeRoot + []Bytes32 ) ``` @@ -453,7 +453,7 @@ Response Content: ) ``` -Requests blocks by their block roots. The response is a list of `BeaconBlock` whose length is less than or equal to the number of requested blocks. It may be less in the case that the responding peer is missing blocks. +Requests blocks by block root (= `signing_root(BeaconBlock)`). The response is a list of `BeaconBlock` whose length is less than or equal to the number of requested blocks. It may be less in the case that the responding peer is missing blocks. `BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 166534031..819f1ac98 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -45,7 +45,6 @@ - [Construct attestation](#construct-attestation) - [Data](#data) - [Aggregation bits](#aggregation-bits) - - [Custody bits](#custody-bits) - [Aggregate signature](#aggregate-signature) - [Broadcast attestation](#broadcast-attestation) - [Attestation aggregation](#attestation-aggregation) @@ -53,7 +52,6 @@ - [Construct aggregate](#construct-aggregate) - [Data](#data-1) - [Aggregation bits](#aggregation-bits-1) - - [Custody bits](#custody-bits-1) - [Aggregate signature](#aggregate-signature-1) - [Broadcast aggregate](#broadcast-aggregate) - [`AggregateAndProof`](#aggregateandproof) @@ -197,7 +195,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). -There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~3 weeks). +There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~6 weeks). #### Block header @@ -231,18 +229,22 @@ def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> The `block.eth1_data` field is for block proposers to vote on recent Eth1 data. This recent data contains an Eth1 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth1 block. If over half of the block proposers in the current Eth1 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. -Let `get_eth1_data(distance: uint64) -> Eth1Data` be the (subjective) function that returns the Eth1 data at distance `distance` relative to the Eth1 head at the start of the current Eth1 voting period. Let `previous_eth1_distance` be the distance relative to the Eth1 block corresponding to `state.eth1_data.block_hash` at the start of the current Eth1 voting period. An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: +Let `get_eth1_data(distance: uint64) -> Eth1Data` be the (subjective) function that returns the Eth1 data at distance `distance` relative to the Eth1 head at the start of the current Eth1 voting period. Let `previous_eth1_distance` be the distance relative to the Eth1 block corresponding to `eth1_data.block_hash` found in the state at the _start_ of the current Eth1 voting period. Note that `eth1_data` can be updated in the middle of a voting period and thus the starting `eth1_data.block_hash` must be stored separately. + +An honest block proposer sets `block.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: ```python def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Data: new_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, 2 * ETH1_FOLLOW_DISTANCE)] all_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, previous_eth1_distance)] - valid_votes = [] - for slot, vote in enumerate(state.eth1_data_votes): - period_tail = slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_squareroot(SLOTS_PER_ETH1_VOTING_PERIOD) - if vote in new_eth1_data or (period_tail and vote in all_eth1_data): - valid_votes.append(vote) + period_tail = state.slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_squareroot(SLOTS_PER_ETH1_VOTING_PERIOD) + if period_tail: + votes_to_consider = all_eth1_data + else: + votes_to_consider = new_eth1_data + + valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] return max( valid_votes, @@ -291,6 +293,8 @@ A validator is expected to create, sign, and broadcast an attestation during eac A validator should create and broadcast the `attestation` to the associated attestation subnet one-third of the way through the `slot` during which the validator is assigned―that is, `SECONDS_PER_SLOT / 3` seconds after the start of `slot`. +*Note*: Although attestations during `GENESIS_EPOCH` do not count toward FFG finality, these initial attestations do give weight to the fork choice, are rewarded fork, and should be made. + #### Attestation data First, the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. @@ -331,25 +335,14 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. -##### Custody bits - -- Let `attestation.custody_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` filled with zeros of length `len(committee)`. - -*Note*: This is a stub for Phase 0. - ##### Aggregate signature Set `attestation.signature = signed_attestation_data` where `signed_attestation_data` is obtained from: ```python def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature: - attestation_data_and_custody_bit = AttestationDataAndCustodyBit( - data=attestation.data, - custody_bit=0b0, - ) - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - return bls_sign(privkey, hash_tree_root(attestation_data_and_custody_bit), domain) + return bls_sign(privkey, hash_tree_root(attestation.data), domain) ``` #### Broadcast attestation @@ -391,12 +384,6 @@ Set `aggregate_attestation.data = attestation_data` where `attestation_data` is Let `aggregate_attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` of length `len(committee)`, where each bit set from each individual attestation is set to `0b1`. -##### Custody bits - -- Let `aggregate_attestation.custody_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` filled with zeros of length `len(committee)`. - -*Note*: This is a stub for Phase 0. - ##### Aggregate signature Set `aggregate_attestation.signature = aggregate_signature` where `aggregate_signature` is obtained from: diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index 77b52e7a2..9cabefb13 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -9,7 +9,7 @@ def test_decoder(): rng = Random(123) # check these types only, Block covers a lot of operation types already. - for typ in [spec.AttestationDataAndCustodyBit, spec.BeaconState, spec.BeaconBlock]: + for typ in [spec.Attestation, spec.BeaconState, spec.BeaconBlock]: # create a random pyspec value original = random_value.get_random_ssz_object(rng, typ, 100, 10, mode=random_value.RandomizationMode.mode_random, diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 70375ef27..8db55cce8 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -1,7 +1,5 @@ -from eth2spec.test.context import with_all_phases, spec_state_test, with_phases - - +from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.state import state_transition_and_sign_block @@ -19,7 +17,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.get_indexed_attestation(state, attestation) spec.on_attestation(store, attestation) assert ( - store.latest_messages[indexed_attestation.custody_bit_0_indices[0]] == + store.latest_messages[indexed_attestation.attesting_indices[0]] == spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, @@ -29,10 +27,9 @@ def run_on_attestation(spec, state, store, attestation, valid=True): @with_all_phases @spec_state_test -def test_on_attestation(spec, state): +def test_on_attestation_current_epoch(spec, state): store = spec.get_genesis_store(state) - time = 100 - spec.on_tick(store, time) + spec.on_tick(store, store.time + spec.SECONDS_PER_SLOT * 2) block = build_empty_block_for_next_slot(spec, state) state_transition_and_sign_block(spec, state, block) @@ -41,9 +38,53 @@ def test_on_attestation(spec, state): spec.on_block(store, block) attestation = get_valid_attestation(spec, state, slot=block.slot) + assert attestation.data.target.epoch == spec.GENESIS_EPOCH + assert spec.compute_epoch_at_slot(spec.get_current_slot(store)) == spec.GENESIS_EPOCH + run_on_attestation(spec, state, store, attestation) +@with_all_phases +@spec_state_test +def test_on_attestation_previous_epoch(spec, state): + store = spec.get_genesis_store(state) + spec.on_tick(store, store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH) + + block = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block) + + # store block in store + spec.on_block(store, block) + + attestation = get_valid_attestation(spec, state, slot=block.slot) + assert attestation.data.target.epoch == spec.GENESIS_EPOCH + assert spec.compute_epoch_at_slot(spec.get_current_slot(store)) == spec.GENESIS_EPOCH + 1 + + run_on_attestation(spec, state, store, attestation) + + +@with_all_phases +@spec_state_test +def test_on_attestation_past_epoch(spec, state): + store = spec.get_genesis_store(state) + + # move time forward 2 epochs + time = store.time + 2 * spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + spec.on_tick(store, time) + + # create and store block from 3 epochs ago + block = build_empty_block_for_next_slot(spec, state) + state_transition_and_sign_block(spec, state, block) + spec.on_block(store, block) + + # create attestation for past block + attestation = get_valid_attestation(spec, state, slot=state.slot) + assert attestation.data.target.epoch == spec.GENESIS_EPOCH + assert spec.compute_epoch_at_slot(spec.get_current_slot(store)) == spec.GENESIS_EPOCH + 2 + + run_on_attestation(spec, state, store, attestation, False) + + @with_all_phases @spec_state_test def test_on_attestation_target_not_in_store(spec, state): @@ -77,8 +118,7 @@ def test_on_attestation_future_epoch(spec, state): spec.on_block(store, block) # move state forward but not store - attestation_slot = block.slot + spec.SLOTS_PER_EPOCH - state.slot = attestation_slot + state.slot = block.slot + spec.SLOTS_PER_EPOCH attestation = get_valid_attestation(spec, state, slot=state.slot) run_on_attestation(spec, state, store, attestation, False) @@ -100,7 +140,7 @@ def test_on_attestation_same_slot(spec, state): run_on_attestation(spec, state, store, attestation, False) -@with_phases(['phase0']) +@with_all_phases @spec_state_test def test_on_attestation_invalid_attestation(spec, state): store = spec.get_genesis_store(state) @@ -113,6 +153,7 @@ def test_on_attestation_invalid_attestation(spec, state): spec.on_block(store, block) attestation = get_valid_attestation(spec, state, slot=block.slot) - # make attestation invalid by setting a phase1-only custody bit - attestation.custody_bits[0] = 1 + # make invalid by using an invalid committee index + attestation.data.index = spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH + run_on_attestation(spec, state, store, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py index 918c0f79e..e42cb1ac4 100644 --- a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py @@ -132,3 +132,74 @@ def test_on_block_before_finalized(spec, state): block = build_empty_block_for_next_slot(spec, state) state_transition_and_sign_block(spec, state, block) run_on_block(spec, store, block, False) + + +@with_all_phases +@spec_state_test +def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state): + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_epoch(spec, state) + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + state, store, last_block = apply_next_epoch_with_attestations(spec, state, store) + next_epoch(spec, state) + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + last_block_root = signing_root(last_block) + + # Mock the justified checkpoint + just_state = store.block_states[last_block_root] + new_justified = spec.Checkpoint( + epoch=just_state.current_justified_checkpoint.epoch + 1, + root=b'\x77' * 32, + ) + just_state.current_justified_checkpoint = new_justified + + block = build_empty_block_for_next_slot(spec, just_state) + state_transition_and_sign_block(spec, deepcopy(just_state), block) + assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH < spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED + run_on_block(spec, store, block) + + assert store.justified_checkpoint == new_justified + + +@with_all_phases +@spec_state_test +def test_on_block_outside_safe_slots_and_old_block(spec, state): + # Initialization + store = spec.get_genesis_store(state) + time = 100 + spec.on_tick(store, time) + + next_epoch(spec, state) + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + state, store, last_block = apply_next_epoch_with_attestations(spec, state, store) + next_epoch(spec, state) + spec.on_tick(store, store.time + state.slot * spec.SECONDS_PER_SLOT) + last_block_root = signing_root(last_block) + + # Mock justified block in store + just_block = build_empty_block_for_next_slot(spec, state) + # Slot is same as justified checkpoint so does not trigger an override in the store + just_block.slot = spec.compute_start_slot_at_epoch(store.justified_checkpoint.epoch) + store.blocks[just_block.hash_tree_root()] = just_block + + # Mock the justified checkpoint + just_state = store.block_states[last_block_root] + new_justified = spec.Checkpoint( + epoch=just_state.current_justified_checkpoint.epoch + 1, + root=just_block.hash_tree_root(), + ) + just_state.current_justified_checkpoint = new_justified + + block = build_empty_block_for_next_slot(spec, just_state) + state_transition_and_sign_block(spec, deepcopy(just_state), block) + + spec.on_tick(store, store.time + spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED * spec.SECONDS_PER_SLOT) + assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH >= spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED + run_on_block(spec, store, block) + + assert store.justified_checkpoint != new_justified + assert store.best_justified_checkpoint == new_justified diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_tick.py b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_tick.py new file mode 100644 index 000000000..77222f65c --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/fork_choice/test_on_tick.py @@ -0,0 +1,105 @@ +from eth2spec.test.context import with_all_phases, spec_state_test + + +def run_on_tick(spec, store, time, new_justified_checkpoint=False): + previous_justified_checkpoint = store.justified_checkpoint + + spec.on_tick(store, time) + + assert store.time == time + + if new_justified_checkpoint: + assert store.justified_checkpoint == store.best_justified_checkpoint + assert store.justified_checkpoint.epoch > previous_justified_checkpoint.epoch + assert store.justified_checkpoint.root != previous_justified_checkpoint.root + else: + assert store.justified_checkpoint == previous_justified_checkpoint + + +@with_all_phases +@spec_state_test +def test_basic(spec, state): + store = spec.get_genesis_store(state) + run_on_tick(spec, store, store.time + 1) + + +@with_all_phases +@spec_state_test +def test_update_justified_single(spec, state): + store = spec.get_genesis_store(state) + seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + store.best_justified_checkpoint = spec.Checkpoint( + epoch=store.justified_checkpoint.epoch + 1, + root=b'\x55' * 32, + ) + + run_on_tick(spec, store, store.time + seconds_per_epoch, True) + + +@with_all_phases +@spec_state_test +def test_no_update_same_slot_at_epoch_boundary(spec, state): + store = spec.get_genesis_store(state) + seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + store.best_justified_checkpoint = spec.Checkpoint( + epoch=store.justified_checkpoint.epoch + 1, + root=b'\x55' * 32, + ) + + # set store time to already be at epoch boundary + store.time = seconds_per_epoch + + run_on_tick(spec, store, store.time + 1) + + +@with_all_phases +@spec_state_test +def test_no_update_not_epoch_boundary(spec, state): + store = spec.get_genesis_store(state) + + store.best_justified_checkpoint = spec.Checkpoint( + epoch=store.justified_checkpoint.epoch + 1, + root=b'\x55' * 32, + ) + + run_on_tick(spec, store, store.time + spec.SECONDS_PER_SLOT) + + +@with_all_phases +@spec_state_test +def test_no_update_new_justified_equal_epoch(spec, state): + store = spec.get_genesis_store(state) + seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + store.best_justified_checkpoint = spec.Checkpoint( + epoch=store.justified_checkpoint.epoch + 1, + root=b'\x55' * 32, + ) + + store.justified_checkpoint = spec.Checkpoint( + epoch=store.best_justified_checkpoint.epoch, + root=b'\44' * 32, + ) + + run_on_tick(spec, store, store.time + seconds_per_epoch) + + +@with_all_phases +@spec_state_test +def test_no_update_new_justified_later_epoch(spec, state): + store = spec.get_genesis_store(state) + seconds_per_epoch = spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + store.best_justified_checkpoint = spec.Checkpoint( + epoch=store.justified_checkpoint.epoch + 1, + root=b'\x55' * 32, + ) + + store.justified_checkpoint = spec.Checkpoint( + epoch=store.best_justified_checkpoint.epoch + 1, + root=b'\44' * 32, + ) + + run_on_tick(spec, store, store.time + seconds_per_epoch) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index d36e62ace..299ca32ff 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -54,11 +54,9 @@ def get_valid_attestation(spec, state, slot=None, index=None, signed=False): committee_size = len(beacon_committee) aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) - custody_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) attestation = spec.Attestation( aggregation_bits=aggregation_bits, data=attestation_data, - custody_bits=custody_bits, ) fill_aggregate_attestation(spec, state, attestation) if signed: @@ -83,7 +81,7 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List def sign_indexed_attestation(spec, state, indexed_attestation): - participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + participants = indexed_attestation.attesting_indices indexed_attestation.signature = sign_aggregate_attestation(spec, state, indexed_attestation.data, participants) @@ -97,14 +95,9 @@ def sign_attestation(spec, state, attestation): attestation.signature = sign_aggregate_attestation(spec, state, attestation.data, participants) -def get_attestation_signature(spec, state, attestation_data, privkey, custody_bit=0b0): - message_hash = spec.AttestationDataAndCustodyBit( - data=attestation_data, - custody_bit=custody_bit, - ).hash_tree_root() - +def get_attestation_signature(spec, state, attestation_data, privkey): return bls_sign( - message_hash=message_hash, + message_hash=attestation_data.hash_tree_root(), privkey=privkey, domain=spec.get_domain( state=state, diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 04fa880e0..f19bc66d3 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -2,7 +2,7 @@ from eth2spec.test.context import ( spec_state_test, expect_assertion_error, always_bls, never_bls, - with_all_phases, with_phases, + with_all_phases, spec_test, low_balances, with_custom_state, @@ -274,35 +274,6 @@ def test_bad_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases -@spec_state_test -def test_inconsistent_bits(spec, state): - attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - custody_bits = attestation.custody_bits[:] - custody_bits.append(False) - - attestation.custody_bits = custody_bits - - sign_attestation(spec, state, attestation) - - yield from run_attestation_processing(spec, state, attestation, False) - - -@with_phases(['phase0']) -@spec_state_test -def test_non_empty_custody_bits(spec, state): - attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.custody_bits = attestation.aggregation_bits[:] - - sign_attestation(spec, state, attestation) - - yield from run_attestation_processing(spec, state, attestation, False) - - @with_all_phases @spec_state_test def test_empty_aggregation_bits(spec, state): @@ -344,32 +315,3 @@ def test_too_few_aggregation_bits(spec, state): attestation.aggregation_bits = attestation.aggregation_bits[:-1] yield from run_attestation_processing(spec, state, attestation, False) - - -@with_all_phases -@spec_state_test -def test_too_many_custody_bits(spec, state): - attestation = get_valid_attestation(spec, state, signed=True) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - # one too many bits - attestation.custody_bits.append(0b0) - - yield from run_attestation_processing(spec, state, attestation, False) - - -@with_all_phases -@spec_state_test -def test_too_few_custody_bits(spec, state): - attestation = get_valid_attestation(spec, state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.custody_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]( - *([0b1] + [0b0] * (len(attestation.custody_bits) - 1))) - - sign_attestation(spec, state, attestation) - - # one too few bits - attestation.custody_bits = attestation.custody_bits[:-1] - - yield from run_attestation_processing(spec, state, attestation, False) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 20a510648..85e807ec0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -25,10 +25,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', None return - slashed_indices = ( - attester_slashing.attestation_1.custody_bit_0_indices - + attester_slashing.attestation_1.custody_bit_1_indices - ) + slashed_indices = attester_slashing.attestation_1.attesting_indices proposer_index = spec.get_beacon_proposer_index(state) pre_proposer_balance = get_balance(state, proposer_index) @@ -112,10 +109,7 @@ def test_success_surround(spec, state): @always_bls def test_success_already_exited_recent(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - slashed_indices = ( - attester_slashing.attestation_1.custody_bit_0_indices - + attester_slashing.attestation_1.custody_bit_1_indices - ) + slashed_indices = attester_slashing.attestation_1.attesting_indices for index in slashed_indices: spec.initiate_validator_exit(state, index) @@ -127,10 +121,7 @@ def test_success_already_exited_recent(spec, state): @always_bls def test_success_already_exited_long_ago(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - slashed_indices = ( - attester_slashing.attestation_1.custody_bit_0_indices - + attester_slashing.attestation_1.custody_bit_1_indices - ) + slashed_indices = attester_slashing.attestation_1.attesting_indices for index in slashed_indices: spec.initiate_validator_exit(state, index) state.validators[index].withdrawable_epoch = spec.get_current_epoch(state) + 2 @@ -190,38 +181,23 @@ def test_participants_already_slashed(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) # set all indices to slashed - attestation_1 = attester_slashing.attestation_1 - validator_indices = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices + validator_indices = attester_slashing.attestation_1.attesting_indices for index in validator_indices: state.validators[index].slashed = True yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases -@spec_state_test -def test_custody_bit_0_and_1_intersect(spec, state): - attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - - attester_slashing.attestation_1.custody_bit_1_indices.append( - attester_slashing.attestation_1.custody_bit_0_indices[0] - ) - - sign_indexed_attestation(spec, state, attester_slashing.attestation_1) - - yield from run_attester_slashing_processing(spec, state, attester_slashing, False) - - @with_all_phases @spec_state_test @always_bls def test_att1_bad_extra_index(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - indices = attester_slashing.attestation_1.custody_bit_0_indices + indices = attester_slashing.attestation_1.attesting_indices options = list(set(range(len(state.validators))) - set(indices)) indices.append(options[len(options) // 2]) # add random index, not previously in attestation. - attester_slashing.attestation_1.custody_bit_0_indices = sorted(indices) + attester_slashing.attestation_1.attesting_indices = sorted(indices) # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), # see if the bad extra index is spotted, and slashing is aborted. @@ -234,10 +210,10 @@ def test_att1_bad_extra_index(spec, state): def test_att1_bad_replaced_index(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - indices = attester_slashing.attestation_1.custody_bit_0_indices + indices = attester_slashing.attestation_1.attesting_indices options = list(set(range(len(state.validators))) - set(indices)) indices[3] = options[len(options) // 2] # replace with random index, not previously in attestation. - attester_slashing.attestation_1.custody_bit_0_indices = sorted(indices) + attester_slashing.attestation_1.attesting_indices = sorted(indices) # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), # see if the bad replaced index is spotted, and slashing is aborted. @@ -250,10 +226,10 @@ def test_att1_bad_replaced_index(spec, state): def test_att2_bad_extra_index(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - indices = attester_slashing.attestation_2.custody_bit_0_indices + indices = attester_slashing.attestation_2.attesting_indices options = list(set(range(len(state.validators))) - set(indices)) indices.append(options[len(options) // 2]) # add random index, not previously in attestation. - attester_slashing.attestation_2.custody_bit_0_indices = sorted(indices) + attester_slashing.attestation_2.attesting_indices = sorted(indices) # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), # see if the bad extra index is spotted, and slashing is aborted. @@ -266,10 +242,10 @@ def test_att2_bad_extra_index(spec, state): def test_att2_bad_replaced_index(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - indices = attester_slashing.attestation_2.custody_bit_0_indices + indices = attester_slashing.attestation_2.attesting_indices options = list(set(range(len(state.validators))) - set(indices)) indices[3] = options[len(options) // 2] # replace with random index, not previously in attestation. - attester_slashing.attestation_2.custody_bit_0_indices = sorted(indices) + attester_slashing.attestation_2.attesting_indices = sorted(indices) # Do not sign the modified attestation (it's ok to slash if attester signed, not if they did not), # see if the bad replaced index is spotted, and slashing is aborted. @@ -278,10 +254,10 @@ def test_att2_bad_replaced_index(spec, state): @with_all_phases @spec_state_test -def test_unsorted_att_1_bit0(spec, state): +def test_unsorted_att_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) - indices = attester_slashing.attestation_1.custody_bit_0_indices + indices = attester_slashing.attestation_1.attesting_indices assert len(indices) >= 3 indices[1], indices[2] = indices[2], indices[1] # unsort second and third index sign_indexed_attestation(spec, state, attester_slashing.attestation_1) @@ -291,15 +267,12 @@ def test_unsorted_att_1_bit0(spec, state): @with_all_phases @spec_state_test -def test_unsorted_att_2_bit0(spec, state): +def test_unsorted_att_2(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=False) - indices = attester_slashing.attestation_2.custody_bit_0_indices + indices = attester_slashing.attestation_2.attesting_indices assert len(indices) >= 3 indices[1], indices[2] = indices[2], indices[1] # unsort second and third index sign_indexed_attestation(spec, state, attester_slashing.attestation_2) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) - - -# note: unsorted indices for custody bit 0 are to be introduced in phase 1 testing. diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index 99ba9d284..7d844b63b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -141,7 +141,7 @@ def test_duplicate_attestation(spec, state): attestation = get_valid_attestation(spec, state, signed=True) indexed_attestation = spec.get_indexed_attestation(state, attestation) - participants = indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices + participants = indexed_attestation.attesting_indices assert len(participants) > 0 diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index bffb1c1fc..bbc4bba3a 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -188,8 +188,7 @@ def test_attester_slashing(spec, state): pre_state = deepcopy(state) attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) - validator_index = (attester_slashing.attestation_1.custody_bit_0_indices - + attester_slashing.attestation_1.custody_bit_1_indices)[0] + validator_index = attester_slashing.attestation_1.attesting_indices[0] assert not state.validators[validator_index].slashed From 3d48a3e9eb6174e53c3bb672e1cdf77c47268eb1 Mon Sep 17 00:00:00 2001 From: ethers <6937903+ethers@users.noreply.github.com> Date: Wed, 11 Dec 2019 18:17:20 -0800 Subject: [PATCH 05/48] Fix dead link to use Internet Archive --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index fca21994f..8cfcd622e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -178,7 +178,7 @@ The following values are (non-configurable) constants used throughout the specif | `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `2**14` (= 16,384) | | `MIN_GENESIS_TIME` | `1578009600` (Jan 3, 2020) | -- For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) +- For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) ### Gwei values From 48317d8ac050252c3d4d4783c143227b6b19d993 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 15 Dec 2019 11:10:43 +0000 Subject: [PATCH 06/48] Minor cosmetic cleanups * remove `BLS_WITHDRAWAL_PREFIX` (it is not used in phase 1, not phase 0) * avoid inline comment (# Validate state root) * simplify header inequality check in `process_proposer_slashing` (using uniqueness of BLS signatures) * add `block = signed_block.message` helper variable for readability * (typo) clarify that the state transition function consumes a signed block (as opposed to a block) * generally make comments more consistent * consistent formatting of container instantiation for `DepositMessage` * avoid using three lines for `rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay)` * introduce `effective_balance` helper variable for readability, and to avoid multi-line statement * consistent ordering of `MIN_EPOCHS_TO_INACTIVITY_PENALTY` in the time parameters table * (typo) "Dequeued validators for activation up to churn limit" => Dequeue validators * "Save current block as the new latest block" => "Cache current block" (for consistent with `process_slot`) * (typo) "Verify the validator has not yet exited" => "Verify exit has not been initiated" * Use Pythonic `default=` for `max()` call in `initiate_validator_exit` --- specs/core/0_beacon-chain.md | 64 +++++++++++++++++------------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 20087c069..2761f973f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -199,7 +199,6 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | -| `BLS_WITHDRAWAL_PREFIX` | `Bytes1(b'\x00')` | ### Time parameters @@ -210,11 +209,11 @@ The following values are (non-configurable) constants used throughout the specif | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | | `MAX_SEED_LOOKAHEAD` | `2**2` (= 4) | epochs | 25.6 minutes | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | | `SLOTS_PER_ETH1_VOTING_PERIOD` | `2**10` (= 1,024) | slots | ~3.4 hours | | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | -| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | ### State list lengths @@ -223,7 +222,7 @@ The following values are (non-configurable) constants used throughout the specif | `EPOCHS_PER_HISTORICAL_VECTOR` | `2**16` (= 65,536) | epochs | ~0.8 years | | `EPOCHS_PER_SLASHINGS_VECTOR` | `2**13` (= 8,192) | epochs | ~36 days | | `HISTORICAL_ROOTS_LIMIT` | `2**24` (= 16,777,216) | historical roots | ~26,131 years | -| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validator spots | +| `VALIDATOR_REGISTRY_LIMIT` | `2**40` (= 1,099,511,627,776) | validators | ### Rewards and penalties @@ -366,7 +365,7 @@ class DepositData(Container): pubkey: BLSPubkey withdrawal_credentials: Bytes32 amount: Gwei - signature: BLSSignature # signing over DepositMessage + signature: BLSSignature # Signing over DepositMessage ``` #### `BeaconBlockHeader` @@ -411,7 +410,7 @@ class Attestation(Container): ```python class Deposit(Container): - proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit data list root + proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root data: DepositData ``` @@ -488,8 +487,6 @@ class BeaconState(Container): ### Signed envelopes -Some messages in the protocol are wrapped in an envelope to better facilitate adding/pruning the signature and to `hash_tree_root` the `message` separate from the signature. - #### `SignedVoluntaryExit` ```python @@ -1016,7 +1013,7 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: # Compute exit queue epoch exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] - exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))]) + exit_queue_epoch = max(exit_epochs, default=compute_activation_exit_epoch(get_current_epoch(state))) exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) if exit_queue_churn >= get_validator_churn_limit(state): exit_queue_epoch += Epoch(1) @@ -1111,19 +1108,21 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. +The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> BeaconState: + block = signed_block.message # Process slots (including those with no blocks) since block - process_slots(state, signed_block.message.slot) + process_slots(state, block.slot) # Verify signature if validate_result: assert verify_block_signature(state, signed_block) # Process block - process_block(state, signed_block.message) + process_block(state, block) + # Verify state root if validate_result: - assert signed_block.message.state_root == hash_tree_root(state) # Validate state root + assert block.state_root == hash_tree_root(state) # Return post-state return state ``` @@ -1299,9 +1298,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) rewards[attestation.proposer_index] += proposer_reward max_attester_reward = get_base_reward(state, index) - proposer_reward - rewards[index] += Gwei( - max_attester_reward // attestation.inclusion_delay - ) + rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) # Inactivity penalty finality_delay = previous_epoch - state.finalized_checkpoint.epoch @@ -1310,9 +1307,8 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence for index in eligible_validator_indices: penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) if index not in matching_target_attesting_indices: - penalties[index] += Gwei( - state.validators[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT - ) + effective_balance = state.validators[index].effective_balance + penalties[index] += Gwei(effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT) return rewards, penalties ``` @@ -1412,13 +1408,14 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: assert block.slot == state.slot # Verify that the parent matches assert block.parent_root == hash_tree_root(state.latest_block_header) - # Save current block as the new latest block + # Cache current block as the new latest block state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, - # `state_root` is zeroed and overwritten in the next `process_slot` call + state_root=Bytes32(), # Overwritten in the next process_slot call body_root=hash_tree_root(block.body), ) + # Verify proposer is not slashed proposer = state.validators[get_beacon_proposer_index(state)] assert not proposer.slashed @@ -1469,14 +1466,14 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: - proposer = state.validators[proposer_slashing.proposer_index] - # Verify slots match + # Verify header slots match assert proposer_slashing.signed_header_1.message.slot == proposer_slashing.signed_header_2.message.slot - # But the headers are different - assert proposer_slashing.signed_header_1.message != proposer_slashing.signed_header_2.message - # Check proposer is slashable + # Verify the headers are different + assert proposer_slashing.signed_header_1 != proposer_slashing.signed_header_2 + # Verify the proposer is slashable + proposer = state.validators[proposer_slashing.proposer_index] assert is_slashable_validator(proposer, get_current_epoch(state)) - # Signatures are valid + # Verify signatures for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) assert bls_verify(proposer.pubkey, hash_tree_root(signed_header.message), signed_header.signature, domain) @@ -1530,7 +1527,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.source == state.previous_justified_checkpoint state.previous_epoch_attestations.append(pending_attestation) - # Check signature + # Verify signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` @@ -1542,7 +1539,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: assert is_valid_merkle_branch( leaf=hash_tree_root(deposit.data), branch=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the `List` length mix-in + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in index=state.eth1_deposit_index, root=state.eth1_data.deposit_root, ) @@ -1554,14 +1551,13 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: amount = deposit.data.amount validator_pubkeys = [v.pubkey for v in state.validators] if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession) for new validators. - # Note: The deposit contract does not check signatures. - # Note: Deposits are valid across forks, thus the deposit domain is retrieved directly from `compute_domain`. - domain = compute_domain(DOMAIN_DEPOSIT) + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract deposit_message = DepositMessage( pubkey=deposit.data.pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, - amount=deposit.data.amount) + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks if not bls_verify(pubkey, hash_tree_root(deposit_message), deposit.data.signature, domain): return @@ -1590,7 +1586,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu validator = state.validators[voluntary_exit.validator_index] # Verify the validator is active assert is_active_validator(validator, get_current_epoch(state)) - # Verify the validator has not yet exited + # Verify exit has not been initiated assert validator.exit_epoch == FAR_FUTURE_EPOCH # Exits must specify an epoch when they become valid; they are not valid before then assert get_current_epoch(state) >= voluntary_exit.epoch From 412a06608c994ad230c008824a465bbd7e86f4b9 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 15 Dec 2019 11:32:11 +0000 Subject: [PATCH 07/48] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2761f973f..f46a4433b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -199,6 +199,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | +| `BLS_WITHDRAWAL_PREFIX` | `Bytes1(b'\x00')` | ### Time parameters From 502ee295379c1f3c5c3649e12330fb5be5d7a83b Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 12:04:56 +0200 Subject: [PATCH 08/48] Phase 0 new BLS --- scripts/build_spec.py | 23 +-- specs/bls_signature.md | 148 ------------------ specs/core/0_beacon-chain.md | 64 +++++--- specs/validator/0_beacon-chain-validator.md | 16 +- .../eth2spec/test/helpers/attestations.py | 17 +- .../pyspec/eth2spec/test/helpers/block.py | 24 +-- .../eth2spec/test/helpers/block_header.py | 10 +- .../pyspec/eth2spec/test/helpers/custody.py | 51 ++---- .../pyspec/eth2spec/test/helpers/deposits.py | 10 +- .../test/helpers/phase1/attestations.py | 18 +-- .../test/helpers/phase1/shard_block.py | 14 +- .../eth2spec/test/helpers/voluntary_exits.py | 14 +- .../eth2spec/test/sanity/test_blocks.py | 23 +-- test_libs/pyspec/eth2spec/utils/bls.py | 26 +-- 14 files changed, 136 insertions(+), 322 deletions(-) delete mode 100644 specs/bls_signature.md diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 834a2dcf7..ca8bbb6dd 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -11,7 +11,7 @@ from typing import ( PHASE0_IMPORTS = '''from typing import ( - Any, Dict, Set, Sequence, Tuple, Optional + Any, Dict, Set, Sequence, Tuple, Optional, TypeVar ) from dataclasses import ( @@ -21,20 +21,23 @@ from dataclasses import ( from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import ( - boolean, Container, List, Vector, uint64, + boolean, Container, List, Vector, uint64, SSZType, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( - bls_aggregate_signatures, + Verify, + Sign, + Aggregate, + FastAggregateVerify, bls_aggregate_pubkeys, - bls_verify, - bls_sign, ) from eth2spec.utils.hash_function import hash + +SSZObject = TypeVar('SSZObject', bound=SSZType) ''' PHASE1_IMPORTS = '''from typing import ( - Any, Dict, Set, Sequence, MutableSequence, NewType, Tuple, Union, + Any, Dict, Set, Sequence, MutableSequence, NewType, Tuple, Union, TypeVar ) from math import ( log2, @@ -56,10 +59,11 @@ from eth2spec.utils.ssz.ssz_typing import ( uint64, bit, boolean, byte, ) from eth2spec.utils.bls import ( + Verify, + Sign, + Aggregate, + FastAggregateVerify, bls_aggregate_pubkeys, - bls_verify, - bls_verify_multiple, - bls_signature_to_G2, ) from eth2spec.utils.hash_function import hash @@ -67,6 +71,7 @@ from eth2spec.utils.hash_function import hash SSZVariableName = str GeneralizedIndex = NewType('GeneralizedIndex', int) +SSZObject = TypeVar('SSZObject', bound=SSZType) ''' SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: uint64) -> int: diff --git a/specs/bls_signature.md b/specs/bls_signature.md deleted file mode 100644 index aafeeb54d..000000000 --- a/specs/bls_signature.md +++ /dev/null @@ -1,148 +0,0 @@ -# BLS signature verification - -**Notice**: This document is a placeholder to facilitate the emergence of cross-client testnets. Substantive changes are postponed until [BLS standardisation](https://github.com/cfrg/draft-irtf-cfrg-bls-signature) is finalized. - -**Warning**: The constructions in this document should not be considered secure. In particular, the `hash_to_G2` function is known to be unsecure. - -## Table of contents - - - - - -- [Curve parameters](#curve-parameters) -- [Point representations](#point-representations) - - [G1 points](#g1-points) - - [G2 points](#g2-points) -- [Helpers](#helpers) - - [`hash_to_G2`](#hash_to_g2) - - [`modular_squareroot`](#modular_squareroot) -- [Aggregation operations](#aggregation-operations) - - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) - - [`bls_aggregate_signatures`](#bls_aggregate_signatures) -- [Signature verification](#signature-verification) - - [`bls_verify`](#bls_verify) - - [`bls_verify_multiple`](#bls_verify_multiple) - - - - -## Curve parameters - -The BLS12-381 curve parameters are defined [here](https://z.cash/blog/new-snark-curve). - -## Point representations - -We represent points in the groups G1 and G2 following [zkcrypto/pairing](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381). We denote by `q` the field modulus and by `i` the imaginary unit. - -### G1 points - -A point in G1 is represented as a 384-bit integer `z` decomposed as a 381-bit integer `x` and three 1-bit flags in the top bits: - -* `x = z % 2**381` -* `a_flag = (z % 2**382) // 2**381` -* `b_flag = (z % 2**383) // 2**382` -* `c_flag = (z % 2**384) // 2**383` - -Respecting bit ordering, `z` is decomposed as `(c_flag, b_flag, a_flag, x)`. - -We require: - -* `x < q` -* `c_flag == 1` -* if `b_flag == 1` then `a_flag == x == 0` and `z` represents the point at infinity -* if `b_flag == 0` then `z` represents the point `(x, y)` where `y` is the valid coordinate such that `(y * 2) // q == a_flag` - -### G2 points - -A point in G2 is represented as a pair of 384-bit integers `(z1, z2)`. We decompose `z1` as above into `x1`, `a_flag1`, `b_flag1`, `c_flag1` and `z2` into `x2`, `a_flag2`, `b_flag2`, `c_flag2`. - -We require: - -* `x1 < q` and `x2 < q` -* `a_flag2 == b_flag2 == c_flag2 == 0` -* `c_flag1 == 1` -* if `b_flag1 == 1` then `a_flag1 == x1 == x2 == 0` and `(z1, z2)` represents the point at infinity -* if `b_flag1 == 0` then `(z1, z2)` represents the point `(x1 * i + x2, y)` where `y` is the valid coordinate such that the imaginary part `y_im` of `y` satisfies `(y_im * 2) // q == a_flag1` - -## Helpers - -### `hash_to_G2` - -```python -G2_cofactor = 305502333931268344200999753193121504214466019254188142667664032982267604182971884026507427359259977847832272839041616661285803823378372096355777062779109 -q = 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 - -def hash_to_G2(message_hash: Bytes32, domain: Bytes8) -> Tuple[uint384, uint384]: - # Initial candidate x coordinate - x_re = int.from_bytes(hash(message_hash + domain + b'\x01'), 'big') - x_im = int.from_bytes(hash(message_hash + domain + b'\x02'), 'big') - x_coordinate = Fq2([x_re, x_im]) # x = x_re + i * x_im - - # Test candidate y coordinates until a one is found - while 1: - y_coordinate_squared = x_coordinate ** 3 + Fq2([4, 4]) # The curve is y^2 = x^3 + 4(i + 1) - y_coordinate = modular_squareroot(y_coordinate_squared) - if y_coordinate is not None: # Check if quadratic residue found - return multiply_in_G2((x_coordinate, y_coordinate), G2_cofactor) - x_coordinate += Fq2([1, 0]) # Add 1 and try again -``` - -### `modular_squareroot` - -`modular_squareroot(x)` returns a solution `y` to `y**2 % q == x`, and `None` if none exists. If there are two solutions, the one with higher imaginary component is favored; if both solutions have equal imaginary component, the one with higher real component is favored (note that this is equivalent to saying that the single solution with either imaginary component > p/2 or imaginary component zero and real component > p/2 is favored). - -The following is a sample implementation; implementers are free to implement modular square roots as they wish. Note that `x2 = -x1` is an _additive modular inverse_ so real and imaginary coefficients remain in `[0 .. q-1]`. `coerce_to_int(element: Fq) -> int` is a function that takes Fq element `element` (i.e. integers `mod q`) and converts it to a regular integer. - -```python -Fq2_order = q ** 2 - 1 -eighth_roots_of_unity = [Fq2([1,1]) ** ((Fq2_order * k) // 8) for k in range(8)] - -def modular_squareroot(value: Fq2) -> Fq2: - candidate_squareroot = value ** ((Fq2_order + 8) // 16) - check = candidate_squareroot ** 2 / value - if check in eighth_roots_of_unity[::2]: - x1 = candidate_squareroot / eighth_roots_of_unity[eighth_roots_of_unity.index(check) // 2] - x2 = -x1 - x1_re, x1_im = coerce_to_int(x1.coeffs[0]), coerce_to_int(x1.coeffs[1]) - x2_re, x2_im = coerce_to_int(x2.coeffs[0]), coerce_to_int(x2.coeffs[1]) - return x1 if (x1_im > x2_im or (x1_im == x2_im and x1_re > x2_re)) else x2 - return None -``` - -## Aggregation operations - -### `bls_aggregate_pubkeys` - -Let `bls_aggregate_pubkeys(pubkeys: List[Bytes48]) -> Bytes48` return `pubkeys[0] + .... + pubkeys[len(pubkeys)-1]`, where `+` is the elliptic curve addition operation over the G1 curve. (When `len(pubkeys) == 0` the empty sum is the G1 point at infinity.) - -### `bls_aggregate_signatures` - -Let `bls_aggregate_signatures(signatures: List[Bytes96]) -> Bytes96` return `signatures[0] + .... + signatures[len(signatures)-1]`, where `+` is the elliptic curve addition operation over the G2 curve. (When `len(signatures) == 0` the empty sum is the G2 point at infinity.) - -## Signature verification - -In the following, `e` is the pairing function and `g` is the G1 generator with the following coordinates (see [here](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381#g1)): - -```python -g_x = 3685416753713387016781088315183077757961620795782546409894578378688607592378376318836054947676345821548104185464507 -g_y = 1339506544944476473020471379941921221584933875938349620426543736416511423956333506472724655353366534992391756441569 -g = Fq2([g_x, g_y]) -``` - -### `bls_verify` - -Let `bls_verify(pubkey: Bytes48, message_hash: Bytes32, signature: Bytes96, domain: Bytes8) -> bool`: - -* Verify that `pubkey` is a valid G1 point. -* Verify that `signature` is a valid G2 point. -* Verify that `e(pubkey, hash_to_G2(message_hash, domain)) == e(g, signature)`. - -### `bls_verify_multiple` - -Let `bls_verify_multiple(pubkeys: List[Bytes48], message_hashes: List[Bytes32], signature: Bytes96, domain: Bytes8) -> bool`: - -* Verify that each `pubkey` in `pubkeys` is a valid G1 point. -* Verify that `signature` is a valid G2 point. -* Verify that `len(pubkeys)` equals `len(message_hashes)` and denote the length `L`. -* Verify that `e(pubkeys[0], hash_to_G2(message_hashes[0], domain)) * ... * e(pubkeys[L-1], hash_to_G2(message_hashes[L-1], domain)) == e(g, signature)`. diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 20087c069..71980e88c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -379,6 +379,14 @@ class BeaconBlockHeader(Container): body_root: Root ``` +#### `DomainWrapper` + +```python +class DomainWrapper(Container): + root: Root + domain: Domain +``` + ### Beacon operations #### `ProposerSlashing` @@ -575,13 +583,17 @@ def bytes_to_int(data: bytes) -> uint64: `def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../simple-serialize.md#merkleization). -#### `bls_verify` +#### BLS Signatures -`bls_verify` is a function for verifying a BLS signature, as defined in the [BLS Signature spec](../bls_signature.md#bls_verify). +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). -#### `bls_aggregate_pubkeys` +Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite where it makes use of the following functions: -`bls_aggregate_pubkeys` is a function for aggregating multiple BLS public keys into a single aggregate key, as defined in the [BLS Signature spec](../bls_signature.md#bls_aggregate_pubkeys). +* `def Sign(SK: int, message: Bytes) -> BLSSignature` +* `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` +* `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` +* `def bls_aggregate_pubkeys(PKs: Sequence[BLSPubkey]) -> BLSPubkey` +* `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool` ### Predicates @@ -664,14 +676,10 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe if not indices == sorted(set(indices)): return False # Verify aggregate signature - if not bls_verify( - pubkey=bls_aggregate_pubkeys([state.validators[i].pubkey for i in indices]), - message_hash=hash_tree_root(indexed_attestation.data), - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch), - ): - return False - return True + pubkeys = [state.validators[i].pubkey for i in indices] + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) + message = compute_domain_wrapper_root(indexed_attestation.data, domain) + return FastAggregateVerify(pubkeys, message, indexed_attestation.signature) ``` #### `is_valid_merkle_branch` @@ -789,6 +797,17 @@ def compute_domain(domain_type: DomainType, fork_version: Version=Version()) -> return Domain(domain_type + fork_version) ``` +### `compute_domain_wrapper_root` + +```python +def compute_domain_wrapper_root(object: SSZObject, domain: Domain) -> Root: + domain_wrapped_object = DomainWrapper( + root=hash_tree_root(object), + domain=domain, + ) + return hash_tree_root(domain_wrapped_object) +``` + ### Beacon state accessors #### `get_current_epoch` @@ -1131,8 +1150,8 @@ def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, valida ```python def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: proposer = state.validators[get_beacon_proposer_index(state)] - domain = get_domain(state, DOMAIN_BEACON_PROPOSER) - return bls_verify(proposer.pubkey, hash_tree_root(signed_block.message), signed_block.signature, domain) + message = compute_domain_wrapper_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) + return Verify(proposer.pubkey, message, signed_block.signature) ``` ```python @@ -1431,7 +1450,8 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: epoch = get_current_epoch(state) # Verify RANDAO reveal proposer = state.validators[get_beacon_proposer_index(state)] - assert bls_verify(proposer.pubkey, hash_tree_root(epoch), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + message = compute_domain_wrapper_root(epoch, get_domain(state, DOMAIN_RANDAO)) + assert Verify(proposer.pubkey, message, body.randao_reveal) # Mix in RANDAO reveal mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix @@ -1478,8 +1498,11 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) - assert bls_verify(proposer.pubkey, hash_tree_root(signed_header.message), signed_header.signature, domain) + message = compute_domain_wrapper_root( + object=signed_header.message, + domain=get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)), + ) + assert Verify(proposer.pubkey, message, signed_header.signature) slash_validator(state, proposer_slashing.proposer_index) ``` @@ -1557,12 +1580,12 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the deposit signature (proof of possession) for new validators. # Note: The deposit contract does not check signatures. # Note: Deposits are valid across forks, thus the deposit domain is retrieved directly from `compute_domain`. - domain = compute_domain(DOMAIN_DEPOSIT) deposit_message = DepositMessage( pubkey=deposit.data.pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, amount=deposit.data.amount) - if not bls_verify(pubkey, hash_tree_root(deposit_message), deposit.data.signature, domain): + message = compute_domain_wrapper_root(deposit_message, compute_domain(DOMAIN_DEPOSIT)) + if not Verify(pubkey, message, deposit.data.signature): return # Add validator and balance entries @@ -1598,7 +1621,8 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) - assert bls_verify(validator.pubkey, hash_tree_root(voluntary_exit), signed_voluntary_exit.signature, domain) + message = compute_domain_wrapper_root(voluntary_exit, domain) + assert Verify(validator.pubkey, message, signed_voluntary_exit.signature) # Initiate exit initiate_validator_exit(state, voluntary_exit.validator_index) ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 76bcc3b7d..5e80e84a6 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -117,7 +117,7 @@ To submit a deposit: - Set `deposit_data.withdrawal_credentials` to `withdrawal_credentials`. - Set `deposit_data.amount` to `amount`. - Let `deposit_message` be a `DepositMessage` with all the `DepositData` contents except the `signature`. -- Let `signature` be the result of `bls_sign` of the `hash_tree_root(deposit_message)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). +- Let `signature` be the result of `Sign` of the `compute_domain_wrapper_root(deposit_message, domain)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). - Let `deposit_data_root` be `hash_tree_root(deposit_data)`. - Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32)` along with a deposit of `amount` Gwei. @@ -234,7 +234,8 @@ Set `block.body.randao_reveal = epoch_signature` where `epoch_signature` is obta ```python def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) - return bls_sign(privkey, hash_tree_root(compute_epoch_at_slot(block.slot)), domain) + message = compute_domain_wrapper_root(compute_epoch_at_slot(block.slot), domain) + return Sign(privkey, message) ``` ##### Eth1 Data @@ -311,7 +312,8 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: ```python def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - return bls_sign(privkey, hash_tree_root(header), domain) + message = compute_domain_wrapper_root(compute_epoch_at_slot(header), domain) + return Sign(privkey, message) ``` ### Attesting @@ -369,7 +371,8 @@ Set `attestation.signature = signed_attestation_data` where `signed_attestation_ ```python def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - return bls_sign(privkey, hash_tree_root(attestation.data), domain) + message = compute_domain_wrapper_root(attestation.data, domain) + return Sign(privkey, message) ``` #### Broadcast attestation @@ -387,7 +390,8 @@ A validator is selected to aggregate based upon the return value of `is_aggregat ```python def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, compute_epoch_at_slot(slot)) - return bls_sign(privkey, hash_tree_root(slot), domain) + message = compute_domain_wrapper_root(slot, domain) + return Sign(privkey, message) ``` ```python @@ -418,7 +422,7 @@ Set `aggregate_attestation.signature = aggregate_signature` where `aggregate_sig ```python def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: signatures = [attestation.signature for attestation in attestations] - return bls_aggregate_signatures(signatures) + return Aggregate(signatures) ``` #### Broadcast aggregate diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 3ed54888f..03ebb7b9b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -3,7 +3,7 @@ from typing import List from eth2spec.test.helpers.block import build_empty_block_for_next_slot, transition_unsigned_block, \ build_empty_block from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.bls import Sign, Aggregate from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -77,8 +77,7 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List privkey ) ) - - return bls_aggregate_signatures(signatures) + return Aggregate(signatures) def sign_indexed_attestation(spec, state, indexed_attestation): @@ -97,15 +96,9 @@ def sign_attestation(spec, state, attestation): def get_attestation_signature(spec, state, attestation_data, privkey): - return bls_sign( - message_hash=attestation_data.hash_tree_root(), - privkey=privkey, - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_BEACON_ATTESTER, - message_epoch=attestation_data.target.epoch, - ) - ) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + message = spec.compute_domain_wrapper_root(attestation_data, domain) + return Sign(privkey, message) def fill_aggregate_attestation(spec, state, attestation, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index b8c514eb4..2dac70d8d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -1,7 +1,7 @@ from copy import deepcopy from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign, only_with_bls +from eth2spec.utils.bls import Sign, only_with_bls from eth2spec.utils.ssz.ssz_impl import hash_tree_root @@ -28,15 +28,9 @@ def apply_randao_reveal(spec, state, block, proposer_index=None): proposer_index = get_proposer_index_maybe(spec, state, block.slot, proposer_index) privkey = privkeys[proposer_index] - block.body.randao_reveal = bls_sign( - privkey=privkey, - message_hash=hash_tree_root(spec.compute_epoch_at_slot(block.slot)), - domain=spec.get_domain( - state, - message_epoch=spec.compute_epoch_at_slot(block.slot), - domain_type=spec.DOMAIN_RANDAO, - ) - ) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) + message = spec.compute_domain_wrapper_root(spec.compute_epoch_at_slot(block.slot), domain) + block.body.randao_reveal = Sign(privkey, message) # Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. @@ -46,14 +40,10 @@ def apply_sig(spec, state, signed_block, proposer_index=None): proposer_index = get_proposer_index_maybe(spec, state, block.slot, proposer_index) privkey = privkeys[proposer_index] + domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) + message = compute_domain_wrapper_root(block, domain) - signed_block.signature = bls_sign( - message_hash=hash_tree_root(block), - privkey=privkey, - domain=spec.get_domain( - state, - spec.DOMAIN_BEACON_PROPOSER, - spec.compute_epoch_at_slot(block.slot))) + signed_block.signature = Sign(privkey, message) def sign_block(spec, state, block, proposer_index=None): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index a9c8145ae..5f439b6a2 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -1,4 +1,4 @@ -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import Sign from eth2spec.utils.ssz.ssz_impl import hash_tree_root @@ -7,8 +7,6 @@ def sign_block_header(spec, state, header, privkey): state=state, domain_type=spec.DOMAIN_BEACON_PROPOSER, ) - return spec.SignedBeaconBlockHeader(message=header, signature=bls_sign( - message_hash=hash_tree_root(header), - privkey=privkey, - domain=domain, - )) + message = spec.compute_domain_wrapper_root(header, domain) + signature = Sign(privkey, message) + return spec.SignedBeaconBlockHeader(message=header, signature=signature) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index f6ca8ecd9..517cd9545 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures +from eth2spec.utils.bls import Sign, Aggregate from eth2spec.utils.hash_function import hash from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector from eth2spec.utils.ssz.ssz_impl import chunkify, pack, hash_tree_root @@ -17,28 +17,15 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING # Generate the secret that is being revealed - reveal = bls_sign( - message_hash=hash_tree_root(spec.Epoch(epoch)), - privkey=privkeys[revealed_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) + message = spec.compute_domain_wrapper_root(spec.Epoch(epoch), domain) + reveal = Sign(privkeys[revealed_index], message) # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask - masker_signature = bls_sign( - message_hash=mask, - privkey=privkeys[masker_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) - masked_reveal = bls_aggregate_signatures([reveal, masker_signature]) + message = spec.compute_domain_wrapper_root(mask, domain) + masker_signature = Sign(privkeys[masker_index], message) + masked_reveal = Aggregate([reveal, masker_signature]) return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, @@ -60,15 +47,9 @@ def get_valid_custody_key_reveal(spec, state, period=None): epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, revealer_index) # Generate the secret that is being revealed - reveal = bls_sign( - message_hash=hash_tree_root(spec.Epoch(epoch_to_sign)), - privkey=privkeys[revealer_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch_to_sign, - ), - ) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) + message = spec.compute_domain_wrapper_root(spec.Epoch(epoch_to_sign), domain) + reveal = Sign(privkeys[revealer_index], message) return spec.CustodyKeyReveal( revealer_index=revealer_index, reveal=reveal, @@ -92,15 +73,9 @@ def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False) responder_index) # Generate the responder key - responder_key = bls_sign( - message_hash=hash_tree_root(spec.Epoch(epoch)), - privkey=privkeys[responder_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) + message = spec.compute_domain_wrapper_root(spec.compute_domain_wrapper_root, domain) + responder_key = Sign(privkeys[responder_index], message) chunk_count = spec.get_custody_chunk_count(attestation.data.crosslink) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index fdab01ca9..90a3ac8f0 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import pubkeys, privkeys -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import Sign from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_proof from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import List @@ -30,12 +30,8 @@ def sign_deposit_data(spec, deposit_data, privkey, state=None): pubkey=deposit_data.pubkey, withdrawal_credentials=deposit_data.withdrawal_credentials, amount=deposit_data.amount) - signature = bls_sign( - message_hash=hash_tree_root(deposit_message), - privkey=privkey, - domain=domain, - ) - deposit_data.signature = signature + message = spec.compute_domain_wrapper_root(deposit_message, domain) + deposit_data.signature = Sign(privkey, message) def build_deposit(spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py index 4f0a9fb0a..c6983e71b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import ( - bls_aggregate_signatures, - bls_sign, + Aggregate, + Sign, ) @@ -25,16 +25,10 @@ def sign_shard_attestation(spec, beacon_state, shard_state, block, participants) ) ) - return bls_aggregate_signatures(signatures) + return Aggregate(signatures) def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): - return bls_sign( - message_hash=message_hash, - privkey=privkey, - domain=spec.get_domain( - state=beacon_state, - domain_type=spec.DOMAIN_SHARD_ATTESTER, - message_epoch=block_epoch, - ) - ) + domain=spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) + message = spec.compute_domain_wrapper(message_hash, domain) + return Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index 8d12110e7..d8a4bcc60 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -2,7 +2,7 @@ from copy import deepcopy from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import ( - bls_sign, + Sign, only_with_bls, ) from eth2spec.utils.ssz.ssz_impl import ( @@ -21,15 +21,9 @@ def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None privkey = privkeys[proposer_index] - block.signature = bls_sign( - message_hash=hash_tree_root(block), - privkey=privkey, - domain=spec.get_domain( - beacon_state, - spec.DOMAIN_SHARD_PROPOSER, - spec.compute_epoch_of_shard_slot(block.slot), - ) - ) + domain=spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) + message = spec.compute_domain_wrapper(block, domain) + block.signature = Sign(privkey, message) def build_empty_shard_block(spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index 62d8f13ba..4f9fc7c30 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -1,17 +1,11 @@ -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import Sign from eth2spec.utils.ssz.ssz_impl import hash_tree_root def sign_voluntary_exit(spec, state, voluntary_exit, privkey): + domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + message = spec.compute_domain_wrapper_root(voluntary_exit, domain) return spec.SignedVoluntaryExit( message=voluntary_exit, - signature=bls_sign( - message_hash=hash_tree_root(voluntary_exit), - privkey=privkey, - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - message_epoch=voluntary_exit.epoch, - ) - ) + signature=Sign(privkey, message) ) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index c2f980ba0..c13bab903 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,7 +1,7 @@ from copy import deepcopy from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.bls import bls_sign +from eth2spec.utils.bls import Sign from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ @@ -104,15 +104,11 @@ def test_zero_block_sig(spec, state): @always_bls def test_invalid_block_sig(spec, state): block = build_empty_block_for_next_slot(spec, state) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) + message = spec.compute_domain_wrapper_root(block, domain) invalid_signed_block = spec.SignedBeaconBlock( message=block, - signature=bls_sign( - message_hash=hash_tree_root(block), - privkey=123456, - domain=spec.get_domain( - state, - spec.DOMAIN_BEACON_PROPOSER, - spec.compute_epoch_at_slot(block.slot))) + signature=Sign(123456, message) ) expect_assertion_error(lambda: spec.state_transition(state, invalid_signed_block)) @@ -417,16 +413,11 @@ def test_voluntary_exit(spec, state): epoch=spec.get_current_epoch(state), validator_index=validator_index, ) + domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) + message = spec.compute_domain_wrapper_root(voluntary_exit, domain) signed_voluntary_exit = spec.SignedVoluntaryExit( message=voluntary_exit, - signature=bls_sign( - message_hash=hash_tree_root(voluntary_exit), - privkey=privkeys[validator_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - ) - ) + signature=Sign(privkeys[validator_index], message) ) # Add to state via block transition diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index f40e5ab04..580d4e75b 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -23,31 +23,35 @@ def only_with_bls(alt_return=None): @only_with_bls(alt_return=True) -def bls_verify(pubkey, message_hash, signature, domain): - return bls.verify(message_hash=message_hash, pubkey=pubkey, - signature=signature, domain=domain) +def Verify(PK, message, signature): + return bls.verify(message_hash=message, pubkey=PK, signature=signature, domain=b'') + + +# @only_with_bls(alt_return=True) +# def AggregateVerify(PKs, messages, signature): +# return bls.verify_multiple(pubkeys=pubkeys, message_hashes=messages, signature=signature, domain=b'') @only_with_bls(alt_return=True) -def bls_verify_multiple(pubkeys, message_hashes, signature, domain): - return bls.verify_multiple(pubkeys=pubkeys, message_hashes=message_hashes, - signature=signature, domain=domain) +def FastAggregateVerify(PKs, message, signature): + aggregate_pubkey = bls.aggregate_pubkeys(PKs) + return bls.verify(pubkey=aggregate_pubkey, message_hash=message, signature=signature, domain=b'') @only_with_bls(alt_return=STUB_PUBKEY) -def bls_aggregate_pubkeys(pubkeys): +def bls_aggregate_pubkeys(PKs): return bls.aggregate_pubkeys(pubkeys) @only_with_bls(alt_return=STUB_SIGNATURE) -def bls_aggregate_signatures(signatures): +def Aggregate(signatures): return bls.aggregate_signatures(signatures) @only_with_bls(alt_return=STUB_SIGNATURE) -def bls_sign(message_hash, privkey, domain): - return bls.sign(message_hash=message_hash, privkey=privkey, - domain=domain) +def Sign(SK, message): + return bls.sign(message_hash=message, privkey=SK, + domain=b'') @only_with_bls(alt_return=STUB_COORDINATES) From 80eb721895339802bbe00b501ce16ea13a9c098d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 12:20:34 +0200 Subject: [PATCH 09/48] Domains are bytes --- scripts/build_spec.py | 2 +- specs/core/0_beacon-chain.md | 12 +++++------- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 4 ++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index ca8bbb6dd..0baa487c3 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -169,7 +169,7 @@ def objects_to_spec(functions: Dict[str, str], functions_spec = '\n\n'.join(functions.values()) for k in list(constants.keys()): if k.startswith('DOMAIN_'): - constants[k] = f"DomainType(({constants[k]}).to_bytes(length=4, byteorder='little'))" + constants[k] = f"DomainType(bytes.fromhex('{constants[k]}'[2:]))" if k == "BLS12_381_Q": constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 71980e88c..271be3ef7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -249,15 +249,13 @@ The following values are (non-configurable) constants used throughout the specif ### Domain types -The following types are defined, mapping into `DomainType` (little endian): - | Name | Value | | - | - | -| `DOMAIN_BEACON_PROPOSER` | `0` | -| `DOMAIN_BEACON_ATTESTER` | `1` | -| `DOMAIN_RANDAO` | `2` | -| `DOMAIN_DEPOSIT` | `3` | -| `DOMAIN_VOLUNTARY_EXIT` | `4` | +| `DOMAIN_BEACON_PROPOSER` | `0x00000000` | +| `DOMAIN_BEACON_ATTESTER` | `0x01000000` | +| `DOMAIN_RANDAO` | `0x02000000` | +| `DOMAIN_DEPOSIT` | `0x03000000` | +| `DOMAIN_VOLUNTARY_EXIT` | `0x04000000` | ## Containers diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 017e2a22a..4f3b26c7c 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -131,7 +131,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | +| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `0x05000000` | ### TODO PLACEHOLDER diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d5964fe7d..9d537a789 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -101,8 +101,8 @@ This document describes the shard transition function (data layer only) and the | Name | Value | | - | - | -| `DOMAIN_SHARD_PROPOSER` | `128` | -| `DOMAIN_SHARD_ATTESTER` | `129` | +| `DOMAIN_SHARD_PROPOSER` | `0x80000000` | +| `DOMAIN_SHARD_ATTESTER` | `0x81000000` | ## Containers From c239ffb78cf2cfc2bf2cafbce2dcfda82f3151aa Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 12:30:46 +0200 Subject: [PATCH 10/48] Linting fixes --- scripts/build_spec.py | 3 +-- specs/core/0_beacon-chain.md | 2 +- test_libs/pyspec/eth2spec/test/helpers/block.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block_header.py | 1 - test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py | 3 +-- test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py | 3 +-- test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py | 1 - test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 1 - test_libs/pyspec/eth2spec/utils/bls.py | 2 +- 9 files changed, 6 insertions(+), 12 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 0baa487c3..caae39533 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -25,11 +25,10 @@ from eth2spec.utils.ssz.ssz_typing import ( Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils.bls import ( - Verify, Sign, + Verify, Aggregate, FastAggregateVerify, - bls_aggregate_pubkeys, ) from eth2spec.utils.hash_function import hash diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 271be3ef7..f2dd8d96c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1499,7 +1499,7 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla message = compute_domain_wrapper_root( object=signed_header.message, domain=get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)), - ) + ) assert Verify(proposer.pubkey, message, signed_header.signature) slash_validator(state, proposer_slashing.proposer_index) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 2dac70d8d..6d22dd545 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -41,7 +41,7 @@ def apply_sig(spec, state, signed_block, proposer_index=None): proposer_index = get_proposer_index_maybe(spec, state, block.slot, proposer_index) privkey = privkeys[proposer_index] domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - message = compute_domain_wrapper_root(block, domain) + message = spec.compute_domain_wrapper_root(block, domain) signed_block.signature = Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index 5f439b6a2..926cebee9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -1,5 +1,4 @@ from eth2spec.utils.bls import Sign -from eth2spec.utils.ssz.ssz_impl import hash_tree_root def sign_block_header(spec, state, header, privkey): diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py index c6983e71b..cbb6cde75 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -24,11 +24,10 @@ def sign_shard_attestation(spec, beacon_state, shard_state, block, participants) privkey, ) ) - return Aggregate(signatures) def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): - domain=spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) + domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) message = spec.compute_domain_wrapper(message_hash, domain) return Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index d8a4bcc60..b0fe39209 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -20,8 +20,7 @@ def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None proposer_index = spec.get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) privkey = privkeys[proposer_index] - - domain=spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) + domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, spec.compute_epoch_of_shard_slot(block.slot)) message = spec.compute_domain_wrapper(block, domain) block.signature = Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index 4f9fc7c30..5c88f07e6 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -1,5 +1,4 @@ from eth2spec.utils.bls import Sign -from eth2spec.utils.ssz.ssz_impl import hash_tree_root def sign_voluntary_exit(spec, state, voluntary_exit, privkey): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index c13bab903..8103f7be2 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,6 +1,5 @@ from copy import deepcopy -from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.bls import Sign from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 580d4e75b..e5f6d8d00 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -40,7 +40,7 @@ def FastAggregateVerify(PKs, message, signature): @only_with_bls(alt_return=STUB_PUBKEY) def bls_aggregate_pubkeys(PKs): - return bls.aggregate_pubkeys(pubkeys) + return bls.aggregate_pubkeys(PKs) @only_with_bls(alt_return=STUB_SIGNATURE) From 995c895b9c27b8bd59ee860b5b501e99dc138b5c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 15:33:37 +0200 Subject: [PATCH 11/48] New BLS in Phase 1 --- scripts/build_spec.py | 7 +++-- specs/core/1_custody-game.md | 36 +++++++------------------- specs/core/1_shard-data-chains.md | 7 ++--- test_libs/pyspec/eth2spec/utils/bls.py | 12 ++++----- 4 files changed, 23 insertions(+), 39 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index caae39533..56b128b45 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -59,10 +59,9 @@ from eth2spec.utils.ssz.ssz_typing import ( ) from eth2spec.utils.bls import ( Verify, - Sign, - Aggregate, + AggregateVerify, FastAggregateVerify, - bls_aggregate_pubkeys, + bls_signature_to_G2, ) from eth2spec.utils.hash_function import hash @@ -86,7 +85,7 @@ def get_eth1_data(distance: uint64) -> Bytes32: return hash(distance) -def hash(x: bytes) -> Bytes32: # type: ignore +def hash(x: bytes) -> Bytes32: if x not in hash_cache: hash_cache[x] = Bytes32(_hash(x)) return hash_cache[x] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 4f3b26c7c..0e3138a29 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -429,16 +429,9 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> assert is_slashable_validator(revealer, get_current_epoch(state)) # Verify signature - assert bls_verify( - pubkey=revealer.pubkey, - message_hash=hash_tree_root(epoch_to_sign), - signature=reveal.reveal, - domain=get_domain( - state=state, - domain_type=DOMAIN_RANDAO, - message_epoch=epoch_to_sign, - ), - ) + domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) + message = compute_domain_wrapper_root(epoch_to_sign, domain) + assert Verify(revealer.pubkey, message, reveal.reveal) # Decrement max reveal lateness if response is timely if epoch_to_sign + EPOCHS_PER_CUSTODY_PERIOD >= get_current_epoch(state): @@ -487,21 +480,12 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived # Verify signature correctness masker = state.validators[reveal.masker_index] pubkeys = [revealed_validator.pubkey, masker.pubkey] - message_hashes = [ - hash_tree_root(reveal.epoch), - reveal.mask, - ] - assert bls_verify_multiple( - pubkeys=pubkeys, - message_hashes=message_hashes, - signature=reveal.reveal, - domain=get_domain( - state=state, - domain_type=DOMAIN_RANDAO, - message_epoch=reveal.epoch, - ), - ) + domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) + messages = [compute_domain_wrapper_root(message, domain) + for message in [hash_tree_root(reveal.epoch), reveal.mask]] + + assert AggregateVerify(pubkeys, messages, reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody @@ -598,7 +582,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> challenger = state.validators[challenge.challenger_index] domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) # TODO incorrect hash-tree-root, but this changes with phase 1 PR #1483 - assert bls_verify(challenger.pubkey, hash_tree_root(challenge), challenge.signature, domain) + assert Verify(challenger.pubkey, compute_domain_wrapper_root(challenge, domain), challenge.signature) # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify attestation @@ -622,7 +606,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> challenge.responder_index, ) domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - assert bls_verify(responder.pubkey, hash_tree_root(epoch_to_sign), challenge.responder_key, domain) + assert Verify(responder.pubkey, compute_domain_wrapper_root(epoch_to_sign, domain), challenge.responder_key) # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert chunk_count == len(challenge.chunk_bits) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9d537a789..806da26f3 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -386,7 +386,7 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert not proposer.slashed # Verify proposer signature domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) - assert bls_verify(proposer.pubkey, hash_tree_root(block), block.signature, domain) + assert Verify(proposer.pubkey, compute_domain_wrapper_root(block, domain), block.signature) ``` #### Attestations @@ -406,8 +406,9 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat assert block.aggregation_bits[i] == 0b0 # Verify attester aggregate signature domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) - message = hash_tree_root(ShardAttestationData(slot=shard_state.slot, parent_root=block.parent_root)) - assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) + shard_attestation_data = ShardAttestationData(slot=shard_state.slot, parent_root=block.parent_root) + message = compute_domain_wrapper_root(shard_attestation_data, domain) + assert FastAggregateVerify(pubkeys, message, block.attestations) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) reward = attestation_count * get_base_reward(beacon_state, proposer_index) // PROPOSER_REWARD_QUOTIENT diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index e5f6d8d00..6aba2c35a 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -27,9 +27,9 @@ def Verify(PK, message, signature): return bls.verify(message_hash=message, pubkey=PK, signature=signature, domain=b'') -# @only_with_bls(alt_return=True) -# def AggregateVerify(PKs, messages, signature): -# return bls.verify_multiple(pubkeys=pubkeys, message_hashes=messages, signature=signature, domain=b'') +@only_with_bls(alt_return=True) +def AggregateVerify(PKs, messages, signature): + return bls.verify_multiple(pubkeys=PKs, message_hashes=messages, signature=signature, domain=b'') @only_with_bls(alt_return=True) @@ -38,9 +38,9 @@ def FastAggregateVerify(PKs, message, signature): return bls.verify(pubkey=aggregate_pubkey, message_hash=message, signature=signature, domain=b'') -@only_with_bls(alt_return=STUB_PUBKEY) -def bls_aggregate_pubkeys(PKs): - return bls.aggregate_pubkeys(PKs) +# @only_with_bls(alt_return=STUB_PUBKEY) +# def bls_aggregate_pubkeys(PKs): +# return bls.aggregate_pubkeys(PKs) @only_with_bls(alt_return=STUB_SIGNATURE) From 177ec939f099bc27d549996ab99ce986ba5566c7 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 15:40:26 +0200 Subject: [PATCH 12/48] Update ToC --- specs/core/0_beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f2dd8d96c..f3e10c9be 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -34,6 +34,7 @@ - [`DepositMessage`](#depositmessage) - [`DepositData`](#depositdata) - [`BeaconBlockHeader`](#beaconblockheader) + - [`DomainWrapper`](#domainwrapper) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -58,8 +59,7 @@ - [Crypto](#crypto) - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - - [`bls_verify`](#bls_verify) - - [`bls_aggregate_pubkeys`](#bls_aggregate_pubkeys) + - [BLS Signatures](#bls-signatures) - [Predicates](#predicates) - [`is_active_validator`](#is_active_validator) - [`is_eligible_for_activation_queue`](#is_eligible_for_activation_queue) @@ -76,6 +76,7 @@ - [`compute_start_slot_at_epoch`](#compute_start_slot_at_epoch) - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - [`compute_domain`](#compute_domain) + - [`compute_domain_wrapper_root`](#compute_domain_wrapper_root) - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) From 3dc78e0905355c2c744d08ccf1ec6cc4513f40a5 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 15:57:00 +0200 Subject: [PATCH 13/48] kick CI cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19ab1543a..5bf94b194 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v4-pyspec + venv_name: v5-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v4-pyspec + venv_name: v5-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: From d3f74ea0d9749065f217607fb85eaecdb6333b5b Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 17 Dec 2019 16:22:46 +0200 Subject: [PATCH 14/48] Realise #1517 was a thing --- .circleci/config.yml | 4 ++-- scripts/build_spec.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bf94b194..19ab1543a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v5-pyspec + venv_name: v4-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v5-pyspec + venv_name: v4-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 56b128b45..dcb530302 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -85,7 +85,7 @@ def get_eth1_data(distance: uint64) -> Bytes32: return hash(distance) -def hash(x: bytes) -> Bytes32: +def hash(x: bytes) -> Bytes32: # type: ignore if x not in hash_cache: hash_cache[x] = Bytes32(_hash(x)) return hash_cache[x] From 4290c1dd1b60b4e2144c09d5f8a6385ff8c32b70 Mon Sep 17 00:00:00 2001 From: Daejun Park Date: Tue, 17 Dec 2019 18:58:56 -0600 Subject: [PATCH 15/48] Vyper b13 -> b13.hotfix1761 --- deposit_contract/contracts/validator_registration.json | 2 +- deposit_contract/contracts/validator_registration.v.py | 2 +- deposit_contract/requirements-testing.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index cd2004233..12cfd74ac 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 95727}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 18283}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}, {"type": "bytes32", "name": "deposit_data_root"}], "constant": false, "payable": true, "type": "function", "gas": 1342680}], "bytecode": "0x740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009857600080fd5b6101406000601f818352015b600061014051602081106100b757600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e357600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012157600080fd5b60c0519050606051600161014051018060405190131561014057600080fd5b809190121561014e57600080fd5b6020811061015b57600080fd5b600260c052602060c02001555b81516001018083528114156100a4575b505061127956600436101561000d576110f4565b600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052600015610286575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100e8578060000360020a82046100ef565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561011a57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610163578060000360020a820461016a565b8060020a82025b905090506101a0525b81516001018083528114156100cb575b5050601860086020820661020001602082840111156101a157600080fd5b60208061022082610180600060046015f15050818152809050905090508051602001806102c0828460006004600a8704601201f16101de57600080fd5b50506102c05160206001820306601f82010390506103206102c0516020818352015b82610320511015156102115761022d565b6000610320516102e001535b8151600101808352811415610200575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b6000610280511115156102625761027e565b602061028051036102a001516020610280510361028052610250565b610160515650005b63c5f2892f600051141561051857341561029f57600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b60016001610180511614156103415760006101a051602081106102e257600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161033357600080fd5b60c0519050610160526103af565b6000610160516020826101c00101526020810190506101a0516020811061036757600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af16103a557600080fd5b60c0519050610160525b61018060026103bd57600080fd5b60028151048152505b81516001018083528114156102c0575b505060006101605160208261046001015260208101905061014051610160516101805163806732896102e0526001546103005261030051600658016100a9565b506103605260006103c0525b6103605160206001820306601f82010390506103c0511015156104445761045d565b6103c05161038001526103c0516020016103c052610422565b61018052610160526101405261036060088060208461046001018260208501600060046012f150508051820191505060006018602082066103e001602082840111156104a857600080fd5b60208061040082610140600060046015f150508181528090509050905060188060208461046001018260208501600060046014f150508051820191505080610460526104609050602060c0825160208401600060025af161050857600080fd5b60c051905060005260206000f350005b63621fd130600051141561062c57341561053157600080fd5b6380673289610140526001546101605261016051600658016100a9565b506101c0526000610220525b6101c05160206001820306601f82010390506102205110151561057c57610595565b610220516101e00152610220516020016102205261055a565b6101c0805160200180610280828460006004600a8704601201f16105b857600080fd5b50506102805160206001820306601f82010390506102e0610280516020818352015b826102e0511015156105eb57610607565b60006102e0516102a001535b81516001018083528114156105da575b5050506020610260526040610280510160206001820306601f8201039050610260f350005b632289511860005114156110f357605060043560040161014037603060043560040135111561065a57600080fd5b60406024356004016101c037602060243560040135111561067a57600080fd5b608060443560040161022037606060443560040135111561069a57600080fd5b63ffffffff600154106106ac57600080fd5b633b9aca006102e0526102e0516106c257600080fd5b6102e05134046102c052633b9aca006102c05110156106e057600080fd5b603061014051146106f057600080fd5b60206101c0511461070057600080fd5b6060610220511461071057600080fd5b610140610360525b6103605151602061036051016103605261036061036051101561073a57610718565b6380673289610380526102c0516103a0526103a051600658016100a9565b50610400526000610460525b6104005160206001820306601f8201039050610460511015156107865761079f565b6104605161042001526104605160200161046052610764565b610340610360525b61036051526020610360510361036052610140610360511015156107ca576107a7565b610400805160200180610300828460006004600a8704601201f16107ed57600080fd5b5050610140610480525b61048051516020610480510161048052610480610480511015610819576107f7565b63806732896104a0526001546104c0526104c051600658016100a9565b50610520526000610580525b6105205160206001820306601f8201039050610580511015156108645761087d565b6105805161054001526105805160200161058052610842565b610460610480525b61048051526020610480510361048052610140610480511015156108a857610885565b6105208051602001806105a0828460006004600a8704601201f16108cb57600080fd5b505060a06106205261062051610660526101408051602001806106205161066001828460006004600a8704601201f161090357600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516040818352015b826106005110151561094057610961565b600061060051610620516106800101535b815160010180835281141561092f575b505050602061062051610660015160206001820306601f82010390506106205101016106205261062051610680526101c08051602001806106205161066001828460006004600a8704601201f16109b757600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516020818352015b82610600511015156109f457610a15565b600061060051610620516106800101535b81516001018083528114156109e3575b505050602061062051610660015160206001820306601f820103905061062051010161062052610620516106a0526103008051602001806106205161066001828460006004600a8704601201f1610a6b57600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516020818352015b8261060051101515610aa857610ac9565b600061060051610620516106800101535b8151600101808352811415610a97575b505050602061062051610660015160206001820306601f820103905061062051010161062052610620516106c0526102208051602001806106205161066001828460006004600a8704601201f1610b1f57600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516060818352015b8261060051101515610b5c57610b7d565b600061060051610620516106800101535b8151600101808352811415610b4b575b505050602061062051610660015160206001820306601f820103905061062051010161062052610620516106e0526105a08051602001806106205161066001828460006004600a8704601201f1610bd357600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516020818352015b8261060051101515610c1057610c31565b600061060051610620516106800101535b8151600101808352811415610bff575b505050602061062051610660015160206001820306601f8201039050610620510101610620527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561062051610660a160006107005260006101406030806020846107c001018260208501600060046016f150508051820191505060006010602082066107400160208284011115610cc757600080fd5b60208061076082610700600060046015f15050818152809050905090506010806020846107c001018260208501600060046013f1505080518201915050806107c0526107c09050602060c0825160208401600060025af1610d2757600080fd5b60c0519050610720526000600060406020820661086001610220518284011115610d5057600080fd5b606080610880826020602088068803016102200160006004601bf1505081815280905090509050602060c0825160208401600060025af1610d9057600080fd5b60c0519050602082610a600101526020810190506000604060206020820661092001610220518284011115610dc457600080fd5b606080610940826020602088068803016102200160006004601bf15050818152809050905090506020806020846109e001018260208501600060046015f1505080518201915050610700516020826109e0010152602081019050806109e0526109e09050602060c0825160208401600060025af1610e4157600080fd5b60c0519050602082610a6001015260208101905080610a6052610a609050602060c0825160208401600060025af1610e7857600080fd5b60c0519050610840526000600061072051602082610b000101526020810190506101c0602080602084610b0001018260208501600060046015f150508051820191505080610b0052610b009050602060c0825160208401600060025af1610ede57600080fd5b60c0519050602082610c800101526020810190506000610300600880602084610c0001018260208501600060046012f15050805182019150506000601860208206610b800160208284011115610f3357600080fd5b602080610ba082610700600060046015f1505081815280905090509050601880602084610c0001018260208501600060046014f150508051820191505061084051602082610c0001015260208101905080610c0052610c009050602060c0825160208401600060025af1610fa657600080fd5b60c0519050602082610c8001015260208101905080610c8052610c809050602060c0825160208401600060025af1610fdd57600080fd5b60c0519050610ae052606435610ae05114610ff757600080fd5b600180546001825401101561100b57600080fd5b6001815401815550600154610d0052610d2060006020818352015b60016001610d005116141561105b57610ae051610d20516020811061104a57600080fd5b600060c052602060c02001556110ef565b6000610d20516020811061106e57600080fd5b600060c052602060c0200154602082610d40010152602081019050610ae051602082610d4001015260208101905080610d4052610d409050602060c0825160208401600060025af16110bf57600080fd5b60c0519050610ae052610d0060026110d657600080fd5b60028151048152505b8151600101808352811415611026575b5050005b5b60006000fd5b61017f6112790361017f60003961017f611279036000f3"} \ No newline at end of file +{"abi": [{"name": "DepositEvent", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "index", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 95628}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 18231}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}, {"type": "bytes32", "name": "deposit_data_root"}], "constant": false, "payable": true, "type": "function", "gas": 1342274}], "bytecode": "0x740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009857600080fd5b6101406000601f818352015b600061014051602081106100b757600080fd5b600260c052602060c020015460208261016001015260208101905061014051602081106100e357600080fd5b600260c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012157600080fd5b60c0519050606051600161014051018060405190131561014057600080fd5b809190121561014e57600080fd5b6020811061015b57600080fd5b600260c052602060c02001555b81516001018083528114156100a4575b505061123556600436101561000d576110b0565b600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a05260001561027f575b6101605261014052600061018052610140516101a0526101c060006008818352015b61018051600860008112156100e8578060000360020a82046100ef565b8060020a82025b905090506101805260ff6101a051166101e052610180516101e0516101805101101561011a57600080fd5b6101e0516101805101610180526101a0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86000811215610163578060000360020a820461016a565b8060020a82025b905090506101a0525b81516001018083528114156100cb575b5050601860086020820661020001602082840111156101a157600080fd5b60208061022082610180600060045af15050818152809050905090508051602001806102c08284600060045af16101d757600080fd5b50506102c05160206001820306601f82010390506103206102c0516020818352015b826103205110151561020a57610226565b6000610320516102e001535b81516001018083528114156101f9575b50505060206102a05260406102c0510160206001820306601f8201039050610280525b60006102805111151561025b57610277565b602061028051036102a001516020610280510361028052610249565b610160515650005b63c5f2892f600051141561050e57341561029857600080fd5b6000610140526101405161016052600154610180526101a060006020818352015b600160016101805116141561033a5760006101a051602081106102db57600080fd5b600060c052602060c02001546020826102400101526020810190506101605160208261024001015260208101905080610240526102409050602060c0825160208401600060025af161032c57600080fd5b60c0519050610160526103a8565b6000610160516020826101c00101526020810190506101a0516020811061036057600080fd5b600260c052602060c02001546020826101c0010152602081019050806101c0526101c09050602060c0825160208401600060025af161039e57600080fd5b60c0519050610160525b61018060026103b657600080fd5b60028151048152505b81516001018083528114156102b9575b505060006101605160208261046001015260208101905061014051610160516101805163806732896102e0526001546103005261030051600658016100a9565b506103605260006103c0525b6103605160206001820306601f82010390506103c05110151561043d57610456565b6103c05161038001526103c0516020016103c05261041b565b61018052610160526101405261036060088060208461046001018260208501600060045af150508051820191505060006018602082066103e001602082840111156104a057600080fd5b60208061040082610140600060045af150508181528090509050905060188060208461046001018260208501600060045af150508051820191505080610460526104609050602060c0825160208401600060025af16104fe57600080fd5b60c051905060005260206000f350005b63621fd130600051141561061c57341561052757600080fd5b6380673289610140526001546101605261016051600658016100a9565b506101c0526000610220525b6101c05160206001820306601f8201039050610220511015156105725761058b565b610220516101e001526102205160200161022052610550565b6101c08051602001806102808284600060045af16105a857600080fd5b50506102805160206001820306601f82010390506102e0610280516020818352015b826102e0511015156105db576105f7565b60006102e0516102a001535b81516001018083528114156105ca575b5050506020610260526040610280510160206001820306601f8201039050610260f350005b632289511860005114156110af57605060043560040161014037603060043560040135111561064a57600080fd5b60406024356004016101c037602060243560040135111561066a57600080fd5b608060443560040161022037606060443560040135111561068a57600080fd5b63ffffffff6001541061069c57600080fd5b633b9aca006102e0526102e0516106b257600080fd5b6102e05134046102c052633b9aca006102c05110156106d057600080fd5b603061014051146106e057600080fd5b60206101c051146106f057600080fd5b6060610220511461070057600080fd5b610140610360525b6103605151602061036051016103605261036061036051101561072a57610708565b6380673289610380526102c0516103a0526103a051600658016100a9565b50610400526000610460525b6104005160206001820306601f8201039050610460511015156107765761078f565b6104605161042001526104605160200161046052610754565b610340610360525b61036051526020610360510361036052610140610360511015156107ba57610797565b6104008051602001806103008284600060045af16107d757600080fd5b5050610140610480525b61048051516020610480510161048052610480610480511015610803576107e1565b63806732896104a0526001546104c0526104c051600658016100a9565b50610520526000610580525b6105205160206001820306601f82010390506105805110151561084e57610867565b610580516105400152610580516020016105805261082c565b610460610480525b61048051526020610480510361048052610140610480511015156108925761086f565b6105208051602001806105a08284600060045af16108af57600080fd5b505060a061062052610620516106605261014080516020018061062051610660018284600060045af16108e157600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516040818352015b826106005110151561091e5761093f565b600061060051610620516106800101535b815160010180835281141561090d575b505050602061062051610660015160206001820306601f82010390506106205101016106205261062051610680526101c080516020018061062051610660018284600060045af161098f57600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516020818352015b82610600511015156109cc576109ed565b600061060051610620516106800101535b81516001018083528114156109bb575b505050602061062051610660015160206001820306601f820103905061062051010161062052610620516106a05261030080516020018061062051610660018284600060045af1610a3d57600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516020818352015b8261060051101515610a7a57610a9b565b600061060051610620516106800101535b8151600101808352811415610a69575b505050602061062051610660015160206001820306601f820103905061062051010161062052610620516106c05261022080516020018061062051610660018284600060045af1610aeb57600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516060818352015b8261060051101515610b2857610b49565b600061060051610620516106800101535b8151600101808352811415610b17575b505050602061062051610660015160206001820306601f820103905061062051010161062052610620516106e0526105a080516020018061062051610660018284600060045af1610b9957600080fd5b505061062051610660015160206001820306601f82010390506106006106205161066001516020818352015b8261060051101515610bd657610bf7565b600061060051610620516106800101535b8151600101808352811415610bc5575b505050602061062051610660015160206001820306601f8201039050610620510101610620527f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c561062051610660a160006107005260006101406030806020846107c001018260208501600060045af150508051820191505060006010602082066107400160208284011115610c8c57600080fd5b60208061076082610700600060045af15050818152809050905090506010806020846107c001018260208501600060045af1505080518201915050806107c0526107c09050602060c0825160208401600060025af1610cea57600080fd5b60c0519050610720526000600060406020820661086001610220518284011115610d1357600080fd5b6060806108808260206020880688030161022001600060045af1505081815280905090509050602060c0825160208401600060025af1610d5257600080fd5b60c0519050602082610a600101526020810190506000604060206020820661092001610220518284011115610d8657600080fd5b6060806109408260206020880688030161022001600060045af15050818152809050905090506020806020846109e001018260208501600060045af1505080518201915050610700516020826109e0010152602081019050806109e0526109e09050602060c0825160208401600060025af1610e0157600080fd5b60c0519050602082610a6001015260208101905080610a6052610a609050602060c0825160208401600060025af1610e3857600080fd5b60c0519050610840526000600061072051602082610b000101526020810190506101c0602080602084610b0001018260208501600060045af150508051820191505080610b0052610b009050602060c0825160208401600060025af1610e9d57600080fd5b60c0519050602082610c800101526020810190506000610300600880602084610c0001018260208501600060045af15050805182019150506000601860208206610b800160208284011115610ef157600080fd5b602080610ba082610700600060045af1505081815280905090509050601880602084610c0001018260208501600060045af150508051820191505061084051602082610c0001015260208101905080610c0052610c009050602060c0825160208401600060025af1610f6257600080fd5b60c0519050602082610c8001015260208101905080610c8052610c809050602060c0825160208401600060025af1610f9957600080fd5b60c0519050610ae052606435610ae05114610fb357600080fd5b6001805460018254011015610fc757600080fd5b6001815401815550600154610d0052610d2060006020818352015b60016001610d005116141561101757610ae051610d20516020811061100657600080fd5b600060c052602060c02001556110ab565b6000610d20516020811061102a57600080fd5b600060c052602060c0200154602082610d40010152602081019050610ae051602082610d4001015260208101905080610d4052610d409050602060c0825160208401600060025af161107b57600080fd5b60c0519050610ae052610d00600261109257600080fd5b60028151048152505b8151600101808352811415610fe2575b5050005b5b60006000fd5b61017f6112350361017f60003961017f611235036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index ca18cbdfd..671252e2e 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -1,4 +1,4 @@ -# Vyper target 0.1.0b13 +# Vyper target 0.1.0b13.hotfix1761 MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 MAX_DEPOSIT_COUNT: constant(uint256) = 4294967295 # 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index 9b07eae6f..fe4b88a23 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -1,5 +1,5 @@ eth-tester[py-evm]==0.1.0b39 -vyper==0.1.0b13 +https://github.com/vyperlang/vyper/archive/1761-HOTFIX-v0.1.0-beta.13.tar.gz web3==5.0.0b2 pytest==3.6.1 ../test_libs/pyspec From 331892ad15ac1a97bd64b1fdde432304988d9cdb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 18 Dec 2019 13:30:02 -0700 Subject: [PATCH 16/48] bump version on despoit cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19ab1543a..9ca22a0e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v6-deposit-contract + venv_name: v7-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v6-deposit-contract + venv_name: v7-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From 01a2ef8158bbb9de7a849bb61fac3129f61254fd Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 18 Dec 2019 16:36:28 -0700 Subject: [PATCH 17/48] fix vyper custom build link and bump pycryptodome version --- deposit_contract/requirements-testing.txt | 2 +- test_libs/pyspec/requirements.txt | 2 +- test_libs/pyspec/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index fe4b88a23..c542121c2 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -1,5 +1,5 @@ eth-tester[py-evm]==0.1.0b39 -https://github.com/vyperlang/vyper/archive/1761-HOTFIX-v0.1.0-beta.13.tar.gz +git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13 web3==5.0.0b2 pytest==3.6.1 ../test_libs/pyspec diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 480602779..8dfbccf66 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -1,6 +1,6 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 -pycryptodome==3.7.3 +pycryptodome==3.9.4 py_ecc==1.7.1 dataclasses==0.6 ssz==0.1.3 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 94575f2a1..d41412eb8 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -7,7 +7,7 @@ setup( install_requires=[ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", - "pycryptodome==3.7.3", + "pycryptodome==3.9.4", "py_ecc==1.7.1", "ssz==0.1.3", "dataclasses==0.6", From 42a3dd4ab1b435cda8f77c8820b93f2545dbb3b7 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 19 Dec 2019 15:47:42 +0200 Subject: [PATCH 18/48] Implement @djrtwo's review suggestions --- specs/core/0_beacon-chain.md | 11 ++++------- specs/light_client/sync_protocol.md | 5 +++-- specs/validator/0_beacon-chain-validator.md | 2 +- test_libs/pyspec/eth2spec/utils/bls.py | 8 +------- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f3e10c9be..d1371483a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -591,7 +591,6 @@ Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuit * `def Sign(SK: int, message: Bytes) -> BLSSignature` * `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` * `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` -* `def bls_aggregate_pubkeys(PKs: Sequence[BLSPubkey]) -> BLSPubkey` * `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool` ### Predicates @@ -799,9 +798,9 @@ def compute_domain(domain_type: DomainType, fork_version: Version=Version()) -> ### `compute_domain_wrapper_root` ```python -def compute_domain_wrapper_root(object: SSZObject, domain: Domain) -> Root: +def compute_domain_wrapper_root(ssz_object: SSZObject, domain: Domain) -> Root: domain_wrapped_object = DomainWrapper( - root=hash_tree_root(object), + root=hash_tree_root(ssz_object), domain=domain, ) return hash_tree_root(domain_wrapped_object) @@ -1497,10 +1496,8 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla assert is_slashable_validator(proposer, get_current_epoch(state)) # Signatures are valid for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): - message = compute_domain_wrapper_root( - object=signed_header.message, - domain=get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)), - ) + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) + message = compute_domain_wrapper_root(signed_header.message, domain) assert Verify(proposer.pubkey, message, signed_header.signature) slash_validator(state, proposer_slashing.proposer_index) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 05180516b..fd3fe2eb5 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -135,9 +135,10 @@ def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: assert 3 * sum(filter(lambda i: update.aggregation_bits[i], balances)) > 2 * sum(balances) # Verify shard attestations - pubkey = bls_aggregate_pubkeys(filter(lambda i: update.aggregation_bits[i], pubkeys)) + pubkeys = filter(lambda i: update.aggregation_bits[i], pubkeys) domain = compute_domain(DOMAIN_SHARD_ATTESTER, update.fork_version) - assert bls_verify(pubkey, update.shard_block_root, update.signature, domain) + message = compute_domain_wrapper_root(update.shard_block_root, domain) + assert FastAggregateVerify(pubkey, message, update.signature) # Update period committees if entering a new period if next_period == current_period + 1: diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 5e80e84a6..bb53946e0 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -312,7 +312,7 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: ```python def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - message = compute_domain_wrapper_root(compute_epoch_at_slot(header), domain) + message = compute_domain_wrapper_root(header, domain) return Sign(privkey, message) ``` diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 6aba2c35a..4b0ed6445 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -38,11 +38,6 @@ def FastAggregateVerify(PKs, message, signature): return bls.verify(pubkey=aggregate_pubkey, message_hash=message, signature=signature, domain=b'') -# @only_with_bls(alt_return=STUB_PUBKEY) -# def bls_aggregate_pubkeys(PKs): -# return bls.aggregate_pubkeys(PKs) - - @only_with_bls(alt_return=STUB_SIGNATURE) def Aggregate(signatures): return bls.aggregate_signatures(signatures) @@ -50,8 +45,7 @@ def Aggregate(signatures): @only_with_bls(alt_return=STUB_SIGNATURE) def Sign(SK, message): - return bls.sign(message_hash=message, privkey=SK, - domain=b'') + return bls.sign(message_hash=message, privkey=SK, domain=b'') @only_with_bls(alt_return=STUB_COORDINATES) From 62cbd0e7da372317a333df6a88337314d74d1cc8 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 19 Dec 2019 15:49:10 +0200 Subject: [PATCH 19/48] Revert domain 6 -> 5 to prevent confusion --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0e3138a29..d6c79c96d 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -131,7 +131,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `0x05000000` | +| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `0x06000000` | ### TODO PLACEHOLDER From 1e410a1290aac7fbe118cfb3b2577fa4f3b4cdc9 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 20 Dec 2019 08:12:43 +0200 Subject: [PATCH 20/48] Fix typo due to change of interface Co-Authored-By: Danny Ryan --- specs/light_client/sync_protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index fd3fe2eb5..ff1ab0f0c 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -138,7 +138,7 @@ def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: pubkeys = filter(lambda i: update.aggregation_bits[i], pubkeys) domain = compute_domain(DOMAIN_SHARD_ATTESTER, update.fork_version) message = compute_domain_wrapper_root(update.shard_block_root, domain) - assert FastAggregateVerify(pubkey, message, update.signature) + assert FastAggregateVerify(pubkeys, message, update.signature) # Update period committees if entering a new period if next_period == current_period + 1: From 7af4429011dd12c3e4102a55f4bee1a0ba85da52 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 20 Dec 2019 08:41:46 +0200 Subject: [PATCH 21/48] Make bls a module --- scripts/build_spec.py | 14 ++------------ specs/core/0_beacon-chain.md | 18 +++++++++--------- specs/core/1_custody-game.md | 10 +++++----- specs/core/1_shard-data-chains.md | 4 ++-- specs/light_client/sync_protocol.md | 2 +- specs/validator/0_beacon-chain-validator.md | 10 +++++----- .../eth2spec/test/helpers/attestations.py | 6 +++--- .../pyspec/eth2spec/test/helpers/block.py | 7 ++++--- .../eth2spec/test/helpers/block_header.py | 4 ++-- .../pyspec/eth2spec/test/helpers/custody.py | 12 ++++++------ .../pyspec/eth2spec/test/helpers/deposits.py | 4 ++-- .../test/helpers/phase1/attestations.py | 9 +++------ .../test/helpers/phase1/shard_block.py | 8 +++----- .../eth2spec/test/helpers/voluntary_exits.py | 4 ++-- .../pyspec/eth2spec/test/sanity/test_blocks.py | 6 +++--- test_libs/pyspec/eth2spec/utils/bls.py | 2 +- 16 files changed, 53 insertions(+), 67 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index dcb530302..42d5a81d0 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -24,12 +24,7 @@ from eth2spec.utils.ssz.ssz_typing import ( boolean, Container, List, Vector, uint64, SSZType, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) -from eth2spec.utils.bls import ( - Sign, - Verify, - Aggregate, - FastAggregateVerify, -) +from eth2spec.utils import bls from eth2spec.utils.hash_function import hash @@ -57,12 +52,7 @@ from eth2spec.utils.ssz.ssz_typing import ( Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, uint64, bit, boolean, byte, ) -from eth2spec.utils.bls import ( - Verify, - AggregateVerify, - FastAggregateVerify, - bls_signature_to_G2, -) +from eth2spec.utils import bls from eth2spec.utils.hash_function import hash diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index d1371483a..9c39801a6 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -584,15 +584,15 @@ def bytes_to_int(data: bytes) -> uint64: #### BLS Signatures -Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). - -Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite where it makes use of the following functions: +Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite which implements the following interfaces: * `def Sign(SK: int, message: Bytes) -> BLSSignature` * `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` * `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` * `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool` +Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. + ### Predicates #### `is_active_validator` @@ -677,7 +677,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe pubkeys = [state.validators[i].pubkey for i in indices] domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) message = compute_domain_wrapper_root(indexed_attestation.data, domain) - return FastAggregateVerify(pubkeys, message, indexed_attestation.signature) + return bls.FastAggregateVerify(pubkeys, message, indexed_attestation.signature) ``` #### `is_valid_merkle_branch` @@ -1149,7 +1149,7 @@ def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, valida def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: proposer = state.validators[get_beacon_proposer_index(state)] message = compute_domain_wrapper_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) - return Verify(proposer.pubkey, message, signed_block.signature) + return bls.Verify(proposer.pubkey, message, signed_block.signature) ``` ```python @@ -1449,7 +1449,7 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: # Verify RANDAO reveal proposer = state.validators[get_beacon_proposer_index(state)] message = compute_domain_wrapper_root(epoch, get_domain(state, DOMAIN_RANDAO)) - assert Verify(proposer.pubkey, message, body.randao_reveal) + assert bls.Verify(proposer.pubkey, message, body.randao_reveal) # Mix in RANDAO reveal mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix @@ -1498,7 +1498,7 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) message = compute_domain_wrapper_root(signed_header.message, domain) - assert Verify(proposer.pubkey, message, signed_header.signature) + assert bls.Verify(proposer.pubkey, message, signed_header.signature) slash_validator(state, proposer_slashing.proposer_index) ``` @@ -1581,7 +1581,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: withdrawal_credentials=deposit.data.withdrawal_credentials, amount=deposit.data.amount) message = compute_domain_wrapper_root(deposit_message, compute_domain(DOMAIN_DEPOSIT)) - if not Verify(pubkey, message, deposit.data.signature): + if not bls.Verify(pubkey, message, deposit.data.signature): return # Add validator and balance entries @@ -1618,7 +1618,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) message = compute_domain_wrapper_root(voluntary_exit, domain) - assert Verify(validator.pubkey, message, signed_voluntary_exit.signature) + assert bls.Verify(validator.pubkey, message, signed_voluntary_exit.signature) # Initiate exit initiate_validator_exit(state, voluntary_exit.validator_index) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d6c79c96d..7f94706e0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -353,7 +353,7 @@ def custody_subchunkify(bytez: bytes) -> Sequence[bytes]: ```python def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: - full_G2_element = bls_signature_to_G2(key) + full_G2_element = bls.signature_to_G2(key) s = full_G2_element[0].coeffs bits = [legendre_bit((i + 1) * s[i % 2] + int.from_bytes(subchunk, "little"), BLS12_381_Q) for i, subchunk in enumerate(custody_subchunkify(chunk))] @@ -431,7 +431,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> # Verify signature domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) message = compute_domain_wrapper_root(epoch_to_sign, domain) - assert Verify(revealer.pubkey, message, reveal.reveal) + assert bls.Verify(revealer.pubkey, message, reveal.reveal) # Decrement max reveal lateness if response is timely if epoch_to_sign + EPOCHS_PER_CUSTODY_PERIOD >= get_current_epoch(state): @@ -485,7 +485,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived messages = [compute_domain_wrapper_root(message, domain) for message in [hash_tree_root(reveal.epoch), reveal.mask]] - assert AggregateVerify(pubkeys, messages, reveal.reveal) + assert bls.AggregateVerify(pubkeys, messages, reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody @@ -582,7 +582,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> challenger = state.validators[challenge.challenger_index] domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) # TODO incorrect hash-tree-root, but this changes with phase 1 PR #1483 - assert Verify(challenger.pubkey, compute_domain_wrapper_root(challenge, domain), challenge.signature) + assert bls.Verify(challenger.pubkey, compute_domain_wrapper_root(challenge, domain), challenge.signature) # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify attestation @@ -606,7 +606,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> challenge.responder_index, ) domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - assert Verify(responder.pubkey, compute_domain_wrapper_root(epoch_to_sign, domain), challenge.responder_key) + assert bls.Verify(responder.pubkey, compute_domain_wrapper_root(epoch_to_sign, domain), challenge.responder_key) # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert chunk_count == len(challenge.chunk_bits) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 806da26f3..44d14ec8c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -386,7 +386,7 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert not proposer.slashed # Verify proposer signature domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) - assert Verify(proposer.pubkey, compute_domain_wrapper_root(block, domain), block.signature) + assert bls.Verify(proposer.pubkey, compute_domain_wrapper_root(block, domain), block.signature) ``` #### Attestations @@ -408,7 +408,7 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) shard_attestation_data = ShardAttestationData(slot=shard_state.slot, parent_root=block.parent_root) message = compute_domain_wrapper_root(shard_attestation_data, domain) - assert FastAggregateVerify(pubkeys, message, block.attestations) + assert bls.FastAggregateVerify(pubkeys, message, block.attestations) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) reward = attestation_count * get_base_reward(beacon_state, proposer_index) // PROPOSER_REWARD_QUOTIENT diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index ff1ab0f0c..de4f0be7c 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -138,7 +138,7 @@ def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: pubkeys = filter(lambda i: update.aggregation_bits[i], pubkeys) domain = compute_domain(DOMAIN_SHARD_ATTESTER, update.fork_version) message = compute_domain_wrapper_root(update.shard_block_root, domain) - assert FastAggregateVerify(pubkeys, message, update.signature) + assert bls.FastAggregateVerify(pubkeys, message, update.signature) # Update period committees if entering a new period if next_period == current_period + 1: diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index bb53946e0..3bd3e1f22 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -235,7 +235,7 @@ Set `block.body.randao_reveal = epoch_signature` where `epoch_signature` is obta def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) message = compute_domain_wrapper_root(compute_epoch_at_slot(block.slot), domain) - return Sign(privkey, message) + return bls.Sign(privkey, message) ``` ##### Eth1 Data @@ -313,7 +313,7 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) message = compute_domain_wrapper_root(header, domain) - return Sign(privkey, message) + return bls.Sign(privkey, message) ``` ### Attesting @@ -372,7 +372,7 @@ Set `attestation.signature = signed_attestation_data` where `signed_attestation_ def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) message = compute_domain_wrapper_root(attestation.data, domain) - return Sign(privkey, message) + return bls.Sign(privkey, message) ``` #### Broadcast attestation @@ -391,7 +391,7 @@ A validator is selected to aggregate based upon the return value of `is_aggregat def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, compute_epoch_at_slot(slot)) message = compute_domain_wrapper_root(slot, domain) - return Sign(privkey, message) + return bls.Sign(privkey, message) ``` ```python @@ -422,7 +422,7 @@ Set `aggregate_attestation.signature = aggregate_signature` where `aggregate_sig ```python def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: signatures = [attestation.signature for attestation in attestations] - return Aggregate(signatures) + return bls.Aggregate(signatures) ``` #### Broadcast aggregate diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 03ebb7b9b..1082b7045 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -3,7 +3,7 @@ from typing import List from eth2spec.test.helpers.block import build_empty_block_for_next_slot, transition_unsigned_block, \ build_empty_block from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import Sign, Aggregate +from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -77,7 +77,7 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List privkey ) ) - return Aggregate(signatures) + return bls.Aggregate(signatures) def sign_indexed_attestation(spec, state, indexed_attestation): @@ -98,7 +98,7 @@ def sign_attestation(spec, state, attestation): def get_attestation_signature(spec, state, attestation_data, privkey): domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) message = spec.compute_domain_wrapper_root(attestation_data, domain) - return Sign(privkey, message) + return bls.Sign(privkey, message) def fill_aggregate_attestation(spec, state, attestation, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 6d22dd545..13669049f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -1,7 +1,8 @@ from copy import deepcopy from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import Sign, only_with_bls +from eth2spec.utils import bls +from eth2spec.utils.bls import only_with_bls from eth2spec.utils.ssz.ssz_impl import hash_tree_root @@ -30,7 +31,7 @@ def apply_randao_reveal(spec, state, block, proposer_index=None): domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) message = spec.compute_domain_wrapper_root(spec.compute_epoch_at_slot(block.slot), domain) - block.body.randao_reveal = Sign(privkey, message) + block.body.randao_reveal = bls.Sign(privkey, message) # Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. @@ -43,7 +44,7 @@ def apply_sig(spec, state, signed_block, proposer_index=None): domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) message = spec.compute_domain_wrapper_root(block, domain) - signed_block.signature = Sign(privkey, message) + signed_block.signature = bls.Sign(privkey, message) def sign_block(spec, state, block, proposer_index=None): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index 926cebee9..4f9f88644 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -1,4 +1,4 @@ -from eth2spec.utils.bls import Sign +from eth2spec.utils import bls def sign_block_header(spec, state, header, privkey): @@ -7,5 +7,5 @@ def sign_block_header(spec, state, header, privkey): domain_type=spec.DOMAIN_BEACON_PROPOSER, ) message = spec.compute_domain_wrapper_root(header, domain) - signature = Sign(privkey, message) + signature = bls.Sign(privkey, message) return spec.SignedBeaconBlockHeader(message=header, signature=signature) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 517cd9545..80b341526 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import Sign, Aggregate +from eth2spec.utils import bls from eth2spec.utils.hash_function import hash from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector from eth2spec.utils.ssz.ssz_impl import chunkify, pack, hash_tree_root @@ -19,13 +19,13 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): # Generate the secret that is being revealed domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) message = spec.compute_domain_wrapper_root(spec.Epoch(epoch), domain) - reveal = Sign(privkeys[revealed_index], message) + reveal = bls.Sign(privkeys[revealed_index], message) # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask message = spec.compute_domain_wrapper_root(mask, domain) - masker_signature = Sign(privkeys[masker_index], message) - masked_reveal = Aggregate([reveal, masker_signature]) + masker_signature = bls.Sign(privkeys[masker_index], message) + masked_reveal = bls.Aggregate([reveal, masker_signature]) return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, @@ -49,7 +49,7 @@ def get_valid_custody_key_reveal(spec, state, period=None): # Generate the secret that is being revealed domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) message = spec.compute_domain_wrapper_root(spec.Epoch(epoch_to_sign), domain) - reveal = Sign(privkeys[revealer_index], message) + reveal = bls.Sign(privkeys[revealer_index], message) return spec.CustodyKeyReveal( revealer_index=revealer_index, reveal=reveal, @@ -75,7 +75,7 @@ def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False) # Generate the responder key domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) message = spec.compute_domain_wrapper_root(spec.compute_domain_wrapper_root, domain) - responder_key = Sign(privkeys[responder_index], message) + responder_key = bls.Sign(privkeys[responder_index], message) chunk_count = spec.get_custody_chunk_count(attestation.data.crosslink) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 90a3ac8f0..a307f524b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.keys import pubkeys, privkeys -from eth2spec.utils.bls import Sign +from eth2spec.utils import bls from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_proof from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import List @@ -31,7 +31,7 @@ def sign_deposit_data(spec, deposit_data, privkey, state=None): withdrawal_credentials=deposit_data.withdrawal_credentials, amount=deposit_data.amount) message = spec.compute_domain_wrapper_root(deposit_message, domain) - deposit_data.signature = Sign(privkey, message) + deposit_data.signature = bls.Sign(privkey, message) def build_deposit(spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py index cbb6cde75..e521bd6c5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -1,8 +1,5 @@ from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import ( - Aggregate, - Sign, -) +from eth2spec.utils import bls def sign_shard_attestation(spec, beacon_state, shard_state, block, participants): @@ -24,10 +21,10 @@ def sign_shard_attestation(spec, beacon_state, shard_state, block, participants) privkey, ) ) - return Aggregate(signatures) + return bls.Aggregate(signatures) def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) message = spec.compute_domain_wrapper(message_hash, domain) - return Sign(privkey, message) + return bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index b0fe39209..f7275b88a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -1,10 +1,8 @@ from copy import deepcopy from eth2spec.test.helpers.keys import privkeys -from eth2spec.utils.bls import ( - Sign, - only_with_bls, -) +from eth2spec.utils import bls +from eth2spec.utils.bls import only_with_bls from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, ) @@ -22,7 +20,7 @@ def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None privkey = privkeys[proposer_index] domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, spec.compute_epoch_of_shard_slot(block.slot)) message = spec.compute_domain_wrapper(block, domain) - block.signature = Sign(privkey, message) + block.signature = bls.Sign(privkey, message) def build_empty_shard_block(spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index 5c88f07e6..286f85e9f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -1,4 +1,4 @@ -from eth2spec.utils.bls import Sign +from eth2spec.utils import bls def sign_voluntary_exit(spec, state, voluntary_exit, privkey): @@ -6,5 +6,5 @@ def sign_voluntary_exit(spec, state, voluntary_exit, privkey): message = spec.compute_domain_wrapper_root(voluntary_exit, domain) return spec.SignedVoluntaryExit( message=voluntary_exit, - signature=Sign(privkey, message) + signature=bls.Sign(privkey, message) ) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 8103f7be2..da89ef070 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.utils.bls import Sign +from eth2spec.utils import bls from eth2spec.test.helpers.state import get_balance, state_transition_and_sign_block, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot, build_empty_block, sign_block, \ @@ -107,7 +107,7 @@ def test_invalid_block_sig(spec, state): message = spec.compute_domain_wrapper_root(block, domain) invalid_signed_block = spec.SignedBeaconBlock( message=block, - signature=Sign(123456, message) + signature=bls.Sign(123456, message) ) expect_assertion_error(lambda: spec.state_transition(state, invalid_signed_block)) @@ -416,7 +416,7 @@ def test_voluntary_exit(spec, state): message = spec.compute_domain_wrapper_root(voluntary_exit, domain) signed_voluntary_exit = spec.SignedVoluntaryExit( message=voluntary_exit, - signature=Sign(privkeys[validator_index], message) + signature=bls.Sign(privkeys[validator_index], message) ) # Add to state via block transition diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 4b0ed6445..3cfc6121e 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -49,5 +49,5 @@ def Sign(SK, message): @only_with_bls(alt_return=STUB_COORDINATES) -def bls_signature_to_G2(signature): +def signature_to_G2(signature): return bls.api.signature_to_G2(signature) From 850d45dae0753d935a29d6fd153bb84468d55d5c Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 19 Dec 2019 09:25:05 +0100 Subject: [PATCH 22/48] Allow empty lists in streamed responses It's possible that block request responses end up not containing any blocks, so we need a way to encode this. --- specs/networking/p2p-interface.md | 37 ++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 84539713d..948ed5ba9 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -92,6 +92,7 @@ It consists of four main sections: - [Why are messages length-prefixed with a protobuf varint in the SSZ-encoding?](#why-are-messages-length-prefixed-with-a-protobuf-varint-in-the-ssz-encoding) - [Why do we version protocol strings with ordinals instead of semver?](#why-do-we-version-protocol-strings-with-ordinals-instead-of-semver) - [Why is it called Req/Resp and not RPC?](#why-is-it-called-reqresp-and-not-rpc) + - [Why do we allow empty responses in block requests?](#why-do-we-allow-empty-responses-in-block-requests) - [Discovery](#discovery) - [Why are we using discv5 and not libp2p Kademlia DHT?](#why-are-we-using-discv5-and-not-libp2p-kademlia-dht) - [What is the difference between an ENR and a multiaddr, and why are we using ENRs?](#what-is-the-difference-between-an-enr-and-a-multiaddr-and-why-are-we-using-enrs) @@ -314,14 +315,14 @@ Request/response messages MUST adhere to the encoding specified in the protocol ``` request ::= | -response ::= + +response ::= * response_chunk ::= | | result ::= “0” | “1” | “2” | [“128” ... ”255”] ``` The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; however, certain encodings like SSZ do, for added security. -A `response` is formed by one or more `response_chunk`s. The exact request determines whether a response consists of a single `response_chunk` or possibly many. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. The encoded-payload of a `response_chunk` has a maximum uncompressed byte size of `MAX_CHUNK_SIZE`. +A `response` is formed by zero or more `response_chunk`s. Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. All other response types (non-Lists) send a single `response_chunk`. The encoded-payload of a `response_chunk` has a maximum uncompressed byte size of `MAX_CHUNK_SIZE`. Clients MUST ensure the each encoded payload of a `response_chunk` is less than or equal to `MAX_CHUNK_SIZE`; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. @@ -344,7 +345,7 @@ The responder MUST: 1. Use the encoding strategy to read the optional header. 2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). Should this not be the case, it should be treated as a failure. 3. Deserialize the expected type, and process the request. -4. Write the response which may consist of one or more `response_chunk`s (result, optional header, payload). +4. Write the response which may consist of zero or more `response_chunk`s (result, optional header, payload). 5. Close their write side of the stream. At this point, the stream will be fully closed. If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. @@ -398,7 +399,7 @@ All messages that contain only a single field MUST be encoded directly as the ty Responses that are SSZ-lists (for example `[]SignedBeaconBlock`) send their constituents individually as `response_chunk`s. For example, the -`[]SignedBeaconBlock` response type sends one or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. +`[]SignedBeaconBlock` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. ### Messages @@ -486,7 +487,7 @@ Requests count beacon blocks from the peer starting from `start_slot` on the cha The request MUST be encoded as an SSZ-container. -The response MUST consist of at least one `response_chunk` and MAY consist of many. Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. +The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. `BeaconBlocksByRange` is primarily used to sync historical blocks. @@ -494,7 +495,7 @@ Clients MUST support requesting blocks since the start of the weak subjectivity Clients MUST support `head_block_root` values since the latest finalized epoch. -Clients MUST respond with at least one block, if they have it. +Clients MUST respond with at least one block, if they have it and it exists in the range. Clients MAY limit the number of blocks in the response. Clients MUST order blocks by increasing slot number. @@ -524,11 +525,11 @@ Requests blocks by block root (= `hash_tree_root(SignedBeaconBlock.message)`). T The request MUST be encoded as an SSZ-field. -The response MUST consist of at least one `response_chunk` and MAY consist of many. Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. +The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. Clients MUST support requesting blocks since the latest finalized epoch. -Clients MUST respond with at least one block, if they have it. +Clients MUST respond with at least one block, if they have it. Clients MAY limit the number of blocks in the response. ## The discovery domain: discv5 @@ -827,6 +828,26 @@ For this reason, we remove and replace semver with ordinals that require explici Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. +### Why do we allow empty responses in block requests? + +When requesting blocks by range or root, it may happen that there are no blocks in the selected range or the responding node does not have the requested blocks. + +Thus, it may happen that we need to transmit an empty list - there are several ways to encode this: + +0) Close the stream without sending any data +1) Add a `null` option to the `success` response, for example by introducing an additional byte +2) Respond with an error result, using a specific error code for "No data" + +Semantically, it is not an error that a block is missing during a slot making option 2 unnatural. + +Option 1 allows allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. + +Under option 0, there is no way for a client to distinguish between a slot without a block and an incomplete response, but given that it already must contain logic to handle the uncertainty of a malicious peer, option 0 was chosen. Clients should mark any slots missing blocks as unknown until they can be verified as not containing a block by successive blocks. + +Assuming option 0 with no special `null` encoding, consider a request for slots `2, 3, 4` - if there was no block produced at slot 4, the response would be `2, 3, EOF`. Now consider the same situation, but where only `4` is requested - closing the stream with only `EOF` (without any `response_chunk`) is consistent. + +Failing to provide blocks that nodes "should" have is reason to trust a peer less - for example, if a particular peer gossips a block, it should have access to its parent. If a request for the parent fails, it's indicative of poor peer quality since peers should validate blocks before gossiping them. + ## Discovery ### Why are we using discv5 and not libp2p Kademlia DHT? From 662a986d042164077fde259247ad99d2fc32e16c Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 25 Dec 2019 09:51:29 -0800 Subject: [PATCH 23/48] Update 0_fork-choice.md --- specs/core/0_fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 34430d092..b286654af 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -196,7 +196,7 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB ```python def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: """ - Retrieve a filtered block true from ``store``, only returning branches + Retrieve a filtered block tree from ``store``, only returning branches whose leaf state's justified/finalized info agrees with that in ``store``. """ base = store.justified_checkpoint.root From 765176ec8ca1dda29a7534d44afb962eb3502fd7 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 27 Dec 2019 09:37:26 +0100 Subject: [PATCH 24/48] PySpec SSZ Bytes instantiated from hex --- scripts/build_spec.py | 2 -- specs/core/0_beacon-chain.md | 14 ++++++-------- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 ++++++- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 67c9a547c..1a719cb29 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -163,8 +163,6 @@ def objects_to_spec(functions: Dict[str, str], del functions[k] functions_spec = '\n\n'.join(functions.values()) for k in list(constants.keys()): - if k.startswith('DOMAIN_'): - constants[k] = f"DomainType(({constants[k]}).to_bytes(length=4, byteorder='little'))" if k == "BLS12_381_Q": constants[k] += " # noqa: E501" constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9c0b9fe66..0c9f49b74 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -199,7 +199,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | -| `BLS_WITHDRAWAL_PREFIX` | `Bytes1(b'\x00')` | +| `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | ### Time parameters @@ -249,15 +249,13 @@ The following values are (non-configurable) constants used throughout the specif ### Domain types -The following types are defined, mapping into `DomainType` (little endian): - | Name | Value | | - | - | -| `DOMAIN_BEACON_PROPOSER` | `0` | -| `DOMAIN_BEACON_ATTESTER` | `1` | -| `DOMAIN_RANDAO` | `2` | -| `DOMAIN_DEPOSIT` | `3` | -| `DOMAIN_VOLUNTARY_EXIT` | `4` | +| `DOMAIN_BEACON_PROPOSER` | `Bytes4('0x00000000')` | +| `DOMAIN_BEACON_ATTESTER` | `Bytes4('0x01000000')` | +| `DOMAIN_RANDAO` | `Bytes4('0x02000000')` | +| `DOMAIN_DEPOSIT` | `Bytes4('0x03000000')` | +| `DOMAIN_VOLUNTARY_EXIT` | `Bytes4('0x04000000')` | ## Containers diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 017e2a22a..829c68ac1 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -131,7 +131,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | +| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `Bytes4('0x06000000')` | ### TODO PLACEHOLDER diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index d5964fe7d..408c9b2a8 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -101,8 +101,8 @@ This document describes the shard transition function (data layer only) and the | Name | Value | | - | - | -| `DOMAIN_SHARD_PROPOSER` | `128` | -| `DOMAIN_SHARD_ATTESTER` | `129` | +| `DOMAIN_SHARD_PROPOSER` | `Bytes4('0x80000000')` | +| `DOMAIN_SHARD_ATTESTER` | `Bytes4('0x81000000')` | ## Containers diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 4bda43360..4ddff1b5e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -451,10 +451,15 @@ class BaseBytes(bytes, Elements, metaclass=BytesType): @classmethod def extract_args(cls, *args): x = args - if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes)): + if len(x) == 1 and isinstance(x[0], (GeneratorType, bytes, str)): x = x[0] if isinstance(x, bytes): # Includes BytesLike return x + if isinstance(x, str): + if x[:2] == '0x': + return bytes.fromhex(x[2:]) + else: + return bytes.fromhex(x) else: return bytes(x) # E.g. GeneratorType put into bytes. From e70863ca936a63b792f690d951eb757576f872cf Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 30 Dec 2019 06:52:00 +0100 Subject: [PATCH 25/48] Bytes4 -> DomainType --- specs/core/0_beacon-chain.md | 10 +++++----- specs/core/1_custody-game.md | 2 +- specs/core/1_shard-data-chains.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0c9f49b74..647d1c9bd 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -251,11 +251,11 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | -| `DOMAIN_BEACON_PROPOSER` | `Bytes4('0x00000000')` | -| `DOMAIN_BEACON_ATTESTER` | `Bytes4('0x01000000')` | -| `DOMAIN_RANDAO` | `Bytes4('0x02000000')` | -| `DOMAIN_DEPOSIT` | `Bytes4('0x03000000')` | -| `DOMAIN_VOLUNTARY_EXIT` | `Bytes4('0x04000000')` | +| `DOMAIN_BEACON_PROPOSER` | `DomainType('0x00000000')` | +| `DOMAIN_BEACON_ATTESTER` | `DomainType('0x01000000')` | +| `DOMAIN_RANDAO` | `DomainType('0x02000000')` | +| `DOMAIN_DEPOSIT` | `DomainType('0x03000000')` | +| `DOMAIN_VOLUNTARY_EXIT` | `DomainType('0x04000000')` | ## Containers diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 829c68ac1..b1f61de2f 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -131,7 +131,7 @@ The following types are defined, mapping into `DomainType` (little endian): | Name | Value | | - | - | -| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `Bytes4('0x06000000')` | +| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `DomainType('0x06000000')` | ### TODO PLACEHOLDER diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 408c9b2a8..93570dbee 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -101,8 +101,8 @@ This document describes the shard transition function (data layer only) and the | Name | Value | | - | - | -| `DOMAIN_SHARD_PROPOSER` | `Bytes4('0x80000000')` | -| `DOMAIN_SHARD_ATTESTER` | `Bytes4('0x81000000')` | +| `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` | +| `DOMAIN_SHARD_ATTESTER` | `DomainType('0x81000000')` | ## Containers From 51bcb29e2820f3e4477f693506569905f92d55f8 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 3 Jan 2020 07:46:27 +0100 Subject: [PATCH 26/48] 'DomainWrapper' -> 'SigningRoot' --- specs/core/0_beacon-chain.md | 38 +++++++++---------- specs/core/1_custody-game.md | 8 ++-- specs/core/1_shard-data-chains.md | 4 +- specs/light_client/sync_protocol.md | 2 +- specs/validator/0_beacon-chain-validator.md | 10 ++--- .../eth2spec/test/helpers/attestations.py | 2 +- .../pyspec/eth2spec/test/helpers/block.py | 4 +- .../eth2spec/test/helpers/block_header.py | 2 +- .../pyspec/eth2spec/test/helpers/custody.py | 8 ++-- .../pyspec/eth2spec/test/helpers/deposits.py | 2 +- .../test/helpers/phase1/attestations.py | 2 +- .../test/helpers/phase1/shard_block.py | 2 +- .../eth2spec/test/helpers/voluntary_exits.py | 2 +- .../eth2spec/test/sanity/test_blocks.py | 4 +- 14 files changed, 45 insertions(+), 45 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 090738bc0..681d4b92b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -34,7 +34,7 @@ - [`DepositMessage`](#depositmessage) - [`DepositData`](#depositdata) - [`BeaconBlockHeader`](#beaconblockheader) - - [`DomainWrapper`](#domainwrapper) + - [`SigningRoot`](#signingroot) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -76,7 +76,7 @@ - [`compute_start_slot_at_epoch`](#compute_start_slot_at_epoch) - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - [`compute_domain`](#compute_domain) - - [`compute_domain_wrapper_root`](#compute_domain_wrapper_root) + - [`compute_signing_root`](#compute_signing_root) - [Beacon state accessors](#beacon-state-accessors) - [`get_current_epoch`](#get_current_epoch) - [`get_previous_epoch`](#get_previous_epoch) @@ -378,11 +378,11 @@ class BeaconBlockHeader(Container): body_root: Root ``` -#### `DomainWrapper` +#### `SigningRoot` ```python -class DomainWrapper(Container): - root: Root +class SigningRoot(Container): + object_root: Root domain: Domain ``` @@ -586,10 +586,10 @@ def bytes_to_int(data: bytes) -> uint64: Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specification](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-00). Specifically, eth2 uses the `BLS_SIG_BLS12381G2-SHA256-SSWU-RO-_POP_` ciphersuite which implements the following interfaces: -* `def Sign(SK: int, message: Bytes) -> BLSSignature` -* `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` -* `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` -* `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool` +- `def Sign(SK: int, message: Bytes) -> BLSSignature` +- `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` +- `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` +- `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. @@ -676,7 +676,7 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify aggregate signature pubkeys = [state.validators[i].pubkey for i in indices] domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) - message = compute_domain_wrapper_root(indexed_attestation.data, domain) + message = compute_signing_root(indexed_attestation.data, domain) return bls.FastAggregateVerify(pubkeys, message, indexed_attestation.signature) ``` @@ -795,12 +795,12 @@ def compute_domain(domain_type: DomainType, fork_version: Version=Version()) -> return Domain(domain_type + fork_version) ``` -### `compute_domain_wrapper_root` +### `compute_signing_root` ```python -def compute_domain_wrapper_root(ssz_object: SSZObject, domain: Domain) -> Root: - domain_wrapped_object = DomainWrapper( - root=hash_tree_root(ssz_object), +def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: + domain_wrapped_object = SigningRoot( + object_root=hash_tree_root(ssz_object), domain=domain, ) return hash_tree_root(domain_wrapped_object) @@ -1148,7 +1148,7 @@ def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, valida ```python def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: proposer = state.validators[get_beacon_proposer_index(state)] - message = compute_domain_wrapper_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) + message = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) return bls.Verify(proposer.pubkey, message, signed_block.signature) ``` @@ -1448,7 +1448,7 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: epoch = get_current_epoch(state) # Verify RANDAO reveal proposer = state.validators[get_beacon_proposer_index(state)] - message = compute_domain_wrapper_root(epoch, get_domain(state, DOMAIN_RANDAO)) + message = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) assert bls.Verify(proposer.pubkey, message, body.randao_reveal) # Mix in RANDAO reveal mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) @@ -1497,7 +1497,7 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla # Signatures are valid for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) - message = compute_domain_wrapper_root(signed_header.message, domain) + message = compute_signing_root(signed_header.message, domain) assert bls.Verify(proposer.pubkey, message, signed_header.signature) slash_validator(state, proposer_slashing.proposer_index) @@ -1580,7 +1580,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: pubkey=deposit.data.pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, amount=deposit.data.amount) - message = compute_domain_wrapper_root(deposit_message, compute_domain(DOMAIN_DEPOSIT)) + message = compute_signing_root(deposit_message, compute_domain(DOMAIN_DEPOSIT)) if not bls.Verify(pubkey, message, deposit.data.signature): return @@ -1617,7 +1617,7 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) - message = compute_domain_wrapper_root(voluntary_exit, domain) + message = compute_signing_root(voluntary_exit, domain) assert bls.Verify(validator.pubkey, message, signed_voluntary_exit.signature) # Initiate exit initiate_validator_exit(state, voluntary_exit.validator_index) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index a87ac17f1..273f27455 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -430,7 +430,7 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> # Verify signature domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - message = compute_domain_wrapper_root(epoch_to_sign, domain) + message = compute_signing_root(epoch_to_sign, domain) assert bls.Verify(revealer.pubkey, message, reveal.reveal) # Decrement max reveal lateness if response is timely @@ -482,7 +482,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived pubkeys = [revealed_validator.pubkey, masker.pubkey] domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) - messages = [compute_domain_wrapper_root(message, domain) + messages = [compute_signing_root(message, domain) for message in [hash_tree_root(reveal.epoch), reveal.mask]] assert bls.AggregateVerify(pubkeys, messages, reveal.reveal) @@ -582,7 +582,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> challenger = state.validators[challenge.challenger_index] domain = get_domain(state, DOMAIN_CUSTODY_BIT_CHALLENGE, get_current_epoch(state)) # TODO incorrect hash-tree-root, but this changes with phase 1 PR #1483 - assert bls.Verify(challenger.pubkey, compute_domain_wrapper_root(challenge, domain), challenge.signature) + assert bls.Verify(challenger.pubkey, compute_signing_root(challenge, domain), challenge.signature) # Verify challenger is slashable assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify attestation @@ -606,7 +606,7 @@ def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> challenge.responder_index, ) domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - assert bls.Verify(responder.pubkey, compute_domain_wrapper_root(epoch_to_sign, domain), challenge.responder_key) + assert bls.Verify(responder.pubkey, compute_signing_root(epoch_to_sign, domain), challenge.responder_key) # Verify the chunk count chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert chunk_count == len(challenge.chunk_bits) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index bae1d2a80..765655f74 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -386,7 +386,7 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert not proposer.slashed # Verify proposer signature domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) - assert bls.Verify(proposer.pubkey, compute_domain_wrapper_root(block, domain), block.signature) + assert bls.Verify(proposer.pubkey, compute_signing_root(block, domain), block.signature) ``` #### Attestations @@ -407,7 +407,7 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat # Verify attester aggregate signature domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) shard_attestation_data = ShardAttestationData(slot=shard_state.slot, parent_root=block.parent_root) - message = compute_domain_wrapper_root(shard_attestation_data, domain) + message = compute_signing_root(shard_attestation_data, domain) assert bls.FastAggregateVerify(pubkeys, message, block.attestations) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index de4f0be7c..744f87ea5 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -137,7 +137,7 @@ def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: # Verify shard attestations pubkeys = filter(lambda i: update.aggregation_bits[i], pubkeys) domain = compute_domain(DOMAIN_SHARD_ATTESTER, update.fork_version) - message = compute_domain_wrapper_root(update.shard_block_root, domain) + message = compute_signing_root(update.shard_block_root, domain) assert bls.FastAggregateVerify(pubkeys, message, update.signature) # Update period committees if entering a new period diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index d6366f920..ec7097617 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -117,7 +117,7 @@ To submit a deposit: - Set `deposit_data.withdrawal_credentials` to `withdrawal_credentials`. - Set `deposit_data.amount` to `amount`. - Let `deposit_message` be a `DepositMessage` with all the `DepositData` contents except the `signature`. -- Let `signature` be the result of `Sign` of the `compute_domain_wrapper_root(deposit_message, domain)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). +- Let `signature` be the result of `Sign` of the `compute_signing_root(deposit_message, domain)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). - Let `deposit_data_root` be `hash_tree_root(deposit_data)`. - Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32)` along with a deposit of `amount` Gwei. @@ -234,7 +234,7 @@ Set `block.body.randao_reveal = epoch_signature` where `epoch_signature` is obta ```python def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) - message = compute_domain_wrapper_root(compute_epoch_at_slot(block.slot), domain) + message = compute_signing_root(compute_epoch_at_slot(block.slot), domain) return bls.Sign(privkey, message) ``` @@ -312,7 +312,7 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: ```python def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - message = compute_domain_wrapper_root(header, domain) + message = compute_signing_root(header, domain) return bls.Sign(privkey, message) ``` @@ -371,7 +371,7 @@ Set `attestation.signature = signed_attestation_data` where `signed_attestation_ ```python def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - message = compute_domain_wrapper_root(attestation.data, domain) + message = compute_signing_root(attestation.data, domain) return bls.Sign(privkey, message) ``` @@ -390,7 +390,7 @@ A validator is selected to aggregate based upon the return value of `is_aggregat ```python def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, compute_epoch_at_slot(slot)) - message = compute_domain_wrapper_root(slot, domain) + message = compute_signing_root(slot, domain) return bls.Sign(privkey, message) ``` diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 1082b7045..52479fd75 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -97,7 +97,7 @@ def sign_attestation(spec, state, attestation): def get_attestation_signature(spec, state, attestation_data, privkey): domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - message = spec.compute_domain_wrapper_root(attestation_data, domain) + message = spec.compute_signing_root(attestation_data, domain) return bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 13669049f..9b2cc5d03 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -30,7 +30,7 @@ def apply_randao_reveal(spec, state, block, proposer_index=None): privkey = privkeys[proposer_index] domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) - message = spec.compute_domain_wrapper_root(spec.compute_epoch_at_slot(block.slot), domain) + message = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) block.body.randao_reveal = bls.Sign(privkey, message) @@ -42,7 +42,7 @@ def apply_sig(spec, state, signed_block, proposer_index=None): proposer_index = get_proposer_index_maybe(spec, state, block.slot, proposer_index) privkey = privkeys[proposer_index] domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - message = spec.compute_domain_wrapper_root(block, domain) + message = spec.compute_signing_root(block, domain) signed_block.signature = bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index 4f9f88644..bb5fe692f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -6,6 +6,6 @@ def sign_block_header(spec, state, header, privkey): state=state, domain_type=spec.DOMAIN_BEACON_PROPOSER, ) - message = spec.compute_domain_wrapper_root(header, domain) + message = spec.compute_signing_root(header, domain) signature = bls.Sign(privkey, message) return spec.SignedBeaconBlockHeader(message=header, signature=signature) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index 80b341526..f75cb9607 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -18,12 +18,12 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): # Generate the secret that is being revealed domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) - message = spec.compute_domain_wrapper_root(spec.Epoch(epoch), domain) + message = spec.compute_signing_root(spec.Epoch(epoch), domain) reveal = bls.Sign(privkeys[revealed_index], message) # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask - message = spec.compute_domain_wrapper_root(mask, domain) + message = spec.compute_signing_root(mask, domain) masker_signature = bls.Sign(privkeys[masker_index], message) masked_reveal = bls.Aggregate([reveal, masker_signature]) @@ -48,7 +48,7 @@ def get_valid_custody_key_reveal(spec, state, period=None): # Generate the secret that is being revealed domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) - message = spec.compute_domain_wrapper_root(spec.Epoch(epoch_to_sign), domain) + message = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) reveal = bls.Sign(privkeys[revealer_index], message) return spec.CustodyKeyReveal( revealer_index=revealer_index, @@ -74,7 +74,7 @@ def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False) # Generate the responder key domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) - message = spec.compute_domain_wrapper_root(spec.compute_domain_wrapper_root, domain) + message = spec.compute_signing_root(spec.compute_signing_root, domain) responder_key = bls.Sign(privkeys[responder_index], message) chunk_count = spec.get_custody_chunk_count(attestation.data.crosslink) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index a307f524b..720704576 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -30,7 +30,7 @@ def sign_deposit_data(spec, deposit_data, privkey, state=None): pubkey=deposit_data.pubkey, withdrawal_credentials=deposit_data.withdrawal_credentials, amount=deposit_data.amount) - message = spec.compute_domain_wrapper_root(deposit_message, domain) + message = spec.compute_signing_root(deposit_message, domain) deposit_data.signature = bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py index e521bd6c5..7947ba811 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -26,5 +26,5 @@ def sign_shard_attestation(spec, beacon_state, shard_state, block, participants) def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) - message = spec.compute_domain_wrapper(message_hash, domain) + message = spec.compute_signing_root(message_hash, domain) return bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index f7275b88a..a72a50649 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -19,7 +19,7 @@ def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None privkey = privkeys[proposer_index] domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, spec.compute_epoch_of_shard_slot(block.slot)) - message = spec.compute_domain_wrapper(block, domain) + message = spec.compute_signing_root(block, domain) block.signature = bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index 286f85e9f..f186f1533 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -3,7 +3,7 @@ from eth2spec.utils import bls def sign_voluntary_exit(spec, state, voluntary_exit, privkey): domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) - message = spec.compute_domain_wrapper_root(voluntary_exit, domain) + message = spec.compute_signing_root(voluntary_exit, domain) return spec.SignedVoluntaryExit( message=voluntary_exit, signature=bls.Sign(privkey, message) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 5a7f654cf..ae02d8c1a 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -108,7 +108,7 @@ def test_invalid_block_sig(spec, state): block = build_empty_block_for_next_slot(spec, state) domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - message = spec.compute_domain_wrapper_root(block, domain) + message = spec.compute_signing_root(block, domain) invalid_signed_block = spec.SignedBeaconBlock( message=block, signature=bls.Sign(123456, message) @@ -417,7 +417,7 @@ def test_voluntary_exit(spec, state): validator_index=validator_index, ) domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) - message = spec.compute_domain_wrapper_root(voluntary_exit, domain) + message = spec.compute_signing_root(voluntary_exit, domain) signed_voluntary_exit = spec.SignedVoluntaryExit( message=voluntary_exit, signature=bls.Sign(privkeys[validator_index], message) From 8580ec33f286895f4d30b56b7377e268e8f1260c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 3 Jan 2020 08:18:34 +0100 Subject: [PATCH 27/48] 'message' -> 'signing_root' --- specs/core/0_beacon-chain.md | 24 ++++++++++----------- specs/core/1_custody-game.md | 4 ++-- specs/core/1_shard-data-chains.md | 4 ++-- specs/light_client/sync_protocol.md | 4 ++-- specs/validator/0_beacon-chain-validator.md | 16 +++++++------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 681d4b92b..4ba4a91bf 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -676,8 +676,8 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe # Verify aggregate signature pubkeys = [state.validators[i].pubkey for i in indices] domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) - message = compute_signing_root(indexed_attestation.data, domain) - return bls.FastAggregateVerify(pubkeys, message, indexed_attestation.signature) + signing_root = compute_signing_root(indexed_attestation.data, domain) + return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature) ``` #### `is_valid_merkle_branch` @@ -1148,8 +1148,8 @@ def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, valida ```python def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: proposer = state.validators[get_beacon_proposer_index(state)] - message = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) - return bls.Verify(proposer.pubkey, message, signed_block.signature) + signing_root = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) + return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) ``` ```python @@ -1448,8 +1448,8 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: epoch = get_current_epoch(state) # Verify RANDAO reveal proposer = state.validators[get_beacon_proposer_index(state)] - message = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) - assert bls.Verify(proposer.pubkey, message, body.randao_reveal) + signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) + assert bls.Verify(proposer.pubkey, signing_root, body.randao_reveal) # Mix in RANDAO reveal mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix @@ -1497,8 +1497,8 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla # Signatures are valid for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) - message = compute_signing_root(signed_header.message, domain) - assert bls.Verify(proposer.pubkey, message, signed_header.signature) + signing_root = compute_signing_root(signed_header.message, domain) + assert bls.Verify(proposer.pubkey, signing_root, signed_header.signature) slash_validator(state, proposer_slashing.proposer_index) ``` @@ -1580,8 +1580,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: pubkey=deposit.data.pubkey, withdrawal_credentials=deposit.data.withdrawal_credentials, amount=deposit.data.amount) - message = compute_signing_root(deposit_message, compute_domain(DOMAIN_DEPOSIT)) - if not bls.Verify(pubkey, message, deposit.data.signature): + signing_root = compute_signing_root(deposit_message, compute_domain(DOMAIN_DEPOSIT)) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): return # Add validator and balance entries @@ -1617,8 +1617,8 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) - message = compute_signing_root(voluntary_exit, domain) - assert bls.Verify(validator.pubkey, message, signed_voluntary_exit.signature) + signing_root = compute_signing_root(voluntary_exit, domain) + assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) # Initiate exit initiate_validator_exit(state, voluntary_exit.validator_index) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 273f27455..23f9f3f4d 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -430,8 +430,8 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> # Verify signature domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) - message = compute_signing_root(epoch_to_sign, domain) - assert bls.Verify(revealer.pubkey, message, reveal.reveal) + signing_root = compute_signing_root(epoch_to_sign, domain) + assert bls.Verify(revealer.pubkey, signing_root, reveal.reveal) # Decrement max reveal lateness if response is timely if epoch_to_sign + EPOCHS_PER_CUSTODY_PERIOD >= get_current_epoch(state): diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 765655f74..e04e3ba5e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -407,8 +407,8 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat # Verify attester aggregate signature domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) shard_attestation_data = ShardAttestationData(slot=shard_state.slot, parent_root=block.parent_root) - message = compute_signing_root(shard_attestation_data, domain) - assert bls.FastAggregateVerify(pubkeys, message, block.attestations) + signing_root = compute_signing_root(shard_attestation_data, domain) + assert bls.FastAggregateVerify(pubkeys, signing_root, block.attestations) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) reward = attestation_count * get_base_reward(beacon_state, proposer_index) // PROPOSER_REWARD_QUOTIENT diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 744f87ea5..5a7410180 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -137,8 +137,8 @@ def update_memory(memory: LightClientMemory, update: LightClientUpdate) -> None: # Verify shard attestations pubkeys = filter(lambda i: update.aggregation_bits[i], pubkeys) domain = compute_domain(DOMAIN_SHARD_ATTESTER, update.fork_version) - message = compute_signing_root(update.shard_block_root, domain) - assert bls.FastAggregateVerify(pubkeys, message, update.signature) + signing_root = compute_signing_root(update.shard_block_root, domain) + assert bls.FastAggregateVerify(pubkeys, signing_root, update.signature) # Update period committees if entering a new period if next_period == current_period + 1: diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ec7097617..063cda12f 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -234,8 +234,8 @@ Set `block.body.randao_reveal = epoch_signature` where `epoch_signature` is obta ```python def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) - message = compute_signing_root(compute_epoch_at_slot(block.slot), domain) - return bls.Sign(privkey, message) + signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) + return bls.Sign(privkey, signing_root) ``` ##### Eth1 Data @@ -312,8 +312,8 @@ def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: ```python def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(header.slot)) - message = compute_signing_root(header, domain) - return bls.Sign(privkey, message) + signing_root = compute_signing_root(header, domain) + return bls.Sign(privkey, signing_root) ``` ### Attesting @@ -371,8 +371,8 @@ Set `attestation.signature = signed_attestation_data` where `signed_attestation_ ```python def get_signed_attestation_data(state: BeaconState, attestation: IndexedAttestation, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - message = compute_signing_root(attestation.data, domain) - return bls.Sign(privkey, message) + signing_root = compute_signing_root(attestation.data, domain) + return bls.Sign(privkey, signing_root) ``` #### Broadcast attestation @@ -390,8 +390,8 @@ A validator is selected to aggregate based upon the return value of `is_aggregat ```python def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: domain = get_domain(state, DOMAIN_BEACON_ATTESTER, compute_epoch_at_slot(slot)) - message = compute_signing_root(slot, domain) - return bls.Sign(privkey, message) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) ``` ```python From 261b6c0d239ebd96005a7e704c2053cb3fc6af96 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Jan 2020 09:37:37 -0700 Subject: [PATCH 28/48] modify get_eth1_data to use timestamp instead of block height --- configs/mainnet.yaml | 2 + configs/minimal.yaml | 2 + specs/validator/0_beacon-chain-validator.md | 49 ++++++++++++++++----- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 0c3c058d5..d2d219562 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -39,6 +39,8 @@ TARGET_AGGREGATORS_PER_COMMITTEE: 16 RANDOM_SUBNETS_PER_VALIDATOR: 1 # 2**8 (= 256) EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 # Deposit contract diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 7adc82eae..aa3756fd3 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -39,6 +39,8 @@ TARGET_AGGREGATORS_PER_COMMITTEE: 16 RANDOM_SUBNETS_PER_VALIDATOR: 1 # 2**8 (= 256) EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 # Deposit contract diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 341fb8e8c..99b23c3fe 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -31,6 +31,8 @@ - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - [Randao reveal](#randao-reveal) - [Eth1 Data](#eth1-data) + - [`Eth1Block`](#eth1block) + - [`get_eth1_data`](#get_eth1_data) - [Proposer slashings](#proposer-slashings) - [Attester slashings](#attester-slashings) - [Attestations](#attestations) @@ -85,6 +87,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | +| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | ## Becoming a validator @@ -239,28 +242,50 @@ def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> ##### Eth1 Data -The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 data. This recent data contains an Eth1 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth1 block. If over half of the block proposers in the current Eth1 voting period vote for the same `eth1_data` then `state.eth1_data` updates at the end of the voting period. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. +The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 data. This recent data contains an Eth1 block hash as well as the associated deposit root (as calculated by the `get_deposit_root()` method of the deposit contract) and deposit count after execution of the corresponding Eth1 block. If over half of the block proposers in the current Eth1 voting period vote for the same `eth1_data` then `state.eth1_data` updates immediately allowing new deposits to be processed. Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. -Let `get_eth1_data(distance: uint64) -> Eth1Data` be the (subjective) function that returns the Eth1 data at distance `distance` relative to the Eth1 head at the start of the current Eth1 voting period. Let `previous_eth1_distance` be the distance relative to the Eth1 block corresponding to `eth1_data.block_hash` found in the state at the _start_ of the current Eth1 voting period. Note that `eth1_data` can be updated in the middle of a voting period and thus the starting `eth1_data.block_hash` must be stored separately. +###### `Eth1Block` -An honest block proposer sets `block.body.eth1_data = get_eth1_vote(state, previous_eth1_distance)` where: +Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` field available. ```python -def get_eth1_vote(state: BeaconState, previous_eth1_distance: uint64) -> Eth1Data: - new_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, 2 * ETH1_FOLLOW_DISTANCE)] - all_eth1_data = [get_eth1_data(distance) for distance in range(ETH1_FOLLOW_DISTANCE, previous_eth1_distance)] +class Eth1Block(Container): + timestamp: uint64 + # All other eth1 block fields +``` - period_tail = state.slot % SLOTS_PER_ETH1_VOTING_PERIOD >= integer_squareroot(SLOTS_PER_ETH1_VOTING_PERIOD) - if period_tail: - votes_to_consider = all_eth1_data - else: - votes_to_consider = new_eth1_data +###### `get_eth1_data` + +Let `get_eth1_data(block: Eth1Block) -> Eth1Data` be the function that returns the Eth1 data for a given Eth1 block. + +An honest block proposer sets `block.body.eth1_data = get_eth1_vote(state)` where: + +```python +def voting_period_start_time(state: BeaconState) -> uint64: + eth1_voting_period_start_slot = state.slot % SLOTS_PER_ETH1_VOTING_PERIOD + return state.genesis_time + eth1_voting_period_start_slot * SECONDS_PER_SLOT +``` + +```python +def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: + return ( + block.timestamp <= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE + and block.timestamp >= period_start - SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 + ) +``` + +```python +def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: + period_start = voting_period_start_time(state) + # `eth1_chain` abstractly represents all blocks in the eth1 chain. + votes_to_consider = [get_eth1_data(block) for block in eth1_chain if + is_candidate_block(block, period_start)] valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] return max( valid_votes, - key=lambda v: (valid_votes.count(v), -all_eth1_data.index(v)), # Tiebreak by smallest distance + key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance default=get_eth1_data(ETH1_FOLLOW_DISTANCE), ) ``` From 6dbc02031db930daf4f8f9d64793048f43aee857 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 3 Jan 2020 17:48:03 -0700 Subject: [PATCH 29/48] add GENESIS_FORK_VERSION and make associated modifications to support configuration of this variable --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/core/0_beacon-chain.md | 9 ++++++++- specs/validator/0_beacon-chain-validator.md | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 0c3c058d5..b61c25e6d 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -63,6 +63,8 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 # --------------------------------------------------------------- # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 +# Mainnet initial fork version, recommend altering for testnets +GENESIS_FORK_VERSION: 0x00000000 BLS_WITHDRAWAL_PREFIX: 0x00 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 7adc82eae..1be5d85ee 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -63,6 +63,8 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 # --------------------------------------------------------------- # 0, GENESIS_EPOCH is derived from this constant GENESIS_SLOT: 0 +# Highest byte set to 0x01 to avoid collisions with mainnet versioning +GENESIS_FORK_VERSION: 0x00000001 BLS_WITHDRAWAL_PREFIX: 0x00 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 647d1c9bd..ffc292059 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -199,6 +199,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `GENESIS_SLOT` | `Slot(0)` | | `GENESIS_EPOCH` | `Epoch(0)` | +| `GENESIS_FORK_VERSION` | `Version('0x00000000')` | | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | ### Time parameters @@ -780,7 +781,7 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: #### `compute_domain` ```python -def compute_domain(domain_type: DomainType, fork_version: Version=Version()) -> Domain: +def compute_domain(domain_type: DomainType, fork_version: Version=GENESIS_FORK_VERSION) -> Domain: """ Return the domain for the ``domain_type`` and ``fork_version``. """ @@ -1063,8 +1064,14 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, eth1_timestamp: uint64, deposits: Sequence[Deposit]) -> BeaconState: + fork = Fork( + previous_version=GENESIS_FORK_VERSION, + current_version=GENESIS_FORK_VERSION, + epoch=GENESIS_EPOCH, + ) state = BeaconState( genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, + fork=fork, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 341fb8e8c..f52bdc495 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -117,7 +117,7 @@ To submit a deposit: - Set `deposit_data.withdrawal_credentials` to `withdrawal_credentials`. - Set `deposit_data.amount` to `amount`. - Let `deposit_message` be a `DepositMessage` with all the `DepositData` contents except the `signature`. -- Let `signature` be the result of `bls_sign` of the `hash_tree_root(deposit_message)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `compute_domain` will default to zeroes there). +- Let `signature` be the result of `bls_sign` of the `hash_tree_root(deposit_message)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (_Warning_: Deposits _must_ be signed with `GENESIS_FORK_VERSION`, calling `compute_domain` without a second argument defaults to the correct version). - Let `deposit_data_root` be `hash_tree_root(deposit_data)`. - Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32)` along with a deposit of `amount` Gwei. From 43d095214c653c8a5ed6243ddb69863034ce1945 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 5 Jan 2020 14:10:06 -0700 Subject: [PATCH 30/48] add note that validator should broadcast attestation immediately when receiving block from expected proposer --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index f52bdc495..5d27eee47 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -318,7 +318,7 @@ def get_block_signature(state: BeaconState, header: BeaconBlockHeader, privkey: A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `index`, and assigned `slot` for which the validator performs this role during an epoch are defined by `get_committee_assignment(state, epoch, validator_index)`. -A validator should create and broadcast the `attestation` to the associated attestation subnet one-third of the way through the `slot` during which the validator is assigned―that is, `SECONDS_PER_SLOT / 3` seconds after the start of `slot`. +A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid block from the expected block proposer for the assigned `slot` or (b) one-third of the `slot` hash transpired (`SECONDS_PER_SLOT / 3` seconds after the start of `slot`) -- whichever comes _first_. *Note*: Although attestations during `GENESIS_EPOCH` do not count toward FFG finality, these initial attestations do give weight to the fork choice, are rewarded fork, and should be made. From 8515aec7aaeb465a61ccdef83fad672bfc69a642 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 5 Jan 2020 15:03:13 -0700 Subject: [PATCH 31/48] move GENESIS_SLOT/EPOCH to constants as they are not truly configurable --- configs/mainnet.yaml | 2 -- configs/minimal.yaml | 2 -- specs/core/0_beacon-chain.md | 4 ++-- specs/validator/0_beacon-chain-validator.md | 3 ++- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index c211f8b43..866b6520b 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -63,8 +63,6 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 # Initial values # --------------------------------------------------------------- -# 0, GENESIS_EPOCH is derived from this constant -GENESIS_SLOT: 0 # Mainnet initial fork version, recommend altering for testnets GENESIS_FORK_VERSION: 0x00000000 BLS_WITHDRAWAL_PREFIX: 0x00 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 7f26879d5..327cadbf3 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -63,8 +63,6 @@ EFFECTIVE_BALANCE_INCREMENT: 1000000000 # Initial values # --------------------------------------------------------------- -# 0, GENESIS_EPOCH is derived from this constant -GENESIS_SLOT: 0 # Highest byte set to 0x01 to avoid collisions with mainnet versioning GENESIS_FORK_VERSION: 0x00000001 BLS_WITHDRAWAL_PREFIX: 0x00 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ffc292059..614002056 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -158,6 +158,8 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | +| `GENESIS_SLOT` | `Slot(0)` | +| `GENESIS_EPOCH` | `Epoch(0)` | | `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | | `BASE_REWARDS_PER_EPOCH` | `4` | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | @@ -197,8 +199,6 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | -| `GENESIS_SLOT` | `Slot(0)` | -| `GENESIS_EPOCH` | `Epoch(0)` | | `GENESIS_FORK_VERSION` | `Version('0x00000000')` | | `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 62e80fe03..492da1aa3 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -263,7 +263,8 @@ An honest block proposer sets `block.body.eth1_data = get_eth1_vote(state)` wher ```python def voting_period_start_time(state: BeaconState) -> uint64: eth1_voting_period_start_slot = state.slot % SLOTS_PER_ETH1_VOTING_PERIOD - return state.genesis_time + eth1_voting_period_start_slot * SECONDS_PER_SLOT + time_since_genesis = (eth1_voting_period_start_slot - GENESIS_SLOT) * SECONDS_PER_SLOT + return state.genesis_time + time_since_genesis ``` ```python From 50c8727ae7f520274864a232417d013aecaf17b5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 5 Jan 2020 15:29:23 -0700 Subject: [PATCH 32/48] make genesis delay configurable --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ specs/core/0_beacon-chain.md | 4 ++-- test_libs/pyspec/eth2spec/test/genesis/test_initialization.py | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index c211f8b43..7713f9863 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -72,6 +72,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- +# 86400 seconds (1 day) +MIN_GENESIS_DELAY: 86400 # 12 seconds SECONDS_PER_SLOT: 12 # 2**0 (= 1) slots 12 seconds diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 7f26879d5..70bb6ca34 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -72,6 +72,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- +# [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis +MIN_GENESIS_DELAY: 300 # [customized] Faster for testing purposes SECONDS_PER_SLOT: 6 # 2**0 (= 1) slots 6 seconds diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ffc292059..9ffce3317 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -161,7 +161,6 @@ The following values are (non-configurable) constants used throughout the specif | `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | | `BASE_REWARDS_PER_EPOCH` | `4` | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | -| `SECONDS_PER_DAY` | `86400` | | `JUSTIFICATION_BITS_LENGTH` | `4` | | `ENDIANNESS` | `'little'` | @@ -206,6 +205,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | +| `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | | `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | @@ -1070,7 +1070,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, epoch=GENESIS_EPOCH, ) state = BeaconState( - genesis_time=eth1_timestamp - eth1_timestamp % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY, + genesis_time=eth1_timestamp - eth1_timestamp % MIN_GENESIS_DELAY + 2 * MIN_GENESIS_DELAY, fork=fork, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py index 462065bb9..9b326590f 100644 --- a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py @@ -20,7 +20,7 @@ def test_initialize_beacon_state_from_eth1(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY + assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY assert len(state.validators) == deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == deposit_count @@ -55,7 +55,7 @@ def test_initialize_beacon_state_some_small_balances(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY + assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY assert len(state.validators) == small_deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == len(deposits) From 71206c9a261aaab60366beaf371095c3406cd2a8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 16:04:34 +0100 Subject: [PATCH 33/48] deposit with other fork version --- .../pyspec/eth2spec/test/helpers/deposits.py | 5 +++ .../block_processing/test_process_deposit.py | 32 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index fdab01ca9..125a9e73c 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -49,6 +49,11 @@ def build_deposit(spec, deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) index = len(deposit_data_list) deposit_data_list.append(deposit_data) + return deposit_from_context(spec, deposit_data_list, index) + + +def deposit_from_context(spec, deposit_data_list, index): + deposit_data = deposit_data_list[index] root = hash_tree_root(List[spec.DepositData, 2**spec.DEPOSIT_CONTRACT_TREE_DEPTH](*deposit_data_list)) tree = calc_merkle_tree_from_leaves(tuple([d.hash_tree_root() for d in deposit_data_list])) proof = list(get_merkle_proof(tree, item_index=index, tree_len=32)) + [(index + 1).to_bytes(32, 'little')] diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 1cef99394..71d23dcba 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -3,9 +3,10 @@ from eth2spec.test.helpers.deposits import ( build_deposit, prepare_state_and_deposit, sign_deposit_data, -) + deposit_from_context) from eth2spec.test.helpers.state import get_balance from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.utils.bls import bls_sign def run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True): @@ -93,6 +94,35 @@ def test_new_deposit_over_max(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) +@with_all_phases +@spec_state_test +@always_bls +def test_invalid_sig_other_version(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] + + deposit_data = spec.DepositData( + pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, + signature=bls_sign( + message_hash=spec.hash_tree_root( + spec.DepositMessage(pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount)), + privkey=privkey, + domain=spec.compute_domain(domain_type=spec.DOMAIN_DEPOSIT, fork_version=spec.Version('0xaabbccdd')), + ) + ) + deposit, root, _ = deposit_from_context(spec, [deposit_data], 0) + + state.eth1_deposit_index = 0 + state.eth1_data.deposit_root = root + state.eth1_data.deposit_count = 1 + + yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=False) + + @with_all_phases @spec_state_test @always_bls From 60954286f9e41f2870721889628ec91494964ae9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 16:16:38 +0100 Subject: [PATCH 34/48] make tests correctly sign for general genesis-domain --- .../pyspec/eth2spec/test/helpers/deposits.py | 22 +++++-------------- .../block_processing/test_process_deposit.py | 15 +++++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 125a9e73c..071e177fd 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -5,27 +5,18 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.utils.ssz.ssz_typing import List -def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=None, signed=False): +def build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, signed=False): deposit_data = spec.DepositData( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, ) if signed: - sign_deposit_data(spec, deposit_data, privkey, state) + sign_deposit_data(spec, deposit_data, privkey) return deposit_data -def sign_deposit_data(spec, deposit_data, privkey, state=None): - if state is None: - # Genesis - domain = spec.compute_domain(spec.DOMAIN_DEPOSIT) - else: - domain = spec.get_domain( - state, - spec.DOMAIN_DEPOSIT, - ) - +def sign_deposit_data(spec, deposit_data, privkey): deposit_message = spec.DepositMessage( pubkey=deposit_data.pubkey, withdrawal_credentials=deposit_data.withdrawal_credentials, @@ -33,20 +24,19 @@ def sign_deposit_data(spec, deposit_data, privkey, state=None): signature = bls_sign( message_hash=hash_tree_root(deposit_message), privkey=privkey, - domain=domain, + domain=spec.compute_domain(spec.DOMAIN_DEPOSIT), ) deposit_data.signature = signature def build_deposit(spec, - state, deposit_data_list, pubkey, privkey, amount, withdrawal_credentials, signed): - deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, state=state, signed=signed) + deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, signed=signed) index = len(deposit_data_list) deposit_data_list.append(deposit_data) return deposit_from_context(spec, deposit_data_list, index) @@ -75,7 +65,6 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] deposit, root, deposit_data_list = build_deposit( spec, - None, deposit_data_list, pubkey, privkey, @@ -103,7 +92,6 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c deposit, root, deposit_data_list = build_deposit( spec, - state, deposit_data_list, pubkey, privkey, diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 71d23dcba..25222664d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -105,6 +105,7 @@ def test_invalid_sig_other_version(spec, state): privkey = privkeys[validator_index] withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] + # Go through the effort of manually signing, not something normally done. This sig domain will be invalid. deposit_data = spec.DepositData( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, signature=bls_sign( @@ -123,6 +124,18 @@ def test_invalid_sig_other_version(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=False) +@with_all_phases +@spec_state_test +@always_bls +def test_valid_sig_but_forked_state(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + # deposits will always be valid, regardless of the current fork + state.fork.current_version = spec.Version('0x1234abcd') + deposit = prepare_state_and_deposit(spec, state, validator_index, amount, signed=True) + yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) + + @with_all_phases @spec_state_test @always_bls @@ -185,7 +198,6 @@ def test_wrong_deposit_for_deposit_count(spec, state): privkey_1 = privkeys[index_1] _, _, deposit_data_leaves = build_deposit( spec, - state, deposit_data_leaves, pubkey_1, privkey_1, @@ -201,7 +213,6 @@ def test_wrong_deposit_for_deposit_count(spec, state): privkey_2 = privkeys[index_2] deposit_2, root_2, deposit_data_leaves = build_deposit( spec, - state, deposit_data_leaves, pubkey_2, privkey_2, From 8391d8ee5b847f45914d86bd073c473172f72fb1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 Jan 2020 16:22:18 +0100 Subject: [PATCH 35/48] missed deposit case, also fix signing here --- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 25222664d..05a40407b 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -238,6 +238,6 @@ def test_bad_merkle_proof(spec, state): # mess up merkle branch deposit.proof[5] = spec.Bytes32() - sign_deposit_data(spec, deposit.data, privkeys[validator_index], state=state) + sign_deposit_data(spec, deposit.data, privkeys[validator_index]) yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) From 9ea03dce6045ed515fdd03d9a66472271c75afe6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 6 Jan 2020 12:13:15 -0700 Subject: [PATCH 36/48] add compute_time_at_slot helper in validator doc --- specs/validator/0_beacon-chain-validator.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 492da1aa3..49261f21a 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -260,11 +260,15 @@ Let `get_eth1_data(block: Eth1Block) -> Eth1Data` be the function that returns t An honest block proposer sets `block.body.eth1_data = get_eth1_vote(state)` where: +```python +def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: + return state.genesis_time + slot * SECONDS_PER_SLOT +``` + ```python def voting_period_start_time(state: BeaconState) -> uint64: - eth1_voting_period_start_slot = state.slot % SLOTS_PER_ETH1_VOTING_PERIOD - time_since_genesis = (eth1_voting_period_start_slot - GENESIS_SLOT) * SECONDS_PER_SLOT - return state.genesis_time + time_since_genesis + eth1_voting_period_start_slot = Slot(state.slot % SLOTS_PER_ETH1_VOTING_PERIOD) + return compute_time_at_slot(state, eth1_voting_period_start_slot) ``` ```python From b637b9ad72cd18619aaf8a7cd7422e31f7b190c6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 6 Jan 2020 17:19:31 -0700 Subject: [PATCH 37/48] fix mod bug for voting period --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 49261f21a..36b4f4ac0 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -267,7 +267,7 @@ def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: ```python def voting_period_start_time(state: BeaconState) -> uint64: - eth1_voting_period_start_slot = Slot(state.slot % SLOTS_PER_ETH1_VOTING_PERIOD) + eth1_voting_period_start_slot = Slot(state.slot - state.slot % SLOTS_PER_ETH1_VOTING_PERIOD) return compute_time_at_slot(state, eth1_voting_period_start_slot) ``` From fbfe024e7ad13b62efd5c2d4c56a34c5b15b45a9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 7 Jan 2020 17:20:17 +0800 Subject: [PATCH 38/48] Rename vyper file from `.v.py` to `.vy` --- .gitattributes | 1 + Makefile | 2 +- .../{validator_registration.v.py => validator_registration.vy} | 0 deposit_contract/tests/contracts/utils.py | 2 +- specs/core/0_deposit-contract.md | 2 +- 5 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 .gitattributes rename deposit_contract/contracts/{validator_registration.v.py => validator_registration.vy} (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..c2b17bf1a --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.vy linguist-language=Python diff --git a/Makefile b/Makefile index 2b165ad7d..51bff90d0 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) compile_deposit_contract: cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \ - python tool/compile_deposit_contract.py contracts/validator_registration.v.py; + python tool/compile_deposit_contract.py contracts/validator_registration.vy; test_deposit_contract: cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \ diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.vy similarity index 100% rename from deposit_contract/contracts/validator_registration.v.py rename to deposit_contract/contracts/validator_registration.vy diff --git a/deposit_contract/tests/contracts/utils.py b/deposit_contract/tests/contracts/utils.py index de7c54489..12eac5832 100644 --- a/deposit_contract/tests/contracts/utils.py +++ b/deposit_contract/tests/contracts/utils.py @@ -5,7 +5,7 @@ DIR = os.path.dirname(__file__) def get_deposit_contract_code(): - file_path = os.path.join(DIR, './../../contracts/validator_registration.v.py') + file_path = os.path.join(DIR, './../../contracts/validator_registration.vy') deposit_contract_code = open(file_path).read() return deposit_contract_code diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index c9f366330..11be41b86 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -61,6 +61,6 @@ Every Ethereum 1.0 deposit emits a `DepositEvent` log for consumption by the bea ## Vyper code -The deposit contract source code, written in Vyper, is available [here](../../deposit_contract/contracts/validator_registration.v.py). +The deposit contract source code, written in Vyper, is available [here](../../deposit_contract/contracts/validator_registration.vy). *Note*: To save on gas, the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. From 8948393e7699cdbee1492f22bbd9a360b8afc807 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 7 Jan 2020 13:07:09 +0100 Subject: [PATCH 39/48] Remove references to BLS messages --- specs/core/0_beacon-chain.md | 7 +++++-- specs/core/1_custody-game.md | 6 ++---- .../pyspec/eth2spec/test/helpers/attestations.py | 4 ++-- test_libs/pyspec/eth2spec/test/helpers/block.py | 8 ++++---- .../pyspec/eth2spec/test/helpers/block_header.py | 4 ++-- .../pyspec/eth2spec/test/helpers/custody.py | 16 ++++++++-------- .../pyspec/eth2spec/test/helpers/deposits.py | 4 ++-- .../eth2spec/test/helpers/phase1/attestations.py | 4 ++-- .../eth2spec/test/helpers/phase1/shard_block.py | 4 ++-- .../eth2spec/test/helpers/voluntary_exits.py | 4 ++-- .../pyspec/eth2spec/test/sanity/test_blocks.py | 8 ++++---- 11 files changed, 35 insertions(+), 34 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4ba4a91bf..2bd7f23b7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -799,6 +799,9 @@ def compute_domain(domain_type: DomainType, fork_version: Version=Version()) -> ```python def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: + """ + Return the signing root of an object by calculating the root of the object-domain tree. + """ domain_wrapped_object = SigningRoot( object_root=hash_tree_root(ssz_object), domain=domain, @@ -959,11 +962,11 @@ def get_total_active_balance(state: BeaconState) -> Gwei: #### `get_domain` ```python -def get_domain(state: BeaconState, domain_type: DomainType, message_epoch: Epoch=None) -> Domain: +def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) -> Domain: """ Return the signature domain (fork version concatenated with domain type) of a message. """ - epoch = get_current_epoch(state) if message_epoch is None else message_epoch + epoch = get_current_epoch(state) if epoch is None else epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version return compute_domain(domain_type, fork_version) ``` diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 23f9f3f4d..4df9c3352 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -482,10 +482,8 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived pubkeys = [revealed_validator.pubkey, masker.pubkey] domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) - messages = [compute_signing_root(message, domain) - for message in [hash_tree_root(reveal.epoch), reveal.mask]] - - assert bls.AggregateVerify(pubkeys, messages, reveal.reveal) + signing_roots = [compute_signing_root(root, domain) for root in [hash_tree_root(reveal.epoch), reveal.mask]] + assert bls.AggregateVerify(pubkeys, signing_roots, reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 52479fd75..cb3e86320 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -97,8 +97,8 @@ def sign_attestation(spec, state, attestation): def get_attestation_signature(spec, state, attestation_data, privkey): domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - message = spec.compute_signing_root(attestation_data, domain) - return bls.Sign(privkey, message) + signing_root = spec.compute_signing_root(attestation_data, domain) + return bls.Sign(privkey, signing_root) def fill_aggregate_attestation(spec, state, attestation, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 9b2cc5d03..dda03cbf1 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -30,8 +30,8 @@ def apply_randao_reveal(spec, state, block, proposer_index=None): privkey = privkeys[proposer_index] domain = spec.get_domain(state, spec.DOMAIN_RANDAO, spec.compute_epoch_at_slot(block.slot)) - message = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) - block.body.randao_reveal = bls.Sign(privkey, message) + signing_root = spec.compute_signing_root(spec.compute_epoch_at_slot(block.slot), domain) + block.body.randao_reveal = bls.Sign(privkey, signing_root) # Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. @@ -42,9 +42,9 @@ def apply_sig(spec, state, signed_block, proposer_index=None): proposer_index = get_proposer_index_maybe(spec, state, block.slot, proposer_index) privkey = privkeys[proposer_index] domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - message = spec.compute_signing_root(block, domain) + signing_root = spec.compute_signing_root(block, domain) - signed_block.signature = bls.Sign(privkey, message) + signed_block.signature = bls.Sign(privkey, signing_root) def sign_block(spec, state, block, proposer_index=None): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index bb5fe692f..c1bc746cc 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -6,6 +6,6 @@ def sign_block_header(spec, state, header, privkey): state=state, domain_type=spec.DOMAIN_BEACON_PROPOSER, ) - message = spec.compute_signing_root(header, domain) - signature = bls.Sign(privkey, message) + signing_root = spec.compute_signing_root(header, domain) + signature = bls.Sign(privkey, signing_root) return spec.SignedBeaconBlockHeader(message=header, signature=signature) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py index f75cb9607..e00d64a17 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/custody.py +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -18,13 +18,13 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): # Generate the secret that is being revealed domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) - message = spec.compute_signing_root(spec.Epoch(epoch), domain) - reveal = bls.Sign(privkeys[revealed_index], message) + signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) + reveal = bls.Sign(privkeys[revealed_index], signing_root) # Generate the mask (any random 32 bytes that don't reveal the masker's secret will do) mask = hash(reveal) # Generate masker's signature on the mask - message = spec.compute_signing_root(mask, domain) - masker_signature = bls.Sign(privkeys[masker_index], message) + signing_root = spec.compute_signing_root(mask, domain) + masker_signature = bls.Sign(privkeys[masker_index], signing_root) masked_reveal = bls.Aggregate([reveal, masker_signature]) return spec.EarlyDerivedSecretReveal( @@ -48,8 +48,8 @@ def get_valid_custody_key_reveal(spec, state, period=None): # Generate the secret that is being revealed domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) - message = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) - reveal = bls.Sign(privkeys[revealer_index], message) + signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) + reveal = bls.Sign(privkeys[revealer_index], signing_root) return spec.CustodyKeyReveal( revealer_index=revealer_index, reveal=reveal, @@ -74,8 +74,8 @@ def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False) # Generate the responder key domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) - message = spec.compute_signing_root(spec.compute_signing_root, domain) - responder_key = bls.Sign(privkeys[responder_index], message) + signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) + responder_key = bls.Sign(privkeys[responder_index], signing_root) chunk_count = spec.get_custody_chunk_count(attestation.data.crosslink) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 720704576..337ad7d82 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -30,8 +30,8 @@ def sign_deposit_data(spec, deposit_data, privkey, state=None): pubkey=deposit_data.pubkey, withdrawal_credentials=deposit_data.withdrawal_credentials, amount=deposit_data.amount) - message = spec.compute_signing_root(deposit_message, domain) - deposit_data.signature = bls.Sign(privkey, message) + signing_root = spec.compute_signing_root(deposit_message, domain) + deposit_data.signature = bls.Sign(privkey, signing_root) def build_deposit(spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py index 7947ba811..622183fe9 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -26,5 +26,5 @@ def sign_shard_attestation(spec, beacon_state, shard_state, block, participants) def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_ATTESTER, block_epoch) - message = spec.compute_signing_root(message_hash, domain) - return bls.Sign(privkey, message) + signing_root = spec.compute_signing_root(message_hash, domain) + return bls.Sign(privkey, signing_root) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index a72a50649..6e1fba8dc 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -19,8 +19,8 @@ def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None privkey = privkeys[proposer_index] domain = spec.get_domain(beacon_state, spec.DOMAIN_SHARD_PROPOSER, spec.compute_epoch_of_shard_slot(block.slot)) - message = spec.compute_signing_root(block, domain) - block.signature = bls.Sign(privkey, message) + signing_root = spec.compute_signing_root(block, domain) + block.signature = bls.Sign(privkey, signing_root) def build_empty_shard_block(spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index f186f1533..55310ef7d 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -3,8 +3,8 @@ from eth2spec.utils import bls def sign_voluntary_exit(spec, state, voluntary_exit, privkey): domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) - message = spec.compute_signing_root(voluntary_exit, domain) + signing_root = spec.compute_signing_root(voluntary_exit, domain) return spec.SignedVoluntaryExit( message=voluntary_exit, - signature=bls.Sign(privkey, message) + signature=bls.Sign(privkey, signing_root) ) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index ae02d8c1a..b386d36b4 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -108,10 +108,10 @@ def test_invalid_block_sig(spec, state): block = build_empty_block_for_next_slot(spec, state) domain = spec.get_domain(state, spec.DOMAIN_BEACON_PROPOSER, spec.compute_epoch_at_slot(block.slot)) - message = spec.compute_signing_root(block, domain) + signing_root = spec.compute_signing_root(block, domain) invalid_signed_block = spec.SignedBeaconBlock( message=block, - signature=bls.Sign(123456, message) + signature=bls.Sign(123456, signing_root) ) expect_assertion_error(lambda: spec.state_transition(state, invalid_signed_block)) @@ -417,10 +417,10 @@ def test_voluntary_exit(spec, state): validator_index=validator_index, ) domain = spec.get_domain(state, spec.DOMAIN_VOLUNTARY_EXIT) - message = spec.compute_signing_root(voluntary_exit, domain) + signing_root = spec.compute_signing_root(voluntary_exit, domain) signed_voluntary_exit = spec.SignedVoluntaryExit( message=voluntary_exit, - signature=bls.Sign(privkeys[validator_index], message) + signature=bls.Sign(privkeys[validator_index], signing_root) ) # Add to state via block transition From 12ea891ce5b6716918e9c65afffcd3e92f427bfb Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 7 Jan 2020 20:52:20 +0100 Subject: [PATCH 40/48] `py_ecc` 2.0.0 baby! --- specs/core/0_beacon-chain.md | 1 + specs/core/1_custody-game.md | 2 +- .../pyspec/eth2spec/test/helpers/keys.py | 4 ++-- test_libs/pyspec/eth2spec/utils/bls.py | 20 +++++++++---------- test_libs/pyspec/requirements.txt | 2 +- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2bd7f23b7..afa1ea657 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -590,6 +590,7 @@ Eth2 makes use of BLS signatures as specified in the [IETF draft BLS specificati - `def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` - `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` - `def FastAggregateVerify(PKs: Sequence[BLSSignature], message: Bytes, signature: BLSSignature) -> bool` +- `def AggregateVerify(pairs: Sequence[PK: BLSSignature, message: Bytes], signature: BLSSignature) -> bool` Within these specifications, BLS signatures are treated as a module for notational clarity, thus to verify a signature `bls.Verify(...)` is used. diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 4df9c3352..fdaf9af42 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -483,7 +483,7 @@ def process_early_derived_secret_reveal(state: BeaconState, reveal: EarlyDerived domain = get_domain(state, DOMAIN_RANDAO, reveal.epoch) signing_roots = [compute_signing_root(root, domain) for root in [hash_tree_root(reveal.epoch), reveal.mask]] - assert bls.AggregateVerify(pubkeys, signing_roots, reveal.reveal) + assert bls.AggregateVerify(zip(pubkeys, signing_roots), reveal.reveal) if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: # Full slashing when the secret was revealed so early it may be a valid custody diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/test_libs/pyspec/eth2spec/test/helpers/keys.py index f47cd7c10..23bb95131 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/keys.py +++ b/test_libs/pyspec/eth2spec/test/helpers/keys.py @@ -1,6 +1,6 @@ -from py_ecc import bls +from py_ecc.bls import G2ProofOfPossession as bls from eth2spec.phase0 import spec privkeys = [i + 1 for i in range(spec.SLOTS_PER_EPOCH * 16)] -pubkeys = [bls.privtopub(privkey) for privkey in privkeys] +pubkeys = [bls.PrivToPub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index 3cfc6121e..83371ac62 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -1,11 +1,12 @@ -from py_ecc import bls +from py_ecc.bls import G2ProofOfPossession as bls +from py_ecc.bls.g2_primatives import signature_to_G2 as _signature_to_G2 # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. bls_active = True STUB_SIGNATURE = b'\x11' * 96 STUB_PUBKEY = b'\x22' * 48 -STUB_COORDINATES = bls.api.signature_to_G2(bls.sign(b"", 0, b"\0" * 8)) +STUB_COORDINATES = _signature_to_G2(bls.Sign(0, b"")) def only_with_bls(alt_return=None): @@ -24,30 +25,29 @@ def only_with_bls(alt_return=None): @only_with_bls(alt_return=True) def Verify(PK, message, signature): - return bls.verify(message_hash=message, pubkey=PK, signature=signature, domain=b'') + return bls.Verify(PK, message, signature) @only_with_bls(alt_return=True) -def AggregateVerify(PKs, messages, signature): - return bls.verify_multiple(pubkeys=PKs, message_hashes=messages, signature=signature, domain=b'') +def AggregateVerify(pairs, signature): + return bls.AggregateVerify(pairs, signature) @only_with_bls(alt_return=True) def FastAggregateVerify(PKs, message, signature): - aggregate_pubkey = bls.aggregate_pubkeys(PKs) - return bls.verify(pubkey=aggregate_pubkey, message_hash=message, signature=signature, domain=b'') + return bls.FastAggregateVerify(PKs, message, signature) @only_with_bls(alt_return=STUB_SIGNATURE) def Aggregate(signatures): - return bls.aggregate_signatures(signatures) + return bls.Aggregate(signatures) @only_with_bls(alt_return=STUB_SIGNATURE) def Sign(SK, message): - return bls.sign(message_hash=message, privkey=SK, domain=b'') + return bls.Sign(SK, message) @only_with_bls(alt_return=STUB_COORDINATES) def signature_to_G2(signature): - return bls.api.signature_to_G2(signature) + return _signature_to_G2(signature) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 8dfbccf66..60acb5d35 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -1,6 +1,6 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.9.4 -py_ecc==1.7.1 +py_ecc==2.0.0 dataclasses==0.6 ssz==0.1.3 From 1f70d3eb975334e1c3ea6dd9ad552172064462a6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 7 Jan 2020 15:12:24 -0700 Subject: [PATCH 41/48] add additional upgrade to py_ecc 2.0.0 to packages --- test_generators/bls/requirements.txt | 2 +- test_libs/pyspec/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 84a28d357..8a72affe0 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ -py_ecc==1.7.1 +py_ecc==2.0.0 eth-utils==1.6.0 ../../test_libs/gen_helpers diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index d41412eb8..5be0db7f8 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -8,7 +8,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", - "py_ecc==1.7.1", + "py_ecc==2.0.0", "ssz==0.1.3", "dataclasses==0.6", ] From 19ca7af5ed65896c4fe3f6effadca0f176cb00cf Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 7 Jan 2020 15:15:51 -0700 Subject: [PATCH 42/48] bump circleci cache versions --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26b259738..f3870d922 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,26 +35,26 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v4-pyspec + venv_name: v5-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v4-pyspec + venv_name: v5-pyspec reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v7-deposit-contract + venv_name: v8-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v7-deposit-contract + venv_name: v8-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: From b8d4ce4c39b4539d696378d2a88a99ce0f23d506 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 9 Jan 2020 12:38:56 -0700 Subject: [PATCH 43/48] fix default value when voting on eth1data --- specs/validator/0_beacon-chain-validator.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 55a061446..473699c5a 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -283,16 +283,20 @@ def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: ```python def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: period_start = voting_period_start_time(state) - # `eth1_chain` abstractly represents all blocks in the eth1 chain. + # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height votes_to_consider = [get_eth1_data(block) for block in eth1_chain if is_candidate_block(block, period_start)] + # Valid votes already cast during this period valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] + # Default vote on latest eth1 block data in the period range unless eth1 chain is not live + default_vote = votes_to_consider[-1] if any(votes_to_consider) else state.eth1_data + return max( valid_votes, key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance - default=get_eth1_data(ETH1_FOLLOW_DISTANCE), + default=default_vote ) ``` From af702d42fd7f8b6f0735e25dfca7272722357be7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 9 Jan 2020 17:14:00 -0700 Subject: [PATCH 44/48] add slot validation condition for beacon block gossip sub channel --- specs/networking/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 0a702ea61..8f61e179d 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -11,7 +11,7 @@ It consists of four main sections: ## Table of contents - + @@ -243,7 +243,9 @@ When processing incoming gossip, clients MAY descore or disconnect peers who fai There are two primary global topics used to propagate beacon blocks and aggregate attestations to all nodes on the network. Their `TopicName`s are: -- `beacon_block` - This topic is used solely for propagating new beacon blocks to all nodes on the networks. Blocks are sent in their entirety. Clients MUST validate the block proposer signature before forwarding it across the network. +- `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network + - The proposer signature, `signed_beacon_block.signature` is valid. + - The block is not from a future slot -- i.e. validate that `signed_beacon_block.message.slot < current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `AggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `aggregate_and_proof` on the network. - The aggregate attestation defined by `hash_tree_root(aggregate_and_proof.aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - The block being voted for (`aggregate_and_proof.aggregate.data.beacon_block_root`) passes validation. From e0cd1090bd02d832ab0303a013e8ea1cde076e5d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 9 Jan 2020 17:38:43 -0700 Subject: [PATCH 45/48] fix voluntary exit to be wrt signed voluntary exit in p2p spec --- specs/networking/p2p-interface.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 8f61e179d..2f0dc6d4a 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -223,15 +223,15 @@ where `base64` is the [URL-safe base64 alphabet](https://tools.ietf.org/html/rfc The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: -| Topic | Message Type | -|----------------------------------------|-------------------| -| beacon_block | SignedBeaconBlock | -| beacon_aggregate_and_proof | AggregateAndProof | -| beacon_attestation\* | Attestation | -| committee_index{subnet_id}\_beacon_attestation | Attestation | -| voluntary_exit | VoluntaryExit | -| proposer_slashing | ProposerSlashing | -| attester_slashing | AttesterSlashing | +| Topic | Message Type | +|------------------------------------------------|----------------------| +| beacon_block | SignedBeaconBlock | +| beacon_aggregate_and_proof | AggregateAndProof | +| beacon_attestation\* | Attestation | +| committee_index{subnet_id}\_beacon_attestation | Attestation | +| voluntary_exit | SignedVoluntaryExit | +| proposer_slashing | ProposerSlashing | +| attester_slashing | AttesterSlashing | Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. @@ -245,7 +245,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - The proposer signature, `signed_beacon_block.signature` is valid. - - The block is not from a future slot -- i.e. validate that `signed_beacon_block.message.slot < current_slot` (a client MAY queue future blocks for processing at the appropriate slot). + - The block is not from a future slot -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `AggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `aggregate_and_proof` on the network. - The aggregate attestation defined by `hash_tree_root(aggregate_and_proof.aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - The block being voted for (`aggregate_and_proof.aggregate.data.beacon_block_root`) passes validation. @@ -257,7 +257,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat Additional global topics are used to propagate lower frequency validator messages. Their `TopicName`s are: -- `voluntary_exit` - This topic is used solely for propagating voluntary validator exits to proposers on the network. Voluntary exits are sent in their entirety. Clients who receive a voluntary exit on this topic MUST validate the conditions within `process_voluntary_exit` before forwarding it across the network. +- `voluntary_exit` - This topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. Clients who receive a signed voluntary exit on this topic MUST validate the conditions within `process_voluntary_exit` before forwarding it across the network. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. Clients who receive a proposer slashing on this topic MUST validate the conditions within `process_proposer_slashing` before forwarding it across the network. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. From 5e96c08f41f61028fe5bcdf954b55dda8afeadf1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 10 Jan 2020 07:11:00 -0700 Subject: [PATCH 46/48] add MAXIMUM_GOSSIP_CLOCK_DISPARITY for gossip subnet validations --- specs/networking/p2p-interface.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/specs/networking/p2p-interface.md b/specs/networking/p2p-interface.md index 2f0dc6d4a..8e060a092 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/networking/p2p-interface.md @@ -10,8 +10,7 @@ It consists of four main sections: 4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. ## Table of contents - - + @@ -82,6 +81,7 @@ It consists of four main sections: - [Why must all clients use the same gossip topic instead of one negotiated between each peer pair?](#why-must-all-clients-use-the-same-gossip-topic-instead-of-one-negotiated-between-each-peer-pair) - [Why are the topics strings and not hashes?](#why-are-the-topics-strings-and-not-hashes) - [Why are we overriding the default libp2p pubsub `message-id`?](#why-are-we-overriding-the-default-libp2p-pubsub-message-id) + - [Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets?](#why-is-there-maximum_gossip_clock_disparity-when-validating-slot-ranges-of-messages-in-gossip-subnets) - [Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?](#why-are-there-attestation_subnet_count-attestation-subnets) - [Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots?](#why-are-attestations-limited-to-be-broadcast-on-gossip-channels-within-slots_per_epoch-slots) - [Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s?](#why-are-aggregate-attestations-broadcast-to-the-global-topic-as-aggregateandproofs-rather-than-just-as-attestations) @@ -105,6 +105,7 @@ It consists of four main sections: - [libp2p implementations matrix](#libp2p-implementations-matrix) + # Network fundamentals @@ -186,6 +187,7 @@ This section outlines constants that are used in this spec. | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | +| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500ms` | The maximum milliseconds of clock disparity assumed between honest nodes. | ## The gossip domain: gossipsub @@ -245,11 +247,11 @@ There are two primary global topics used to propagate beacon blocks and aggregat - `beacon_block` - This topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. The following validations MUST pass before forwarding the `signed_beacon_block` on the network - The proposer signature, `signed_beacon_block.signature` is valid. - - The block is not from a future slot -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). + - The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `signed_beacon_block.message.slot <= current_slot` (a client MAY queue future blocks for processing at the appropriate slot). - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `AggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `aggregate_and_proof` on the network. - The aggregate attestation defined by `hash_tree_root(aggregate_and_proof.aggregate)` has _not_ already been seen (via aggregate gossip, within a block, or through the creation of an equivalent aggregate locally). - The block being voted for (`aggregate_and_proof.aggregate.data.beacon_block_root`) passes validation. - - `aggregate_and_proof.aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (`aggregate_and_proof.aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate_and_proof.aggregate.data.slot`). + - `aggregate_and_proof.aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate_and_proof.aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate_and_proof.aggregate.data.slot`. - The validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate_and_proof.aggregate.data, aggregate_and_proof.aggregate.aggregation_bits)`. - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate_and_proof.aggregate.data.slot, aggregate_and_proof.aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. - The `aggregate_and_proof.selection_proof` is a valid signature of the `aggregate_and_proof.aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. @@ -269,7 +271,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio - The attestation's committee index (`attestation.data.index`) is for the correct subnet. - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`). - The block being voted for (`attestation.data.beacon_block_root`) passes validation. - - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (`attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot`). + - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot`. - The signature of `attestation` is valid. #### Interop @@ -767,6 +769,14 @@ Some examples of where messages could be duplicated: * Attestation aggregation strategies where clients partially aggregate attestations and propagate them. Partial aggregates could be duplicated * Clients re-publishing seen messages +### Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets? + +For some gossip channels (e.g. those for Attestations and BeaconBlocks), there are designated ranges of slots during which particular messages can be sent, limiting messages gossiped to those that can be reasonably used in the consensus at the current time/slot. This is to reduce optionality in DoS attacks. + +`MAXIMUM_GOSSIP_CLOCK_DISPARITY` provides some leeway in validating slot ranges to prevent the gossip network from becoming overly brittle with respect to clock disparity. For minimum and maximum allowable slot broadcast times, `MAXIMUM_GOSSIP_CLOCK_DISPARITY` MUST be subtracted and added respectively, marginally extending the valid range. Although messages can at times be eagerly gossiped to the network, the node's fork choice prevents integration of these messages into the actual consensus until the _actual local start_ of the designated slot. + +The value of this constant is currently a placeholder and will be tuned based on data observed in testnets. + ### Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets? Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. The exact grouping will be dependent on more involved network tests. This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). The value is currently set to to be equal `MAX_COMMITTEES_PER_SLOT` until network tests indicate otherwise. From 676e216beb0fd95932793ac271802fd48b07e4c5 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 10 Jan 2020 11:42:55 -0700 Subject: [PATCH 47/48] reorg specs by fork and move ssz out to own folder. make all of the build and link changes to support move --- .circleci/config.yml | 38 +++++++++---------- .gitignore | 6 +-- Makefile | 17 +++++---- README.md | 29 +++++++------- deposit_contract/requirements-testing.txt | 2 +- scripts/build_spec.py | 24 ++++++------ .../beacon-chain.md} | 4 +- .../deposit-contract.md} | 2 +- .../fork-choice.md} | 4 +- specs/{networking => phase0}/p2p-interface.md | 4 +- .../validator.md} | 32 ++++++++-------- .../beacon-chain-misc.md} | 0 .../custody-game.md} | 2 +- .../light-client-sync.md} | 0 .../shard-data-chains.md} | 0 .../merkle_proofs.md => ssz/merkle-proofs.md | 0 {specs => ssz}/simple-serialize.md | 2 +- .../epoch_processing/requirements.txt | 3 -- test_generators/genesis/requirements.txt | 3 -- test_generators/operations/requirements.txt | 4 -- test_generators/sanity/requirements.txt | 3 -- test_generators/shuffling/requirements.txt | 4 -- test_generators/ssz_generic/requirements.txt | 4 -- test_generators/ssz_static/requirements.txt | 3 -- .../core}/config_helpers/README.md | 4 +- .../config_helpers/preset_loader}/__init__.py | 0 .../config_helpers/preset_loader/loader.py | 0 .../core}/config_helpers/requirements.txt | 0 .../core}/config_helpers/setup.py | 0 .../core}/gen_helpers/README.md | 2 +- .../core/gen_helpers/gen_base}/__init__.py | 0 .../core}/gen_helpers/gen_base/gen_runner.py | 0 .../core}/gen_helpers/gen_base/gen_typing.py | 0 .../gen_helpers/gen_from_tests}/__init__.py | 0 .../core}/gen_helpers/gen_from_tests/gen.py | 0 .../core}/gen_helpers/requirements.txt | 0 .../core}/gen_helpers/setup.py | 0 {test_libs => tests/core}/pyspec/README.md | 2 +- .../core/pyspec}/__init__.py | 0 .../core/pyspec/eth2spec}/__init__.py | 0 .../core/pyspec/eth2spec/debug}/__init__.py | 0 .../core}/pyspec/eth2spec/debug/decode.py | 0 .../core}/pyspec/eth2spec/debug/encode.py | 0 .../pyspec/eth2spec/debug/random_value.py | 0 .../core/pyspec/eth2spec/fuzzing}/__init__.py | 0 .../core}/pyspec/eth2spec/fuzzing/decoder.py | 0 .../pyspec/eth2spec/fuzzing/test_decoder.py | 0 .../core/pyspec/eth2spec/phase0}/__init__.py | 0 .../core/pyspec/eth2spec/phase1}/__init__.py | 0 .../core/pyspec/eth2spec/test}/__init__.py | 0 .../core}/pyspec/eth2spec/test/conftest.py | 2 +- .../core}/pyspec/eth2spec/test/context.py | 0 .../test/fork_choice/test_get_head.py | 0 .../test/fork_choice/test_on_attestation.py | 0 .../test/fork_choice/test_on_block.py | 0 .../eth2spec/test/fork_choice/test_on_tick.py | 0 .../pyspec/eth2spec/test/genesis}/__init__.py | 0 .../test/genesis/test_initialization.py | 0 .../eth2spec/test/genesis/test_validity.py | 0 .../pyspec/eth2spec/test/helpers}/__init__.py | 0 .../eth2spec/test/helpers/attestations.py | 0 .../test/helpers/attester_slashings.py | 0 .../pyspec/eth2spec/test/helpers/block.py | 0 .../eth2spec/test/helpers/block_header.py | 0 .../pyspec/eth2spec/test/helpers/custody.py | 0 .../pyspec/eth2spec/test/helpers/deposits.py | 0 .../pyspec/eth2spec/test/helpers/genesis.py | 0 .../pyspec/eth2spec/test/helpers/keys.py | 0 .../eth2spec/test/helpers/phase1}/__init__.py | 0 .../test/helpers/phase1/attestations.py | 0 .../test/helpers/phase1/shard_block.py | 0 .../test/helpers/phase1/shard_state.py | 0 .../test/helpers/proposer_slashings.py | 0 .../pyspec/eth2spec/test/helpers/state.py | 0 .../eth2spec/test/helpers/voluntary_exits.py | 0 .../eth2spec/test/merkle_proofs}/__init__.py | 0 .../test/merkle_proofs/test_merkle_proofs.py | 0 .../pyspec/eth2spec/test/phase_0}/__init__.py | 0 .../phase_0/block_processing}/__init__.py | 0 .../test_process_attestation.py | 0 .../test_process_attester_slashing.py | 0 .../test_process_block_header.py | 0 .../block_processing/test_process_deposit.py | 0 .../test_process_proposer_slashing.py | 0 .../test_process_voluntary_exit.py | 0 .../phase_0/epoch_processing}/__init__.py | 0 .../run_epoch_process_base.py | 0 .../test_process_final_updates.py | 0 ..._process_justification_and_finalization.py | 0 .../test_process_registry_updates.py | 0 .../test_process_rewards_and_penalties.py | 0 .../test_process_slashings.py | 0 .../pyspec/eth2spec/test/phase_1}/__init__.py | 0 .../phase_1/block_processing}/__init__.py | 0 .../test_process_bit_challenge.py | 0 .../test_process_custody_key_reveal.py | 0 ...est_process_early_derived_secret_reveal.py | 0 .../test/phase_1/sanity/test_shard_blocks.py | 0 .../pyspec/eth2spec/test/sanity}/__init__.py | 0 .../eth2spec/test/sanity/test_blocks.py | 0 .../pyspec/eth2spec/test/sanity/test_slots.py | 0 .../pyspec/eth2spec/test/test_finality.py | 0 .../core}/pyspec/eth2spec/test/utils.py | 0 .../core/pyspec/eth2spec/utils}/__init__.py | 0 .../core}/pyspec/eth2spec/utils/bls.py | 0 .../pyspec/eth2spec/utils/hash_function.py | 0 .../pyspec/eth2spec/utils/merkle_minimal.py | 0 .../pyspec/eth2spec/utils/ssz}/__init__.py | 0 .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 0 .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 0 .../eth2spec/utils/ssz/test_ssz_impl.py | 0 .../eth2spec/utils/ssz/test_ssz_typing.py | 0 .../eth2spec/utils/test_merkle_minimal.py | 0 .../core}/pyspec/requirements-testing.txt | 0 .../core}/pyspec/requirements.txt | 0 {test_libs => tests/core}/pyspec/setup.py | 0 .../test_formats => tests/formats}/README.md | 0 .../formats}/bls/README.md | 0 .../formats}/bls/aggregate_pubkeys.md | 0 .../formats}/bls/aggregate_sigs.md | 0 .../formats}/bls/msg_hash_g2_compressed.md | 0 .../formats}/bls/msg_hash_g2_uncompressed.md | 0 .../formats}/bls/priv_to_pub.md | 0 .../formats}/bls/sign_msg.md | 0 .../formats}/epoch_processing/README.md | 0 .../formats}/genesis/README.md | 0 .../formats}/genesis/initialization.md | 0 .../formats}/genesis/validity.md | 0 .../formats}/operations/README.md | 0 .../formats}/sanity/README.md | 0 .../formats}/sanity/blocks.md | 0 .../formats}/sanity/slots.md | 0 .../formats}/shuffling/README.md | 2 +- .../formats}/ssz_generic/README.md | 0 .../formats}/ssz_static/README.md | 2 +- .../formats}/ssz_static/core.md | 4 +- .../generators}/README.md | 8 ++-- .../generators}/bls/README.md | 0 .../generators}/bls/main.py | 0 .../generators}/bls/requirements.txt | 2 +- .../generators}/epoch_processing/README.md | 0 .../generators}/epoch_processing/main.py | 0 .../epoch_processing/requirements.txt | 3 ++ .../generators}/genesis/README.md | 4 +- .../generators}/genesis/main.py | 0 tests/generators/genesis/requirements.txt | 3 ++ .../generators}/operations/README.md | 0 .../generators}/operations/main.py | 0 tests/generators/operations/requirements.txt | 4 ++ .../generators}/sanity/README.md | 0 .../generators}/sanity/main.py | 0 tests/generators/sanity/requirements.txt | 3 ++ .../generators}/shuffling/README.md | 0 .../generators}/shuffling/main.py | 0 tests/generators/shuffling/requirements.txt | 4 ++ .../generators/ssz_generic}/__init__.py | 0 .../generators}/ssz_generic/main.py | 0 tests/generators/ssz_generic/requirements.txt | 4 ++ .../ssz_generic/ssz_basic_vector.py | 0 .../generators}/ssz_generic/ssz_bitlist.py | 0 .../generators}/ssz_generic/ssz_bitvector.py | 0 .../generators}/ssz_generic/ssz_boolean.py | 0 .../generators}/ssz_generic/ssz_container.py | 0 .../generators}/ssz_generic/ssz_test_case.py | 0 .../generators}/ssz_generic/ssz_uints.py | 0 .../ssz_generic/uint_test_cases.py | 0 .../generators}/ssz_static/README.md | 0 .../generators/ssz_static}/__init__.py | 0 .../generators}/ssz_static/main.py | 0 tests/generators/ssz_static/requirements.txt | 3 ++ 170 files changed, 123 insertions(+), 123 deletions(-) rename specs/{core/0_beacon-chain.md => phase0/beacon-chain.md} (99%) rename specs/{core/0_deposit-contract.md => phase0/deposit-contract.md} (94%) rename specs/{core/0_fork-choice.md => phase0/fork-choice.md} (97%) rename specs/{networking => phase0}/p2p-interface.md (99%) rename specs/{validator/0_beacon-chain-validator.md => phase0/validator.md} (87%) rename specs/{core/1_beacon-chain-misc.md => phase1/beacon-chain-misc.md} (100%) rename specs/{core/1_custody-game.md => phase1/custody-game.md} (99%) rename specs/{light_client/sync_protocol.md => phase1/light-client-sync.md} (100%) rename specs/{core/1_shard-data-chains.md => phase1/shard-data-chains.md} (100%) rename specs/light_client/merkle_proofs.md => ssz/merkle-proofs.md (100%) rename {specs => ssz}/simple-serialize.md (97%) delete mode 100644 test_generators/epoch_processing/requirements.txt delete mode 100644 test_generators/genesis/requirements.txt delete mode 100644 test_generators/operations/requirements.txt delete mode 100644 test_generators/sanity/requirements.txt delete mode 100644 test_generators/shuffling/requirements.txt delete mode 100644 test_generators/ssz_generic/requirements.txt delete mode 100644 test_generators/ssz_static/requirements.txt rename {test_libs => tests/core}/config_helpers/README.md (72%) rename {test_generators/ssz_generic => tests/core/config_helpers/preset_loader}/__init__.py (100%) rename {test_libs => tests/core}/config_helpers/preset_loader/loader.py (100%) rename {test_libs => tests/core}/config_helpers/requirements.txt (100%) rename {test_libs => tests/core}/config_helpers/setup.py (100%) rename {test_libs => tests/core}/gen_helpers/README.md (96%) rename {test_generators/ssz_static => tests/core/gen_helpers/gen_base}/__init__.py (100%) rename {test_libs => tests/core}/gen_helpers/gen_base/gen_runner.py (100%) rename {test_libs => tests/core}/gen_helpers/gen_base/gen_typing.py (100%) rename {test_libs/config_helpers/preset_loader => tests/core/gen_helpers/gen_from_tests}/__init__.py (100%) rename {test_libs => tests/core}/gen_helpers/gen_from_tests/gen.py (100%) rename {test_libs => tests/core}/gen_helpers/requirements.txt (100%) rename {test_libs => tests/core}/gen_helpers/setup.py (100%) rename {test_libs => tests/core}/pyspec/README.md (95%) rename {test_libs/gen_helpers/gen_base => tests/core/pyspec}/__init__.py (100%) rename {test_libs/gen_helpers/gen_from_tests => tests/core/pyspec/eth2spec}/__init__.py (100%) rename {test_libs/pyspec => tests/core/pyspec/eth2spec/debug}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/debug/decode.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/debug/encode.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/debug/random_value.py (100%) rename {test_libs/pyspec/eth2spec => tests/core/pyspec/eth2spec/fuzzing}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/fuzzing/decoder.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/fuzzing/test_decoder.py (100%) rename {test_libs/pyspec/eth2spec/debug => tests/core/pyspec/eth2spec/phase0}/__init__.py (100%) rename {test_libs/pyspec/eth2spec/fuzzing => tests/core/pyspec/eth2spec/phase1}/__init__.py (100%) rename {test_libs/pyspec/eth2spec/phase0 => tests/core/pyspec/eth2spec/test}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/conftest.py (93%) rename {test_libs => tests/core}/pyspec/eth2spec/test/context.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/fork_choice/test_get_head.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/fork_choice/test_on_attestation.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/fork_choice/test_on_block.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/fork_choice/test_on_tick.py (100%) rename {test_libs/pyspec/eth2spec/phase1 => tests/core/pyspec/eth2spec/test/genesis}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/genesis/test_initialization.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/genesis/test_validity.py (100%) rename {test_libs/pyspec/eth2spec/test => tests/core/pyspec/eth2spec/test/helpers}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/attestations.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/attester_slashings.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/block.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/block_header.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/custody.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/deposits.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/genesis.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/keys.py (100%) rename {test_libs/pyspec/eth2spec/test/genesis => tests/core/pyspec/eth2spec/test/helpers/phase1}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/phase1/attestations.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/phase1/shard_block.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/phase1/shard_state.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/proposer_slashings.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/state.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/helpers/voluntary_exits.py (100%) rename {test_libs/pyspec/eth2spec/test/helpers => tests/core/pyspec/eth2spec/test/merkle_proofs}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py (100%) rename {test_libs/pyspec/eth2spec/test/helpers/phase1 => tests/core/pyspec/eth2spec/test/phase_0}/__init__.py (100%) rename {test_libs/pyspec/eth2spec/test/merkle_proofs => tests/core/pyspec/eth2spec/test/phase_0/block_processing}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py (100%) rename {test_libs/pyspec/eth2spec/test/phase_0 => tests/core/pyspec/eth2spec/test/phase_0/epoch_processing}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py (100%) rename {test_libs/pyspec/eth2spec/test/phase_0/block_processing => tests/core/pyspec/eth2spec/test/phase_1}/__init__.py (100%) rename {test_libs/pyspec/eth2spec/test/phase_0/epoch_processing => tests/core/pyspec/eth2spec/test/phase_1/block_processing}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py (100%) rename {test_libs/pyspec/eth2spec/test/phase_1 => tests/core/pyspec/eth2spec/test/sanity}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/sanity/test_blocks.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/sanity/test_slots.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/test_finality.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/test/utils.py (100%) rename {test_libs/pyspec/eth2spec/test/phase_1/block_processing => tests/core/pyspec/eth2spec/utils}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/bls.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/hash_function.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/merkle_minimal.py (100%) rename {test_libs/pyspec/eth2spec/test/sanity => tests/core/pyspec/eth2spec/utils/ssz}/__init__.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/ssz/ssz_impl.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/ssz/ssz_typing.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/ssz/test_ssz_impl.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/ssz/test_ssz_typing.py (100%) rename {test_libs => tests/core}/pyspec/eth2spec/utils/test_merkle_minimal.py (100%) rename {test_libs => tests/core}/pyspec/requirements-testing.txt (100%) rename {test_libs => tests/core}/pyspec/requirements.txt (100%) rename {test_libs => tests/core}/pyspec/setup.py (100%) rename {specs/test_formats => tests/formats}/README.md (100%) rename {specs/test_formats => tests/formats}/bls/README.md (100%) rename {specs/test_formats => tests/formats}/bls/aggregate_pubkeys.md (100%) rename {specs/test_formats => tests/formats}/bls/aggregate_sigs.md (100%) rename {specs/test_formats => tests/formats}/bls/msg_hash_g2_compressed.md (100%) rename {specs/test_formats => tests/formats}/bls/msg_hash_g2_uncompressed.md (100%) rename {specs/test_formats => tests/formats}/bls/priv_to_pub.md (100%) rename {specs/test_formats => tests/formats}/bls/sign_msg.md (100%) rename {specs/test_formats => tests/formats}/epoch_processing/README.md (100%) rename {specs/test_formats => tests/formats}/genesis/README.md (100%) rename {specs/test_formats => tests/formats}/genesis/initialization.md (100%) rename {specs/test_formats => tests/formats}/genesis/validity.md (100%) rename {specs/test_formats => tests/formats}/operations/README.md (100%) rename {specs/test_formats => tests/formats}/sanity/README.md (100%) rename {specs/test_formats => tests/formats}/sanity/blocks.md (100%) rename {specs/test_formats => tests/formats}/sanity/slots.md (100%) rename {specs/test_formats => tests/formats}/shuffling/README.md (96%) rename {specs/test_formats => tests/formats}/ssz_generic/README.md (100%) rename {specs/test_formats => tests/formats}/ssz_static/README.md (81%) rename {specs/test_formats => tests/formats}/ssz_static/core.md (92%) rename {test_generators => tests/generators}/README.md (97%) rename {test_generators => tests/generators}/bls/README.md (100%) rename {test_generators => tests/generators}/bls/main.py (100%) rename {test_generators => tests/generators}/bls/requirements.txt (52%) rename {test_generators => tests/generators}/epoch_processing/README.md (100%) rename {test_generators => tests/generators}/epoch_processing/main.py (100%) create mode 100644 tests/generators/epoch_processing/requirements.txt rename {test_generators => tests/generators}/genesis/README.md (71%) rename {test_generators => tests/generators}/genesis/main.py (100%) create mode 100644 tests/generators/genesis/requirements.txt rename {test_generators => tests/generators}/operations/README.md (100%) rename {test_generators => tests/generators}/operations/main.py (100%) create mode 100644 tests/generators/operations/requirements.txt rename {test_generators => tests/generators}/sanity/README.md (100%) rename {test_generators => tests/generators}/sanity/main.py (100%) create mode 100644 tests/generators/sanity/requirements.txt rename {test_generators => tests/generators}/shuffling/README.md (100%) rename {test_generators => tests/generators}/shuffling/main.py (100%) create mode 100644 tests/generators/shuffling/requirements.txt rename {test_libs/pyspec/eth2spec/utils => tests/generators/ssz_generic}/__init__.py (100%) rename {test_generators => tests/generators}/ssz_generic/main.py (100%) create mode 100644 tests/generators/ssz_generic/requirements.txt rename {test_generators => tests/generators}/ssz_generic/ssz_basic_vector.py (100%) rename {test_generators => tests/generators}/ssz_generic/ssz_bitlist.py (100%) rename {test_generators => tests/generators}/ssz_generic/ssz_bitvector.py (100%) rename {test_generators => tests/generators}/ssz_generic/ssz_boolean.py (100%) rename {test_generators => tests/generators}/ssz_generic/ssz_container.py (100%) rename {test_generators => tests/generators}/ssz_generic/ssz_test_case.py (100%) rename {test_generators => tests/generators}/ssz_generic/ssz_uints.py (100%) rename {test_generators => tests/generators}/ssz_generic/uint_test_cases.py (100%) rename {test_generators => tests/generators}/ssz_static/README.md (100%) rename {test_libs/pyspec/eth2spec/utils/ssz => tests/generators/ssz_static}/__init__.py (100%) rename {test_generators => tests/generators}/ssz_static/main.py (100%) create mode 100644 tests/generators/ssz_static/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index f3870d922..04d5bf799 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,27 +35,27 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v5-pyspec - reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} + venv_name: v6-pyspec + reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v5-pyspec - reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} - venv_path: ./test_libs/pyspec/venv + venv_name: v6-pyspec + reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} + venv_path: ./tests/core/pyspec/venv restore_deposit_contract_cached_venv: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v8-deposit-contract - reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} + venv_name: v9-deposit-contract + reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v8-deposit-contract - reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} + venv_name: v9-deposit-contract + reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: checkout_specs: @@ -66,16 +66,16 @@ jobs: # Restore git repo at point close to target branch/revision, to speed up checkout - restore_cache: keys: - - v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - v1-specs-repo-{{ .Branch }}- - - v1-specs-repo- + - v2-specs-repo-{{ .Branch }}-{{ .Revision }} + - v2-specs-repo-{{ .Branch }}- + - v2-specs-repo- - checkout - run: name: Clean up git repo to reduce cache size command: git gc # Save the git checkout as a cache, to make cloning next time faster. - save_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo install_pyspec_test: @@ -84,7 +84,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: name: Install pyspec requirements @@ -96,13 +96,13 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: name: Run py-tests command: make citest - store_test_results: - path: test_libs/pyspec/test-reports + path: tests/core/pyspec/test-reports table_of_contents: docker: - image: circleci/node:10.16.3 @@ -127,7 +127,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_pyspec_cached_venv - run: name: Run linter @@ -138,7 +138,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_deposit_contract_cached_venv - run: name: Install deposit contract requirements @@ -150,7 +150,7 @@ jobs: working_directory: ~/specs-repo steps: - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + key: v2-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_deposit_contract_cached_venv - run: name: Run deposit contract test diff --git a/.gitignore b/.gitignore index 4dff5fbcb..ff1f2d9f8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,12 +14,12 @@ eth2.0-spec-tests/ .mypy_cache # Dynamically built from Markdown spec -test_libs/pyspec/eth2spec/phase0/spec.py -test_libs/pyspec/eth2spec/phase1/spec.py +tests/core/pyspec/eth2spec/phase0/spec.py +tests/core/pyspec/eth2spec/phase1/spec.py # coverage reports .htmlcov .coverage # local CI testing output -test_libs/pyspec/test-reports +tests/core/pyspec/test-reports diff --git a/Makefile b/Makefile index 51bff90d0..e91a686f1 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ SPEC_DIR = ./specs +SSZ_DIR = ./ssz SCRIPT_DIR = ./scripts -TEST_LIBS_DIR = ./test_libs +TEST_LIBS_DIR = ./tests/core PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests -GENERATOR_DIR = ./test_generators +GENERATOR_DIR = ./tests/generators DEPOSIT_CONTRACT_DIR = ./deposit_contract CONFIGS_DIR = ./configs @@ -16,17 +17,19 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER # To check generator matching: #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}]) +PHASE0_SPEC_DIR = $(SPEC_DIR)/phase0 PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py -PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/core/0_*.md) +PY_SPEC_PHASE_0_DEPS = $(wildcard $(SPEC_DIR)/phase0/*.md) +PHASE1_SPEC_DIR = $(SPEC_DIR)/phase1 PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py -PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/core/1_*.md) +PY_SPEC_PHASE_1_DEPS = $(wildcard $(SPEC_DIR)/phase1/*.md) PY_SPEC_ALL_DEPS = $(PY_SPEC_PHASE_0_DEPS) $(PY_SPEC_PHASE_1_DEPS) PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) -MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SPEC_DIR)/light_client/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) +MARKDOWN_FILES = $(PY_SPEC_ALL_DEPS) $(wildcard $(SPEC_DIR)/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md) COV_HTML_OUT=.htmlcov COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html @@ -101,10 +104,10 @@ test_deposit_contract: pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(PHASE0_SPEC_DIR)/validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(PHASE0_SPEC_DIR)/beacon-chain.md $(PHASE0_SPEC_DIR)/fork-choice.md $(SSZ_DIR)/merkle-proofs.md $(PHASE1_SPEC_DIR)/custody-game.md $(PHASE1_SPEC_DIR)/shard-data-chains.md $(PHASE1_SPEC_DIR)/beacon-chain-misc.md $@ CURRENT_DIR = ${CURDIR} diff --git a/README.md b/README.md index fa103394d..d08cc0bb5 100644 --- a/README.md +++ b/README.md @@ -9,18 +9,19 @@ This repository hosts the current Eth2 specifications. Discussions about design ## Specs -Core specifications for Eth2 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: +Core specifications for Eth2 clients be found in [specs/](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: ### Phase 0 -* [The Beacon Chain](specs/core/0_beacon-chain.md) -* [Fork Choice](specs/core/0_fork-choice.md) -* [Deposit Contract](specs/core/0_deposit-contract.md) -* [Honest Validator](specs/validator/0_beacon-chain-validator.md) +* [The Beacon Chain](specs/phase0/beacon-chain.md) +* [Fork Choice](specs/phase0/fork-choice.md) +* [Deposit Contract](specs/phase0/deposit-contract.md) +* [Honest Validator](specs/phase0/validator.md) ### Phase 1 -* [Custody Game](specs/core/1_custody-game.md) -* [Shard Data Chains](specs/core/1_shard-data-chains.md) -* [Misc beacon chain updates](specs/core/1_beacon-chain-misc.md) +* [Custody Game](specs/phase1/custody-game.md) +* [Shard Data Chains](specs/phase1/shard-data-chains.md) +* [Misc beacon chain updates](specs/phase1/beacon-chain-misc.md) +* [Light client syncing protocol](specs/phase1/light-client-sync.md) ### Phase 2 @@ -30,11 +31,9 @@ See the [Eth2 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for c ### Accompanying documents can be found in [specs](specs) and include: -* [SimpleSerialize (SSZ) spec](specs/simple-serialize.md) -* [BLS signature verification](specs/bls_signature.md) -* [General test format](specs/test_formats/README.md) -* [Merkle proof formats](specs/light_client/merkle_proofs.md) -* [Light client syncing protocol](specs/light_client/sync_protocol.md) +* [SimpleSerialize (SSZ) spec](ssz/simple-serialize.md) +* [Merkle proof formats](ssz/merkle-proofs.md) +* [General test format](tests/formats/README.md) ## Additional specifications for client implementers @@ -63,6 +62,6 @@ The following are the broad design goals for Ethereum 2.0: ## For spec contributors Documentation on the different components used during spec writing can be found here: -* [YAML Test Generators](test_generators/README.md) -* [Executable Python Spec, with Py-tests](test_libs/pyspec/README.md) +* [YAML Test Generators](tests/generators/README.md) +* [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md) diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index c542121c2..cbf6983c1 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -2,4 +2,4 @@ eth-tester[py-evm]==0.1.0b39 git+https://github.com/vyperlang/vyper@1761-HOTFIX-v0.1.0-beta.13 web3==5.0.0b2 pytest==3.6.1 -../test_libs/pyspec +../tests/core/pyspec diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 50f57960f..114832bc2 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -308,18 +308,18 @@ if __name__ == '__main__': description = ''' Build the specs from the md docs. If building phase 0: - 1st argument is input /core/0_beacon-chain.md - 2nd argument is input /core/0_fork-choice.md - 3rd argument is input /core/0_beacon-chain-validator.md + 1st argument is input phase0/beacon-chain.md + 2nd argument is input phase0/fork-choice.md + 3rd argument is input phase0/validator.md 4th argument is output spec.py If building phase 1: - 1st argument is input /core/0_beacon-chain.md - 2nd argument is input /core/0_fork-choice.md - 3rd argument is input /light_client/merkle_proofs.md - 4th argument is input /core/1_custody-game.md - 5th argument is input /core/1_shard-data-chains.md - 6th argument is input /core/1_beacon-chain-misc.md + 1st argument is input phase0/beacon-chain.md + 2nd argument is input phase0/fork-choice.md + 3rd argument is input ssz/merkle-proofs.md + 4th argument is input phase1/custody-game.md + 5th argument is input phase1/shard-data-chains.md + 6th argument is input phase1/beacon-chain-misc.md 7th argument is output spec.py ''' parser = ArgumentParser(description=description) @@ -338,9 +338,9 @@ If building phase 1: else: print( " Phase 1 requires input files as well as an output file:\n" - "\t core/phase_0: (0_beacon-chain.md, 0_fork-choice.md)\n" - "\t light_client: (merkle_proofs.md)\n" - "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md, 1_beacon-chain-misc.md)\n" + "\t phase0: (beacon-chain.md, fork-choice.md)\n" + "\t ssz: (merkle-proofs.md)\n" + "\t phase1: (custody-game.md, shard-data-chains.md, beacon-chain-misc.md)\n" "\t and output.py" ) else: diff --git a/specs/core/0_beacon-chain.md b/specs/phase0/beacon-chain.md similarity index 99% rename from specs/core/0_beacon-chain.md rename to specs/phase0/beacon-chain.md index 6fe947ba8..503286e48 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -261,7 +261,7 @@ The following values are (non-configurable) constants used throughout the specif ## Containers -The following types are [SimpleSerialize (SSZ)](../simple-serialize.md) containers. +The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. *Note*: The definitions are ordered topologically to facilitate execution of the spec. @@ -579,7 +579,7 @@ def bytes_to_int(data: bytes) -> uint64: #### `hash_tree_root` -`def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../simple-serialize.md#merkleization). +`def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../../ssz/simple-serialize.md#merkleization). #### BLS Signatures diff --git a/specs/core/0_deposit-contract.md b/specs/phase0/deposit-contract.md similarity index 94% rename from specs/core/0_deposit-contract.md rename to specs/phase0/deposit-contract.md index 11be41b86..9c5137a32 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/phase0/deposit-contract.md @@ -40,7 +40,7 @@ The initial deployment phases of Ethereum 2.0 are implemented without consensus ### `deposit` function -The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32`. The first three arguments populate a [`DepositData`](./0_beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata. +The deposit contract has a public `deposit` function to make deposits. It takes as arguments `pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32`. The first three arguments populate a [`DepositData`](./beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata. #### Deposit amount diff --git a/specs/core/0_fork-choice.md b/specs/phase0/fork-choice.md similarity index 97% rename from specs/core/0_fork-choice.md rename to specs/phase0/fork-choice.md index b286654af..0d9823fcd 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -34,7 +34,7 @@ ## Introduction -This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md). +This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./beacon-chain.md). ## Fork choice @@ -48,7 +48,7 @@ The head block root associated with a `store` is defined as `get_head(store)`. A 1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). 2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. -3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](../validator/0_beacon-chain-validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](./validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. 4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. 5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). diff --git a/specs/networking/p2p-interface.md b/specs/phase0/p2p-interface.md similarity index 99% rename from specs/networking/p2p-interface.md rename to specs/phase0/p2p-interface.md index 8e060a092..c04d2ae4c 100644 --- a/specs/networking/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -398,12 +398,12 @@ Here, `result` represents the 1-byte response code. The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time: -- `ssz`: the contents are [SSZ-encoded](../simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Bytes32`'s. +- `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Bytes32`'s. - `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet. #### SSZ-encoding strategy (with or without Snappy) -The [SimpleSerialize (SSZ) specification](../simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialized form to the Snappy compressor on encoding. The inverse happens on decoding. +The [SimpleSerialize (SSZ) specification](../../ssz/simple-serialize.md) outlines how objects are SSZ-encoded. If the Snappy variant is selected, we feed the serialized form to the Snappy compressor on encoding. The inverse happens on decoding. **Encoding-dependent header:** Req/Resp protocols using the `ssz` or `ssz_snappy` encoding strategies MUST prefix all encoded and compressed (if applicable) payloads with an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/phase0/validator.md similarity index 87% rename from specs/validator/0_beacon-chain-validator.md rename to specs/phase0/validator.md index 473699c5a..75b9f9311 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/phase0/validator.md @@ -1,6 +1,6 @@ # Ethereum 2.0 Phase 0 -- Honest Validator -**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](../core/0_beacon-chain.md), which describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. +**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](./beacon-chain.md), which describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. ## Table of contents @@ -75,7 +75,7 @@ A validator is an entity that participates in the consensus of the Ethereum 2.0 ## Prerequisites -All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](../core/0_beacon-chain.md) and [Phase 0 -- Deposit Contract](../core/0_deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. +All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. ## Constants @@ -110,11 +110,11 @@ The validator constructs their `withdrawal_credentials` via the following: ### Submit deposit -In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 proof-of-work chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. +In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 proof-of-work chain. Deposits are made to the [deposit contract](./deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. To submit a deposit: -- Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. +- Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](./beacon-chain.md#depositdata) SSZ object. - Let `amount` be the amount in Gwei to be deposited by the validator where `amount >= MIN_DEPOSIT_AMOUNT`. - Set `deposit_data.pubkey` to validator's `pubkey`. - Set `deposit_data.withdrawal_credentials` to `withdrawal_credentials`. @@ -132,13 +132,13 @@ Deposits cannot be processed into the beacon chain until the Eth1 block in which ### Validator index -Once a validator has been processed and added to the beacon state's `validators`, the validator's `validator_index` is defined by the index into the registry at which the [`ValidatorRecord`](../core/0_beacon-chain.md#validator) contains the `pubkey` specified in the validator's deposit. A validator's `validator_index` is guaranteed to not change from the time of initial deposit until the validator exits and fully withdraws. This `validator_index` is used throughout the specification to dictate validator roles and responsibilities at any point and should be stored locally. +Once a validator has been processed and added to the beacon state's `validators`, the validator's `validator_index` is defined by the index into the registry at which the [`ValidatorRecord`](./beacon-chain.md#validator) contains the `pubkey` specified in the validator's deposit. A validator's `validator_index` is guaranteed to not change from the time of initial deposit until the validator exits and fully withdraws. This `validator_index` is used throughout the specification to dictate validator roles and responsibilities at any point and should be stored locally. ### Activation In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `MAX_SEED_LOOKAHEAD` epochs (25.6 minutes). -The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: +The function [`is_active_validator`](./beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: ```python def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: @@ -209,7 +209,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo ### Block proposal -A validator is expected to propose a [`SignedBeaconBlock`](../core/0_beacon-chain.md#signedbeaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`SignedBeaconBlock`](./beacon-chain.md#signedbeaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](./beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~6 weeks). @@ -302,25 +302,25 @@ def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Da ##### Proposer slashings -Up to `MAX_PROPOSER_SLASHINGS`, [`ProposerSlashing`](../core/0_beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](../core/0_beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. +Up to `MAX_PROPOSER_SLASHINGS`, [`ProposerSlashing`](./beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](./beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. ##### Attester slashings -Up to `MAX_ATTESTER_SLASHINGS`, [`AttesterSlashing`](../core/0_beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [attester slashings processing](../core/0_beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. +Up to `MAX_ATTESTER_SLASHINGS`, [`AttesterSlashing`](./beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [attester slashings processing](./beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. ##### Attestations -Up to `MAX_ATTESTATIONS`, aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](../core/0_beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. +Up to `MAX_ATTESTATIONS`, aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](./beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. ##### Deposits -If there are any unprocessed deposits for the existing `state.eth1_data` (i.e. `state.eth1_data.deposit_count > state.eth1_deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, eth1_data.deposit_count - state.eth1_deposit_index)`. These [`deposits`](../core/0_beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth1 deposit contract](../core/0_deposit-contract.md) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](../core/0_beacon-chain.md#deposits). +If there are any unprocessed deposits for the existing `state.eth1_data` (i.e. `state.eth1_data.deposit_count > state.eth1_deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, eth1_data.deposit_count - state.eth1_deposit_index)`. These [`deposits`](./beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth1 deposit contract](./deposit-contract.md) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](./beacon-chain.md#deposits). The `proof` for each deposit must be constructed against the deposit root contained in `state.eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation. ##### Voluntary exits -Up to `MAX_VOLUNTARY_EXITS`, [`VoluntaryExit`](../core/0_beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](../core/0_beacon-chain.md#voluntary-exits). +Up to `MAX_VOLUNTARY_EXITS`, [`VoluntaryExit`](./beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](./beacon-chain.md#voluntary-exits). #### Packaging into a `SignedBeaconBlock` @@ -360,7 +360,7 @@ A validator should create and broadcast the `attestation` to the associated atte #### Attestation data -First, the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. +First, the validator should construct `attestation_data`, an [`AttestationData`](./beacon-chain.md#attestationdata) object based upon the state at the assigned slot. - Let `head_block` be the result of running the fork choice during the assigned slot. - Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. @@ -386,7 +386,7 @@ Set `attestation_data.beacon_block_root = hash_tree_root(head_block)`. #### Construct attestation -Next, the validator creates `attestation`, an [`Attestation`](../core/0_beacon-chain.md#attestation) object. +Next, the validator creates `attestation`, an [`Attestation`](./beacon-chain.md#attestation) object. ##### Data @@ -495,7 +495,7 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ### Proposer slashing -To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same epoch. +To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](./beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same epoch. *In Phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings.* @@ -508,7 +508,7 @@ If the software crashes at some point within this routine, then when the validat ### Attester slashing -To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](../core/0_beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](../core/0_beacon-chain.md#is_slashable_attestation_data). +To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](./beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](./beacon-chain.md#is_slashable_attestation_data). Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: diff --git a/specs/core/1_beacon-chain-misc.md b/specs/phase1/beacon-chain-misc.md similarity index 100% rename from specs/core/1_beacon-chain-misc.md rename to specs/phase1/beacon-chain-misc.md diff --git a/specs/core/1_custody-game.md b/specs/phase1/custody-game.md similarity index 99% rename from specs/core/1_custody-game.md rename to specs/phase1/custody-game.md index fdaf9af42..c09fa1867 100644 --- a/specs/core/1_custody-game.md +++ b/specs/phase1/custody-game.md @@ -59,7 +59,7 @@ ## Introduction -This document details the beacon chain additions and changes in Phase 1 of Ethereum 2.0 to support the shard data custody game, building upon the [Phase 0](0_beacon-chain.md) specification. +This document details the beacon chain additions and changes in Phase 1 of Ethereum 2.0 to support the shard data custody game, building upon the [Phase 0](../phase0/beacon-chain.md) specification. ## Terminology diff --git a/specs/light_client/sync_protocol.md b/specs/phase1/light-client-sync.md similarity index 100% rename from specs/light_client/sync_protocol.md rename to specs/phase1/light-client-sync.md diff --git a/specs/core/1_shard-data-chains.md b/specs/phase1/shard-data-chains.md similarity index 100% rename from specs/core/1_shard-data-chains.md rename to specs/phase1/shard-data-chains.md diff --git a/specs/light_client/merkle_proofs.md b/ssz/merkle-proofs.md similarity index 100% rename from specs/light_client/merkle_proofs.md rename to ssz/merkle-proofs.md diff --git a/specs/simple-serialize.md b/ssz/simple-serialize.md similarity index 97% rename from specs/simple-serialize.md rename to ssz/simple-serialize.md index 860a27f01..1c4f588eb 100644 --- a/specs/simple-serialize.md +++ b/ssz/simple-serialize.md @@ -239,7 +239,7 @@ We now define Merkleization `hash_tree_root(value)` of an object `value` recursi Let `A` be an object derived from another object `B` by replacing some of the (possibly nested) values of `B` by their `hash_tree_root`. We say `A` is a "summary" of `B`, and that `B` is an "expansion" of `A`. Notice `hash_tree_root(A) == hash_tree_root(B)`. -We similarly define "summary types" and "expansion types". For example, [`BeaconBlock`](./core/0_beacon-chain.md#beaconblock) is an expansion type of [`BeaconBlockHeader`](./core/0_beacon-chain.md#beaconblockheader). Notice that objects expand to at most one object of a given expansion type. For example, `BeaconBlockHeader` objects uniquely expand to `BeaconBlock` objects. +We similarly define "summary types" and "expansion types". For example, [`BeaconBlock`](../specs/phase0/beacon-chain.md#beaconblock) is an expansion type of [`BeaconBlockHeader`](../specs/phase0/beacon-chain.md#beaconblockheader). Notice that objects expand to at most one object of a given expansion type. For example, `BeaconBlockHeader` objects uniquely expand to `BeaconBlock` objects. ## Implementations diff --git a/test_generators/epoch_processing/requirements.txt b/test_generators/epoch_processing/requirements.txt deleted file mode 100644 index 3314093d3..000000000 --- a/test_generators/epoch_processing/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/genesis/requirements.txt b/test_generators/genesis/requirements.txt deleted file mode 100644 index 3314093d3..000000000 --- a/test_generators/genesis/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/operations/requirements.txt b/test_generators/operations/requirements.txt deleted file mode 100644 index 595cee69c..000000000 --- a/test_generators/operations/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -eth-utils==1.6.0 -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/sanity/requirements.txt b/test_generators/sanity/requirements.txt deleted file mode 100644 index 3314093d3..000000000 --- a/test_generators/sanity/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/shuffling/requirements.txt b/test_generators/shuffling/requirements.txt deleted file mode 100644 index 595cee69c..000000000 --- a/test_generators/shuffling/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -eth-utils==1.6.0 -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/ssz_generic/requirements.txt b/test_generators/ssz_generic/requirements.txt deleted file mode 100644 index c540f26b5..000000000 --- a/test_generators/ssz_generic/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -eth-utils==1.6.0 -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec diff --git a/test_generators/ssz_static/requirements.txt b/test_generators/ssz_static/requirements.txt deleted file mode 100644 index 3314093d3..000000000 --- a/test_generators/ssz_static/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec \ No newline at end of file diff --git a/test_libs/config_helpers/README.md b/tests/core/config_helpers/README.md similarity index 72% rename from test_libs/config_helpers/README.md rename to tests/core/config_helpers/README.md index 20dfcf7bf..85a9304d7 100644 --- a/test_libs/config_helpers/README.md +++ b/tests/core/config_helpers/README.md @@ -1,7 +1,7 @@ # Eth2 config helpers -`preset_loader`: A util to load constants-presets with. -See [Constants-presets documentation](../../configs/constants_presets/README.md). +`preset_loader`: A util to load config-presets with. +See [Configs documentation](../../../configs/README.md). Usage: diff --git a/test_generators/ssz_generic/__init__.py b/tests/core/config_helpers/preset_loader/__init__.py similarity index 100% rename from test_generators/ssz_generic/__init__.py rename to tests/core/config_helpers/preset_loader/__init__.py diff --git a/test_libs/config_helpers/preset_loader/loader.py b/tests/core/config_helpers/preset_loader/loader.py similarity index 100% rename from test_libs/config_helpers/preset_loader/loader.py rename to tests/core/config_helpers/preset_loader/loader.py diff --git a/test_libs/config_helpers/requirements.txt b/tests/core/config_helpers/requirements.txt similarity index 100% rename from test_libs/config_helpers/requirements.txt rename to tests/core/config_helpers/requirements.txt diff --git a/test_libs/config_helpers/setup.py b/tests/core/config_helpers/setup.py similarity index 100% rename from test_libs/config_helpers/setup.py rename to tests/core/config_helpers/setup.py diff --git a/test_libs/gen_helpers/README.md b/tests/core/gen_helpers/README.md similarity index 96% rename from test_libs/gen_helpers/README.md rename to tests/core/gen_helpers/README.md index 1d74a31d3..20b48db83 100644 --- a/test_libs/gen_helpers/README.md +++ b/tests/core/gen_helpers/README.md @@ -4,7 +4,7 @@ A util to quickly write new test suite generators with. -See [Generators documentation](../../test_generators/README.md) for integration details. +See [Generators documentation](../../generators/README.md) for integration details. Options: diff --git a/test_generators/ssz_static/__init__.py b/tests/core/gen_helpers/gen_base/__init__.py similarity index 100% rename from test_generators/ssz_static/__init__.py rename to tests/core/gen_helpers/gen_base/__init__.py diff --git a/test_libs/gen_helpers/gen_base/gen_runner.py b/tests/core/gen_helpers/gen_base/gen_runner.py similarity index 100% rename from test_libs/gen_helpers/gen_base/gen_runner.py rename to tests/core/gen_helpers/gen_base/gen_runner.py diff --git a/test_libs/gen_helpers/gen_base/gen_typing.py b/tests/core/gen_helpers/gen_base/gen_typing.py similarity index 100% rename from test_libs/gen_helpers/gen_base/gen_typing.py rename to tests/core/gen_helpers/gen_base/gen_typing.py diff --git a/test_libs/config_helpers/preset_loader/__init__.py b/tests/core/gen_helpers/gen_from_tests/__init__.py similarity index 100% rename from test_libs/config_helpers/preset_loader/__init__.py rename to tests/core/gen_helpers/gen_from_tests/__init__.py diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/tests/core/gen_helpers/gen_from_tests/gen.py similarity index 100% rename from test_libs/gen_helpers/gen_from_tests/gen.py rename to tests/core/gen_helpers/gen_from_tests/gen.py diff --git a/test_libs/gen_helpers/requirements.txt b/tests/core/gen_helpers/requirements.txt similarity index 100% rename from test_libs/gen_helpers/requirements.txt rename to tests/core/gen_helpers/requirements.txt diff --git a/test_libs/gen_helpers/setup.py b/tests/core/gen_helpers/setup.py similarity index 100% rename from test_libs/gen_helpers/setup.py rename to tests/core/gen_helpers/setup.py diff --git a/test_libs/pyspec/README.md b/tests/core/pyspec/README.md similarity index 95% rename from test_libs/pyspec/README.md rename to tests/core/pyspec/README.md index ea994c71b..2e596520c 100644 --- a/test_libs/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -63,4 +63,4 @@ The pyspec is not a replacement. ## License -Same as the spec itself; see [LICENSE](../../LICENSE) file in the specs repository root. +Same as the spec itself; see [LICENSE](../../../LICENSE) file in the specs repository root. diff --git a/test_libs/gen_helpers/gen_base/__init__.py b/tests/core/pyspec/__init__.py similarity index 100% rename from test_libs/gen_helpers/gen_base/__init__.py rename to tests/core/pyspec/__init__.py diff --git a/test_libs/gen_helpers/gen_from_tests/__init__.py b/tests/core/pyspec/eth2spec/__init__.py similarity index 100% rename from test_libs/gen_helpers/gen_from_tests/__init__.py rename to tests/core/pyspec/eth2spec/__init__.py diff --git a/test_libs/pyspec/__init__.py b/tests/core/pyspec/eth2spec/debug/__init__.py similarity index 100% rename from test_libs/pyspec/__init__.py rename to tests/core/pyspec/eth2spec/debug/__init__.py diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/tests/core/pyspec/eth2spec/debug/decode.py similarity index 100% rename from test_libs/pyspec/eth2spec/debug/decode.py rename to tests/core/pyspec/eth2spec/debug/decode.py diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/tests/core/pyspec/eth2spec/debug/encode.py similarity index 100% rename from test_libs/pyspec/eth2spec/debug/encode.py rename to tests/core/pyspec/eth2spec/debug/encode.py diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/tests/core/pyspec/eth2spec/debug/random_value.py similarity index 100% rename from test_libs/pyspec/eth2spec/debug/random_value.py rename to tests/core/pyspec/eth2spec/debug/random_value.py diff --git a/test_libs/pyspec/eth2spec/__init__.py b/tests/core/pyspec/eth2spec/fuzzing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/__init__.py rename to tests/core/pyspec/eth2spec/fuzzing/__init__.py diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/tests/core/pyspec/eth2spec/fuzzing/decoder.py similarity index 100% rename from test_libs/pyspec/eth2spec/fuzzing/decoder.py rename to tests/core/pyspec/eth2spec/fuzzing/decoder.py diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/tests/core/pyspec/eth2spec/fuzzing/test_decoder.py similarity index 100% rename from test_libs/pyspec/eth2spec/fuzzing/test_decoder.py rename to tests/core/pyspec/eth2spec/fuzzing/test_decoder.py diff --git a/test_libs/pyspec/eth2spec/debug/__init__.py b/tests/core/pyspec/eth2spec/phase0/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/debug/__init__.py rename to tests/core/pyspec/eth2spec/phase0/__init__.py diff --git a/test_libs/pyspec/eth2spec/fuzzing/__init__.py b/tests/core/pyspec/eth2spec/phase1/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/fuzzing/__init__.py rename to tests/core/pyspec/eth2spec/phase1/__init__.py diff --git a/test_libs/pyspec/eth2spec/phase0/__init__.py b/tests/core/pyspec/eth2spec/test/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/phase0/__init__.py rename to tests/core/pyspec/eth2spec/test/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py similarity index 93% rename from test_libs/pyspec/eth2spec/test/conftest.py rename to tests/core/pyspec/eth2spec/test/conftest.py index 5713c3470..1d299042a 100644 --- a/test_libs/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -34,6 +34,6 @@ def pytest_addoption(parser): def config(request): config_name = request.config.getoption("--config") from preset_loader import loader - presets = loader.load_presets('../../configs/', config_name) + presets = loader.load_presets('../../../configs/', config_name) spec_phase0.apply_constants_preset(presets) spec_phase1.apply_constants_preset(presets) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/context.py rename to tests/core/pyspec/eth2spec/test/context.py diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/fork_choice/test_get_head.py rename to tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/fork_choice/test_on_attestation.py rename to tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/fork_choice/test_on_block.py rename to tests/core/pyspec/eth2spec/test/fork_choice/test_on_block.py diff --git a/test_libs/pyspec/eth2spec/test/fork_choice/test_on_tick.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/fork_choice/test_on_tick.py rename to tests/core/pyspec/eth2spec/test/fork_choice/test_on_tick.py diff --git a/test_libs/pyspec/eth2spec/phase1/__init__.py b/tests/core/pyspec/eth2spec/test/genesis/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/phase1/__init__.py rename to tests/core/pyspec/eth2spec/test/genesis/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/genesis/test_initialization.py rename to tests/core/pyspec/eth2spec/test/genesis/test_initialization.py diff --git a/test_libs/pyspec/eth2spec/test/genesis/test_validity.py b/tests/core/pyspec/eth2spec/test/genesis/test_validity.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/genesis/test_validity.py rename to tests/core/pyspec/eth2spec/test/genesis/test_validity.py diff --git a/test_libs/pyspec/eth2spec/test/__init__.py b/tests/core/pyspec/eth2spec/test/helpers/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/__init__.py rename to tests/core/pyspec/eth2spec/test/helpers/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/attestations.py rename to tests/core/pyspec/eth2spec/test/helpers/attestations.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/attester_slashings.py rename to tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/tests/core/pyspec/eth2spec/test/helpers/block.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/block.py rename to tests/core/pyspec/eth2spec/test/helpers/block.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/tests/core/pyspec/eth2spec/test/helpers/block_header.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/block_header.py rename to tests/core/pyspec/eth2spec/test/helpers/block_header.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/custody.py rename to tests/core/pyspec/eth2spec/test/helpers/custody.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/deposits.py rename to tests/core/pyspec/eth2spec/test/helpers/deposits.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/genesis.py rename to tests/core/pyspec/eth2spec/test/helpers/genesis.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/keys.py b/tests/core/pyspec/eth2spec/test/helpers/keys.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/keys.py rename to tests/core/pyspec/eth2spec/test/helpers/keys.py diff --git a/test_libs/pyspec/eth2spec/test/genesis/__init__.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/genesis/__init__.py rename to tests/core/pyspec/eth2spec/test/helpers/phase1/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py rename to tests/core/pyspec/eth2spec/test/helpers/phase1/attestations.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_block.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py rename to tests/core/pyspec/eth2spec/test/helpers/phase1/shard_block.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_state.py b/tests/core/pyspec/eth2spec/test/helpers/phase1/shard_state.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/phase1/shard_state.py rename to tests/core/pyspec/eth2spec/test/helpers/phase1/shard_state.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/proposer_slashings.py rename to tests/core/pyspec/eth2spec/test/helpers/proposer_slashings.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/state.py rename to tests/core/pyspec/eth2spec/test/helpers/state.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py rename to tests/core/pyspec/eth2spec/test/helpers/voluntary_exits.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/__init__.py b/tests/core/pyspec/eth2spec/test/merkle_proofs/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/__init__.py rename to tests/core/pyspec/eth2spec/test/merkle_proofs/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py b/tests/core/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py rename to tests/core/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/__init__.py b/tests/core/pyspec/eth2spec/test/phase_0/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/helpers/phase1/__init__.py rename to tests/core/pyspec/eth2spec/test/phase_0/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/merkle_proofs/__init__.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/merkle_proofs/__init__.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py rename to tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/__init__.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/__init__.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/run_epoch_process_base.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_final_updates.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py rename to tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/block_processing/__init__.py rename to tests/core/pyspec/eth2spec/test/phase_1/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py rename to tests/core/pyspec/eth2spec/test/phase_1/block_processing/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py rename to tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_bit_challenge.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py rename to tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py rename to tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py rename to tests/core/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/__init__.py b/tests/core/pyspec/eth2spec/test/sanity/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_1/__init__.py rename to tests/core/pyspec/eth2spec/test/sanity/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/sanity/test_blocks.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/sanity/test_blocks.py rename to tests/core/pyspec/eth2spec/test/sanity/test_blocks.py diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py b/tests/core/pyspec/eth2spec/test/sanity/test_slots.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/sanity/test_slots.py rename to tests/core/pyspec/eth2spec/test/sanity/test_slots.py diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/tests/core/pyspec/eth2spec/test/test_finality.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/test_finality.py rename to tests/core/pyspec/eth2spec/test/test_finality.py diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/tests/core/pyspec/eth2spec/test/utils.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/utils.py rename to tests/core/pyspec/eth2spec/test/utils.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/__init__.py b/tests/core/pyspec/eth2spec/utils/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/phase_1/block_processing/__init__.py rename to tests/core/pyspec/eth2spec/utils/__init__.py diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/bls.py rename to tests/core/pyspec/eth2spec/utils/bls.py diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/tests/core/pyspec/eth2spec/utils/hash_function.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/hash_function.py rename to tests/core/pyspec/eth2spec/utils/hash_function.py diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/tests/core/pyspec/eth2spec/utils/merkle_minimal.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/merkle_minimal.py rename to tests/core/pyspec/eth2spec/utils/merkle_minimal.py diff --git a/test_libs/pyspec/eth2spec/test/sanity/__init__.py b/tests/core/pyspec/eth2spec/utils/ssz/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/sanity/__init__.py rename to tests/core/pyspec/eth2spec/utils/ssz/__init__.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py rename to tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py rename to tests/core/pyspec/eth2spec/utils/ssz/ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/test_ssz_impl.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/ssz/test_ssz_impl.py rename to tests/core/pyspec/eth2spec/utils/ssz/test_ssz_impl.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py b/tests/core/pyspec/eth2spec/utils/ssz/test_ssz_typing.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/ssz/test_ssz_typing.py rename to tests/core/pyspec/eth2spec/utils/ssz/test_ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/tests/core/pyspec/eth2spec/utils/test_merkle_minimal.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py rename to tests/core/pyspec/eth2spec/utils/test_merkle_minimal.py diff --git a/test_libs/pyspec/requirements-testing.txt b/tests/core/pyspec/requirements-testing.txt similarity index 100% rename from test_libs/pyspec/requirements-testing.txt rename to tests/core/pyspec/requirements-testing.txt diff --git a/test_libs/pyspec/requirements.txt b/tests/core/pyspec/requirements.txt similarity index 100% rename from test_libs/pyspec/requirements.txt rename to tests/core/pyspec/requirements.txt diff --git a/test_libs/pyspec/setup.py b/tests/core/pyspec/setup.py similarity index 100% rename from test_libs/pyspec/setup.py rename to tests/core/pyspec/setup.py diff --git a/specs/test_formats/README.md b/tests/formats/README.md similarity index 100% rename from specs/test_formats/README.md rename to tests/formats/README.md diff --git a/specs/test_formats/bls/README.md b/tests/formats/bls/README.md similarity index 100% rename from specs/test_formats/bls/README.md rename to tests/formats/bls/README.md diff --git a/specs/test_formats/bls/aggregate_pubkeys.md b/tests/formats/bls/aggregate_pubkeys.md similarity index 100% rename from specs/test_formats/bls/aggregate_pubkeys.md rename to tests/formats/bls/aggregate_pubkeys.md diff --git a/specs/test_formats/bls/aggregate_sigs.md b/tests/formats/bls/aggregate_sigs.md similarity index 100% rename from specs/test_formats/bls/aggregate_sigs.md rename to tests/formats/bls/aggregate_sigs.md diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/tests/formats/bls/msg_hash_g2_compressed.md similarity index 100% rename from specs/test_formats/bls/msg_hash_g2_compressed.md rename to tests/formats/bls/msg_hash_g2_compressed.md diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/tests/formats/bls/msg_hash_g2_uncompressed.md similarity index 100% rename from specs/test_formats/bls/msg_hash_g2_uncompressed.md rename to tests/formats/bls/msg_hash_g2_uncompressed.md diff --git a/specs/test_formats/bls/priv_to_pub.md b/tests/formats/bls/priv_to_pub.md similarity index 100% rename from specs/test_formats/bls/priv_to_pub.md rename to tests/formats/bls/priv_to_pub.md diff --git a/specs/test_formats/bls/sign_msg.md b/tests/formats/bls/sign_msg.md similarity index 100% rename from specs/test_formats/bls/sign_msg.md rename to tests/formats/bls/sign_msg.md diff --git a/specs/test_formats/epoch_processing/README.md b/tests/formats/epoch_processing/README.md similarity index 100% rename from specs/test_formats/epoch_processing/README.md rename to tests/formats/epoch_processing/README.md diff --git a/specs/test_formats/genesis/README.md b/tests/formats/genesis/README.md similarity index 100% rename from specs/test_formats/genesis/README.md rename to tests/formats/genesis/README.md diff --git a/specs/test_formats/genesis/initialization.md b/tests/formats/genesis/initialization.md similarity index 100% rename from specs/test_formats/genesis/initialization.md rename to tests/formats/genesis/initialization.md diff --git a/specs/test_formats/genesis/validity.md b/tests/formats/genesis/validity.md similarity index 100% rename from specs/test_formats/genesis/validity.md rename to tests/formats/genesis/validity.md diff --git a/specs/test_formats/operations/README.md b/tests/formats/operations/README.md similarity index 100% rename from specs/test_formats/operations/README.md rename to tests/formats/operations/README.md diff --git a/specs/test_formats/sanity/README.md b/tests/formats/sanity/README.md similarity index 100% rename from specs/test_formats/sanity/README.md rename to tests/formats/sanity/README.md diff --git a/specs/test_formats/sanity/blocks.md b/tests/formats/sanity/blocks.md similarity index 100% rename from specs/test_formats/sanity/blocks.md rename to tests/formats/sanity/blocks.md diff --git a/specs/test_formats/sanity/slots.md b/tests/formats/sanity/slots.md similarity index 100% rename from specs/test_formats/sanity/slots.md rename to tests/formats/sanity/slots.md diff --git a/specs/test_formats/shuffling/README.md b/tests/formats/shuffling/README.md similarity index 96% rename from specs/test_formats/shuffling/README.md rename to tests/formats/shuffling/README.md index a2184020b..15bfe6996 100644 --- a/specs/test_formats/shuffling/README.md +++ b/tests/formats/shuffling/README.md @@ -30,7 +30,7 @@ The `count` specifies the validator registry size. One should compute the shuffl The `seed` is the raw shuffling seed, passed to permute-index (or optimized shuffling approach). The `mapping` is a look up array, constructed as `[spec.compute_shuffled_index(i, count, seed) for i in range(count)]` -I.e. `mapping[i]` is the shuffled location of `i`. +I.e. `mapping[i]` is the shuffled location of `i`. ## Condition diff --git a/specs/test_formats/ssz_generic/README.md b/tests/formats/ssz_generic/README.md similarity index 100% rename from specs/test_formats/ssz_generic/README.md rename to tests/formats/ssz_generic/README.md diff --git a/specs/test_formats/ssz_static/README.md b/tests/formats/ssz_static/README.md similarity index 81% rename from specs/test_formats/ssz_static/README.md rename to tests/formats/ssz_static/README.md index 78df2fc9a..1ac91cb13 100644 --- a/specs/test_formats/ssz_static/README.md +++ b/tests/formats/ssz_static/README.md @@ -3,6 +3,6 @@ This set of test-suites provides static testing for SSZ: to instantiate just the known Eth2 SSZ types from binary data. -This series of tests is based on the spec-maintained `eth2spec/utils/ssz/ssz_impl.py`, i.e. fully consistent with the SSZ spec. +This series of tests is based on the spec-maintained `eth2spec/utils/ssz/ssz_impl.py`, i.e. fully consistent with the SSZ spec. Test format documentation can be found here: [core test format](./core.md). diff --git a/specs/test_formats/ssz_static/core.md b/tests/formats/ssz_static/core.md similarity index 92% rename from specs/test_formats/ssz_static/core.md rename to tests/formats/ssz_static/core.md index ddb370983..d6f99a32b 100644 --- a/specs/test_formats/ssz_static/core.md +++ b/tests/formats/ssz_static/core.md @@ -49,5 +49,5 @@ A test-runner can implement the following assertions: ## References -**`serialized`**—[SSZ serialization](../../simple-serialize.md#serialization) -**`root`**—[hash_tree_root](../../simple-serialize.md#merkleization) function +**`serialized`**—[SSZ serialization](../../../ssz/simple-serialize.md#serialization) +**`root`**—[hash_tree_root](../../../ssz/simple-serialize.md#merkleization) function diff --git a/test_generators/README.md b/tests/generators/README.md similarity index 97% rename from test_generators/README.md rename to tests/generators/README.md index 9b1aab29c..77a50606b 100644 --- a/test_generators/README.md +++ b/tests/generators/README.md @@ -78,9 +78,9 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` -../../test_libs/gen_helpers -../../test_libs/config_helpers -../../test_libs/pyspec +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec ``` The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests. @@ -200,7 +200,7 @@ Recommendations: - Use config `minimal` for performance and simplicity, but also implement a suite with the `mainnet` config where necessary. - You may be able to write your test case provider in a way where it does not make assumptions on constants. If so, you can generate test cases with different configurations for the same scenario (see example). -- See [`test_libs/gen_helpers/README.md`](../test_libs/gen_helpers/README.md) for command line options for generators. +- See [`tests/core/gen_helpers/README.md`](../core/gen_helpers/README.md) for command line options for generators. ## How to add a new test generator diff --git a/test_generators/bls/README.md b/tests/generators/bls/README.md similarity index 100% rename from test_generators/bls/README.md rename to tests/generators/bls/README.md diff --git a/test_generators/bls/main.py b/tests/generators/bls/main.py similarity index 100% rename from test_generators/bls/main.py rename to tests/generators/bls/main.py diff --git a/test_generators/bls/requirements.txt b/tests/generators/bls/requirements.txt similarity index 52% rename from test_generators/bls/requirements.txt rename to tests/generators/bls/requirements.txt index 8a72affe0..7b1d2f4a9 100644 --- a/test_generators/bls/requirements.txt +++ b/tests/generators/bls/requirements.txt @@ -1,3 +1,3 @@ py_ecc==2.0.0 eth-utils==1.6.0 -../../test_libs/gen_helpers +../../core/gen_helpers diff --git a/test_generators/epoch_processing/README.md b/tests/generators/epoch_processing/README.md similarity index 100% rename from test_generators/epoch_processing/README.md rename to tests/generators/epoch_processing/README.md diff --git a/test_generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py similarity index 100% rename from test_generators/epoch_processing/main.py rename to tests/generators/epoch_processing/main.py diff --git a/tests/generators/epoch_processing/requirements.txt b/tests/generators/epoch_processing/requirements.txt new file mode 100644 index 000000000..3c318f56b --- /dev/null +++ b/tests/generators/epoch_processing/requirements.txt @@ -0,0 +1,3 @@ +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec \ No newline at end of file diff --git a/test_generators/genesis/README.md b/tests/generators/genesis/README.md similarity index 71% rename from test_generators/genesis/README.md rename to tests/generators/genesis/README.md index 8a2b01c62..e270f6e35 100644 --- a/test_generators/genesis/README.md +++ b/tests/generators/genesis/README.md @@ -2,7 +2,5 @@ Genesis tests cover the initialization and validity-based launch trigger for the Beacon Chain genesis state. -Information on the format of the tests can be found in the [genesis test formats documentation](../../specs/test_formats/genesis/README.md). - - +Information on the format of the tests can be found in the [genesis test formats documentation](../../formats/genesis/README.md). diff --git a/test_generators/genesis/main.py b/tests/generators/genesis/main.py similarity index 100% rename from test_generators/genesis/main.py rename to tests/generators/genesis/main.py diff --git a/tests/generators/genesis/requirements.txt b/tests/generators/genesis/requirements.txt new file mode 100644 index 000000000..3c318f56b --- /dev/null +++ b/tests/generators/genesis/requirements.txt @@ -0,0 +1,3 @@ +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec \ No newline at end of file diff --git a/test_generators/operations/README.md b/tests/generators/operations/README.md similarity index 100% rename from test_generators/operations/README.md rename to tests/generators/operations/README.md diff --git a/test_generators/operations/main.py b/tests/generators/operations/main.py similarity index 100% rename from test_generators/operations/main.py rename to tests/generators/operations/main.py diff --git a/tests/generators/operations/requirements.txt b/tests/generators/operations/requirements.txt new file mode 100644 index 000000000..f34243cf4 --- /dev/null +++ b/tests/generators/operations/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.6.0 +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec \ No newline at end of file diff --git a/test_generators/sanity/README.md b/tests/generators/sanity/README.md similarity index 100% rename from test_generators/sanity/README.md rename to tests/generators/sanity/README.md diff --git a/test_generators/sanity/main.py b/tests/generators/sanity/main.py similarity index 100% rename from test_generators/sanity/main.py rename to tests/generators/sanity/main.py diff --git a/tests/generators/sanity/requirements.txt b/tests/generators/sanity/requirements.txt new file mode 100644 index 000000000..3c318f56b --- /dev/null +++ b/tests/generators/sanity/requirements.txt @@ -0,0 +1,3 @@ +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec \ No newline at end of file diff --git a/test_generators/shuffling/README.md b/tests/generators/shuffling/README.md similarity index 100% rename from test_generators/shuffling/README.md rename to tests/generators/shuffling/README.md diff --git a/test_generators/shuffling/main.py b/tests/generators/shuffling/main.py similarity index 100% rename from test_generators/shuffling/main.py rename to tests/generators/shuffling/main.py diff --git a/tests/generators/shuffling/requirements.txt b/tests/generators/shuffling/requirements.txt new file mode 100644 index 000000000..f34243cf4 --- /dev/null +++ b/tests/generators/shuffling/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.6.0 +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/tests/generators/ssz_generic/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/__init__.py rename to tests/generators/ssz_generic/__init__.py diff --git a/test_generators/ssz_generic/main.py b/tests/generators/ssz_generic/main.py similarity index 100% rename from test_generators/ssz_generic/main.py rename to tests/generators/ssz_generic/main.py diff --git a/tests/generators/ssz_generic/requirements.txt b/tests/generators/ssz_generic/requirements.txt new file mode 100644 index 000000000..6b11d61af --- /dev/null +++ b/tests/generators/ssz_generic/requirements.txt @@ -0,0 +1,4 @@ +eth-utils==1.6.0 +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec diff --git a/test_generators/ssz_generic/ssz_basic_vector.py b/tests/generators/ssz_generic/ssz_basic_vector.py similarity index 100% rename from test_generators/ssz_generic/ssz_basic_vector.py rename to tests/generators/ssz_generic/ssz_basic_vector.py diff --git a/test_generators/ssz_generic/ssz_bitlist.py b/tests/generators/ssz_generic/ssz_bitlist.py similarity index 100% rename from test_generators/ssz_generic/ssz_bitlist.py rename to tests/generators/ssz_generic/ssz_bitlist.py diff --git a/test_generators/ssz_generic/ssz_bitvector.py b/tests/generators/ssz_generic/ssz_bitvector.py similarity index 100% rename from test_generators/ssz_generic/ssz_bitvector.py rename to tests/generators/ssz_generic/ssz_bitvector.py diff --git a/test_generators/ssz_generic/ssz_boolean.py b/tests/generators/ssz_generic/ssz_boolean.py similarity index 100% rename from test_generators/ssz_generic/ssz_boolean.py rename to tests/generators/ssz_generic/ssz_boolean.py diff --git a/test_generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py similarity index 100% rename from test_generators/ssz_generic/ssz_container.py rename to tests/generators/ssz_generic/ssz_container.py diff --git a/test_generators/ssz_generic/ssz_test_case.py b/tests/generators/ssz_generic/ssz_test_case.py similarity index 100% rename from test_generators/ssz_generic/ssz_test_case.py rename to tests/generators/ssz_generic/ssz_test_case.py diff --git a/test_generators/ssz_generic/ssz_uints.py b/tests/generators/ssz_generic/ssz_uints.py similarity index 100% rename from test_generators/ssz_generic/ssz_uints.py rename to tests/generators/ssz_generic/ssz_uints.py diff --git a/test_generators/ssz_generic/uint_test_cases.py b/tests/generators/ssz_generic/uint_test_cases.py similarity index 100% rename from test_generators/ssz_generic/uint_test_cases.py rename to tests/generators/ssz_generic/uint_test_cases.py diff --git a/test_generators/ssz_static/README.md b/tests/generators/ssz_static/README.md similarity index 100% rename from test_generators/ssz_static/README.md rename to tests/generators/ssz_static/README.md diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/tests/generators/ssz_static/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/utils/ssz/__init__.py rename to tests/generators/ssz_static/__init__.py diff --git a/test_generators/ssz_static/main.py b/tests/generators/ssz_static/main.py similarity index 100% rename from test_generators/ssz_static/main.py rename to tests/generators/ssz_static/main.py diff --git a/tests/generators/ssz_static/requirements.txt b/tests/generators/ssz_static/requirements.txt new file mode 100644 index 000000000..3c318f56b --- /dev/null +++ b/tests/generators/ssz_static/requirements.txt @@ -0,0 +1,3 @@ +../../core/gen_helpers +../../core/config_helpers +../../core/pyspec \ No newline at end of file From 55fb97d8b1c1005709a6ed073e77984fe45537ed Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 10 Jan 2020 12:03:05 -0700 Subject: [PATCH 48/48] bump cache versions --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04d5bf799..45cb958a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: - venv_name: v6-pyspec + venv_name: v7-pyspec reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} save_pyspec_cached_venv: description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: - venv_name: v6-pyspec + venv_name: v7-pyspec reqs_checksum: cache-{{ checksum "tests/core/pyspec/requirements.txt" }}-{{ checksum "tests/core/pyspec/requirements-testing.txt" }} venv_path: ./tests/core/pyspec/venv restore_deposit_contract_cached_venv: