From ccace9a0342c5fd22c7896b5f8c5a9d384d5b7c9 Mon Sep 17 00:00:00 2001 From: Mamy Ratsimbazafy Date: Thu, 19 Mar 2020 17:18:48 +0100 Subject: [PATCH 01/13] Fix shortlog crypto (#818) * fix shortlogs of crypto types * Valid signature without a "real: " prefix tag * remove 0x prefix --- beacon_chain/spec/crypto.nim | 14 +++++++++++--- beacon_chain/spec/state_transition_block.nim | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index 2138439ce..095b2b0fb 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -202,9 +202,17 @@ proc newKeyPair*(): tuple[pub: ValidatorPubKey, priv: ValidatorPrivKey] {.noInit # ---------------------------------------------------------------------- func shortLog*(x: BlsValue): string = - ($x)[0..7] + ## Logging for wrapped BLS types + ## that may contain valid or non-validated data + # The prefix must be short + # due to the mechanics of the `shortLog` function. + if x.kind == Real: + x.blsValue.toHex()[0..7] + else: + "raw: " & x.blob.toHex(lowercase = true)[0..7] func shortLog*(x: BlsCurveType): string = + ## Logging for raw unwrapped BLS types ($x)[0..7] proc toGaugeValue*(hash: Eth2Digest): int64 = @@ -220,9 +228,9 @@ func `$`*(x: BlsValue): string = # The prefix must be short # due to the mechanics of the `shortLog` function. if x.kind == Real: - "real: 0x" & x.blsValue.toHex() + x.blsValue.toHex() else: - "raw: 0x" & x.blob.toHex(lowercase = true) + "raw: " & x.blob.toHex(lowercase = true) func getBytes*(x: BlsValue): auto = if x.kind == Real: diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index b0b4d2bb8..f86b37646 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -111,9 +111,9 @@ proc process_randao( let signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) if skipBLSValidation notin flags: if not blsVerify(proposer.pubkey, signing_root.data, body.randao_reveal): - notice "Randao mismatch", proposer_pubkey = proposer.pubkey, + notice "Randao mismatch", proposer_pubkey = shortLog(proposer.pubkey), message = epoch, - signature = body.randao_reveal, + signature = shortLog(body.randao_reveal), slot = state.slot return false From 689bcf71c4e9d050bad28e125e43fa24de2de9ab Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 20 Mar 2020 00:48:03 +0100 Subject: [PATCH 02/13] clean up block creation * consistently use state at new block slot to produce block * factor out signature funcs * fix missing block application test --- beacon_chain/attestation_aggregation.nim | 12 +- beacon_chain/attestation_pool.nim | 9 +- beacon_chain/beacon_node.nim | 63 ++++------- beacon_chain/extras.nim | 8 -- beacon_chain/spec/beaconstate.nim | 8 +- beacon_chain/spec/datatypes.nim | 1 + beacon_chain/spec/state_transition_block.nim | 84 ++++++++++++++ beacon_chain/validator_pool.nim | 23 +--- research/state_sim.nim | 6 +- tests/mocking/mock_attestations.nim | 19 +--- tests/mocking/mock_blocks.nim | 27 +---- tests/test_attestation_pool.nim | 66 +++++------ tests/test_block_pool.nim | 17 ++- tests/test_state_transition.nim | 10 +- tests/testblockutil.nim | 110 ++++++++----------- 15 files changed, 224 insertions(+), 239 deletions(-) diff --git a/beacon_chain/attestation_aggregation.nim b/beacon_chain/attestation_aggregation.nim index c9801642a..f6c2bf663 100644 --- a/beacon_chain/attestation_aggregation.nim +++ b/beacon_chain/attestation_aggregation.nim @@ -22,7 +22,8 @@ import options, - ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator], + ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator, + state_transition_block], ./attestation_pool, ./beacon_node_types, ./ssz # TODO gossipsub validation lives somewhere, maybe here @@ -33,13 +34,6 @@ const # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/p2p-interface.md#configuration ATTESTATION_PROPAGATION_SLOT_RANGE = 32 -# https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/validator/0_beacon-chain-validator.md#aggregation-selection -func get_slot_signature(state: BeaconState, slot: Slot, privkey: ValidatorPrivKey): - ValidatorSig = - let domain = - get_domain(state, DOMAIN_BEACON_ATTESTER, compute_epoch_at_slot(slot)) - bls_sign(privkey, hash_tree_root(slot).data, domain) - # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/validator.md#aggregation-selection func is_aggregator(state: BeaconState, slot: Slot, index: uint64, slot_signature: ValidatorSig): bool = @@ -58,7 +52,7 @@ proc aggregate_attestations*( let slot = state.slot - 2 - slot_signature = get_slot_signature(state, slot, privkey) + slot_signature = get_slot_signature(state.fork, slot, privkey) if slot < 0: return none(AggregateAndProof) diff --git a/beacon_chain/attestation_pool.nim b/beacon_chain/attestation_pool.nim index 1e145a3a8..cf2260f98 100644 --- a/beacon_chain/attestation_pool.nim +++ b/beacon_chain/attestation_pool.nim @@ -266,10 +266,12 @@ proc add*(pool: var AttestationPool, attestation: Attestation) = pool.addResolved(blck, attestation) proc getAttestationsForBlock*( - pool: AttestationPool, state: BeaconState, - newBlockSlot: Slot): seq[Attestation] = + pool: AttestationPool, state: BeaconState): seq[Attestation] = + ## Retrieve attestations that may be added to a new block at the slot of the + ## given state logScope: pcs = "retrieve_attestation" + let newBlockSlot = state.slot if newBlockSlot < (GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY): debug "Too early for attestations", newBlockSlot = shortLog(newBlockSlot), @@ -327,8 +329,7 @@ proc getAttestationsForBlock*( # TODO we're checking signatures here every time which is very slow - this # is needed because validate does nothing for now and we don't want # to include a broken attestation - if not check_attestation( - state, attestation, {nextSlot}, cache): + if not check_attestation(state, attestation, {}, cache): continue for v in a.validations[1..^1]: diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index cd8555e4f..9eaa2acc3 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -10,7 +10,8 @@ import eth/p2p/enode, eth/[keys, async_utils], eth/p2p/discoveryv5/enr, # Local modules - spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network], + spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network, + state_transition_block], conf, time, state_transition, beacon_chain_db, validator_pool, extras, attestation_pool, block_pool, eth2_network, eth2_discovery, beacon_node_types, mainchain_monitor, version, ssz, ssz/dynamic_navigator, @@ -381,8 +382,8 @@ proc proposeBlock(node: BeaconNode, cat = "fastforward" return head - # Advance state to the slot immediately preceding the one we're creating a - # block for - potentially we will be processing empty slots along the way. + # Advance state to the slot that we're proposing for - this is the equivalent + # of running `process_slots` up to the slot of the new block. let (nroot, nblck) = node.blockPool.withState( node.blockPool.tmpState, head.atSlot(slot)): let (eth1data, deposits) = @@ -394,45 +395,27 @@ proc proposeBlock(node: BeaconNode, (node.mainchainMonitor.eth1Data, node.mainchainMonitor.getPendingDeposits()) - # To create a block, we'll first apply a partial block to the state, skipping - # some validations. - let - fork = state.fork - blockBody = BeaconBlockBody( - randao_reveal: validator.genRandaoReveal(fork, slot), - eth1_data: eth1data, - attestations: - node.attestationPool.getAttestationsForBlock(state, slot), - deposits: deposits) - - var cache = get_empty_per_epoch_cache() - let proposer_index = get_beacon_proposer_index(state, cache) - if proposer_index.isNone: - doAssert false, "proposeBlock: missing proposer index" + let message = makeBeaconBlock( + state, + head.root, + validator.genRandaoReveal(state.fork, slot), + eth1data, + Eth2Digest(), + node.attestationPool.getAttestationsForBlock(state), + deposits) + if not message.isSome(): + return head # already logged elsewhere! var newBlock = SignedBeaconBlock( - message: BeaconBlock( - slot: slot, - proposer_index: proposer_index.get.uint64, - parent_root: head.root, - body: blockBody)) - tmpState = hashedState - discard state_transition(tmpState, newBlock, {skipStateRootValidation}) - # TODO only enable in fast-fail debugging situations - # otherwise, bad attestations can bring down network - # doAssert ok # TODO: err, could this fail somehow? - - newBlock.message.state_root = tmpState.root + message: message.get() + ) let blockRoot = hash_tree_root(newBlock.message) - # Careful, state no longer valid after here.. - # We use the fork from the pre-newBlock state which should be fine because - # fork carries two epochs, so even if it's a fork block, the right thing - # will happen here + # Careful, state no longer valid after here because of the await.. newBlock.signature = - await validator.signBlockProposal(fork, slot, blockRoot) + await validator.signBlockProposal(state.fork, slot, blockRoot) (blockRoot, newBlock) @@ -454,10 +437,12 @@ proc proposeBlock(node: BeaconNode, SSZ.saveFile( node.config.dumpDir / "block-" & $newBlock.message.slot & "-" & shortLog(newBlockRef.root) & ".ssz", newBlock) - SSZ.saveFile( - node.config.dumpDir / "state-" & $tmpState.data.slot & "-" & - shortLog(newBlockRef.root) & "-" & shortLog(tmpState.root) & ".ssz", - tmpState.data) + node.blockPool.withState( + node.blockPool.tmpState, newBlockRef.atSlot(newBlockRef.slot)): + SSZ.saveFile( + node.config.dumpDir / "state-" & $state.slot & "-" & + shortLog(newBlockRef.root) & "-" & shortLog(root()) & ".ssz", + state()) node.network.broadcast(topicBeaconBlocks, newBlock) diff --git a/beacon_chain/extras.nim b/beacon_chain/extras.nim index 3a5790c75..ccda6f41f 100644 --- a/beacon_chain/extras.nim +++ b/beacon_chain/extras.nim @@ -16,16 +16,8 @@ # improved coverage, and to avoid unnecessary validation when replaying trusted # (previously validated) blocks. - type UpdateFlag* = enum - nextSlot ##\ - ## Perform the operation as if the next slot was being processed - this is - ## useful when using the state to verify data that will go in the next slot, - ## for example when proposing - ## TODO need to be careful here, easy to assume that slot number change is - ## enough, vs advancing the state - however, making a full state copy - ## is expensive also :/ skipMerkleValidation ##\ ## When processing deposits, skip verifying the Merkle proof trees of each ## deposit. This is a holdover from both interop issues with the malformed diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 6e056fbb2..08ba2774f 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -444,11 +444,9 @@ proc check_attestation*( ## at the current slot. When acting as a proposer, the same rules need to ## be followed! - let stateSlot = - if nextSlot in flags: state.slot + 1 - else: state.slot - - let data = attestation.data + let + stateSlot = state.slot + data = attestation.data trace "process_attestation: beginning", attestation=attestation diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index cb8495619..9476886ee 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -592,6 +592,7 @@ func shortLog*(e: Epoch): uint64 = func shortLog*(v: BeaconBlock): auto = ( slot: shortLog(v.slot), + proposer_index: v.proposer_index, parent_root: shortLog(v.parent_root), state_root: shortLog(v.state_root), proposer_slashings_len: v.body.proposer_slashings.len(), diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index f86b37646..7167dc83a 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -436,3 +436,87 @@ proc process_block*( return false true + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md +# TODO There's more to do here - the spec has helpers that deal set up some of +# the fields in here! +proc makeBeaconBlock*( + state: BeaconState, + parent_root: Eth2Digest, + randao_reveal: ValidatorSig, + eth1_data: Eth1Data, + graffiti: Eth2Digest, + attestations: seq[Attestation], + deposits: seq[Deposit]): Option[BeaconBlock] = + ## Create a block for the given state. The last block applied to it must be + ## the one identified by parent_root and process_slots must be called up to + ## the slot for which a block is to be created. + var cache = get_empty_per_epoch_cache() + let proposer_index = get_beacon_proposer_index(state, cache) + doAssert proposer_index.isSome, "Unable to get proposer index when proposing!" + + # To create a block, we'll first apply a partial block to the state, skipping + # some validations. + + var blck = BeaconBlock( + slot: state.slot, + proposer_index: proposer_index.get().uint64, + parent_root: parent_root, + body: BeaconBlockBody( + randao_reveal: randao_reveal, + eth1_data: eth1data, + graffiti: graffiti, + attestations: attestations, + deposits: deposits) + ) + + var tmpState = state + let ok = process_block(tmpState, blck, {skipBlsValidation}, cache) + + if not ok: + warn "Unable to apply new block to state", blck = shortLog(blck) + return + + blck.state_root = hash_tree_root(tmpState) + + some(blck) + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md +func get_slot_signature*( + fork: Fork, slot: Slot, privkey: ValidatorPrivKey): ValidatorSig = + let + domain = + get_domain(fork, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + + blsSign(privKey, signing_root.data) + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md +func get_epoch_signature*( + fork: Fork, slot: Slot, privkey: ValidatorPrivKey): ValidatorSig = + let + domain = + get_domain(fork, DOMAIN_RANDAO, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(compute_epoch_at_slot(slot), domain) + + blsSign(privKey, signing_root.data) + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md +func get_block_signature*( + fork: Fork, slot: Slot, root: Eth2Digest, privkey: ValidatorPrivKey): ValidatorSig = + let + domain = + get_domain(fork, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(root, domain) + + blsSign(privKey, signing_root.data) + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md +func get_attestation_signature*( + fork: Fork, attestation: AttestationData, privkey: ValidatorPrivKey): ValidatorSig = + let + attestationRoot = hash_tree_root(attestation) + domain = get_domain(fork, DOMAIN_BEACON_ATTESTER, attestation.target.epoch) + signing_root = compute_signing_root(attestationRoot, domain) + + blsSign(privKey, signing_root.data) diff --git a/beacon_chain/validator_pool.nim b/beacon_chain/validator_pool.nim index da528616b..a79e5d235 100644 --- a/beacon_chain/validator_pool.nim +++ b/beacon_chain/validator_pool.nim @@ -1,7 +1,7 @@ import tables, chronos, chronicles, - spec/[datatypes, crypto, digest, helpers], ssz, + spec/[datatypes, crypto, digest, state_transition_block], ssz, beacon_node_types func init*(T: type ValidatorPool): T = @@ -29,18 +29,12 @@ proc signBlockProposal*(v: AttachedValidator, fork: Fork, slot: Slot, blockRoot: Eth2Digest): Future[ValidatorSig] {.async.} = if v.kind == inProcess: - # TODO state might become invalid after any async calls - it's fragile to - # care about this in here - let - domain = - get_domain(fork, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(slot)) # TODO this is an ugly hack to fake a delay and subsequent async reordering # for the purpose of testing the external validator delay - to be # replaced by something more sensible await sleepAsync(chronos.milliseconds(1)) - let signing_root = compute_signing_root(blockRoot, domain) - result = blsSign(v.privKey, signing_root.data) + result = get_block_signature(fork, slot, blockRoot, v.privKey) else: error "Unimplemented" quit 1 @@ -49,17 +43,12 @@ proc signAttestation*(v: AttachedValidator, attestation: AttestationData, fork: Fork): Future[ValidatorSig] {.async.} = if v.kind == inProcess: - let - attestationRoot = hash_tree_root(attestation) - domain = get_domain(fork, DOMAIN_BEACON_ATTESTER, attestation.target.epoch) - # TODO this is an ugly hack to fake a delay and subsequent async reordering # for the purpose of testing the external validator delay - to be # replaced by something more sensible await sleepAsync(chronos.milliseconds(1)) - let signing_root = compute_signing_root(attestationRoot, domain) - result = blsSign(v.privKey, signing_root.data) + result = get_attestation_signature(fork, attestation, v.privKey) else: error "Unimplemented" quit 1 @@ -67,11 +56,7 @@ proc signAttestation*(v: AttachedValidator, # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/validator.md#randao-reveal func genRandaoReveal*(k: ValidatorPrivKey, fork: Fork, slot: Slot): ValidatorSig = - let - domain = get_domain(fork, DOMAIN_RANDAO, compute_epoch_at_slot(slot)) - signing_root = compute_signing_root(compute_epoch_at_slot(slot).uint64, domain) - - bls_sign(k, signing_root.data) + get_epoch_signature(fork, slot, k) func genRandaoReveal*(v: AttachedValidator, fork: Fork, slot: Slot): ValidatorSig = diff --git a/research/state_sim.nim b/research/state_sim.nim index b6c0b79c1..0b6f7bb4e 100644 --- a/research/state_sim.nim +++ b/research/state_sim.nim @@ -121,8 +121,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6, let attestations_idx = state.slot - body = BeaconBlockBody( - attestations: attestations.getOrDefault(attestations_idx)) + blockAttestations = attestations.getOrDefault(attestations_idx) attestations.del attestations_idx doAssert len(attestations) <= @@ -134,7 +133,8 @@ cli do(slots = SLOTS_PER_EPOCH * 6, else: tBlock withTimer(timers[t]): - signedBlock = addBlock(state, latest_block_root, body, flags) + signedBlock = addTestBlock( + state, latest_block_root, attestations = blockAttestations, flags = flags) latest_block_root = withTimerRet(timers[tHashBlock]): hash_tree_root(signedBlock.message) diff --git a/tests/mocking/mock_attestations.nim b/tests/mocking/mock_attestations.nim index 1130096c2..36e3552e7 100644 --- a/tests/mocking/mock_attestations.nim +++ b/tests/mocking/mock_attestations.nim @@ -12,7 +12,8 @@ import # Standard library sets, # Specs - ../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto], + ../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto, + state_transition_block], # Internals ../../beacon_chain/[ssz, extras, state_transition], # Mocking procs @@ -53,20 +54,6 @@ proc mockAttestationData( epoch: target_epoch, root: epoch_boundary_root ) -proc get_attestation_signature( - state: BeaconState, - attestation_data: AttestationData, - privkey: ValidatorPrivKey - ): ValidatorSig = - let domain = get_domain( - state = state, - domain_type = DOMAIN_BEACON_ATTESTER, - message_epoch = attestation_data.target.epoch - ) - let signing_root = compute_signing_root(attestation_data, domain) - - return blsSign(privkey, signing_root.data) - proc signMockAttestation*(state: BeaconState, attestation: var Attestation) = var cache = get_empty_per_epoch_cache() let participants = get_attesting_indices( @@ -79,7 +66,7 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) = var first_iter = true # Can't do while loop on hashset for validator_index in participants: let sig = get_attestation_signature( - state, attestation.data, MockPrivKeys[validator_index] + state.fork, attestation.data, MockPrivKeys[validator_index] ) if first_iter: attestation.signature = sig diff --git a/tests/mocking/mock_blocks.nim b/tests/mocking/mock_blocks.nim index 92487d93f..80b011c81 100644 --- a/tests/mocking/mock_blocks.nim +++ b/tests/mocking/mock_blocks.nim @@ -8,7 +8,7 @@ import options, # Specs - ../../beacon_chain/spec/[datatypes, crypto, helpers, validator], + ../../beacon_chain/spec/[datatypes, crypto, validator, state_transition_block], # Internals ../../beacon_chain/[ssz, extras, state_transition], # Mock helpers @@ -27,27 +27,10 @@ proc signMockBlockImpl( let privkey = MockPrivKeys[proposer_index] - - block: - let domain = get_domain( - state, - DOMAIN_RANDAO, - message_epoch = block_slot.compute_epoch_at_slot(), - ) - let signing_root = compute_signing_root( - block_slot.compute_epoch_at_slot(), - domain - ) - signedBlock.message.body.randao_reveal = bls_sign(privkey, signing_root.data) - - block: - let domain = get_domain( - state, - DOMAIN_BEACON_PROPOSER, - message_epoch = block_slot.compute_epoch_at_slot(), - ) - let signing_root = compute_signing_root(signedBlock.message, domain) - signedBlock.signature = bls_sign(privkey, signing_root.data) + signedBlock.message.body.randao_reveal = get_epoch_signature( + state.fork, block_slot, privkey) + signedBlock.signature = get_block_signature( + state.fork, block_slot, hash_tree_root(signedBlock.message), privkey) proc signMockBlock*( state: BeaconState, diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim index 3aa69fe2c..d3659e27d 100644 --- a/tests/test_attestation_pool.nim +++ b/tests/test_attestation_pool.nim @@ -35,17 +35,16 @@ when const_preset == "minimal": # Too much stack space used on mainnet var cache = get_empty_per_epoch_cache() let # Create an attestation for slot 1! - beacon_committee = get_beacon_committee(state.data.data, - state.data.data.slot, 0, cache) + beacon_committee = get_beacon_committee( + state.data.data, state.data.data.slot, 0, cache) attestation = makeAttestation( state.data.data, state.blck.root, beacon_committee[0], cache) pool.add(attestation) - process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot) # minus 1? + process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot + 1) - let attestations = pool.getAttestationsForBlock( - state.data.data, state.data.data.slot + 1) + let attestations = pool.getAttestationsForBlock(state.data.data) check: attestations.len == 1 @@ -54,8 +53,8 @@ when const_preset == "minimal": # Too much stack space used on mainnet var cache = get_empty_per_epoch_cache() let # Create an attestation for slot 1! - bc0 = get_beacon_committee(state.data.data, - state.data.data.slot, 0, cache) + bc0 = get_beacon_committee( + state.data.data, state.data.data.slot, 0, cache) attestation0 = makeAttestation( state.data.data, state.blck.root, bc0[0], cache) @@ -71,10 +70,9 @@ when const_preset == "minimal": # Too much stack space used on mainnet pool.add(attestation1) pool.add(attestation0) - process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot) # minus 1? + process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot + 1) - let attestations = pool.getAttestationsForBlock( - state.data.data, state.data.data.slot + 1) + let attestations = pool.getAttestationsForBlock(state.data.data) check: attestations.len == 1 @@ -83,8 +81,8 @@ when const_preset == "minimal": # Too much stack space used on mainnet var cache = get_empty_per_epoch_cache() let # Create an attestation for slot 1! - bc0 = get_beacon_committee(state.data.data, - state.data.data.slot, 0, cache) + bc0 = get_beacon_committee( + state.data.data, state.data.data.slot, 0, cache) attestation0 = makeAttestation( state.data.data, state.blck.root, bc0[0], cache) attestation1 = makeAttestation( @@ -93,10 +91,9 @@ when const_preset == "minimal": # Too much stack space used on mainnet pool.add(attestation0) pool.add(attestation1) - process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot) # minus 1? + process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot + 1) - let attestations = pool.getAttestationsForBlock( - state.data.data, state.data.data.slot + 1) + let attestations = pool.getAttestationsForBlock(state.data.data) check: attestations.len == 1 @@ -106,8 +103,8 @@ when const_preset == "minimal": # Too much stack space used on mainnet var # Create an attestation for slot 1! - bc0 = get_beacon_committee(state.data.data, - state.data.data.slot, 0, cache) + bc0 = get_beacon_committee( + state.data.data, state.data.data.slot, 0, cache) attestation0 = makeAttestation( state.data.data, state.blck.root, bc0[0], cache) attestation1 = makeAttestation( @@ -118,10 +115,9 @@ when const_preset == "minimal": # Too much stack space used on mainnet pool.add(attestation0) pool.add(attestation1) - process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot) # minus 1? + process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot + 1) - let attestations = pool.getAttestationsForBlock( - state.data.data, state.data.data.slot + 1) + let attestations = pool.getAttestationsForBlock(state.data.data) check: attestations.len == 1 @@ -142,17 +138,16 @@ when const_preset == "minimal": # Too much stack space used on mainnet pool.add(attestation1) pool.add(attestation0) - process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot) # minus 1? + process_slots(state.data, MIN_ATTESTATION_INCLUSION_DELAY.Slot + 1) - let attestations = pool.getAttestationsForBlock( - state.data.data, state.data.data.slot + 1) + let attestations = pool.getAttestationsForBlock(state.data.data) check: attestations.len == 1 timedTest "Fork choice returns latest block with no attestations": let - b1 = addBlock(state.data.data, blockPool.tail.root, BeaconBlockBody()) + b1 = addTestBlock(state.data.data, blockPool.tail.root) b1Root = hash_tree_root(b1.message) b1Add = blockPool.add(b1Root, b1) head = pool.selectHead() @@ -161,7 +156,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet head == b1Add let - b2 = addBlock(state.data.data, b1Root, BeaconBlockBody()) + b2 = addTestBlock(state.data.data, b1Root) b2Root = hash_tree_root(b2.message) b2Add = blockPool.add(b2Root, b2) head2 = pool.selectHead() @@ -172,7 +167,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet timedTest "Fork choice returns block with attestation": var cache = get_empty_per_epoch_cache() let - b10 = makeBlock(state.data.data, blockPool.tail.root, BeaconBlockBody()) + b10 = makeTestBlock(state.data.data, blockPool.tail.root) b10Root = hash_tree_root(b10.message) b10Add = blockPool.add(b10Root, b10) head = pool.selectHead() @@ -181,16 +176,15 @@ when const_preset == "minimal": # Too much stack space used on mainnet head == b10Add let - b11 = makeBlock(state.data.data, blockPool.tail.root, BeaconBlockBody( - graffiti: Eth2Digest(data: [1'u8, 0, 0, 0 ,0 ,0 ,0 ,0 ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - )) + b11 = makeTestBlock(state.data.data, blockPool.tail.root, + graffiti = Eth2Digest(data: [1'u8, 0, 0, 0 ,0 ,0 ,0 ,0 ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + ) b11Root = hash_tree_root(b11.message) b11Add = blockPool.add(b11Root, b11) - bc1 = get_beacon_committee(state.data.data, - state.data.data.slot, 1, cache) - attestation0 = makeAttestation( - state.data.data, b10Root, bc1[0], cache) + bc1 = get_beacon_committee( + state.data.data, state.data.data.slot, 1, cache) + attestation0 = makeAttestation(state.data.data, b10Root, bc1[0], cache) pool.add(attestation0) @@ -201,10 +195,8 @@ when const_preset == "minimal": # Too much stack space used on mainnet head2 == b10Add let - attestation1 = makeAttestation( - state.data.data, b11Root, bc1[1], cache) - attestation2 = makeAttestation( - state.data.data, b11Root, bc1[2], cache) + attestation1 = makeAttestation(state.data.data, b11Root, bc1[1], cache) + attestation2 = makeAttestation(state.data.data, b11Root, bc1[2], cache) pool.add(attestation1) let head3 = pool.selectHead() diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim index 1239c09c9..67968ce10 100644 --- a/tests/test_block_pool.nim +++ b/tests/test_block_pool.nim @@ -89,9 +89,9 @@ when const_preset == "minimal": # Too much stack space used on mainnet db = makeTestDB(SLOTS_PER_EPOCH) pool = BlockPool.init(db) state = pool.loadTailState().data.data - b1 = addBlock(state, pool.tail.root, BeaconBlockBody()) + b1 = addTestBlock(state, pool.tail.root) b1Root = hash_tree_root(b1.message) - b2 = addBlock(state, b1Root, BeaconBlockBody()) + b2 = addTestBlock(state, b1Root) b2Root {.used.} = hash_tree_root(b2.message) timedTest "getRef returns nil for missing blocks": @@ -246,8 +246,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet block: # Create a fork that will not be taken var - blck = makeBlock(pool.headState.data.data, pool.head.blck.root, - BeaconBlockBody()) + blck = makeTestBlock(pool.headState.data.data, pool.head.blck.root) discard pool.add(hash_tree_root(blck.message), blck) for i in 0 ..< (SLOTS_PER_EPOCH * 6): @@ -258,11 +257,11 @@ when const_preset == "minimal": # Too much stack space used on mainnet pool.heads.len == 2 var cache = get_empty_per_epoch_cache() - blck = makeBlock(pool.headState.data.data, pool.head.blck.root, - BeaconBlockBody( - attestations: makeFullAttestations( - pool.headState.data.data, pool.head.blck.root, - pool.headState.data.data.slot, cache, {}))) + blck = makeTestBlock( + pool.headState.data.data, pool.head.blck.root, + attestations = makeFullAttestations( + pool.headState.data.data, pool.head.blck.root, + pool.headState.data.data.slot, cache, {})) let added = pool.add(hash_tree_root(blck.message), blck) pool.updateHead(added) diff --git a/tests/test_state_transition.nim b/tests/test_state_transition.nim index 855c992af..a48254300 100644 --- a/tests/test_state_transition.nim +++ b/tests/test_state_transition.nim @@ -38,7 +38,7 @@ suiteReport "Block processing" & preset(): var state = genesisState previous_block_root = hash_tree_root(genesisBlock.message) - new_block = makeBlock(state, previous_block_root, BeaconBlockBody()) + new_block = makeTestBlock(state, previous_block_root) let block_ok = state_transition(state, new_block, {}) @@ -62,7 +62,7 @@ suiteReport "Block processing" & preset(): previous_block_root = genesisRoot for i in 1..SLOTS_PER_EPOCH.int: - var new_block = makeBlock(state, previous_block_root, BeaconBlockBody()) + let new_block = makeTestBlock(state, previous_block_root) let block_ok = state_transition(state, new_block, {}) @@ -95,9 +95,9 @@ suiteReport "Block processing" & preset(): process_slots(state, GENESIS_SLOT + MIN_ATTESTATION_INCLUSION_DELAY + 1) let - new_block = makeBlock(state, previous_block_root, BeaconBlockBody( - attestations: @[attestation] - )) + new_block = makeTestBlock(state, previous_block_root, + attestations = @[attestation] + ) discard state_transition(state, new_block, {}) check: diff --git a/tests/testblockutil.nim b/tests/testblockutil.nim index 8f7a8c15c..e12c3b006 100644 --- a/tests/testblockutil.nim +++ b/tests/testblockutil.nim @@ -11,7 +11,7 @@ import ../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, state_transition, validator_pool], ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, - helpers, validator] + helpers, validator, state_transition_block] func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey = # 0 is not a valid BLS private key - 1000 helps interop with rust BLS library, @@ -62,81 +62,70 @@ func makeInitialDeposits*( for i in 0..= 1 # Initial attestation var attestation = Attestation( aggregation_bits: CommitteeValidatorsBits.init(committee.len), data: data, - signature: bls_sign( - hackPrivKey(state.validators[committee[0]]), - signing_root.data - ) + signature: get_attestation_signature( + state.fork, data, + hackPrivKey(state.validators[committee[0]])) ) # Aggregate the remainder attestation.aggregation_bits.setBit 0 for j in 1 ..< committee.len(): attestation.aggregation_bits.setBit j if skipBLSValidation notin flags: - attestation.signature.aggregate(bls_sign( - hackPrivKey(state.validators[committee[j]]), - signing_root.data + attestation.signature.aggregate(get_attestation_signature( + state.fork, data, + hackPrivKey(state.validators[committee[j]]) )) result.add attestation From 53e4b0a26cadb02809890b392602b85d0a070b2f Mon Sep 17 00:00:00 2001 From: tersec Date: Sun, 22 Mar 2020 16:03:07 +0000 Subject: [PATCH 03/13] complete (except for get_domain(...)) 0.11.0 beacon chain spec update (#822) * complete (except for get_domain(...)) 0.11.0 beacon chain spec update * mark compute_start_slot_at_epoch(...), is_active_validator(...), compute_signing_root(...), and get_seed(...) as 0.11.0 --- beacon_chain/spec/helpers.nim | 39 +++++++++++++++---- beacon_chain/spec/network.nim | 6 +-- beacon_chain/spec/state_transition_epoch.nim | 16 ++++---- tests/test_honest_validator.nim | 40 ++++++++++---------- 4 files changed, 63 insertions(+), 38 deletions(-) diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index ebe3955be..33baddee4 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -36,12 +36,12 @@ func compute_epoch_at_slot*(slot: Slot|uint64): Epoch = template epoch*(slot: Slot): Epoch = compute_epoch_at_slot(slot) -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#compute_start_slot_at_epoch +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#compute_start_slot_at_epoch func compute_start_slot_at_epoch*(epoch: Epoch): Slot = # Return the start slot of ``epoch``. (epoch * SLOTS_PER_EPOCH).Slot -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#is_active_validator +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#is_active_validator func is_active_validator*(validator: Validator, epoch: Epoch): bool = ### Check if ``validator`` is active validator.activation_epoch <= epoch and epoch < validator.exit_epoch @@ -114,13 +114,38 @@ func int_to_bytes4*(x: uint64): array[4, byte] = result[2] = ((x shr 16) and 0xff).byte result[3] = ((x shr 24) and 0xff).byte -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#compute_domain +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#compute_fork_data_root +func compute_fork_data_root(current_version: array[4, byte], + genesis_validators_root: Eth2Digest): Eth2Digest = + # Return the 32-byte fork data root for the ``current_version`` and + # ``genesis_validators_root``. + # This is used primarily in signature domains to avoid collisions across + # forks/chains. + hash_tree_root(ForkData( + current_version: current_version, + genesis_validators_root: genesis_validators_root + )) + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#compute_fork_digest +func compute_fork_digest(current_version: array[4, byte], + genesis_validators_root: Eth2Digest): array[4, byte] = + # Return the 4-byte fork digest for the ``current_version`` and + # ``genesis_validators_root``. + # This is a digest primarily used for domain separation on the p2p layer. + # 4-bytes suffices for practical separation of forks/chains. + result[0..3] = + compute_fork_data_root(current_version, genesis_validators_root).data[0..3] + +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#compute_domain func compute_domain*( domain_type: DomainType, - fork_version: array[4, byte] = [0'u8, 0, 0, 0]): Domain = + fork_version: array[4, byte] = [0'u8, 0, 0, 0], + genesis_validators_root: Eth2Digest = ZERO_HASH): Domain = # Return the domain for the ``domain_type`` and ``fork_version``. + let fork_data_root = + compute_fork_data_root(fork_version, genesis_validators_root) result[0..3] = int_to_bytes4(domain_type.uint64) - result[4..7] = fork_version + result[4..31] = fork_data_root.data[0..27] # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#get_domain func get_domain*( @@ -144,7 +169,7 @@ func get_domain*( func get_domain*(state: BeaconState, domain_type: DomainType): Domain = get_domain(state, domain_type, get_current_epoch(state)) -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#compute_signing_root +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#compute_signing_root func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest = # Return the signing root of an object by calculating the root of the # object-domain tree. @@ -154,7 +179,7 @@ func compute_signing_root*(ssz_object: auto, domain: Domain): Eth2Digest = ) hash_tree_root(domain_wrapped_object) -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#get_seed +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#get_seed func get_seed*(state: BeaconState, epoch: Epoch, domain_type: DomainType): Eth2Digest = # Return the seed at ``epoch``. diff --git a/beacon_chain/spec/network.nim b/beacon_chain/spec/network.nim index 5b52fe94e..8825e67ae 100644 --- a/beacon_chain/spec/network.nim +++ b/beacon_chain/spec/network.nim @@ -14,10 +14,10 @@ const topicProposerSlashings* = "/eth2/proposer_slashing/ssz" topicAttesterSlashings* = "/eth2/attester_slashing/ssz" - # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/p2p-interface.md#configuration + # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/p2p-interface.md#configuration ATTESTATION_SUBNET_COUNT* = 64 func getAttestationTopic*(committeeIndex: uint64): string = - # https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/validator.md#broadcast-attestation + # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md#broadcast-attestation let topicIndex = committeeIndex mod ATTESTATION_SUBNET_COUNT - &"/eth2/index{topicIndex}{topicAttestationSuffix}" + &"/eth2/committee_index{topicIndex}{topicAttestationSuffix}" diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 2c34da407..eb70fd002 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -71,7 +71,7 @@ func get_total_active_balance*(state: BeaconState): Gwei = state, get_active_validator_indices(state, get_current_epoch(state))) -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#helper-functions-1 +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#helper-functions-1 func get_matching_source_attestations(state: BeaconState, epoch: Epoch): seq[PendingAttestation] = doAssert epoch in [get_current_epoch(state), get_previous_epoch(state)] @@ -233,7 +233,7 @@ proc process_justification_and_finalization*( checkpoint = shortLog(state.finalized_checkpoint), cat = "finalization" -# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#rewards-and-penalties-1 +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#rewards-and-penalties-1 func get_base_reward(state: BeaconState, index: ValidatorIndex, total_balance: auto): Gwei = # Spec function recalculates total_balance every time, which creates an @@ -242,7 +242,7 @@ func get_base_reward(state: BeaconState, index: ValidatorIndex, effective_balance * BASE_REWARD_FACTOR div integer_squareroot(total_balance) div BASE_REWARDS_PER_EPOCH -# https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/core/0_beacon-chain.md#rewards-and-penalties-1 +# https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#rewards-and-penalties-1 func get_attestation_deltas(state: BeaconState, stateCache: var StateCache): tuple[a: seq[Gwei], b: seq[Gwei]] {.nbench.}= let @@ -276,9 +276,11 @@ func get_attestation_deltas(state: BeaconState, stateCache: var StateCache): attesting_balance = get_total_balance(state, unslashed_attesting_indices) for index in eligible_validator_indices: if index in unslashed_attesting_indices: - rewards[index] += - get_base_reward(state, index, total_balance) * attesting_balance div - total_balance + # Factored out from balance totals to avoid uint64 overflow + const increment = EFFECTIVE_BALANCE_INCREMENT + let reward_numerator = get_base_reward(state, index, total_balance) * + (attesting_balance div increment) + rewards[index] = reward_numerator div (total_balance div increment) else: penalties[index] += get_base_reward(state, index, total_balance) @@ -290,8 +292,6 @@ func get_attestation_deltas(state: BeaconState, stateCache: var StateCache): matching_source_attestations, get_attesting_indices(state, it.data, it.aggregation_bits, stateCache)) - ## TODO if this is still a profiling issue, do higher-level semantic - ## translation for index in get_unslashed_attesting_indices( state, matching_source_attestations, stateCache): # Translation of attestation = min([...]) diff --git a/tests/test_honest_validator.nim b/tests/test_honest_validator.nim index 5bae259ca..30b7584bc 100644 --- a/tests/test_honest_validator.nim +++ b/tests/test_honest_validator.nim @@ -7,23 +7,23 @@ import suiteReport "Honest validator": timedTest "Attestation topics": check: - getAttestationTopic(0) == "/eth2/index0_beacon_attestation/ssz" - getAttestationTopic(9) == "/eth2/index9_beacon_attestation/ssz" - getAttestationTopic(10) == "/eth2/index10_beacon_attestation/ssz" - getAttestationTopic(11) == "/eth2/index11_beacon_attestation/ssz" - getAttestationTopic(14) == "/eth2/index14_beacon_attestation/ssz" - getAttestationTopic(22) == "/eth2/index22_beacon_attestation/ssz" - getAttestationTopic(34) == "/eth2/index34_beacon_attestation/ssz" - getAttestationTopic(46) == "/eth2/index46_beacon_attestation/ssz" - getAttestationTopic(60) == "/eth2/index60_beacon_attestation/ssz" - getAttestationTopic(63) == "/eth2/index63_beacon_attestation/ssz" - getAttestationTopic(200) == "/eth2/index8_beacon_attestation/ssz" - getAttestationTopic(400) == "/eth2/index16_beacon_attestation/ssz" - getAttestationTopic(469) == "/eth2/index21_beacon_attestation/ssz" - getAttestationTopic(550) == "/eth2/index38_beacon_attestation/ssz" - getAttestationTopic(600) == "/eth2/index24_beacon_attestation/ssz" - getAttestationTopic(613) == "/eth2/index37_beacon_attestation/ssz" - getAttestationTopic(733) == "/eth2/index29_beacon_attestation/ssz" - getAttestationTopic(775) == "/eth2/index7_beacon_attestation/ssz" - getAttestationTopic(888) == "/eth2/index56_beacon_attestation/ssz" - getAttestationTopic(995) == "/eth2/index35_beacon_attestation/ssz" + getAttestationTopic(0) == "/eth2/committee_index0_beacon_attestation/ssz" + getAttestationTopic(9) == "/eth2/committee_index9_beacon_attestation/ssz" + getAttestationTopic(10) == "/eth2/committee_index10_beacon_attestation/ssz" + getAttestationTopic(11) == "/eth2/committee_index11_beacon_attestation/ssz" + getAttestationTopic(14) == "/eth2/committee_index14_beacon_attestation/ssz" + getAttestationTopic(22) == "/eth2/committee_index22_beacon_attestation/ssz" + getAttestationTopic(34) == "/eth2/committee_index34_beacon_attestation/ssz" + getAttestationTopic(46) == "/eth2/committee_index46_beacon_attestation/ssz" + getAttestationTopic(60) == "/eth2/committee_index60_beacon_attestation/ssz" + getAttestationTopic(63) == "/eth2/committee_index63_beacon_attestation/ssz" + getAttestationTopic(200) == "/eth2/committee_index8_beacon_attestation/ssz" + getAttestationTopic(400) == "/eth2/committee_index16_beacon_attestation/ssz" + getAttestationTopic(469) == "/eth2/committee_index21_beacon_attestation/ssz" + getAttestationTopic(550) == "/eth2/committee_index38_beacon_attestation/ssz" + getAttestationTopic(600) == "/eth2/committee_index24_beacon_attestation/ssz" + getAttestationTopic(613) == "/eth2/committee_index37_beacon_attestation/ssz" + getAttestationTopic(733) == "/eth2/committee_index29_beacon_attestation/ssz" + getAttestationTopic(775) == "/eth2/committee_index7_beacon_attestation/ssz" + getAttestationTopic(888) == "/eth2/committee_index56_beacon_attestation/ssz" + getAttestationTopic(995) == "/eth2/committee_index35_beacon_attestation/ssz" From adcec61081e29b867e40f7d5bc5967f71aa192fa Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 17 Mar 2020 00:28:54 +0200 Subject: [PATCH 04/13] Initial implementation of a JSON-RPC service --- beacon_chain/beacon_node.nim | 142 +++++++++++++++--- beacon_chain/block_pool.nim | 11 ++ beacon_chain/conf.nim | 61 +++++--- beacon_chain/libp2p_backend.nim | 3 + beacon_chain/peer_pool.nim | 4 + .../spec/eth2_apis/beacon_callsigs.nim | 13 ++ .../spec/eth2_apis/validator_callsigs.nim | 36 +++++ beacon_chain/spec/network.nim | 5 + tests/simulation/run_node.sh | 11 +- tests/simulation/vars.sh | 1 + 10 files changed, 242 insertions(+), 45 deletions(-) create mode 100644 beacon_chain/spec/eth2_apis/beacon_callsigs.nim create mode 100644 beacon_chain/spec/eth2_apis/validator_callsigs.nim diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 9eaa2acc3..17d0ca17a 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -1,13 +1,13 @@ import # Standard library - os, net, tables, random, strutils, times, sequtils, + os, tables, random, strutils, times, sequtils, # Nimble packages stew/[objects, bitseqs, byteutils], - chronos, chronicles, confutils, metrics, - json_serialization/std/[options, sets], serialization/errors, + chronos, chronicles, confutils, metrics, json_rpc/[rpcserver, jsonmarshal], + json_serialization/std/[options, sets, net], serialization/errors, kvstore, kvstore_sqlite3, - eth/p2p/enode, eth/[keys, async_utils], eth/p2p/discoveryv5/enr, + eth/p2p/enode, eth/[keys, async_utils], eth/p2p/discoveryv5/[protocol, enr], # Local modules spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network, @@ -24,6 +24,10 @@ const type KeyPair = eth2_network.KeyPair + RpcServer = RpcHttpServer + +template init(T: type RpcHttpServer, ip: IpAddress, port: Port): T = + newRpcHttpServer([initTAddress(ip, port)]) # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#interop-metrics declareGauge beacon_slot, @@ -61,6 +65,7 @@ type attestationPool: AttestationPool mainchainMonitor: MainchainMonitor beaconClock: BeaconClock + rpcServer: RpcServer proc onBeaconBlock*(node: BeaconNode, signedBlock: SignedBeaconBlock) {.gcsafe.} proc updateHead(node: BeaconNode): BlockRef @@ -203,6 +208,11 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async addressFile = string(conf.dataDir) / "beacon_node.address" network.saveConnectionAddressFile(addressFile) + let rpcServer = if conf.rpcEnabled: + RpcServer.init(conf.rpcAddress, conf.rpcPort) + else: + nil + var res = BeaconNode( nickname: nickname, network: network, @@ -218,6 +228,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async attestationPool: AttestationPool.init(blockPool), mainchainMonitor: mainchainMonitor, beaconClock: BeaconClock.init(blockPool.headState.data.data), + rpcServer: rpcServer, ) # TODO sync is called when a remote peer is connected - is that the right @@ -845,7 +856,107 @@ proc onSecond(node: BeaconNode, moment: Moment) {.async.} = addTimer(nextSecond) do (p: pointer): asyncCheck node.onSecond(nextSecond) +# TODO: Should we move these to other modules? +# This would require moving around other type definitions +proc installValidatorApiHandlers(rpcServer: RpcServer, node: BeaconNode) = + discard + +func slotOrZero(time: BeaconTime): Slot = + let exSlot = time.toSlot + if exSlot.afterGenesis: exSlot.slot + else: Slot(0) + +func currentSlot(node: BeaconNode): Slot = + node.beaconClock.now.slotOrZero + +proc connectedPeersCount(node: BeaconNode): int = + libp2p_peers.value.int + +proc fromJson(n: JsonNode; argName: string; result: var Slot) = + var i: int + fromJson(n, argName, i) + result = Slot(i) + +proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) = + rpcServer.rpc("getBeaconHead") do () -> Slot: + return node.currentSlot + + template requireOneOf(x, y: distinct Option) = + if x.isNone xor y.isNone: + raise newException(CatchableError, + "Please specify one of " & astToStr(x) & " or " & astToStr(y)) + + template jsonResult(x: auto): auto = + # TODO, yes this is silly, but teching json-rpc about + # all beacon node types will require quite a lot of work. + # A minor refactoring in json-rpc can solve this. We need + # to allow the handlers to return raw/literal json strings. + parseJson(Json.encode(x)) + + rpcServer.rpc("getBeaconBlock") do (slot: Option[Slot], + root: Option[Eth2Digest]) -> JsonNode: + requireOneOf(slot, root) + var blockHash: Eth2Digest + if root.isSome: + blockHash = root.get + else: + let foundRef = node.blockPool.getBlockByPreciseSlot(slot.get) + if foundRef.isSome: + blockHash = foundRef.get.root + else: + return newJNull() + + let dbBlock = node.db.getBlock(blockHash) + if dbBlock.isSome: + return jsonResult(dbBlock.get) + else: + return newJNull() + + rpcServer.rpc("getBeaconState") do (slot: Option[Slot], + root: Option[Eth2Digest]) -> JsonNode: + requireOneOf(slot, root) + if slot.isSome: + let blk = node.blockPool.head.blck.atSlot(slot.get) + var tmpState: StateData + node.blockPool.withState(tmpState, blk): + return jsonResult(state) + else: + let state = node.db.getState(root.get) + if state.isSome: + return jsonResult(state.get) + else: + return newJNull() + + rpcServer.rpc("getNetworkPeerId") do () -> string: + when networkBackend != libp2p: + raise newException(CatchableError, "Unsupported operation") + else: + return $publicKey(node.network) + + rpcServer.rpc("getNetworkPeers") do () -> seq[string]: + when networkBackend != libp2p: + if true: + raise newException(CatchableError, "Unsupported operation") + + for peerId, peer in node.network.peerPool: + result.add $peerId + + rpcServer.rpc("getNetworkEnr") do () -> string: + return $node.network.discovery.localNode.record + +proc installDebugApiHandlers(rpcServer: RpcServer, node: BeaconNode) = + discard + +proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) = + rpcServer.installValidatorApiHandlers(node) + rpcServer.installBeaconApiHandlers(node) + rpcServer.installDebugApiHandlers(node) + proc run*(node: BeaconNode) = + if node.rpcServer != nil: + node.rpcServer.installRpcHandlers(node) + node.rpcServer.start() + waitFor node.network.subscribe(topicBeaconBlocks) do (signedBlock: SignedBeaconBlock): onBeaconBlock(node, signedBlock) @@ -955,11 +1066,6 @@ when hasPrompt: else: p[].writeLine("Unknown command: " & cmd) - proc slotOrZero(time: BeaconTime): Slot = - let exSlot = time.toSlot - if exSlot.afterGenesis: exSlot.slot - else: Slot(0) - proc initPrompt(node: BeaconNode) = if isatty(stdout) and node.config.statusBarEnabled: enableTrueColors() @@ -982,7 +1088,7 @@ when hasPrompt: # arbitrary expression that is resolvable through this API. case expr.toLowerAscii of "connected_peers": - $(libp2p_peers.value.int) + $(node.connectedPeersCount) of "last_finalized_epoch": var head = node.blockPool.finalizedHead @@ -999,7 +1105,7 @@ when hasPrompt: $SLOTS_PER_EPOCH of "slot": - $node.beaconClock.now.slotOrZero + $node.currentSlot of "slot_trailing_digits": var slotStr = $node.beaconClock.now.slotOrZero @@ -1115,9 +1221,9 @@ when isMainModule: let networkKeys = getPersistentNetKeys(config) bootstrapAddress = enode.Address( - ip: parseIpAddress(config.bootstrapAddress), - tcpPort: Port config.bootstrapPort, - udpPort: Port config.bootstrapPort) + ip: config.bootstrapAddress, + tcpPort: config.bootstrapPort, + udpPort: config.bootstrapPort) bootstrapEnr = enr.Record.init( 1, # sequence number @@ -1151,11 +1257,11 @@ when isMainModule: initPrompt(node) when useInsecureFeatures: - if config.metricsServer: - let metricsAddress = config.metricsServerAddress + if config.metricsEnabled: + let metricsAddress = config.metricsAddress info "Starting metrics HTTP server", - address = metricsAddress, port = config.metricsServerPort - metrics.startHttpServer(metricsAddress, Port(config.metricsServerPort)) + address = metricsAddress, port = config.metricsPort + metrics.startHttpServer($metricsAddress, config.metricsPort) if node.nickname != "": dynamicLogScope(node = node.nickname): node.start() diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim index c3e5111a6..1b450bda7 100644 --- a/beacon_chain/block_pool.nim +++ b/beacon_chain/block_pool.nim @@ -514,6 +514,17 @@ proc getBlockRange*(pool: BlockPool, headBlock: Eth2Digest, trace "getBlockRange result", position = result, blockSlot = b.slot skip skipStep +func getBlockBySlot*(pool: BlockPool, slot: Slot): BlockRef = + ## Retrieves the first block in the current canonical chain + ## with slot number less or equal to `slot`. + pool.head.blck.findAncestorBySlot(slot).blck + +func getBlockByPreciseSlot*(pool: BlockPool, slot: Slot): Option[BlockRef] = + ## Retrieves a block from the canonical chain with a slot + ## number equal to `slot`. + let found = pool.getBlockBySlot(slot) + if found.slot != slot: some(found) else: none(BlockRef) + proc get*(pool: BlockPool, blck: BlockRef): BlockData = ## Retrieve the associated block body of a block reference doAssert (not blck.isNil), "Trying to get nil BlockRef" diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index a4fba6db9..fd22ad006 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -1,14 +1,12 @@ import os, options, strformat, strutils, chronicles, confutils, json_serialization, - confutils/defs, chronicles/options as chroniclesOptions, + confutils/defs, confutils/std/net, + chronicles/options as chroniclesOptions, spec/[crypto] export - defs, enabledLogLevel - -const - DEFAULT_NETWORK* {.strdefine.} = "testnet0" + defs, enabledLogLevel, parseCmdArg, completeCmdArg type ValidatorKeyPath* = TypedInputFile[ValidatorPrivKey, Txt, "privkey"] @@ -75,6 +73,21 @@ type desc: "Textual template for the contents of the status bar." name: "status-bar-contents" }: string + rpcEnabled* {. + defaultValue: false + desc: "Enable the JSON-RPC server" + name: "rpc" }: bool + + rpcPort* {. + defaultValue: defaultEth2RpcPort + desc: "HTTP port for the JSON-RPC service." + name: "rpc-port" }: Port + + rpcAddress* {. + defaultValue: defaultListenAddress(config) + desc: "Listening address of the RPC server" + name: "rpc-address" }: IpAddress + case cmd* {. command defaultValue: noCommand }: StartUpCmd @@ -91,14 +104,14 @@ type name: "bootstrap-file" }: InputFile tcpPort* {. - defaultValue: defaultPort(config) + defaultValue: defaultEth2TcpPort desc: "TCP listening port." - name: "tcp-port" }: int + name: "tcp-port" }: Port udpPort* {. - defaultValue: defaultPort(config) + defaultValue: defaultEth2TcpPort desc: "UDP listening port." - name: "udp-port" }: int + name: "udp-port" }: Port maxPeers* {. defaultValue: 10 @@ -137,20 +150,20 @@ type desc: "A positive epoch selects the epoch at which to stop." name: "stop-at-epoch" }: uint64 - metricsServer* {. + metricsEnabled* {. defaultValue: false desc: "Enable the metrics server." - name: "metrics-server" }: bool + name: "metrics" }: bool - metricsServerAddress* {. - defaultValue: "0.0.0.0" + metricsAddress* {. + defaultValue: defaultListenAddress(config) desc: "Listening address of the metrics server." - name: "metrics-server-address" }: string # TODO: use a validated type here + name: "metrics-address" }: IpAddress - metricsServerPort* {. + metricsPort* {. defaultValue: 8008 desc: "Listening HTTP port of the metrics server." - name: "metrics-server-port" }: uint16 + name: "metrics-port" }: Port dump* {. defaultValue: false @@ -178,14 +191,14 @@ type name: "last-user-validator" }: uint64 bootstrapAddress* {. - defaultValue: "127.0.0.1" + defaultValue: parseIpAddress("127.0.0.1") desc: "The public IP address that will be advertised as a bootstrap node for the testnet." - name: "bootstrap-address" }: string + name: "bootstrap-address" }: IpAddress bootstrapPort* {. - defaultValue: defaultPort(config) + defaultValue: defaultEth2TcpPort desc: "The TCP/UDP port that will be used by the bootstrap node." - name: "bootstrap-port" }: int + name: "bootstrap-port" }: Port genesisOffset* {. defaultValue: 5 @@ -248,9 +261,6 @@ type argument desc: "REST API path to evaluate" }: string -proc defaultPort*(config: BeaconNodeConf): int = - 9000 - proc defaultDataDir*(conf: BeaconNodeConf): string = let dataDir = when defined(windows): "AppData" / "Roaming" / "Nimbus" @@ -274,6 +284,11 @@ func localValidatorsDir*(conf: BeaconNodeConf): string = func databaseDir*(conf: BeaconNodeConf): string = conf.dataDir / "db" +func defaultListenAddress*(conf: BeaconNodeConf): IpAddress = + # TODO: How should we select between IPv4 and IPv6 + # Maybe there should be a config option for this. + parseIpAddress("0.0.0.0") + iterator validatorKeys*(conf: BeaconNodeConf): ValidatorPrivKey = for validatorKeyFile in conf.validators: try: diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index b3268414d..069626986 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -277,6 +277,9 @@ proc init*(T: type Eth2Node, conf: BeaconNodeConf, if msg.protocolMounter != nil: msg.protocolMounter result +template publicKey*(node: Eth2Node): keys.PublicKey = + node.discovery.privKey.getPublicKey + template addKnownPeer*(node: Eth2Node, peer: ENode|enr.Record) = node.discovery.addNode peer diff --git a/beacon_chain/peer_pool.nim b/beacon_chain/peer_pool.nim index 0042bff1c..1c5c1b8f1 100644 --- a/beacon_chain/peer_pool.nim +++ b/beacon_chain/peer_pool.nim @@ -55,6 +55,10 @@ proc fireNotFullEvent[A, B](pool: PeerPool[A, B], elif item.peerType == PeerType.Outgoing: pool.outNotFullEvent.fire() +iterator pairs*[A, B](pool: PeerPool[A, B]): (B, A) = + for peerId, peerIdx in pool.registry: + yield (peerId, pool.storage[peerIdx.data].data) + proc waitNotEmptyEvent[A, B](pool: PeerPool[A, B], filter: set[PeerType]) {.async.} = if filter == {PeerType.Incoming, PeerType.Outgoing} or filter == {}: diff --git a/beacon_chain/spec/eth2_apis/beacon_callsigs.nim b/beacon_chain/spec/eth2_apis/beacon_callsigs.nim new file mode 100644 index 000000000..0bba59fba --- /dev/null +++ b/beacon_chain/spec/eth2_apis/beacon_callsigs.nim @@ -0,0 +1,13 @@ +import + options, + ../datatypes + +# https://github.com/ethereum/eth2.0-APIs/blob/master/apis/beacon/basic.md +# +proc getBeaconHead(): Slot +proc getBeaconBlock(slot = none(Slot), root = none(Eth2Digest)): BeaconBlock +proc getBeaconState(slot = none(Slot), root = none(Eth2Digest)): BeaconState +proc getNetworkPeerId() +proc getNetworkPeers() +proc getNetworkEnr() + diff --git a/beacon_chain/spec/eth2_apis/validator_callsigs.nim b/beacon_chain/spec/eth2_apis/validator_callsigs.nim new file mode 100644 index 000000000..209f7e9a9 --- /dev/null +++ b/beacon_chain/spec/eth2_apis/validator_callsigs.nim @@ -0,0 +1,36 @@ +import + options, + ../datatypes + +# https://github.com/ethereum/eth2.0-APIs/tree/master/apis/validator + +type + SyncStatus* = object + starting_slot*: Slot + current_slot*: Slot + highest_slot*: Slot + + SyncingStatusResponse* = object + is_syncing*: bool + sync_status*: SyncStatus + + ValidatorDuty* = object + validator_pubkey: ValidatorPubKey + attestation_slot: Slot + attestation_shard: uint + block_proposal_slot: Slot + +proc getNodeVersion(): string +proc getGenesisTime(): uint64 +proc getSyncingStatus(): SyncingStatusResponse +proc getValidator(key: ValidatorPubKey): Validator +proc getValidatorDuties(validators: openarray[ValidatorPubKey], epoch: Epoch): seq[ValidatorDuty] +proc getBlockForSigning(slot: Slot, randaoReveal: string): BeaconBlock +proc postBlock(blk: BeaconBlock) +proc getAttestationForSigning(validatorKey: ValidatorPubKey, pocBit: int, slot: Slot, shard: uint): Attestation +proc postAttestation(attestation: Attestation) + +# Optional RPCs + +proc getForkId() + diff --git a/beacon_chain/spec/network.nim b/beacon_chain/spec/network.nim index 8825e67ae..719237d0c 100644 --- a/beacon_chain/spec/network.nim +++ b/beacon_chain/spec/network.nim @@ -17,6 +17,11 @@ const # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/p2p-interface.md#configuration ATTESTATION_SUBNET_COUNT* = 64 + defaultEth2TcpPort* = 9000 + + # This is not part of the spec yet! + defaultEth2RpcPort* = 9090 + func getAttestationTopic*(committeeIndex: uint64): string = # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/validator.md#broadcast-attestation let topicIndex = committeeIndex mod ATTESTATION_SUBNET_COUNT diff --git a/tests/simulation/run_node.sh b/tests/simulation/run_node.sh index e2e275474..fbaccc068 100755 --- a/tests/simulation/run_node.sh +++ b/tests/simulation/run_node.sh @@ -64,9 +64,12 @@ cd "$DATA_DIR" && $NODE_BIN \ --state-snapshot=$SNAPSHOT_FILE \ $DEPOSIT_WEB3_URL_ARG \ --deposit-contract=$DEPOSIT_CONTRACT_ADDRESS \ - --verify-finalization=on \ - --metrics-server=on \ - --metrics-server-address="127.0.0.1" \ - --metrics-server-port="$(( $BASE_METRICS_PORT + $NODE_ID ))" \ + --verify-finalization \ + --rpc \ + --rpc-address="127.0.0.1" \ + --rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))" \ + --metrics \ + --metrics-address="127.0.0.1" \ + --metrics-port="$(( $BASE_METRICS_PORT + $NODE_ID ))" \ "$@" diff --git a/tests/simulation/vars.sh b/tests/simulation/vars.sh index 1f715a5e9..a1dc352b8 100644 --- a/tests/simulation/vars.sh +++ b/tests/simulation/vars.sh @@ -41,6 +41,7 @@ DEPLOY_DEPOSIT_CONTRACT_BIN="${SIMULATION_DIR}/deploy_deposit_contract" MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-${MASTER_NODE}/beacon_node.address" BASE_P2P_PORT=30000 +BASE_RPC_PORT=7000 BASE_METRICS_PORT=8008 # Set DEPOSIT_WEB3_URL_ARG to empty to get genesis state from file, not using web3 # DEPOSIT_WEB3_URL_ARG=--web3-url=ws://localhost:8545 From 5a2a52869eb93077ed0713ee7e021af306ec91df Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 17 Mar 2020 01:49:53 +0200 Subject: [PATCH 05/13] Add a RPC proc similar to /spec/eth2_config --- beacon_chain/beacon_node.nim | 16 +++++++++++++--- beacon_chain/spec/datatypes.nim | 5 +---- beacon_chain/spec/presets/custom.nim | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 17d0ca17a..1f79ad9e9 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -3,7 +3,7 @@ import os, tables, random, strutils, times, sequtils, # Nimble packages - stew/[objects, bitseqs, byteutils], + stew/[objects, bitseqs, byteutils], stew/shims/macros, chronos, chronicles, confutils, metrics, json_rpc/[rpcserver, jsonmarshal], json_serialization/std/[options, sets, net], serialization/errors, kvstore, kvstore_sqlite3, @@ -11,7 +11,7 @@ import # Local modules spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network, - state_transition_block], + state_transition_block], spec/presets/custom, conf, time, state_transition, beacon_chain_db, validator_pool, extras, attestation_pool, block_pool, eth2_network, eth2_discovery, beacon_node_types, mainchain_monitor, version, ssz, ssz/dynamic_navigator, @@ -945,7 +945,17 @@ proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) = return $node.network.discovery.localNode.record proc installDebugApiHandlers(rpcServer: RpcServer, node: BeaconNode) = - discard + rpcServer.rpc("getSpecPreset") do () -> JsonNode: + var res = newJObject() + genCode: + for setting in BeaconChainConstants: + let + settingSym = ident($setting) + settingKey = newLit(toLowerAscii($setting)) + yield quote do: + res[`settingKey`] = %`settingSym` + + return res proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) = rpcServer.installValidatorApiHandlers(node) diff --git a/beacon_chain/spec/datatypes.nim b/beacon_chain/spec/datatypes.nim index 9476886ee..262caf5a0 100644 --- a/beacon_chain/spec/datatypes.nim +++ b/beacon_chain/spec/datatypes.nim @@ -82,7 +82,7 @@ type # Domains # --------------------------------------------------------------- # https://github.com/ethereum/eth2.0-specs/blob/v0.11.0/specs/phase0/beacon-chain.md#domain-types - DomainType* {.pure.} = enum + DomainType* = enum DOMAIN_BEACON_PROPOSER = 0 DOMAIN_BEACON_ATTESTER = 1 DOMAIN_RANDAO = 2 @@ -510,9 +510,6 @@ proc writeValue*(writer: var JsonWriter, value: ValidatorIndex) = proc readValue*(reader: var JsonReader, value: var ValidatorIndex) = value = ValidatorIndex reader.readValue(uint32) -proc `%`*(i: uint64): JsonNode = - % int(i) - # `ValidatorIndex` seq handling. proc max*(a: ValidatorIndex, b: int) : auto = max(a.int, b) diff --git a/beacon_chain/spec/presets/custom.nim b/beacon_chain/spec/presets/custom.nim index 3d988c38a..45010324d 100644 --- a/beacon_chain/spec/presets/custom.nim +++ b/beacon_chain/spec/presets/custom.nim @@ -2,7 +2,7 @@ import macros, strutils, tables type - BeaconChainConstants* = enum + BeaconChainConstants* {.pure.} = enum BASE_REWARDS_PER_EPOCH BASE_REWARD_FACTOR BLS_WITHDRAWAL_PREFIX From 19847bbd204ecba40698ec7be09c0c69604a2c76 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 17 Mar 2020 01:50:08 +0200 Subject: [PATCH 06/13] Remove some warnings --- beacon_chain/eth2_discovery.nim | 2 +- beacon_chain/eth2_network.nim | 2 +- beacon_chain/version.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_chain/eth2_discovery.nim b/beacon_chain/eth2_discovery.nim index 094e89e4a..b228aaad7 100644 --- a/beacon_chain/eth2_discovery.nim +++ b/beacon_chain/eth2_discovery.nim @@ -25,7 +25,7 @@ proc new*(T: type Eth2DiscoveryProtocol, pk = initPrivateKey(rawPrivKeyBytes) db = DiscoveryDB.init(newMemoryDB()) - newProtocol(pk, db, ip, Port conf.tcpPort, Port conf.udpPort) + newProtocol(pk, db, ip, conf.tcpPort, conf.udpPort) proc toENode*(a: MultiAddress): Result[ENode, cstring] = if not IPFS.match(a): diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index 799bd89f4..a7daed141 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -141,7 +141,7 @@ when networkBackend in [libp2p, libp2pDaemon]: bootstrapNodes: seq[ENode]): Future[Eth2Node] {.async.} = var (extIp, extTcpPort, _) = setupNat(conf) - hostAddress = tcpEndPoint(globalListeningAddr, Port conf.tcpPort) + hostAddress = tcpEndPoint(globalListeningAddr, conf.tcpPort) announcedAddresses = if extIp == globalListeningAddr: @[] else: @[tcpEndPoint(extIp, extTcpPort)] diff --git a/beacon_chain/version.nim b/beacon_chain/version.nim index 17b0f1f4a..06b9eb2fb 100644 --- a/beacon_chain/version.nim +++ b/beacon_chain/version.nim @@ -4,7 +4,7 @@ type libp2pDaemon const - NETWORK_TYPE {.strdefine.} = "libp2p_daemon" + NETWORK_TYPE {.strdefine.} = "libp2p" networkBackend* = when NETWORK_TYPE == "libp2p": libp2p elif NETWORK_TYPE == "libp2p_daemon": libp2pDaemon From 7d2381240fb1e2e551f3ed1e85706d65b0e93a06 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Tue, 17 Mar 2020 22:06:26 +0200 Subject: [PATCH 07/13] Bump nim-eth and nim-json-rpc --- beacon_chain/beacon_node.nim | 16 ++++++---------- vendor/nim-json-rpc | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 1f79ad9e9..2ce4c50a4 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -887,14 +887,10 @@ proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) = "Please specify one of " & astToStr(x) & " or " & astToStr(y)) template jsonResult(x: auto): auto = - # TODO, yes this is silly, but teching json-rpc about - # all beacon node types will require quite a lot of work. - # A minor refactoring in json-rpc can solve this. We need - # to allow the handlers to return raw/literal json strings. - parseJson(Json.encode(x)) + StringOfJson(Json.encode(x)) rpcServer.rpc("getBeaconBlock") do (slot: Option[Slot], - root: Option[Eth2Digest]) -> JsonNode: + root: Option[Eth2Digest]) -> StringOfJson: requireOneOf(slot, root) var blockHash: Eth2Digest if root.isSome: @@ -904,16 +900,16 @@ proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) = if foundRef.isSome: blockHash = foundRef.get.root else: - return newJNull() + return StringOfJson("null") let dbBlock = node.db.getBlock(blockHash) if dbBlock.isSome: return jsonResult(dbBlock.get) else: - return newJNull() + return StringOfJson("null") rpcServer.rpc("getBeaconState") do (slot: Option[Slot], - root: Option[Eth2Digest]) -> JsonNode: + root: Option[Eth2Digest]) -> StringOfJson: requireOneOf(slot, root) if slot.isSome: let blk = node.blockPool.head.blck.atSlot(slot.get) @@ -925,7 +921,7 @@ proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) = if state.isSome: return jsonResult(state.get) else: - return newJNull() + return StringOfJson("null") rpcServer.rpc("getNetworkPeerId") do () -> string: when networkBackend != libp2p: diff --git a/vendor/nim-json-rpc b/vendor/nim-json-rpc index 4d1d2577e..6fbaeb61c 160000 --- a/vendor/nim-json-rpc +++ b/vendor/nim-json-rpc @@ -1 +1 @@ -Subproject commit 4d1d2577e9a6634f567899b927b6dc9021624588 +Subproject commit 6fbaeb61cab889f74870372bc00cc99371e16794 From d589cb79640227a579972093c21ca6154b5db51e Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 18 Mar 2020 18:22:16 +0200 Subject: [PATCH 08/13] Fix the build with Json logging enabled --- vendor/nim-json-serialization | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index 88b79e230..ab53e009b 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit 88b79e230005d8301c3ae950abdbf8ad55e37f19 +Subproject commit ab53e009b980f93283ada1f4e496e1785cfc5496 From d2b7ee27de7d8d91ccafd3a2f873b81134392107 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 18 Mar 2020 21:36:22 +0200 Subject: [PATCH 09/13] The RPC service listens on localhost by default --- beacon_chain/beacon_node.nim | 4 +- beacon_chain/conf.nim | 74 ++++++++++++++++++--------------- beacon_chain/eth2_network.nim | 2 +- scripts/connect_to_testnet.nims | 2 +- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 2ce4c50a4..72dd129d8 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -363,7 +363,7 @@ proc sendAttestation(node: BeaconNode, node.network.broadcast( getAttestationTopic(attestationData.index), attestation) - if node.config.dump: + if node.config.dumpEnabled: SSZ.saveFile( node.config.dumpDir / "att-" & $attestationData.slot & "-" & $attestationData.index & "-" & validator.pubKey.shortLog & @@ -444,7 +444,7 @@ proc proposeBlock(node: BeaconNode, validator = shortLog(validator), cat = "consensus" - if node.config.dump: + if node.config.dumpEnabled: SSZ.saveFile( node.config.dumpDir / "block-" & $newBlock.message.slot & "-" & shortLog(newBlockRef.root) & ".ssz", newBlock) diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index fd22ad006..5ef9a3ed1 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -60,34 +60,6 @@ type desc: "Address of the deposit contract." name: "deposit-contract" }: string - statusBarEnabled* {. - defaultValue: true - desc: "Display a status bar at the bottom of the terminal screen." - name: "status-bar" }: bool - - statusBarContents* {. - defaultValue: "peers: $connected_peers; " & - "epoch: $epoch, slot: $epoch_slot/$slots_per_epoch ($slot); " & - "finalized epoch: $last_finalized_epoch |" & - "ETH: $attached_validators_balance" - desc: "Textual template for the contents of the status bar." - name: "status-bar-contents" }: string - - rpcEnabled* {. - defaultValue: false - desc: "Enable the JSON-RPC server" - name: "rpc" }: bool - - rpcPort* {. - defaultValue: defaultEth2RpcPort - desc: "HTTP port for the JSON-RPC service." - name: "rpc-port" }: Port - - rpcAddress* {. - defaultValue: defaultListenAddress(config) - desc: "Listening address of the RPC server" - name: "rpc-address" }: IpAddress - case cmd* {. command defaultValue: noCommand }: StartUpCmd @@ -103,14 +75,19 @@ type desc: "Specifies a line-delimited file of bootsrap Ethereum network addresses." name: "bootstrap-file" }: InputFile + libp2pAddress* {. + defaultValue: defaultListenAddress(config) + desc: "Listening address for the Ethereum LibP2P traffic." + name: "listen-address"}: IpAddress + tcpPort* {. defaultValue: defaultEth2TcpPort - desc: "TCP listening port." + desc: "Listening TCP port for Ethereum LibP2P traffic." name: "tcp-port" }: Port udpPort* {. defaultValue: defaultEth2TcpPort - desc: "UDP listening port." + desc: "Listening UDP port for node discovery." name: "udp-port" }: Port maxPeers* {. @@ -156,7 +133,7 @@ type name: "metrics" }: bool metricsAddress* {. - defaultValue: defaultListenAddress(config) + defaultValue: defaultAdminListenAddress(config) desc: "Listening address of the metrics server." name: "metrics-address" }: IpAddress @@ -165,10 +142,38 @@ type desc: "Listening HTTP port of the metrics server." name: "metrics-port" }: Port - dump* {. + statusBarEnabled* {. + defaultValue: true + desc: "Display a status bar at the bottom of the terminal screen." + name: "status-bar" }: bool + + statusBarContents* {. + defaultValue: "peers: $connected_peers; " & + "epoch: $epoch, slot: $epoch_slot/$slots_per_epoch ($slot); " & + "finalized epoch: $last_finalized_epoch |" & + "ETH: $attached_validators_balance" + desc: "Textual template for the contents of the status bar." + name: "status-bar-contents" }: string + + rpcEnabled* {. + defaultValue: false + desc: "Enable the JSON-RPC server" + name: "rpc" }: bool + + rpcPort* {. + defaultValue: defaultEth2RpcPort + desc: "HTTP port for the JSON-RPC service." + name: "rpc-port" }: Port + + rpcAddress* {. + defaultValue: defaultAdminListenAddress(config) + desc: "Listening address of the RPC server" + name: "rpc-address" }: IpAddress + + dumpEnabled* {. defaultValue: false desc: "Write SSZ dumps of blocks, attestations and states to data dir" - .}: bool + name: "dump" }: bool of createTestnet: validatorsDir* {. @@ -289,6 +294,9 @@ func defaultListenAddress*(conf: BeaconNodeConf): IpAddress = # Maybe there should be a config option for this. parseIpAddress("0.0.0.0") +func defaultAdminListenAddress*(conf: BeaconNodeConf): IpAddress = + parseIpAddress("127.0.0.1") + iterator validatorKeys*(conf: BeaconNodeConf): ValidatorPrivKey = for validatorKeyFile in conf.validators: try: diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim index a7daed141..bb71ef73b 100644 --- a/beacon_chain/eth2_network.nim +++ b/beacon_chain/eth2_network.nim @@ -141,7 +141,7 @@ when networkBackend in [libp2p, libp2pDaemon]: bootstrapNodes: seq[ENode]): Future[Eth2Node] {.async.} = var (extIp, extTcpPort, _) = setupNat(conf) - hostAddress = tcpEndPoint(globalListeningAddr, conf.tcpPort) + hostAddress = tcpEndPoint(conf.libp2pAddress, conf.tcpPort) announcedAddresses = if extIp == globalListeningAddr: @[] else: @[tcpEndPoint(extIp, extTcpPort)] diff --git a/scripts/connect_to_testnet.nims b/scripts/connect_to_testnet.nims index bff824dab..f30d4c2cf 100644 --- a/scripts/connect_to_testnet.nims +++ b/scripts/connect_to_testnet.nims @@ -142,7 +142,7 @@ cli do (skipGoerliKey {. mode = Verbose execIgnoringExitCode replace(&"""{beaconNodeBinary} --data-dir="{dataDir}" - --dump=true + --dump --web3-url={web3Url} {bootstrapFileOpt} {logLevelOpt} From 1d78a43724589e08806b712532dab3dab3ac8ca0 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 18 Mar 2020 21:38:34 +0200 Subject: [PATCH 10/13] Don't return Option[BlockRef] because BlockRef is a ref type --- beacon_chain/beacon_node.nim | 4 ++-- beacon_chain/block_pool.nim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 72dd129d8..db3f3694d 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -897,8 +897,8 @@ proc installBeaconApiHandlers(rpcServer: RpcServer, node: BeaconNode) = blockHash = root.get else: let foundRef = node.blockPool.getBlockByPreciseSlot(slot.get) - if foundRef.isSome: - blockHash = foundRef.get.root + if foundRef != nil: + blockHash = foundRef.root else: return StringOfJson("null") diff --git a/beacon_chain/block_pool.nim b/beacon_chain/block_pool.nim index 1b450bda7..7d758e7e2 100644 --- a/beacon_chain/block_pool.nim +++ b/beacon_chain/block_pool.nim @@ -519,11 +519,11 @@ func getBlockBySlot*(pool: BlockPool, slot: Slot): BlockRef = ## with slot number less or equal to `slot`. pool.head.blck.findAncestorBySlot(slot).blck -func getBlockByPreciseSlot*(pool: BlockPool, slot: Slot): Option[BlockRef] = +func getBlockByPreciseSlot*(pool: BlockPool, slot: Slot): BlockRef = ## Retrieves a block from the canonical chain with a slot ## number equal to `slot`. let found = pool.getBlockBySlot(slot) - if found.slot != slot: some(found) else: none(BlockRef) + if found.slot != slot: found else: nil proc get*(pool: BlockPool, blck: BlockRef): BlockData = ## Retrieve the associated block body of a block reference From 286d6e14da6bec6fe3d362a92bedbdc50b027370 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Sun, 22 Mar 2020 18:20:46 +0200 Subject: [PATCH 11/13] Attempt to fix the Travis ARM build --- beacon_chain/beacon_node.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index db3f3694d..6b6ac170f 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -866,7 +866,7 @@ func slotOrZero(time: BeaconTime): Slot = if exSlot.afterGenesis: exSlot.slot else: Slot(0) -func currentSlot(node: BeaconNode): Slot = +proc currentSlot(node: BeaconNode): Slot = node.beaconClock.now.slotOrZero proc connectedPeersCount(node: BeaconNode): int = From 5868afe341d4d454d1048674afb9736141d4daaa Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Wed, 18 Mar 2020 20:48:36 +0200 Subject: [PATCH 12/13] Restore compilation with -d:p2pdump --- beacon_chain/libp2p_backend.nim | 6 +++++- beacon_chain/libp2p_backends_common.nim | 9 +++++---- beacon_chain/libp2p_daemon_backend.nim | 7 ++++++- vendor/nim-eth | 2 +- vendor/nim-json-serialization | 2 +- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/beacon_chain/libp2p_backend.nim b/beacon_chain/libp2p_backend.nim index 069626986..b9f320be8 100644 --- a/beacon_chain/libp2p_backend.nim +++ b/beacon_chain/libp2p_backend.nim @@ -107,6 +107,9 @@ const template `$`*(peer: Peer): string = id(peer.info) chronicles.formatIt(Peer): $it +template remote*(peer: Peer): untyped = + peer.info.peerId + # TODO: This exists only as a compatibility layer between the daemon # APIs and the native LibP2P ones. It won't be necessary once the # daemon is removed. @@ -351,6 +354,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = msgName = $msg.ident msgNameLit = newLit msgName MsgRecName = msg.recName + MsgStrongRecName = msg.strongRecName codecNameLit = getRequestProtoName(msg.procDef) if msg.procDef.body.kind != nnkEmpty and msg.kind == msgRequest: @@ -388,7 +392,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = proc thunk(`streamVar`: `P2PStream`, proto: string): Future[void] {.gcsafe.} = return handleIncomingStream(`networkVar`, `streamVar`, - `MsgRecName`, `Format`) + `MsgStrongRecName`, `Format`) mount `networkVar`.switch, LPProtocol(codec: `codecNameLit`, handler: thunk) diff --git a/beacon_chain/libp2p_backends_common.nim b/beacon_chain/libp2p_backends_common.nim index 7de5ab87a..500ab194f 100644 --- a/beacon_chain/libp2p_backends_common.nim +++ b/beacon_chain/libp2p_backends_common.nim @@ -358,7 +358,7 @@ proc implementSendProcBody(sendProc: SendProc) = proc handleIncomingStream(network: Eth2Node, stream: P2PStream, MsgType, Format: distinct type) {.async, gcsafe.} = - mixin callUserHandler + mixin callUserHandler, RecType const msgName = typetraits.name(MsgType) ## Uncomment this to enable tracing on all incoming requests @@ -384,9 +384,10 @@ proc handleIncomingStream(network: Eth2Node, stream: P2PStream, await sendErrorResponse(peer, stream, ServerError, readTimeoutErrorMsg) return - var msg: MsgType + type MsgRec = RecType(MsgType) + var msg: MsgRec try: - msg = decode(Format, msgBytes, MsgType) + msg = decode(Format, msgBytes, MsgRec) except SerializationError as err: await sendErrorResponse(peer, stream, err, msgName, msgBytes) return @@ -399,7 +400,7 @@ proc handleIncomingStream(network: Eth2Node, stream: P2PStream, raise err try: - logReceivedMsg(peer, msg) + logReceivedMsg(peer, MsgType(msg)) await callUserHandler(peer, stream, msg) except CatchableError as err: await sendErrorResponse(peer, stream, ServerError, err.msg) diff --git a/beacon_chain/libp2p_daemon_backend.nim b/beacon_chain/libp2p_daemon_backend.nim index 2ce39c9ab..4d57d927a 100644 --- a/beacon_chain/libp2p_daemon_backend.nim +++ b/beacon_chain/libp2p_daemon_backend.nim @@ -101,6 +101,10 @@ template openStream(node: Eth2Node, peer: Peer, protocolId: string): untyped = proc init*(T: type Peer, network: Eth2Node, id: PeerID): Peer {.gcsafe.} +template remote*(peer: Peer): untyped = + # TODO: Can we get a proper address here? + peer.id + proc getPeer*(node: Eth2Node, peerId: PeerID): Peer {.gcsafe.} = result = node.peers.getOrDefault(peerId) if result == nil: @@ -211,6 +215,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = msgName = $msg.ident msgNameLit = newLit msgName MsgRecName = msg.recName + MsgStrongRecName = msg.strongRecName if msg.procDef.body.kind != nnkEmpty and msg.kind == msgRequest: # Request procs need an extra param - the stream where the response @@ -235,7 +240,7 @@ proc p2pProtocolBackendImpl*(p: P2PProtocol): Backend = proc `thunkName`(`daemonVar`: `DaemonAPI`, `streamVar`: `P2PStream`): Future[void] {.gcsafe.} = return handleIncomingStream(`Eth2Node`(`daemonVar`.userData), `streamVar`, - `MsgRecName`, `Format`) + `MsgStrongRecName`, `Format`) else: thunkName = newNilLit() diff --git a/vendor/nim-eth b/vendor/nim-eth index b3b585400..9c442bf65 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit b3b5854003ebb10b99efa451d4391ef050f5951e +Subproject commit 9c442bf65b52a4c857cc6e51efe901352e8b6ebf diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index ab53e009b..6350b72b5 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit ab53e009b980f93283ada1f4e496e1785cfc5496 +Subproject commit 6350b72b5eda69f7ccfa57a94fd420509dbf6f49 From 1752e1c2b6e4aa724a2d7351a11227c6a1326c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Mon, 23 Mar 2020 13:25:24 +0100 Subject: [PATCH 13/13] bump submodules --- vendor/nim-chronicles | 2 +- vendor/nim-libp2p | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles index e4de3a900..8da0e30c5 160000 --- a/vendor/nim-chronicles +++ b/vendor/nim-chronicles @@ -1 +1 @@ -Subproject commit e4de3a90019a5f621c265729ac9071769a061a79 +Subproject commit 8da0e30c526ab1c6c825e16546fc5db972c5408d diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index b1a34f478..0a3e4a764 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit b1a34f478efe6d1212cfccecaf4dd8bf89fc2b1f +Subproject commit 0a3e4a764b718d13fc330f228fce60ed265cfde2