status-im/nimbus-eth2
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth2.git synced 2025-01-11 14:54:12 +00:00
Code Issues Projects Releases Wiki Activity

- extracted the commands to run a VC into a separate run_validator.sh script

Browse Source 
- each BN gets half of its previous validators as inProcess and the other half goes to the respective VC for that BN - using separate data dirs where the keys are copied
    - also removed a few command line options which are no longer necessary
- block proposals originating from a VC are propagated from one BN to the rest properly
- other cleanup & moving code back to  since it is no longer used elsewhere
This commit is contained in:
Viktor Kirilov 2020-06-05 12:57:40 +03:00
parent ce897fe83f
commit 3bae40ae91
13 changed files with 363 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
beacon_chain/beacon_node.nim
View File

@ -24,7 +24,7 @@ import
attestation_pool, block_pool, eth2_network, eth2_discovery,
beacon_node_common, beacon_node_types, block_pools/block_pools_types,
nimbus_binary_common,
mainchain_monitor, version, ssz/[merkleization],
mainchain_monitor, version, ssz/[merkleization], sszdump,
sync_protocol, request_manager, validator_keygen, interop, statusbar,
sync_manager, state_transition,
validator_duties, validator_api, attestation_aggregation
@ -56,12 +56,16 @@ declareGauge beacon_head_slot,
# Metrics for tracking attestation and beacon block loss
declareCounter beacon_attestations_received,
"Number of beacon chain attestations received by this peer"
declareCounter beacon_blocks_received,
"Number of beacon chain blocks received by this peer"
declareHistogram beacon_attestation_received_seconds_from_slot_start,
"Interval between slot start and attestation receival", buckets = [2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, Inf]
logScope: topics = "beacnde"
proc onBeaconBlock(node: BeaconNode, signedBlock: SignedBeaconBlock) {.gcsafe.}
proc getStateFromSnapshot(conf: BeaconNodeConf): NilableBeaconStateRef =
var
genesisPath = conf.dataDir/genesisFile
@ -253,6 +257,81 @@ proc connectToNetwork(node: BeaconNode) {.async.} =
let addressFile = node.config.dataDir / "beacon_node.address"
writeFile(addressFile, node.network.announcedENR.toURI)
proc onAttestation(node: BeaconNode, attestation: Attestation) =
# We received an attestation from the network but don't know much about it
# yet - in particular, we haven't verified that it belongs to particular chain
# we're on, or that it follows the rules of the protocol
logScope: pcs = "on_attestation"
let
wallSlot = node.beaconClock.now().toSlot()
head = node.blockPool.head
debug "Attestation received",
attestation = shortLog(attestation),
headRoot = shortLog(head.blck.root),
headSlot = shortLog(head.blck.slot),
wallSlot = shortLog(wallSlot.slot),
cat = "consensus" # Tag "consensus|attestation"?
if not wallSlot.afterGenesis or wallSlot.slot < head.blck.slot:
warn "Received attestation before genesis or head - clock is wrong?",
afterGenesis = wallSlot.afterGenesis,
wallSlot = shortLog(wallSlot.slot),
headSlot = shortLog(head.blck.slot),
cat = "clock_drift" # Tag "attestation|clock_drift"?
return
if attestation.data.slot > head.blck.slot and
(attestation.data.slot - head.blck.slot) > MaxEmptySlotCount:
warn "Ignoring attestation, head block too old (out of sync?)",
attestationSlot = attestation.data.slot, headSlot = head.blck.slot
return
node.attestationPool.add(attestation)
proc storeBlock(
node: BeaconNode, signedBlock: SignedBeaconBlock): Result[void, BlockError] =
let blockRoot = hash_tree_root(signedBlock.message)
debug "Block received",
signedBlock = shortLog(signedBlock.message),
blockRoot = shortLog(blockRoot),
cat = "block_listener",
pcs = "receive_block"
if node.config.dumpEnabled:
dump(node.config.dumpDir / "incoming", signedBlock, blockRoot)
beacon_blocks_received.inc()
let blck = node.blockPool.add(blockRoot, signedBlock)
if blck.isErr:
if blck.error == Invalid and node.config.dumpEnabled:
let parent = node.blockPool.getRef(signedBlock.message.parent_root)
if parent != nil:
node.blockPool.withState(
node.blockPool.tmpState, parent.atSlot(signedBlock.message.slot - 1)):
dump(node.config.dumpDir / "invalid", hashedState, parent)
dump(node.config.dumpDir / "invalid", signedBlock, blockRoot)
return err(blck.error)
# The block we received contains attestations, and we might not yet know about
# all of them. Let's add them to the attestation pool - in case the block
# is not yet resolved, neither will the attestations be!
# But please note that we only care about recent attestations.
# TODO shouldn't add attestations if the block turns out to be invalid..
let currentSlot = node.beaconClock.now.toSlot
if currentSlot.afterGenesis and
signedBlock.message.slot.epoch + 1 >= currentSlot.slot.epoch:
for attestation in signedBlock.message.body.attestations:
node.onAttestation(attestation)
ok()
proc onBeaconBlock(node: BeaconNode, signedBlock: SignedBeaconBlock) =
# We received a block but don't know much about it yet - in particular, we
# don't know if it's part of the chain we're currently building.
discard node.storeBlock(signedBlock)
func verifyFinalization(node: BeaconNode, slot: Slot) =
# Epoch must be >= 4 to check finalization
const SETTLING_TIME_OFFSET = 1'u64
@ -621,9 +700,7 @@ proc installDebugApiHandlers(rpcServer: RpcServer, node: BeaconNode) =
return res
proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) =
# TODO: remove this if statement later - here just to test the config option for now
if node.config.validatorApi:
rpcServer.installValidatorApiHandlers(node)
rpcServer.installValidatorApiHandlers(node)
rpcServer.installBeaconApiHandlers(node)
rpcServer.installDebugApiHandlers(node)
@ -638,6 +715,14 @@ proc installAttestationHandlers(node: BeaconNode) =
node.onAttestation(attestation)
proc attestationValidator(attestation: Attestation,
committeeIndex: uint64): bool =
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#attestation-subnets
let (afterGenesis, slot) = node.beaconClock.now().toSlot()
if not afterGenesis:
return false
node.attestationPool.isValidAttestation(attestation, slot, committeeIndex, {})
var attestationSubscriptions: seq[Future[void]] = @[]
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#mainnet-3
@ -648,25 +733,16 @@ proc installAttestationHandlers(node: BeaconNode) =
getMainnetAttestationTopic(node.forkDigest, ci), attestationHandler,
# This proc needs to be within closureScope; don't lift out of loop.
proc(attestation: Attestation): bool =
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#attestation-subnets
let (afterGenesis, slot) = node.beaconClock.now().toSlot()
if not afterGenesis:
return false
node.attestationPool.isValidAttestation(attestation, slot, ci)))
attestationValidator(attestation, ci)
when ETH2_SPEC == "v0.11.3":
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#interop-3
attestationSubscriptions.add(node.network.subscribe(
getInteropAttestationTopic(node.forkDigest), attestationHandler,
proc(attestation: Attestation): bool =
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#attestation-subnets
let (afterGenesis, slot) = node.beaconClock.now().toSlot()
if not afterGenesis:
return false
# isValidAttestation checks attestation.data.index == topicCommitteeIndex
# which doesn't make sense here, so rig that check to vacuously pass.
node.attestationPool.isValidAttestation(
attestation, slot, attestation.data.index)))
attestationValidator(attestation, attestation.data.index)
waitFor allFutures(attestationSubscriptions)

86
beacon_chain/beacon_node_common.nim
View File

@ -9,16 +9,15 @@
import
# Standard library
os, tables,
tables,
# Nimble packages
chronos, json_rpc/rpcserver, metrics,
chronicles,
# Local modules
spec/[datatypes, crypto, digest, helpers],
conf, time, beacon_chain_db, sszdump,
ssz/merkleization,
spec/[datatypes, crypto, digest],
conf, time, beacon_chain_db,
attestation_pool, block_pool, eth2_network,
beacon_node_types, mainchain_monitor, request_manager,
sync_manager
@ -57,85 +56,6 @@ const
declareGauge beacon_head_root,
"Root of the head block of the beacon chain"
# Metrics for tracking attestation and beacon block loss
declareCounter beacon_blocks_received,
"Number of beacon chain blocks received by this peer"
proc onAttestation*(node: BeaconNode, attestation: Attestation) =
# We received an attestation from the network but don't know much about it
# yet - in particular, we haven't verified that it belongs to particular chain
# we're on, or that it follows the rules of the protocol
logScope: pcs = "on_attestation"
let
wallSlot = node.beaconClock.now().toSlot()
head = node.blockPool.head
debug "Attestation received",
attestation = shortLog(attestation),
headRoot = shortLog(head.blck.root),
headSlot = shortLog(head.blck.slot),
wallSlot = shortLog(wallSlot.slot),
cat = "consensus" # Tag "consensus|attestation"?
if not wallSlot.afterGenesis or wallSlot.slot < head.blck.slot:
warn "Received attestation before genesis or head - clock is wrong?",
afterGenesis = wallSlot.afterGenesis,
wallSlot = shortLog(wallSlot.slot),
headSlot = shortLog(head.blck.slot),
cat = "clock_drift" # Tag "attestation|clock_drift"?
return
if attestation.data.slot > head.blck.slot and
(attestation.data.slot - head.blck.slot) > MaxEmptySlotCount:
warn "Ignoring attestation, head block too old (out of sync?)",
attestationSlot = attestation.data.slot, headSlot = head.blck.slot
return
node.attestationPool.add(attestation)
proc storeBlock*(
node: BeaconNode, signedBlock: SignedBeaconBlock): Result[void, BlockError] =
let blockRoot = hash_tree_root(signedBlock.message)
debug "Block received",
signedBlock = shortLog(signedBlock.message),
blockRoot = shortLog(blockRoot),
cat = "block_listener",
pcs = "receive_block"
if node.config.dumpEnabled:
dump(node.config.dumpDir / "incoming", signedBlock, blockRoot)
beacon_blocks_received.inc()
let blck = node.blockPool.add(blockRoot, signedBlock)
if blck.isErr:
if blck.error == Invalid and node.config.dumpEnabled:
let parent = node.blockPool.getRef(signedBlock.message.parent_root)
if parent != nil:
node.blockPool.withState(
node.blockPool.tmpState, parent.atSlot(signedBlock.message.slot - 1)):
dump(node.config.dumpDir / "invalid", hashedState, parent)
dump(node.config.dumpDir / "invalid", signedBlock, blockRoot)
return err(blck.error)
# The block we received contains attestations, and we might not yet know about
# all of them. Let's add them to the attestation pool - in case they block
# is not yet resolved, neither will the attestations be!
# But please note that we only care about recent attestations.
# TODO shouldn't add attestations if the block turns out to be invalid..
let currentSlot = node.beaconClock.now.toSlot
if currentSlot.afterGenesis and
signedBlock.message.slot.epoch + 1 >= currentSlot.slot.epoch:
for attestation in signedBlock.message.body.attestations:
node.onAttestation(attestation)
ok()
proc onBeaconBlock*(node: BeaconNode, signedBlock: SignedBeaconBlock) =
# We received a block but don't know much about it yet - in particular, we
# don't know if it's part of the chain we're currently building.
discard node.storeBlock(signedBlock)
proc updateHead*(node: BeaconNode): BlockRef =
# Check pending attestations - maybe we found some blocks for them
node.attestationPool.resolve()

11
beacon_chain/conf.nim
View File

@ -106,11 +106,6 @@ type
abbr: "v"
name: "validator" }: seq[ValidatorKeyPath]
validatorApi* {.
defaultValue: false
desc: "Specify whether the validator API should be enabled which would allow for external validators (validator clients) to use this beacon node."
name: "validator-api" }: bool
stateSnapshot* {.
desc: "Json file specifying a recent state snapshot."
abbr: "s"
@ -299,12 +294,6 @@ type
abbr: "v"
name: "validator" }: seq[ValidatorKeyPath]
delayStart* {.
defaultValue: 0
desc: "Seconds from now to delay the starting of the validator client (useful for debug purposes when starting before the beacon node in a script)."
abbr: "g"
name: "delay-start" }: int
proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string =
let dataDir = when defined(windows):
"AppData" / "Roaming" / "Nimbus"

18
beacon_chain/nimbus_binary_common.nim
View File

@ -16,7 +16,7 @@ import
chronicles, chronicles/helpers as chroniclesHelpers,
# Local modules
spec/[datatypes, crypto], eth2_network
spec/[datatypes, crypto], eth2_network, time
proc setupMainProc*(logLevel: string) =
when compiles(defaultChroniclesStream.output.writer):
@ -57,3 +57,19 @@ template ctrlCHandling*(extraCode: untyped) =
template makeBannerAndConfig*(clientId: string, ConfType: type): untyped =
let banner = clientId & "\p" & copyrights & "\p\p" & nimBanner
ConfType.load(version = banner, copyrightBanner = banner)
# TODO not sure if this belongs here but it doesn't belong in `time.nim` either
proc sleepToSlotOffset*(clock: BeaconClock, extra: chronos.Duration,
slot: Slot, msg: static string): Future[bool] {.async.} =
let
fromNow = clock.fromNow(slot.toBeaconTime(extra))
if fromNow.inFuture:
trace msg,
slot = shortLog(slot),
fromNow = shortLog(fromNow.offset),
cat = "scheduling"
await sleepAsync(fromNow.offset)
return true
return false

11
beacon_chain/spec/eth2_apis/validator_callsigs.nim
View File

@ -16,19 +16,22 @@ proc get_v1_beacon_states_fork(stateId: string): Fork
# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
proc get_v1_beacon_genesis(): BeaconGenesisTuple
# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
proc post_v1_beacon_pool_attestations(attestation: Attestation): bool
proc get_v1_validator_blocks(slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig): BeaconBlock
# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
proc post_v1_beacon_blocks(body: SignedBeaconBlock): bool
proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeIndex): AttestationData
proc get_v1_validator_aggregate_attestation(query: Eth2Digest): Attestation
proc get_v1_validator_aggregate_attestation(attestation_data_root: Eth2Digest): Attestation
proc post_v1_validator_aggregate_and_proof(payload: SignedAggregateAndProof)
# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
proc post_v1_validator_aggregate_and_proof(payload: SignedAggregateAndProof): bool
# TODO this should perhaps be a GET instead of a POST?
# this is a POST instead of a GET because of this: https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAJk5rbKA
proc post_v1_validator_duties_attester(epoch: Epoch, public_keys: seq[ValidatorPubKey]): seq[AttesterDuties]
proc get_v1_validator_duties_proposer(epoch: Epoch): seq[ValidatorPubkeySlotPair]

98
beacon_chain/validator_api.nim
View File

@ -12,12 +12,13 @@ import
# Nimble packages
stew/[objects],
chronos, metrics, json_rpc/[rpcserver, jsonmarshal],
chronicles,
# Local modules
spec/[datatypes, digest, crypto, validator],
block_pool,
beacon_node_common,
validator_duties,
spec/[datatypes, digest, crypto, validator, beaconstate],
block_pool, ssz/merkleization,
beacon_node_common, beacon_node_types,
validator_duties, eth2_network,
spec/eth2_apis/validator_callsigs_types,
eth2_json_rpc_serialization
@ -36,7 +37,7 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
of "genesis":
Fork(previous_version: Version(GENESIS_FORK_VERSION),
current_version: Version(GENESIS_FORK_VERSION),
epoch: 0.Epoch)
epoch: GENESIS_EPOCH)
of "finalized":
# TODO
Fork()
@ -54,16 +55,24 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
genesis_validators_root: node.blockPool.headState.data.data.genesis_validators_root,
genesis_fork_version: Version(GENESIS_FORK_VERSION))
rpcServer.rpc("get_v1_validator_blocks") do (slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig) -> BeaconBlock:
rpcServer.rpc("post_v1_beacon_pool_attestations") do (attestation: Attestation) -> bool:
#notice "== post_v1_beacon_pool_attestations"
node.sendAttestation(attestation)
return true
rpcServer.rpc("get_v1_validator_blocks") do (
slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig) -> BeaconBlock:
notice "== get_v1_validator_blocks", slot = slot
var head = node.updateHead()
let head = node.updateHead()
let proposer = node.blockPool.getProposer(head, slot)
# TODO how do we handle the case when we cannot return a meaningful block? 404...
doAssert(proposer.isSome())
let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viRandao_reveal, randao_reveal: randao_reveal)
let res = makeBeaconBlockForHeadAndSlot(node, valInfo, proposer.get()[0], graffiti, head, slot)
let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viRandao_reveal,
randao_reveal: randao_reveal)
let res = makeBeaconBlockForHeadAndSlot(
node, valInfo, proposer.get()[0], graffiti, head, slot)
# TODO how do we handle the case when we cannot return a meaningful block? 404...
# currently this fails often - perhaps because the block has already been
@ -71,28 +80,53 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
# doAssert(res.message.isSome())
return res.message.get(BeaconBlock()) # returning a default if empty
rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock) -> bool :
rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock) -> bool:
notice "== post_v1_beacon_blocks"
# TODO make onBeaconBlock return a result and discard it wherever its unnecessary
onBeaconBlock(node, body)
logScope: pcs = "block_proposal"
let head = node.updateHead()
if head.slot >= body.message.slot:
warn "Skipping proposal, have newer head already",
headSlot = shortLog(head.slot),
headBlockRoot = shortLog(head.root),
slot = shortLog(body.message.slot),
cat = "fastforward"
return false
return head != await proposeSignedBlock(node, head, AttachedValidator(),
body, hash_tree_root(body.message))
rpcServer.rpc("get_v1_validator_attestation_data") do (
slot: Slot, committee_index: CommitteeIndex) -> AttestationData:
#notice "== get_v1_validator_attestation_data"
# Obtain the data to form an attestation
let head = node.updateHead()
let attestationHead = head.atSlot(slot)
node.blockPool.withState(node.blockPool.tmpState, attestationHead):
return makeAttestationData(state, slot, committee_index.uint64, blck.root)
rpcServer.rpc("get_v1_validator_aggregate_attestation") do (
attestation_data_root: Eth2Digest)-> Attestation:
notice "== get_v1_validator_aggregate_attestation"
# TODO look at attestation.data.beacon_block_root
rpcServer.rpc("post_v1_validator_aggregate_and_proof") do (
payload: SignedAggregateAndProof) -> bool:
notice "== post_v1_validator_aggregate_and_proof"
node.network.broadcast(node.topicAggregateAndProofs, payload)
return true
rpcServer.rpc("get_v1_validator_attestation_data") do (slot: Slot, committee_index: CommitteeIndex) -> AttestationData:
discard
rpcServer.rpc("get_v1_validator_aggregate_attestation") do (query: Eth2Digest)-> Attestation:
# TODO look at attestation.data.beacon_block_root
discard
rpcServer.rpc("post_v1_validator_aggregate_and_proof") do (payload: SignedAggregateAndProof):
discard
rpcServer.rpc("post_v1_validator_duties_attester") do (epoch: Epoch, public_keys: seq[ValidatorPubKey]) -> seq[AttesterDuties]:
rpcServer.rpc("post_v1_validator_duties_attester") do (
epoch: Epoch, public_keys: seq[ValidatorPubKey]) -> seq[AttesterDuties]:
notice "== post_v1_validator_duties_attester", epoch = epoch
discard node.updateHead() # TODO do we need this?
for pubkey in public_keys:
let idx = node.blockPool.headState.data.data.validators.asSeq.findIt(it.pubKey == pubkey)
if idx != -1:
let res = node.blockPool.headState.data.data.get_committee_assignment(epoch, idx.ValidatorIndex)
# TODO this might crash if the requested epoch is further than the BN epoch
# because of this: `doAssert epoch <= next_epoch`
let res = node.blockPool.headState.data.data.get_committee_assignment(
epoch, idx.ValidatorIndex)
if res.isSome:
result.add(AttesterDuties(public_key: pubkey,
committee_index: res.get.b,
@ -100,18 +134,18 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
validator_committee_index: res.get.a.find(idx.ValidatorIndex).uint64,
slot: res.get.c))
rpcServer.rpc("get_v1_validator_duties_proposer") do (epoch: Epoch) -> seq[ValidatorPubkeySlotPair]:
rpcServer.rpc("get_v1_validator_duties_proposer") do (
epoch: Epoch) -> seq[ValidatorPubkeySlotPair]:
notice "== get_v1_validator_duties_proposer", epoch = epoch
discard node.updateHead() # TODO do we need this?
var cache = get_empty_per_epoch_cache()
result = get_beacon_proposer_indexes_for_epoch(node.blockPool.headState.data.data, epoch, cache).mapIt(ValidatorPubkeySlotPair(
result = get_beacon_proposer_indexes_for_epoch(node.blockPool.headState.data.data,
epoch, cache).mapIt(ValidatorPubkeySlotPair(
public_key: node.blockPool.headState.data.data.validators[it.i].pubkey,
slot: it.s
))
rpcServer.rpc("post_v1_validator_beacon_committee_subscription") do (
committee_index: CommitteeIndex,
slot: Slot,
aggregator: bool,
validator_pubkey: ValidatorPubKey,
slot_signature: ValidatorSig):
discard
committee_index: CommitteeIndex, slot: Slot, aggregator: bool,
validator_pubkey: ValidatorPubKey, slot_signature: ValidatorSig):
notice "== post_v1_validator_beacon_committee_subscription"

61
beacon_chain/validator_client.nim
View File

@ -38,18 +38,16 @@ type
attachedValidators: ValidatorPool
fork: Fork
proposalsForEpoch: Table[Slot, ValidatorPubKey]
attestationsForEpoch: Table[Slot, AttesterDuties]
attestationsForEpoch: Table[Slot, seq[AttesterDuties]]
beaconGenesis: BeaconGenesisTuple
# TODO remove this and move to real logging once done experimenting
# it's much easier to distinguish such output from the one with timestamps
# TODO remove this and move to real logging once done experimenting - it's much
# easier to distinguish such output from the one from chronicles with timestamps
proc port_logged(vc: ValidatorClient, msg: string) =
echo "== ", vc.config.rpcPort, " ", msg
proc getValidatorDutiesForEpoch(vc: ValidatorClient, epoch: Epoch) {.gcsafe, async.} =
vc.port_logged "await 1"
let proposals = await vc.client.get_v1_validator_duties_proposer(epoch)
vc.port_logged "await 2"
# update the block proposal duties this VC should do during this epoch
vc.proposalsForEpoch.clear()
for curr in proposals:
@ -61,16 +59,17 @@ proc getValidatorDutiesForEpoch(vc: ValidatorClient, epoch: Epoch) {.gcsafe, asy
for key in vc.attachedValidators.validators.keys:
validatorPubkeys.add key
# update the attestation duties this VC should do during this epoch
let attestations = await vc.client.post_v1_validator_duties_attester(epoch, validatorPubkeys)
let attestations = await vc.client.post_v1_validator_duties_attester(
epoch, validatorPubkeys)
vc.attestationsForEpoch.clear()
for a in attestations:
vc.attestationsForEpoch.add(a.slot, a)
if vc.attestationsForEpoch.hasKeyOrPut(a.slot, @[a]):
vc.attestationsForEpoch[a.slot].add(a)
# for now we will get the fork each time we update the validator duties for each epoch
vc.fork = await vc.client.get_v1_beacon_states_fork("head")
proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, async.} =
vc.port_logged "WAKE UP! slot " & $scheduledSlot
let
# The slot we should be at, according to the clock
@ -81,12 +80,16 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
slot = wallSlot.slot # afterGenesis == true!
nextSlot = slot + 1
vc.port_logged "WAKE UP! scheduledSlot " & $scheduledSlot & " slot " & $slot
try:
# at the start of each epoch - request all validator duties for the current epoch
# TODO perhaps call this not on the first slot of each Epoch but perhaps 1 slot earlier
# because there are a few back-and-forth requests which could take up time for attesting...
if scheduledSlot.isEpoch:
await getValidatorDutiesForEpoch(vc, scheduledSlot.compute_epoch_at_slot)
# at the start of each epoch - request all validator duties
# TODO perhaps call this not on the first slot of each Epoch but perhaps
# 1 slot earlier because there are a few back-and-forth requests which
# could take up time for attesting... Perhaps this should be called more
# than once per epoch because of forks & other events...
if slot.isEpoch:
await getValidatorDutiesForEpoch(vc, slot.compute_epoch_at_slot)
# check if we have a validator which needs to propose on this slot
if vc.proposalsForEpoch.contains slot:
@ -96,32 +99,38 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
let randao_reveal = validator.genRandaoReveal(
vc.fork, vc.beaconGenesis.genesis_validators_root, slot)
vc.port_logged "await 3"
var newBlock = SignedBeaconBlock(
message: await vc.client.get_v1_validator_blocks(slot, Eth2Digest(), randao_reveal)
)
vc.port_logged "await 4"
let blockRoot = hash_tree_root(newBlock.message)
newBlock.signature = await validator.signBlockProposal(
vc.fork, vc.beaconGenesis.genesis_validators_root, slot, blockRoot)
vc.port_logged "about to await for the last time!"
discard await vc.client.post_v1_beacon_blocks(newBlock)
vc.port_logged "did we do it?"
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#attesting
# A validator should create and broadcast the attestation to the associated
# attestation subnet when either (a) the validator has received a valid
# block from the expected block proposer for the assigned slot or
# (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / 3` seconds
# after the start of slot) -- whichever comes first.
discard await vc.beaconClock.sleepToSlotOffset(
seconds(int64(SECONDS_PER_SLOT)) div 3, slot, "Waiting to send attestations")
# check if we have a validator which needs to propose on this slot
# check if we have validators which need to attest on this slot
if vc.attestationsForEpoch.contains slot:
let a = vc.attestationsForEpoch[slot]
let validator = vc.attachedValidators.validators[a.public_key]
for a in vc.attestationsForEpoch[slot]:
let validator = vc.attachedValidators.validators[a.public_key]
discard validator
let ad = await vc.client.get_v1_validator_attestation_data(slot, a.committee_index)
vc.port_logged("attestation: " & $a.committee_index.int64 & " " & $a.validator_committee_index)
# TODO I don't like these (u)int64-to-int conversions...
let attestation = await validator.produceAndSignAttestation(
ad, a.committee_length.int, a.validator_committee_index.int,
vc.fork, vc.beaconGenesis.genesis_validators_root)
discard await vc.client.post_v1_beacon_pool_attestations(attestation)
except CatchableError as err:
error "Caught an unexpected error", err = err.msg
@ -137,8 +146,6 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
programMain:
let config = makeBannerAndConfig("Nimbus validator client v" & fullVersionStr, ValidatorClientConf)
sleep(config.delayStart * 1000)
setupMainProc(config.logLevel)
# TODO figure out how to re-enable this without the VCs continuing

123
beacon_chain/validator_duties.nim
View File

@ -21,7 +21,7 @@ import
spec/[datatypes, digest, crypto, beaconstate, helpers, validator, network],
conf, time, validator_pool, state_transition,
attestation_pool, block_pool, eth2_network,
beacon_node_common, beacon_node_types,
beacon_node_common, beacon_node_types, nimbus_binary_common,
mainchain_monitor, version, ssz/merkleization, interop,
attestation_aggregation, sync_manager, sszdump
@ -91,34 +91,31 @@ proc isSynced(node: BeaconNode, head: BlockRef): bool =
else:
true
proc sendAttestation(node: BeaconNode,
fork: Fork,
genesis_validators_root: Eth2Digest,
validator: AttachedValidator,
attestationData: AttestationData,
committeeLen: int,
indexInCommittee: int) {.async.} =
proc sendAttestation*(node: BeaconNode, attestation: Attestation) =
logScope: pcs = "send_attestation"
let validatorSignature = await validator.signAttestation(attestationData,
fork, genesis_validators_root)
var aggregationBits = CommitteeValidatorsBits.init(committeeLen)
aggregationBits.setBit indexInCommittee
var attestation = Attestation(
data: attestationData,
signature: validatorSignature,
aggregation_bits: aggregationBits
)
# https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/validator.md#broadcast-attestation
node.network.broadcast(
getMainnetAttestationTopic(node.forkDigest, attestationData.index),
getMainnetAttestationTopic(node.forkDigest, attestation.data.index),
attestation)
beacon_attestations_sent.inc()
proc createAndSendAttestation(node: BeaconNode,
fork: Fork,
genesis_validators_root: Eth2Digest,
validator: AttachedValidator,
attestationData: AttestationData,
committeeLen: int,
indexInCommittee: int) {.async.} =
logScope: pcs = "send_attestation"
var attestation = await validator.produceAndSignAttestation(attestationData, committeeLen, indexInCommittee, fork, genesis_validators_root)
node.sendAttestation(attestation)
if node.config.dumpEnabled:
dump(node.config.dumpDir, attestationData, validator.pubKey)
dump(node.config.dumpDir, attestation.data, validator.pubKey)
info "Attestation sent",
attestation = shortLog(attestation),
@ -126,8 +123,6 @@ proc sendAttestation(node: BeaconNode,
indexInCommittee = indexInCommittee,
cat = "consensus"
beacon_attestations_sent.inc()
type
ValidatorInfoForMakeBeaconBlockKind* = enum
viValidator
@ -198,6 +193,38 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
return (message, state.fork, state.genesis_validators_root)
proc proposeSignedBlock*(node: BeaconNode,
head: BlockRef,
validator: AttachedValidator,
newBlock: SignedBeaconBlock,
blockRoot: Eth2Digest): Future[BlockRef] {.async.} =
let newBlockRef = node.blockPool.add(blockRoot, newBlock)
if newBlockRef.isErr:
warn "Unable to add proposed block to block pool",
newBlock = shortLog(newBlock.message),
blockRoot = shortLog(blockRoot),
cat = "bug"
return head
info "Block proposed",
blck = shortLog(newBlock.message),
blockRoot = shortLog(newBlockRef[].root),
validator = shortLog(validator),
cat = "consensus"
if node.config.dumpEnabled:
dump(node.config.dumpDir, newBlock, newBlockRef[])
node.blockPool.withState(
node.blockPool.tmpState, newBlockRef[].atSlot(newBlockRef[].slot)):
dump(node.config.dumpDir, hashedState, newBlockRef[])
node.network.broadcast(node.topicBeaconBlocks, newBlock)
beacon_blocks_proposed.inc()
return newBlockRef[]
proc proposeBlock(node: BeaconNode,
validator: AttachedValidator,
validator_index: ValidatorIndex,
@ -232,33 +259,7 @@ proc proposeBlock(node: BeaconNode,
newBlock.signature = await validator.signBlockProposal(
beaconBlockTuple.fork, beaconBlockTuple.genesis_validators_root, slot, blockRoot)
let newBlockRef = node.blockPool.add(blockRoot, newBlock)
if newBlockRef.isErr:
warn "Unable to add proposed block to block pool",
newBlock = shortLog(newBlock.message),
blockRoot = shortLog(blockRoot),
cat = "bug"
return head
info "Block proposed",
blck = shortLog(newBlock.message),
blockRoot = shortLog(newBlockRef[].root),
validator = shortLog(validator),
cat = "consensus"
if node.config.dumpEnabled:
dump(node.config.dumpDir, newBlock, newBlockRef[])
node.blockPool.withState(
node.blockPool.tmpState, newBlockRef[].atSlot(newBlockRef[].slot)):
dump(node.config.dumpDir, hashedState, newBlockRef[])
node.network.broadcast(node.topicBeaconBlocks, newBlock)
beacon_blocks_proposed.inc()
return newBlockRef[]
return await node.proposeSignedBlock(head, validator, newBlock, blockRoot)
proc handleAttestations(node: BeaconNode, head: BlockRef, slot: Slot) =
## Perform all attestations that the validators attached to this node should
@ -320,7 +321,7 @@ proc handleAttestations(node: BeaconNode, head: BlockRef, slot: Slot) =
attestations.add((ad, committee.len, index_in_committee, validator))
for a in attestations:
traceAsyncErrors sendAttestation(
traceAsyncErrors createAndSendAttestation(
node, state.fork, state.genesis_validators_root, a.validator, a.data,
a.committeeLen, a.indexInCommittee)
@ -455,22 +456,12 @@ proc handleValidatorDuties*(
# block from the expected block proposer for the assigned slot or
# (b) one-third of the slot has transpired (`SECONDS_PER_SLOT / 3` seconds
# after the start of slot) -- whichever comes first.
template sleepToSlotOffset(extra: chronos.Duration, msg: static string) =
let
fromNow = node.beaconClock.fromNow(slot.toBeaconTime(extra))
if fromNow.inFuture:
trace msg,
slot = shortLog(slot),
fromNow = shortLog(fromNow.offset),
cat = "scheduling"
await sleepAsync(fromNow.offset)
template sleepToSlotOffsetWithHeadUpdate(extra: chronos.Duration, msg: static string) =
if await node.beaconClock.sleepToSlotOffset(extra, slot, msg):
# Time passed - we might need to select a new head in that case
head = node.updateHead()
sleepToSlotOffset(
sleepToSlotOffsetWithHeadUpdate(
seconds(int64(SECONDS_PER_SLOT)) div 3, "Waiting to send attestations")
handleAttestations(node, head, slot)
@ -482,7 +473,7 @@ proc handleValidatorDuties*(
# through the slot-that is, SECONDS_PER_SLOT * 2 / 3 seconds after the start
# of slot.
if slot > 2:
sleepToSlotOffset(
sleepToSlotOffsetWithHeadUpdate(
seconds(int64(SECONDS_PER_SLOT * 2) div 3),
"Waiting to aggregate attestations")

13
beacon_chain/validator_pool.nim
View File

@ -57,6 +57,19 @@ proc signAttestation*(v: AttachedValidator,
error "Unimplemented"
quit 1
proc produceAndSignAttestation*(validator: AttachedValidator,
attestationData: AttestationData,
committeeLen: int, indexInCommittee: int,
fork: Fork, genesis_validators_root: Eth2Digest):
Future[Attestation] {.async.} =
let validatorSignature = await validator.signAttestation(attestationData,
fork, genesis_validators_root)
var aggregationBits = CommitteeValidatorsBits.init(committeeLen)
aggregationBits.setBit indexInCommittee
return Attestation(data: attestationData, signature: validatorSignature, aggregation_bits: aggregationBits)
proc signAggregateAndProof*(v: AttachedValidator,
aggregate_and_proof: AggregateAndProof,
fork: Fork, genesis_validators_root: Eth2Digest): ValidatorSig =

32
tests/simulation/run_node.sh
View File

@ -43,8 +43,16 @@ mkdir -p "$DATA_DIR/validators"
rm -f $DATA_DIR/validators/*
if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
FIRST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * NODE_ID ))
LAST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * (NODE_ID + 1) - 1 ))
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
VALIDATORS_PER_NODE_HALF=$((VALIDATORS_PER_NODE / 2))
FIRST_VALIDATOR_IDX=$(( VALIDATORS_PER_NODE * NODE_ID ))
# if using validator client binaries in addition to beacon nodes
# we will split the keys for this instance in half between the BN and the VC
if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then
LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE_HALF - 1 ))
else
LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE - 1 ))
fi
pushd "$VALIDATORS_DIR" >/dev/null
cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$DATA_DIR/validators"
@ -61,25 +69,6 @@ fi
cd "$DATA_DIR"
# uncomment to force always using an external VC binary for VC duties
# TODO remove this when done with implementing the VC - here just for convenience during dev
#VALIDATOR_API="yes"
VALIDATOR_API_ARG=""
if [ "${VALIDATOR_API:-}" == "yes" ]; then
VALIDATOR_API_ARG="--validator-api"
# we lass a few seconds as delay for the start ==> that way we can start the
# beacon node before the VC - otherwise we would have to add "&" conditionally to
# the command which starts the BN - makes the shell script much more complicated
# TODO launch the VC through the start.sh script in order to address this comment:
# https://github.com/status-im/nim-beacon-chain/pull/1055#discussion_r429540155
$VALIDATOR_CLIENT_BIN \
--log-level=${LOG_LEVEL:-DEBUG} \
--data-dir=$DATA_DIR \
--rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))" \
--delay-start=5 &
fi
# if you want tracing messages, add "--log-level=TRACE" below
$BEACON_NODE_BIN \
--log-level=${LOG_LEVEL:-DEBUG} \
@ -89,7 +78,6 @@ $BEACON_NODE_BIN \
--tcp-port=$PORT \
--udp-port=$PORT \
$SNAPSHOT_ARG \
$VALIDATOR_API_ARG \
$NAT_ARG \
$WEB3_ARG \
--deposit-contract=$DEPOSIT_CONTRACT_ADDRESS \

40
tests/simulation/run_validator.sh Executable file
View File

@ -0,0 +1,40 @@
#!/bin/bash
set -e
NODE_ID=${1}
shift
# Read in variables
# shellcheck source=/dev/null
source "$(dirname "$0")/vars.sh"
# set up the environment
# shellcheck source=/dev/null
source "${SIM_ROOT}/../../env.sh"
cd "$GIT_ROOT"
VC_DATA_DIR="${SIMULATION_DIR}/validator-$NODE_ID"
mkdir -p "$VC_DATA_DIR/validators"
rm -f $VC_DATA_DIR/validators/*
if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
# we will split the keys for this instance in half between the BN and the VC
VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
VALIDATORS_PER_NODE_HALF=$((VALIDATORS_PER_NODE / 2))
FIRST_VALIDATOR_IDX=$(( VALIDATORS_PER_NODE * NODE_ID + VALIDATORS_PER_NODE_HALF))
LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE_HALF - 1 ))
pushd "$VALIDATORS_DIR" >/dev/null
cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$VC_DATA_DIR/validators"
popd >/dev/null
fi
cd "$VC_DATA_DIR"
$VALIDATOR_CLIENT_BIN \
--log-level=${LOG_LEVEL:-DEBUG} \
--data-dir=$VC_DATA_DIR \
--rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))"

37
tests/simulation/start.sh
View File

@ -179,20 +179,9 @@ fi
LAST_WAITING_NODE=0
for i in $(seq $MASTER_NODE -1 $TOTAL_USER_NODES); do
if [[ "$i" != "$MASTER_NODE" && "$USE_MULTITAIL" == "no" ]]; then
# Wait for the master node to write out its address file
while [ ! -f "${MASTER_NODE_ADDRESS_FILE}" ]; do
if (( LAST_WAITING_NODE != i )); then
echo Waiting for $MASTER_NODE_ADDRESS_FILE to appear...
LAST_WAITING_NODE=i
fi
sleep 0.1
done
fi
CMD="${SIM_ROOT}/run_node.sh ${i} --verify-finalization"
function run_cmd {
i=$1
CMD=$2
if [[ "$USE_TMUX" != "no" ]]; then
echo "Starting node $i..."
echo $TMUX split-window -t "${TMUX_SESSION_NAME}" "$CMD"
@ -209,6 +198,26 @@ for i in $(seq $MASTER_NODE -1 $TOTAL_USER_NODES); do
else
eval "${CMD}" &
fi
}
for i in $(seq $MASTER_NODE -1 $TOTAL_USER_NODES); do
if [[ "$i" != "$MASTER_NODE" && "$USE_MULTITAIL" == "no" ]]; then
# Wait for the master node to write out its address file
while [ ! -f "${MASTER_NODE_ADDRESS_FILE}" ]; do
if (( LAST_WAITING_NODE != i )); then
echo Waiting for $MASTER_NODE_ADDRESS_FILE to appear...
LAST_WAITING_NODE=i
fi
sleep 0.1
done
fi
run_cmd $i "${SIM_ROOT}/run_node.sh ${i} --verify-finalization"
if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then
# start the VC with a few seconds of delay so that we can connect through RPC
run_cmd $i "sleep 3 && ${SIM_ROOT}/run_validator.sh ${i}"
fi
done
if [[ "$USE_TMUX" != "no" ]]; then

2
tests/simulation/vars.sh
View File

@ -46,3 +46,5 @@ else
DEPOSIT_CONTRACT_ADDRESS="0x"
fi
# uncomment to enable the use of VCs in addition to BNs - will split the validators equally
#SPLIT_VALIDATORS_BETWEEN_BN_AND_VC="yes"