From bb83817c2a3e44954cac52c206f590d361ca7947 Mon Sep 17 00:00:00 2001 From: Viktor Kirilov Date: Tue, 1 Sep 2020 16:44:40 +0300 Subject: [PATCH] use a separate process for the private keys (Off by default) - there is a new signing_process binary which loads all validators of the beacon node and the BN dictates through stdin of the signing process what to be signed and when and reads from stdout of the process --- Makefile | 23 ++--- beacon_chain/attestation_aggregation.nim | 9 +- beacon_chain/beacon_node.nim | 12 ++- beacon_chain/beacon_node_common.nim | 3 +- beacon_chain/beacon_node_types.nim | 5 +- beacon_chain/conf.nim | 46 ++++++---- beacon_chain/keystore_management.nim | 25 ++++-- beacon_chain/signing_process.nim | 32 +++++++ beacon_chain/spec/crypto.nim | 3 + beacon_chain/spec/signatures.nim | 76 +++++++++------- beacon_chain/validator_api.nim | 2 +- beacon_chain/validator_client.nim | 6 +- beacon_chain/validator_duties.nim | 106 +++++++++++++++-------- beacon_chain/validator_pool.nim | 73 +++++++++++----- docker/shared_testnet/entry_point.sh | 2 +- docs/the_nimbus_book/src/beacon_node.md | 3 + scripts/launch_local_testnet.sh | 2 +- scripts/reset_testnet.sh | 2 +- tests/simulation/start.sh | 2 +- 19 files changed, 288 insertions(+), 144 deletions(-) create mode 100644 beacon_chain/signing_process.nim diff --git a/Makefile b/Makefile index c554931c0..89ef00a97 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,8 @@ TOOLS := \ process_dashboard \ stack_sizes \ state_sim \ - validator_client + validator_client \ + signing_process # bench_bls_sig_agggregation TODO reenable after bls v0.10.1 changes @@ -173,14 +174,14 @@ clean-testnet0: clean-testnet1: rm -rf build/data/testnet1* -testnet0 testnet1: | beacon_node +testnet0 testnet1: | beacon_node signing_process build/beacon_node \ --network=$@ \ --log-level="$(LOG_LEVEL)" \ --data-dir=build/data/$@_$(NODE_ID) \ $(GOERLI_TESTNETS_PARAMS) $(NODE_PARAMS) -medalla: | beacon_node +medalla: | beacon_node signing_process mkdir -p build/data/shared_medalla_$(NODE_ID) scripts/make_prometheus_config.sh \ @@ -195,7 +196,7 @@ medalla: | beacon_node --data-dir=build/data/shared_medalla_$(NODE_ID) \ $(GOERLI_TESTNETS_PARAMS) $(NODE_PARAMS) -medalla-vc: | beacon_node validator_client +medalla-vc: | beacon_node signing_process validator_client # if launching a VC as well - send the BN looking nowhere for validators/secrets mkdir -p build/data/shared_medalla_$(NODE_ID)/empty_dummy_folder @@ -225,7 +226,7 @@ ifneq ($(LOG_LEVEL), TRACE) medalla-dev: + $(MAKE) LOG_LEVEL=TRACE $@ else -medalla-dev: | beacon_node +medalla-dev: | beacon_node signing_process mkdir -p build/data/shared_medalla_$(NODE_ID) scripts/make_prometheus_config.sh \ @@ -240,7 +241,7 @@ medalla-dev: | beacon_node $(GOERLI_TESTNETS_PARAMS) --dump $(NODE_PARAMS) endif -medalla-deposit-data: | beacon_node deposit_contract +medalla-deposit-data: | beacon_node signing_process deposit_contract build/beacon_node deposits create \ --network=medalla \ --new-wallet-file=build/data/shared_medalla_$(NODE_ID)/wallet.json \ @@ -249,7 +250,7 @@ medalla-deposit-data: | beacon_node deposit_contract --out-deposits-file=medalla-deposits_data-$$(date +"%Y%m%d%H%M%S").json \ --count=$(VALIDATORS) -medalla-deposit: | beacon_node deposit_contract +medalla-deposit: | beacon_node signing_process deposit_contract build/beacon_node deposits create \ --network=medalla \ --out-deposits-file=nbc-medalla-deposits.json \ @@ -270,7 +271,7 @@ clean-medalla: rm -rf build/data/shared_medalla*/dump rm -rf build/data/shared_medalla*/*.log -altona: | beacon_node +altona: | beacon_node signing_process $(CPU_LIMIT_CMD) build/beacon_node \ --network=altona \ --log-level="$(LOG_LEVEL)" \ @@ -278,7 +279,7 @@ altona: | beacon_node --data-dir=build/data/shared_altona_$(NODE_ID) \ $(GOERLI_TESTNETS_PARAMS) $(NODE_PARAMS) -altona-vc: | beacon_node validator_client +altona-vc: | beacon_node signing_process validator_client # if launching a VC as well - send the BN looking nowhere for validators/secrets mkdir -p build/data/shared_altona_$(NODE_ID)/empty_dummy_folder $(CPU_LIMIT_CMD) build/beacon_node \ @@ -296,14 +297,14 @@ altona-vc: | beacon_node validator_client --data-dir=build/data/shared_altona_$(NODE_ID) \ --rpc-port=$$(( $(BASE_RPC_PORT) +$(NODE_ID) )) -altona-dev: | beacon_node +altona-dev: | beacon_node signing_process $(CPU_LIMIT_CMD) build/beacon_node \ --network=altona \ --log-level="DEBUG; TRACE:discv5,networking; REQUIRED:none; DISABLED:none" \ --data-dir=build/data/shared_altona_$(NODE_ID) \ $(GOERLI_TESTNETS_PARAMS) --dump $(NODE_PARAMS) -altona-deposit: | beacon_node deposit_contract +altona-deposit: | beacon_node signing_process deposit_contract build/beacon_node deposits create \ --network=altona \ --out-deposits-file=nbc-altona-deposits.json \ diff --git a/beacon_chain/attestation_aggregation.nim b/beacon_chain/attestation_aggregation.nim index 7e1726262..ba98b98e2 100644 --- a/beacon_chain/attestation_aggregation.nim +++ b/beacon_chain/attestation_aggregation.nim @@ -8,7 +8,7 @@ {.push raises: [Defect].} import - options, chronicles, + options, chronos, chronicles, ./spec/[ beaconstate, datatypes, crypto, digest, helpers, network, validator, signatures], @@ -32,13 +32,10 @@ func is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, proc aggregate_attestations*( pool: AttestationPool, state: BeaconState, index: CommitteeIndex, - validatorIndex: ValidatorIndex, privkey: ValidatorPrivKey, - cache: var StateCache): - Option[AggregateAndProof] = + validatorIndex: ValidatorIndex, slot_signature: ValidatorSig, + cache: var StateCache): Option[AggregateAndProof] = let slot = state.slot - slot_signature = get_slot_signature( - state.fork, state.genesis_validators_root, slot, privkey) doAssert validatorIndex in get_beacon_committee(state, slot, index, cache) doAssert index.uint64 < get_committee_count_per_slot(state, slot.epoch, cache) diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index 59412b47c..83d75d733 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -8,7 +8,7 @@ import # Standard library std/[algorithm, os, tables, strutils, sequtils, times, math, terminal], - std/random, + std/[osproc, random], # Nimble packages stew/[objects, byteutils, endians2], stew/shims/macros, @@ -277,7 +277,13 @@ proc init*(T: type BeaconNode, res.requestManager = RequestManager.init( network, res.processor.blocksQueue) - res.addLocalValidators() + if res.config.inProcessValidators: + res.addLocalValidators() + else: + res.vcProcess = startProcess(getAppDir() & "/signing_process".addFileExt(ExeExt), + getCurrentDir(), [$res.config.validatorsDir, + $res.config.secretsDir]) + res.addRemoteValidators() # This merely configures the BeaconSync # The traffic will be started when we join the network. @@ -787,6 +793,8 @@ proc removeMessageHandlers(node: BeaconNode) = proc stop*(node: BeaconNode) = status = BeaconNodeStatus.Stopping info "Graceful shutdown" + if not node.config.inProcessValidators: + node.vcProcess.close() waitFor node.network.stop() proc run*(node: BeaconNode) = diff --git a/beacon_chain/beacon_node_common.nim b/beacon_chain/beacon_node_common.nim index 07352719e..93037d42e 100644 --- a/beacon_chain/beacon_node_common.nim +++ b/beacon_chain/beacon_node_common.nim @@ -9,7 +9,7 @@ import # Standard library - tables, + tables, osproc, # Nimble packages chronos, json_rpc/rpcserver, metrics, @@ -46,6 +46,7 @@ type mainchainMonitor*: MainchainMonitor beaconClock*: BeaconClock rpcServer*: RpcServer + vcProcess*: Process forkDigest*: ForkDigest requestManager*: RequestManager syncManager*: SyncManager[Peer, PeerID] diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim index c65d28065..8f651790e 100644 --- a/beacon_chain/beacon_node_types.nim +++ b/beacon_chain/beacon_node_types.nim @@ -1,7 +1,7 @@ {.push raises: [Defect].} import - deques, tables, + deques, tables, streams, stew/endians2, spec/[datatypes, crypto], block_pools/block_pools_types, @@ -74,6 +74,9 @@ type remote ValidatorConnection* = object + inStream*: Stream + outStream*: Stream + pubKeyStr*: string AttachedValidator* = ref object pubKey*: ValidatorPubKey diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 315a843c9..1d8f89043 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -204,6 +204,11 @@ type desc: "Listening address of the RPC server" name: "rpc-address" }: ValidIpAddress + inProcessValidators* {. + defaultValue: true # the use of the signing_process binary by default will be delayed until async I/O over stdin/stdout is developed for the child process. + desc: "Disable the push model (the beacon node tells a signing process with the private keys of the validators what to sign and when) and load the validators in the beacon node itself" + name: "in-process-validators" }: bool + discv5Enabled* {. defaultValue: true desc: "Enable Discovery v5" @@ -356,10 +361,24 @@ type desc: "Do not display interative prompts. Quit on missing configuration" name: "non-interactive" }: bool + validators* {. + required + desc: "Attach a validator by supplying a keystore path" + abbr: "v" + name: "validator" }: seq[ValidatorKeyPath] + + validatorsDirFlag* {. + desc: "A directory containing validator keystores" + name: "validators-dir" }: Option[InputDir] + + secretsDirFlag* {. + desc: "A directory containing validator keystore passwords" + name: "secrets-dir" }: Option[InputDir] + case cmd* {. command defaultValue: VCNoCommand }: VCStartUpCmd - + of VCNoCommand: graffiti* {. desc: "The graffiti value that will appear in proposed blocks. " & @@ -373,27 +392,18 @@ type rpcPort* {. defaultValue: defaultEth2RpcPort - desc: "HTTP port of the server to connect to for RPC" + desc: "HTTP port of the server to connect to for RPC - for the validator duties in the pull model" name: "rpc-port" }: Port rpcAddress* {. defaultValue: defaultAdminListenAddress(config) - desc: "Address of the server to connect to for RPC" + desc: "Address of the server to connect to for RPC - for the validator duties in the pull model" name: "rpc-address" }: ValidIpAddress - - validators* {. - required - desc: "Attach a validator by supplying a keystore path" - abbr: "v" - name: "validator" }: seq[ValidatorKeyPath] - - validatorsDirFlag* {. - desc: "A directory containing validator keystores" - name: "validators-dir" }: Option[InputDir] - - secretsDirFlag* {. - desc: "A directory containing validator keystore passwords" - name: "secrets-dir" }: Option[InputDir] + + retryDelay* {. + defaultValue: 10 + desc: "Delay in seconds between retries after unsuccessful attempts to connect to a beacon node" + name: "retry-delay" }: int proc defaultDataDir*(conf: BeaconNodeConf|ValidatorClientConf): string = let dataDir = when defined(windows): @@ -451,7 +461,7 @@ func validatorsDir*(conf: BeaconNodeConf|ValidatorClientConf): string = func secretsDir*(conf: BeaconNodeConf|ValidatorClientConf): string = string conf.secretsDirFlag.get(InputDir(conf.dataDir / "secrets")) -func walletsDir*(conf: BeaconNodeConf|ValidatorClientConf): string = +func walletsDir*(conf: BeaconNodeConf): string = if conf.walletsDirFlag.isSome: conf.walletsDirFlag.get.string else: diff --git a/beacon_chain/keystore_management.nim b/beacon_chain/keystore_management.nim index 53dc408ef..044f1e4c5 100644 --- a/beacon_chain/keystore_management.nim +++ b/beacon_chain/keystore_management.nim @@ -22,8 +22,8 @@ type walletPath*: WalletPathPair mnemonic*: Mnemonic -proc loadKeystore(conf: BeaconNodeConf|ValidatorClientConf, - validatorsDir, keyName: string): Option[ValidatorPrivKey] = +proc loadKeystore(validatorsDir, secretsDir, keyName: string, + nonInteractive: bool): Option[ValidatorPrivKey] = let keystorePath = validatorsDir / keyName / keystoreFileName keystore = @@ -35,7 +35,7 @@ proc loadKeystore(conf: BeaconNodeConf|ValidatorClientConf, error "Invalid keystore", err = err.formatMsg(keystorePath) return - let passphrasePath = conf.secretsDir / keyName + let passphrasePath = secretsDir / keyName if fileExists(passphrasePath): let passphrase = KeystorePass: @@ -51,9 +51,9 @@ proc loadKeystore(conf: BeaconNodeConf|ValidatorClientConf, error "Failed to decrypt keystore", keystorePath, passphrasePath return - if conf.nonInteractive: + if nonInteractive: error "Unable to load validator key store. Please ensure matching passphrase exists in the secrets dir", - keyName, validatorsDir, secretsDir = conf.secretsDir + keyName, validatorsDir, secretsDir = secretsDir return var remainingAttempts = 3 @@ -72,6 +72,19 @@ proc loadKeystore(conf: BeaconNodeConf|ValidatorClientConf, prompt = "Keystore decryption failed. Please try again" dec remainingAttempts +iterator validatorKeysFromDirs*(validatorsDir, secretsDir: string): ValidatorPrivKey = + try: + for kind, file in walkDir(validatorsDir): + if kind == pcDir: + let keyName = splitFile(file).name + let key = loadKeystore(validatorsDir, secretsDir, keyName, true) + if key.isSome: + yield key.get + else: + quit 1 + except OSError: + quit 1 + iterator validatorKeys*(conf: BeaconNodeConf|ValidatorClientConf): ValidatorPrivKey = for validatorKeyFile in conf.validators: try: @@ -86,7 +99,7 @@ iterator validatorKeys*(conf: BeaconNodeConf|ValidatorClientConf): ValidatorPriv for kind, file in walkDir(validatorsDir): if kind == pcDir: let keyName = splitFile(file).name - let key = loadKeystore(conf, validatorsDir, keyName) + let key = loadKeystore(validatorsDir, conf.secretsDir, keyName, conf.nonInteractive) if key.isSome: yield key.get else: diff --git a/beacon_chain/signing_process.nim b/beacon_chain/signing_process.nim new file mode 100644 index 000000000..422dd8ef6 --- /dev/null +++ b/beacon_chain/signing_process.nim @@ -0,0 +1,32 @@ +# beacon_chain +# Copyright (c) 2018-2020 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + # Standard library + os, strutils, tables, + + # Local modules + spec/[digest, crypto], + keystore_management + +programMain: + var validators: Table[ValidatorPubKey, ValidatorPrivKey] + # load and send all public keys so the BN knows for which ones to ping us + doAssert paramCount() == 2 + for curr in validatorKeysFromDirs(paramStr(1), paramStr(2)): + validators[curr.toPubKey.initPubKey] = curr + echo curr.toPubKey + echo "end" + + # simple format: ` ` => `` + while true: + let args = stdin.readLine.split(" ") + doAssert args.len == 2 + + let privKey = validators[ValidatorPubKey.fromHex(args[0]).get().initPubKey()] + + echo blsSign(privKey, Eth2Digest.fromHex(args[1]).data) diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index 2ded59e9f..2b854b8a0 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -108,6 +108,9 @@ proc toRealPubKey(pubkey: ValidatorPubKey): Option[ValidatorPubKey] = none ValidatorPubKey return validatorKeyCache.mGetOrPut(pubkey.blob, maybeRealKey) +# TODO this needs a massive comment explaining the reasoning along with every +# seemingly ad-hoc place where it's called - one shouldn't have to git-blame +# commits and PRs for information which ought to be inplace here in the code proc initPubKey*(pubkey: ValidatorPubKey): ValidatorPubKey = let key = toRealPubKey(pubkey) if key.isNone: diff --git a/beacon_chain/spec/signatures.nim b/beacon_chain/spec/signatures.nim index f37401d0a..1ca11341d 100644 --- a/beacon_chain/spec/signatures.nim +++ b/beacon_chain/spec/signatures.nim @@ -17,17 +17,20 @@ template withTrust(sig: SomeSig, body: untyped): bool = else: body -# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#aggregation-selection -func get_slot_signature*( - fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - privkey: ValidatorPrivKey): ValidatorSig = +func compute_slot_root*( + fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot + ): Eth2Digest = let epoch = compute_epoch_at_slot(slot) domain = get_domain( fork, DOMAIN_SELECTION_PROOF, epoch, genesis_validators_root) - signing_root = compute_signing_root(slot, domain) + result = compute_signing_root(slot, domain) - blsSign(privKey, signing_root.data) +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#aggregation-selection +func get_slot_signature*( + fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, + privkey: ValidatorPrivKey): ValidatorSig = + blsSign(privKey, compute_slot_root(fork, genesis_validators_root, slot).data) proc verify_slot_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, @@ -41,15 +44,18 @@ proc verify_slot_signature*( blsVerify(pubkey, signing_root.data, signature) +func compute_epoch_root*( + fork: Fork, genesis_validators_root: Eth2Digest, epoch: Epoch + ): Eth2Digest = + let + domain = get_domain(fork, DOMAIN_RANDAO, epoch, genesis_validators_root) + result = compute_signing_root(epoch, domain) + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#randao-reveal func get_epoch_signature*( fork: Fork, genesis_validators_root: Eth2Digest, epoch: Epoch, privkey: ValidatorPrivKey): ValidatorSig = - let - domain = get_domain(fork, DOMAIN_RANDAO, epoch, genesis_validators_root) - signing_root = compute_signing_root(epoch, domain) - - blsSign(privKey, signing_root.data) + blsSign(privKey, compute_epoch_root(fork, genesis_validators_root, epoch).data) proc verify_epoch_signature*( fork: Fork, genesis_validators_root: Eth2Digest, epoch: Epoch, @@ -61,17 +67,20 @@ proc verify_epoch_signature*( blsVerify(pubkey, signing_root.data, signature) -# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#signature -func get_block_signature*( +func compute_block_root*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, - root: Eth2Digest, privkey: ValidatorPrivKey): ValidatorSig = + root: Eth2Digest): Eth2Digest = let epoch = compute_epoch_at_slot(slot) domain = get_domain( fork, DOMAIN_BEACON_PROPOSER, epoch, genesis_validators_root) - signing_root = compute_signing_root(root, domain) + result = compute_signing_root(root, domain) - blsSign(privKey, signing_root.data) +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#signature +func get_block_signature*( + fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, + root: Eth2Digest, privkey: ValidatorPrivKey): ValidatorSig = + blsSign(privKey, compute_block_root(fork, genesis_validators_root, slot, root).data) proc verify_block_signature*( fork: Fork, genesis_validators_root: Eth2Digest, slot: Slot, @@ -87,17 +96,21 @@ proc verify_block_signature*( blsVerify(pubKey, signing_root.data, signature) -# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#broadcast-aggregate -func get_aggregate_and_proof_signature*(fork: Fork, genesis_validators_root: Eth2Digest, - aggregate_and_proof: AggregateAndProof, - privKey: ValidatorPrivKey): ValidatorSig = +func compute_aggregate_and_proof_root*(fork: Fork, genesis_validators_root: Eth2Digest, + aggregate_and_proof: AggregateAndProof, + ): Eth2Digest = let epoch = compute_epoch_at_slot(aggregate_and_proof.aggregate.data.slot) domain = get_domain( fork, DOMAIN_AGGREGATE_AND_PROOF, epoch, genesis_validators_root) - signing_root = compute_signing_root(aggregate_and_proof, domain) + result = compute_signing_root(aggregate_and_proof, domain) - blsSign(privKey, signing_root.data) +# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#broadcast-aggregate +func get_aggregate_and_proof_signature*(fork: Fork, genesis_validators_root: Eth2Digest, + aggregate_and_proof: AggregateAndProof, + privKey: ValidatorPrivKey): ValidatorSig = + blsSign(privKey, compute_aggregate_and_proof_root(fork, genesis_validators_root, + aggregate_and_proof).data) proc verify_aggregate_and_proof_signature*(fork: Fork, genesis_validators_root: Eth2Digest, aggregate_and_proof: AggregateAndProof, @@ -111,18 +124,23 @@ proc verify_aggregate_and_proof_signature*(fork: Fork, genesis_validators_root: blsVerify(pubKey, signing_root.data, signature) +func compute_attestation_root*( + fork: Fork, genesis_validators_root: Eth2Digest, + attestation_data: AttestationData + ): Eth2Digest = + let + epoch = attestation_data.target.epoch + domain = get_domain( + fork, DOMAIN_BEACON_ATTESTER, epoch, genesis_validators_root) + result = compute_signing_root(attestation_data, domain) + # https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/validator.md#aggregate-signature func get_attestation_signature*( fork: Fork, genesis_validators_root: Eth2Digest, attestation_data: AttestationData, privkey: ValidatorPrivKey): ValidatorSig = - let - epoch = attestation_data.target.epoch - domain = get_domain( - fork, DOMAIN_BEACON_ATTESTER, epoch, genesis_validators_root) - signing_root = compute_signing_root(attestation_data, domain) - - blsSign(privKey, signing_root.data) + blsSign(privKey, compute_attestation_root(fork, genesis_validators_root, + attestation_data).data) proc verify_attestation_signature*( fork: Fork, genesis_validators_root: Eth2Digest, diff --git a/beacon_chain/validator_api.nim b/beacon_chain/validator_api.nim index c139356c3..3a31cf73a 100644 --- a/beacon_chain/validator_api.nim +++ b/beacon_chain/validator_api.nim @@ -308,7 +308,7 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) = raise newException(CatchableError, "could not retrieve block for slot: " & $slot) let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viRandao_reveal, randao_reveal: randao_reveal) - let res = makeBeaconBlockForHeadAndSlot( + let res = await makeBeaconBlockForHeadAndSlot( node, valInfo, proposer.get()[0], graffiti, head, slot) if res.message.isNone(): raise newException(CatchableError, "could not retrieve block for slot: " & $slot) diff --git a/beacon_chain/validator_client.nim b/beacon_chain/validator_client.nim index 92e10e225..999ab9ea9 100644 --- a/beacon_chain/validator_client.nim +++ b/beacon_chain/validator_client.nim @@ -7,7 +7,7 @@ import # Standard library - os, strutils, json, times, + os, strutils, json, # Nimble packages stew/shims/[tables, macros], @@ -52,7 +52,7 @@ template attemptUntilSuccess(vc: ValidatorClient, body: untyped) = break except CatchableError as err: warn "Caught an unexpected error", err = err.msg - waitFor sleepAsync(chronos.seconds(1)) # 1 second before retrying + waitFor sleepAsync(chronos.seconds(vc.config.retryDelay)) proc getValidatorDutiesForEpoch(vc: ValidatorClient, epoch: Epoch) {.gcsafe, async.} = info "Getting validator duties for epoch", epoch = epoch @@ -136,7 +136,7 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a info "Proposing block", slot = slot, public_key = public_key - let randao_reveal = validator.genRandaoReveal( + let randao_reveal = await validator.genRandaoReveal( vc.fork, vc.beaconGenesis.genesis_validators_root, slot) var newBlock = SignedBeaconBlock( diff --git a/beacon_chain/validator_duties.nim b/beacon_chain/validator_duties.nim index 97ae4ff65..8a23fb4d0 100644 --- a/beacon_chain/validator_duties.nim +++ b/beacon_chain/validator_duties.nim @@ -7,7 +7,7 @@ import # Standard library - std/[os, tables, strutils], + std/[os, tables, strutils, sequtils, osproc, streams], # Nimble packages stew/[objects], stew/shims/macros, @@ -41,25 +41,43 @@ proc saveValidatorKey*(keyName, key: string, conf: BeaconNodeConf) = writeFile(outputFile, key) info "Imported validator key", file = outputFile -proc addLocalValidator*(node: BeaconNode, - state: BeaconState, - privKey: ValidatorPrivKey) = - let pubKey = privKey.toPubKey() - +proc checkValidatorInRegistry(state: BeaconState, + pubKey: ValidatorPubKey) = let idx = state.validators.asSeq.findIt(it.pubKey == pubKey) if idx == -1: # We allow adding a validator even if its key is not in the state registry: # it might be that the deposit for this validator has not yet been processed warn "Validator not in registry (yet?)", pubKey +proc addLocalValidator*(node: BeaconNode, + state: BeaconState, + privKey: ValidatorPrivKey) = + let pubKey = privKey.toPubKey() + state.checkValidatorInRegistry(pubKey) node.attachedValidators.addLocalValidator(pubKey, privKey) proc addLocalValidators*(node: BeaconNode) = for validatorKey in node.config.validatorKeys: node.addLocalValidator node.chainDag.headState.data.data, validatorKey - info "Local validators attached ", count = node.attachedValidators.count +proc addRemoteValidators*(node: BeaconNode) = + # load all the validators from the child process - loop until `end` + var line = newStringOfCap(120).TaintedString + while line != "end" and running(node.vcProcess): + if node.vcProcess.outputStream.readLine(line) and line != "end": + let key = ValidatorPubKey.fromHex(line).get().initPubKey() + node.chainDag.headState.data.data.checkValidatorInRegistry(key) + + let v = AttachedValidator(pubKey: key, + kind: ValidatorKind.remote, + connection: ValidatorConnection( + inStream: node.vcProcess.inputStream, + outStream: node.vcProcess.outputStream, + pubKeyStr: $key)) + node.attachedValidators.addRemoteValidator(key, v) + info "Remote validators attached ", count = node.attachedValidators.count + proc getAttachedValidator*(node: BeaconNode, pubkey: ValidatorPubKey): AttachedValidator = node.attachedValidators.getValidator(pubkey) @@ -172,7 +190,8 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode, graffiti: GraffitiBytes, head: BlockRef, slot: Slot): - tuple[message: Option[BeaconBlock], fork: Fork, genesis_validators_root: Eth2Digest] = + Future[tuple[message: Option[BeaconBlock], fork: Fork, + genesis_validators_root: Eth2Digest]] {.async.} = # Advance state to the slot that we're proposing for - this is the equivalent # of running `process_slots` up to the slot of the new block. node.chainDag.withState( @@ -189,9 +208,11 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode, # need for the discriminated union)... but we need the `state` from `withState` # in order to get the fork/root for the specific head/slot for the randao_reveal # and it's causing problems when the function becomes a generic for 2 types... - proc getRandaoReveal(val_info: ValidatorInfoForMakeBeaconBlock): ValidatorSig = + proc getRandaoReveal(val_info: ValidatorInfoForMakeBeaconBlock): + Future[ValidatorSig] {.async.} = if val_info.kind == viValidator: - return val_info.validator.genRandaoReveal(state.fork, state.genesis_validators_root, slot) + return await val_info.validator.genRandaoReveal( + state.fork, state.genesis_validators_root, slot) elif val_info.kind == viRandao_reveal: return val_info.randao_reveal @@ -210,7 +231,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode, hashedState, validator_index, head.root, - getRandaoReveal(val_info), + await getRandaoReveal(val_info), eth1data, graffiti, node.attestationPool[].getAttestationsForBlock(state), @@ -280,7 +301,8 @@ proc proposeBlock(node: BeaconNode, return head let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viValidator, validator: validator) - let beaconBlockTuple = makeBeaconBlockForHeadAndSlot(node, valInfo, validator_index, node.graffitiBytes, head, slot) + let beaconBlockTuple = await makeBeaconBlockForHeadAndSlot( + node, valInfo, validator_index, node.graffitiBytes, head, slot) if not beaconBlockTuple.message.isSome(): return head # already logged elsewhere! var @@ -388,7 +410,7 @@ proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot): return head proc broadcastAggregatedAttestations( - node: BeaconNode, aggregationHead: BlockRef, aggregationSlot: Slot) = + node: BeaconNode, aggregationHead: BlockRef, aggregationSlot: Slot) {.async.} = # The index is via a # locally attested validator. Unlike in handleAttestations(...) there's a # single one at most per slot (because that's how aggregation attestation @@ -402,6 +424,13 @@ proc broadcastAggregatedAttestations( let committees_per_slot = get_committee_count_per_slot(state, aggregationSlot.epoch, cache) + + var + slotSigs: seq[Future[ValidatorSig]] = @[] + slotSigsData: seq[tuple[committee_index: uint64, + validator_idx: ValidatorIndex, + v: AttachedValidator]] = @[] + for committee_index in 0'u64..