Implement the 'deposits exit' command; Remove 'deposits create'

beacon_chain/conf.nim

@@ -1,7 +1,7 @@
 {.push raises: [Defect].}
 
 import
-  strutils, os, options, unicode,
+  strutils, os, options, unicode, uri,
   chronicles, chronicles/options as chroniclesOptions,
   confutils, confutils/defs, confutils/std/net, stew/shims/net as stewNet,
   stew/io2, unicodedb/properties, normalize,
@@ -12,12 +12,11 @@ import
   network_metadata, filepath
 
 export
+  uri,
   defaultEth2TcpPort, enabledLogLevel, ValidIpAddress,
   defs, parseCmdArg, completeCmdArg, network_metadata
 
 type
-  ValidatorKeyPath* = TypedInputFile[ValidatorPrivKey, Txt, "privkey"]
-
   BNStartUpCmd* = enum
     noCommand
     createTestnet
@@ -31,9 +30,10 @@ type
     list    = "Lists details about all wallets"
 
   DepositsCmd* {.pure.} = enum
-    create   = "Creates validator keystores and deposits"
+    # create   = "Creates validator keystores and deposits"
     `import` = "Imports password-protected keystores interactively"
-    status   = "Displays status information about all deposits"
+    # status   = "Displays status information about all deposits"
+    exit     = "Submits a validator voluntary exit"
 
   VCStartUpCmd* = enum
     VCNoCommand
@@ -325,6 +325,7 @@ type
 
     of deposits:
       case depositsCmd* {.command.}: DepositsCmd
+      #[
       of DepositsCmd.create:
         totalDeposits* {.
           defaultValue: 1
@@ -357,13 +358,28 @@ type
           desc: "Output wallet file"
           name: "new-wallet-file" }: Option[OutFile]
 
+      of DepositsCmd.status:
+        discard
+      #]#
+
       of DepositsCmd.`import`:
         importedDepositsDir* {.
           argument
           desc: "A directory with keystores to import" }: Option[InputDir]
 
-      of DepositsCmd.status:
+      of DepositsCmd.exit:
-        discard
+        exitedValidator* {.
+          name: "validator"
+          desc: "Validator index or a public key of the exited validator" }: string
+
+        rpcUrlForExit* {.
+          name: "rpc-url"
+          defaultValue: parseUri("wss://localhost:" & $defaultEth2RpcPort)
+          desc: "URL of the beacon node JSON-RPC service" }: Uri
+
+        exitAtEpoch* {.
+          name: "epoch"
+          desc: "The desired exit epoch" }: Option[uint64]
 
     of record:
       case recordCmd* {.command.}: RecordCmd
@@ -501,6 +517,13 @@ func parseCmdArg*(T: type BlockHashOrNumber, input: TaintedString): T
 func completeCmdArg*(T: type BlockHashOrNumber, input: TaintedString): seq[string] =
   return @[]
 
+func parseCmdArg*(T: type Uri, input: TaintedString): T
+                 {.raises: [ValueError, Defect].} =
+  parseUri(input.string)
+
+func completeCmdArg*(T: type Uri, input: TaintedString): seq[string] =
+  return @[]
+
 func parseCmdArg*(T: type Checkpoint, input: TaintedString): T
                  {.raises: [ValueError, Defect].} =
   let sepIdx = find(input.string, ':')
@@ -569,9 +592,11 @@ func outWalletName*(conf: BeaconNodeConf): Option[WalletName] =
     of WalletsCmd.restore: conf.restoredWalletNameFlag
     of WalletsCmd.list: fail()
   of deposits:
-    case conf.depositsCmd
+    # TODO: Uncomment when the deposits create command is restored
-    of DepositsCmd.create: conf.newWalletNameFlag
+    #case conf.depositsCmd
-    else: fail()
+    #of DepositsCmd.create: conf.newWalletNameFlag
+    #else: fail()
+    fail()
   else:
     fail()
 
@@ -586,9 +611,11 @@ func outWalletFile*(conf: BeaconNodeConf): Option[OutFile] =
     of WalletsCmd.restore: conf.restoredWalletFileFlag
     of WalletsCmd.list: fail()
   of deposits:
-    case conf.depositsCmd
+    # TODO: Uncomment when the deposits create command is restored
-    of DepositsCmd.create: conf.newWalletFileFlag
+    #case conf.depositsCmd
-    else: fail()
+    #of DepositsCmd.create: conf.newWalletFileFlag
+    #else: fail()
+    fail()
   else:
     fail()
 

beacon_chain/eth2_json_rpc_serialization.nim

@@ -48,9 +48,19 @@ template genFromJsonForIntType(T: untyped) =
   proc fromJson*(n: JsonNode, argName: string, result: var T) =
     n.kind.expect(JInt, argName)
     let asInt = n.getBiggestInt()
-    # signed -> unsigned conversions are unchecked
+    when T is Epoch:
-    # https://github.com/nim-lang/RFCs/issues/175
+      if asInt == -1:
+        # TODO: This is a major hack here. Since the json library
+        # cannot handle properly 0xffffffff when serializing and
+        # deserializing uint64 values, we detect one known wrong
+        # result, appering in most `Validator` records. To fix
+        # this issue, we'll have to switch to nim-json-serialization
+        # in nim-json-rpc or work towards implementing a fix upstream.
+        result = FAR_FUTURE_EPOCH
+        return
     if asInt < 0:
+      # signed -> unsigned conversions are unchecked
+      # https://github.com/nim-lang/RFCs/issues/175
       raise newException(
         ValueError, "JSON-RPC input is an unexpected negative value")
     result = T(asInt)

beacon_chain/exit_pool.nim

@@ -213,7 +213,7 @@ proc validateProposerSlashing*(
 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/p2p-interface.md#voluntary_exit
 proc validateVoluntaryExit*(
     pool: var ExitPool, signed_voluntary_exit: SignedVoluntaryExit):
-    Result[bool, (ValidationResult, cstring)] =
+    Result[void, (ValidationResult, cstring)] =
   # [IGNORE] The voluntary exit is the first valid voluntary exit received for
   # the validator with index signed_voluntary_exit.message.validator_index.
   if signed_voluntary_exit.message.validator_index >=
@@ -241,4 +241,4 @@ proc validateVoluntaryExit*(
   pool.voluntary_exits.addExitMessage(
     signed_voluntary_exit, VOLUNTARY_EXITS_BOUND)
 
-  ok(true)
+  ok()

beacon_chain/keystore_management.nim

@@ -38,7 +38,7 @@ const
       "../vendor/nimbus-security-resources/passwords/10-million-password-list-top-100000.txt",
     minWordLen = minPasswordLen)
 
-proc echoP(msg: string) =
+proc echoP*(msg: string) =
   ## Prints a paragraph aligned to 80 columns
   echo ""
   echo wrapWords(msg, 80)
@@ -213,8 +213,8 @@ proc keyboardGetPassword[T](prompt: string, attempts: int,
       dec(remainingAttempts)
   err("Failed to decrypt keystore")
 
-proc loadKeystore(validatorsDir, secretsDir, keyName: string,
+proc loadKeystore*(validatorsDir, secretsDir, keyName: string,
-                  nonInteractive: bool): Option[ValidatorPrivKey] =
+                   nonInteractive: bool): Option[ValidatorPrivKey] =
   let
     keystorePath = validatorsDir / keyName / keystoreFileName
     keystore =

beacon_chain/network_metadata.nim

@@ -4,10 +4,10 @@ import
   eth/common/eth_types as commonEthTypes,
   web3/[ethtypes, conversions],
   chronicles,
-  spec/presets,
-  spec/datatypes,
   json_serialization,
-  json_serialization/std/[options, sets, net], serialization/errors
+  json_serialization/std/[options, sets, net], serialization/errors,
+  ssz/navigator,
+  spec/[presets, datatypes, digest]
 
 # ATTENTION! This file will produce a large C file, because we are inlining
 # genesis states as C literals in the generated code (and blobs in the final
@@ -219,3 +219,7 @@ proc getRuntimePresetForNetwork*(eth2Network: Option[string]): RuntimePreset =
   if eth2Network.isSome:
     return getMetadataForNetwork(eth2Network.get).runtimePreset
   return defaultRuntimePreset
+
+proc extractGenesisValidatorRootFromSnapshop*(snapshot: string): Eth2Digest =
+  sszMount(snapshot, BeaconState).genesis_validators_root[]
+

beacon_chain/nimbus_beacon_node.nim

@@ -7,12 +7,12 @@
 
 import
   # Standard library
-  std/[os, tables, strutils, strformat, sequtils, times, math, terminal, osproc,
+  std/[os, tables, strutils, strformat, sequtils, times, math,
-    random],
+       terminal, osproc, random],
 
   # Nimble packages
   stew/[objects, byteutils, endians2, io2], stew/shims/macros,
-  chronos, confutils, metrics, json_rpc/[rpcserver, jsonmarshal],
+  chronos, confutils, metrics, json_rpc/[rpcclient, rpcserver, jsonmarshal],
   chronicles, bearssl, blscurve,
   json_serialization/std/[options, sets, net], serialization/errors,
 
@@ -24,13 +24,14 @@ import
   ./rpc/[beacon_api, config_api, debug_api, event_api, nimbus_api, node_api,
     validator_api],
   spec/[datatypes, digest, crypto, beaconstate, helpers, network, presets],
-  spec/[weak_subjectivity],
+  spec/[weak_subjectivity, signatures],
+  spec/eth2_apis/beacon_rpc_client,
   conf, time, beacon_chain_db, validator_pool, extras,
   attestation_pool, exit_pool, eth2_network, eth2_discovery,
   beacon_node_common, beacon_node_types, beacon_node_status,
   block_pools/[chain_dag, quarantine, clearance, block_pools_types],
   nimbus_binary_common, network_metadata,
-  eth1_monitor, version, ssz/[navigator, merkleization],
+  eth1_monitor, version, ssz/merkleization,
   sync_protocol, request_manager, keystore_management, interop, statusbar,
   sync_manager, validator_duties, filepath,
   validator_slashing_protection, ./eth2_processor
@@ -220,7 +221,7 @@ proc init*(T: type BeaconNode,
   if genesisStateContents != nil:
     let
       networkGenesisValidatorsRoot =
-        sszMount(genesisStateContents[], BeaconState).genesis_validators_root[]
+        extractGenesisValidatorRootFromSnapshop(genesisStateContents[])
 
     if networkGenesisValidatorsRoot != databaseGenesisValidatorsRoot:
       fatal "The specified --data-dir contains data for a different network",
@@ -1025,6 +1026,143 @@ when hasPrompt:
       # var t: Thread[ptr Prompt]
       # createThread(t, processPromptCommands, addr p)
 
+proc handleValidatorExitCommand(config: BeaconNodeConf) {.async.} =
+  let port = try:
+    let value = parseInt(config.rpcUrlForExit.port)
+    if value < Port.low.int or value > Port.high.int:
+      raise newException(ValueError,
+        "The port number must be between " & $Port.low & " and " & $Port.high)
+    Port value
+  except CatchableError as err:
+    fatal "Invalid port number", err = err.msg
+    quit 1
+
+  let rpcClient = newRpcHttpClient()
+
+  try:
+    await connect(rpcClient, config.rpcUrlForExit.hostname, port)
+  except CatchableError as err:
+    fatal "Failed to connect to the beacon node RPC service", err = err.msg
+    quit 1
+
+  let (validator, validatorIdx, status, balance) = try:
+    await rpcClient.get_v1_beacon_states_stateId_validators_validatorId(
+      "head", config.exitedValidator)
+  except CatchableError as err:
+    fatal "Failed to obtain information for validator", err = err.msg
+    quit 1
+
+  let exitAtEpoch = if config.exitAtEpoch.isSome:
+    Epoch config.exitAtEpoch.get
+  else:
+    let headSlot = try:
+      await rpcClient.getBeaconHead()
+    except CatchableError as err:
+      fatal "Failed to obtain the current head slot", err = err.msg
+      quit 1
+    headSlot.epoch
+
+  let
+    validatorsDir = config.validatorsDir
+    validatorKeyAsStr = "0x" & $validator.pubkey
+    keystoreDir = validatorsDir / validatorKeyAsStr
+
+  if not dirExists(keystoreDir):
+    echo "The validator keystores directory '" & config.validatorsDir.string &
+         "' does not contain a keystore for the selected validator with public " &
+         "key '" & validatorKeyAsStr & "'."
+    quit 1
+
+  let signingKey = loadKeystore(
+    validatorsDir,
+    config.secretsDir,
+    validatorKeyAsStr,
+    config.nonInteractive)
+
+  if signingKey.isNone:
+    fatal "Unable to continue without decrypted signing key"
+    quit 1
+
+  let fork = try:
+    await rpcClient.get_v1_beacon_states_fork("head")
+  except CatchableError as err:
+    fatal "Failed to obtain the fork id of the head state", err = err.msg
+    quit 1
+
+  let genesisValidatorsRoot = try:
+    (await rpcClient.get_v1_beacon_genesis()).genesis_validators_root
+  except CatchableError as err:
+    fatal "Failed to obtain the genesis validators root of the network",
+           err = err.msg
+    quit 1
+
+  var signedExit = SignedVoluntaryExit(
+    message: VoluntaryExit(
+      epoch: exitAtEpoch,
+      validator_index: validatorIdx))
+
+  signedExit.signature = get_voluntary_exit_signature(
+    fork, genesisValidatorsRoot, signedExit.message, signingKey.get)
+
+  template ask(prompt: string): string =
+    try:
+      stdout.write prompt, ": "
+      stdin.readLine()
+    except IOError as err:
+      fatal "Failed to read user input from stdin"
+      quit 1
+
+  try:
+    echoP "PLEASE BEWARE!"
+
+    echoP "Publishing a voluntary exit is an irreversible operation! " &
+          "You won't be able to restart again with the same validator."
+
+    echoP "By requesting an exit now, you'll be exempt from penalties " &
+          "stemming from not performing your validator duties, but you " &
+          "won't be able to withdraw your deposited funds at the time " &
+          "being. This means that your funds will be effectively frozen " &
+          "until withdrawals are enabled in a future phase of the Eth2 " &
+          "rollout."
+
+    echoP "To understand more about the Eth2 roadmap, we recommend you " &
+          "have a look at\n" &
+          "https://ethereum.org/en/eth2/#roadmap"
+
+    echoP "You must not shut down your validator for at least 5 epochs " &
+          "(32 minutes) after requesting a validator exit, as you will " &
+          "still be required to perform validator duties until your exit " &
+          "has been processed. The number of epochs could be significantly " &
+          "higher depending on how many other validators are queued to exit."
+
+    echoP "As such, we recommend you keep track of your validator's status " &
+          "using an Eth2 block explorer before shutting down your beacon node."
+
+    const
+      confirmation = "I understand the implications of submitting a voluntary exit"
+
+    while true:
+      echoP "To proceed to submitting your voluntary exit, please type '" &
+            confirmation & "' (without the quotes) in the prompt below and " &
+            "press ENTER or type 'q' to quit."
+      echo ""
+
+      let choice = ask "Your choice"
+      if choice == "q":
+        quit 0
+      elif choice == confirmation:
+        let success = await rpcClient.post_v1_beacon_pool_voluntary_exits(signedExit)
+        if success:
+          echo "Successfully published voluntary exit for validator " &
+                $validatorIdx & "(" & validatorKeyAsStr[0..9] & ")."
+          quit 0
+        else:
+          echo "The voluntary exit was not submitted successfully. Please try again."
+          quit 1
+  except CatchableError as err:
+    fatal "Failed to send the signed exit message to the beacon node RPC"
+    quit 1
+
 programMain:
   var
     config = makeBannerAndConfig(clientId, BeaconNodeConf)
@@ -1210,6 +1348,7 @@ programMain:
 
   of deposits:
     case config.depositsCmd
+    #[
     of DepositsCmd.create:
       var seed: KeySeed
       defer: burnMem(seed)
@@ -1287,6 +1426,11 @@ programMain:
         fatal "Failed to create launchpad deposit data file", err = err.msg
         quit 1
 
+    of DepositsCmd.status:
+      echo "The status command is not implemented yet"
+      quit 1
+
+    #]#
     of DepositsCmd.`import`:
       let validatorKeysDir = config.importedDepositsDir.get:
         let cwd = os.getCurrentDir()
@@ -1304,9 +1448,8 @@ programMain:
         validatorKeysDir.string,
         config.validatorsDir, config.secretsDir)
 
-    of DepositsCmd.status:
+    of DepositsCmd.exit:
-      echo "The status command is not implemented yet"
+      waitFor handleValidatorExitCommand(config)
-      quit 1
 
   of wallets:
     case config.walletsCmd:

beacon_chain/rpc/beacon_api.nim

@@ -11,7 +11,7 @@ import
   chronicles,
   ../beacon_node_common, ../eth2_json_rpc_serialization, ../eth2_network,
   ../validator_duties,
-  ../block_pools/chain_dag,
+  ../block_pools/chain_dag, ../exit_pool,
   ../spec/[crypto, digest, datatypes, validator],
   ../spec/eth2_apis/callsigs_types,
   ../ssz/merkleization,
@@ -26,8 +26,11 @@ template unimplemented() =
   raise (ref CatchableError)(msg: "Unimplemented")
 
 proc parsePubkey(str: string): ValidatorPubKey =
-  if str.len != RawPubKeySize + 2: # +2 because of the `0x` prefix
+  const expectedLen = RawPubKeySize * 2 + 2
-    raise newException(CatchableError, "Not a valid public key (too short)")
+  if str.len != expectedLen: # +2 because of the `0x` prefix
+    raise newException(ValueError,
+      "A hex public key should be exactly " & $expectedLen & " characters. " &
+      $str.len & " provided")
   let pubkeyRes = fromHex(ValidatorPubKey, str)
   if pubkeyRes.isErr:
     raise newException(CatchableError, "Not a valid public key")
@@ -46,19 +49,20 @@ proc getValidatorInfoFromValidatorId(
   if status notin allowedStatuses:
     raise newException(CatchableError, "Invalid status requested")
 
+  var validatorIdx: uint64
   let validator = if validatorId.startsWith("0x"):
     let pubkey = parsePubkey(validatorId)
     let idx = state.validators.asSeq.findIt(it.pubKey == pubkey)
     if idx == -1:
       raise newException(CatchableError, "Could not find validator")
+    validatorIdx = idx.uint64
     state.validators[idx]
   else:
-    var valIdx: BiggestUInt
+    if parseBiggestUInt(validatorId, validatorIdx) != validatorId.len:
-    if parseBiggestUInt(validatorId, valIdx) != validatorId.len:
       raise newException(CatchableError, "Not a valid index")
-    if valIdx > state.validators.lenu64:
+    if validatorIdx > state.validators.lenu64:
       raise newException(CatchableError, "Index out of bounds")
-    state.validators[valIdx]
+    state.validators[validatorIdx]
 
   # time to determine the status of the validator - the code mimics
   # whatever is detailed here: https://hackmd.io/ofFJ5gOmQpu1jjHilHbdQQ
@@ -102,8 +106,10 @@ proc getValidatorInfoFromValidatorId(
   if status != "" and status notin actual_status:
     return none(BeaconStatesValidatorsTuple)
 
-  return some((validator: validator, status: actual_status,
+  return some((validator: validator,
-                balance: validator.effective_balance))
+               index: validatorIdx,
+               status: actual_status,
+               balance: validator.effective_balance))
 
 proc getBlockDataFromBlockId(node: BeaconNode, blockId: string): BlockData =
   result = case blockId:
@@ -254,5 +260,13 @@ proc installBeaconApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
   rpcServer.rpc("get_v1_beacon_pool_voluntary_exits") do () -> JsonNode:
     unimplemented()
 
-  rpcServer.rpc("post_v1_beacon_pool_voluntary_exits") do () -> JsonNode:
+  rpcServer.rpc("post_v1_beacon_pool_voluntary_exits") do (
-    unimplemented()
+      exit: SignedVoluntaryExit) -> bool:
+    doAssert node.exitPool != nil
+    let validity = node.exitPool[].validateVoluntaryExit(exit)
+    if validity.isOk:
+      node.sendVoluntaryExit(exit)
+    else:
+      raise newException(ValueError, $(validity.error[1]))
+    return true
+

beacon_chain/spec/eth2_apis/beacon_callsigs.nim

@@ -54,8 +54,8 @@ proc get_v1_beacon_blocks_blockId_attestations(blockId: string): seq[Attestation
 # TODO GET  /v1/beacon/pool/attestations
 
 
-
 proc post_v1_beacon_pool_attestations(attestation: Attestation): bool
+proc post_v1_beacon_pool_voluntary_exits(exit: SignedVoluntaryExit): bool
 
 proc get_v1_config_fork_schedule(): seq[Fork]
 

beacon_chain/spec/eth2_apis/beacon_rpc_client.nim

@@ -0,0 +1,9 @@
+import
+  os, json,
+  json_rpc/[rpcclient, jsonmarshal],
+  ../../eth2_json_rpc_serialization,
+  ../digest, ../datatypes,
+  callsigs_types
+
+createRpcSigs(RpcClient, currentSourcePath.parentDir / "beacon_callsigs.nim")
+

beacon_chain/spec/eth2_apis/callsigs_types.nim

@@ -30,6 +30,7 @@ type
 
   BeaconStatesValidatorsTuple* = tuple
     validator: Validator
+    index: uint64
     status: string
     balance: uint64
 

beacon_chain/spec/signatures.nim

@@ -178,10 +178,24 @@ proc verify_deposit_signature*(preset: RuntimePreset,
 
   blsVerify(deposit.pubkey, signing_root.data, deposit.signature)
 
-proc verify_voluntary_exit_signature*(
+func get_voluntary_exit_signature*(
-    fork: Fork, genesis_validators_root: Eth2Digest,
+    fork: Fork,
+    genesis_validators_root: Eth2Digest,
     voluntary_exit: VoluntaryExit,
-    pubkey: ValidatorPubKey, signature: SomeSig): bool =
+    privkey: ValidatorPrivKey): ValidatorSig =
+  let
+    domain = get_domain(
+      fork, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch, genesis_validators_root)
+    signing_root = compute_signing_root(voluntary_exit, domain)
+
+  blsSign(privKey, signing_root.data)
+
+proc verify_voluntary_exit_signature*(
+    fork: Fork,
+    genesis_validators_root: Eth2Digest,
+    voluntary_exit: VoluntaryExit,
+    pubkey: ValidatorPubKey,
+    signature: SomeSig): bool =
   withTrust(signature):
     let
       domain = get_domain(

beacon_chain/validator_duties.nim

@@ -157,6 +157,11 @@ proc sendAttestation*(
 
   beacon_attestations_sent.inc()
 
+proc sendVoluntaryExit*(node: BeaconNode, exit: SignedVoluntaryExit) =
+  node.network.broadcast(
+    getVoluntaryExitsTopic(node.forkDIgest),
+    exit)
+
 proc sendAttestation*(node: BeaconNode, attestation: Attestation) =
   # For the validator API, which doesn't supply num_active_validators.
   let attestationBlck =