web3signer refactoring and test suite. (#4775)

* Refactor nimbus_signing_node to support Unix signals.

* Fix SN unable to close REST server properly.

* Fix `keys`, `deposit` and `validator_registration` endpoints issues.
Add getValidatorExitSignature() and getDepositMessageSignature() to validator_pool.

* Add /reload endpoint and implementation.
Fix signData to not cancel `timer`.
Fix validator_pool should clear attachedValidators table.

* Diva protocol enhancement implementation.

AllTests-mainnet.md

@@ -364,6 +364,42 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + Voluntary exit signatures                                                                  OK
 ```
 OK: 8/8 Fail: 0/8 Skip: 0/8
+## Nimbus remote signer/signing test (web3signer)
+```diff
++ Connection timeout test                                                                    OK
++ Connections pool stress test                                                               OK
++ Idle connection test                                                                       OK
++ Public keys enumeration (/api/v1/eth2/publicKeys) test                                     OK
++ Public keys reload (/reload) test                                                          OK
++ Signing BeaconBlock (getBlockSignature(altair))                                            OK
++ Signing BeaconBlock (getBlockSignature(bellatrix))                                         OK
++ Signing BeaconBlock (getBlockSignature(capella))                                           OK
++ Signing BeaconBlock (getBlockSignature(deneb))                                             OK
++ Signing BeaconBlock (getBlockSignature(phase0))                                            OK
++ Signing SC contribution and proof (getContributionAndProofSignature())                     OK
++ Signing SC message (getSyncCommitteeMessage())                                             OK
++ Signing SC selection proof (getSyncCommitteeSelectionProof())                              OK
++ Signing aggregate and proof (getAggregateAndProofSignature())                              OK
++ Signing aggregation slot (getSlotSignature())                                              OK
++ Signing attestation (getAttestationSignature())                                            OK
++ Signing deposit message (getDepositMessageSignature())                                     OK
++ Signing phase0 block                                                                       OK
++ Signing randao reveal (getEpochSignature())                                                OK
++ Signing validator registration (getBuilderSignature())                                     OK
++ Signing voluntary exit (getValidatorExitSignature())                                       OK
++ Waiting for signing node (/upcheck) test                                                   OK
+```
+OK: 22/22 Fail: 0/22 Skip: 0/22
+## Nimbus remote signer/signing test (web3signer-diva)
+```diff
++ Signing BeaconBlock (getBlockSignature(altair))                                            OK
++ Signing BeaconBlock (getBlockSignature(bellatrix))                                         OK
++ Signing BeaconBlock (getBlockSignature(capella))                                           OK
++ Signing BeaconBlock (getBlockSignature(deneb))                                             OK
++ Signing BeaconBlock (getBlockSignature(phase0))                                            OK
++ Waiting for signing node (/upcheck) test                                                   OK
+```
+OK: 6/6 Fail: 0/6 Skip: 0/6
 ## Old database versions [Preset: mainnet]
 ```diff
 + pre-1.1.0                                                                                  OK
@@ -640,4 +676,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 357/362 Fail: 0/362 Skip: 5/362
+OK: 385/390 Fail: 0/390 Skip: 5/390

Makefile

@@ -340,7 +340,7 @@ FORCE_BUILD_ALONE_ALL_TESTS_DEPS :=
 endif
 force_build_alone_all_tests: | $(FORCE_BUILD_ALONE_ALL_TESTS_DEPS)
 
-all_tests: | build deps force_build_alone_all_tests
+all_tests: | build deps nimbus_signing_node force_build_alone_all_tests
 	+ echo -e $(BUILD_MSG) "build/$@" && \
 		MAKE="$(MAKE)" V="$(V)" $(ENV_SCRIPT) scripts/compile_nim_program.sh \
 			$@ \

beacon_chain/conf.nim

@@ -986,6 +986,11 @@ type
       desc: "A directory containing validator keystore passwords"
       name: "secrets-dir" .}: Option[InputDir]
 
+    expectedFeeRecipient* {.
+      desc: "Signatures for blocks will require proofs of the specified " &
+            "fee recipient"
+      name: "expected-fee-recipient".}: Option[Address]
+
     serverIdent* {.
       desc: "Server identifier which will be used in HTTP Host header"
       name: "server-ident" .}: Option[string]

beacon_chain/nimbus_signing_node.nim

@@ -8,7 +8,7 @@ import std/[tables, os, strutils]
 import serialization, json_serialization,
        json_serialization/std/[options, net],
        chronos, presto, presto/secureserver, chronicles, confutils,
-       stew/[base10, results, byteutils, io2]
+       stew/[base10, results, byteutils, io2, bitops2]
 import "."/spec/datatypes/[base, altair, phase0],
        "."/spec/[crypto, digest, network, signatures, forks],
        "."/spec/eth2_apis/[rest_types, eth2_rest_serialization],
@@ -36,36 +36,51 @@ type
     signingServer: SigningNodeServer
     keystoreCache: KeystoreCacheRef
     keysList: string
+    runKeystoreCachePruningLoopFut: Future[void]
+    sigintHandleFut: Future[void]
+    sigtermHandleFut: Future[void]
 
-proc getRouter*(): RestRouter
+  SigningNodeRef* = ref SigningNode
 
-proc router(sn: SigningNode): RestRouter =
+  SigningNodeError* = object of CatchableError
+
+proc validate(key: string, value: string): int =
+  case key
+  of "{validator_key}":
+    0
+  else:
+    1
+
+proc getRouter*(): RestRouter =
+  RestRouter.init(validate)
+
+proc router(sn: SigningNodeRef): RestRouter =
   case sn.signingServer.kind
   of SigningNodeKind.Secure:
     sn.signingServer.sserver.router
   of SigningNodeKind.NonSecure:
     sn.signingServer.nserver.router
 
-proc start(sn: SigningNode) =
+proc start(sn: SigningNodeRef) =
   case sn.signingServer.kind
   of SigningNodeKind.Secure:
     sn.signingServer.sserver.start()
   of SigningNodeKind.NonSecure:
     sn.signingServer.nserver.start()
 
-proc stop(sn: SigningNode) {.async.} =
+proc stop(sn: SigningNodeRef) {.async.} =
   case sn.signingServer.kind
   of SigningNodeKind.Secure:
     await sn.signingServer.sserver.stop()
   of SigningNodeKind.NonSecure:
     await sn.signingServer.nserver.stop()
 
-proc close(sn: SigningNode) {.async.} =
+proc close(sn: SigningNodeRef) {.async.} =
   case sn.signingServer.kind
   of SigningNodeKind.Secure:
-    await sn.signingServer.sserver.stop()
+    await sn.signingServer.sserver.closeWait()
   of SigningNodeKind.NonSecure:
-    await sn.signingServer.nserver.stop()
+    await sn.signingServer.nserver.closeWait()
 
 proc loadTLSCert(pathName: InputFile): Result[TLSCertificate, cstring] =
   let data =
@@ -95,106 +110,49 @@ proc loadTLSKey(pathName: InputFile): Result[TLSPrivateKey, cstring] =
       return err("Invalid private key or incorrect file format")
   ok(key)
 
-proc initValidators(sn: var SigningNode): bool =
+proc new(t: typedesc[SigningNodeRef], config: SigningNodeConf): SigningNodeRef =
-  info "Initializaing validators", path = sn.config.validatorsDir()
+  when declared(waitSignal):
-  var publicKeyIdents: seq[string]
+    SigningNodeRef(
-  for keystore in listLoadableKeystores(sn.config, sn.keystoreCache):
+      config: config,
-    # Not relevant in signing node
+      sigintHandleFut: waitSignal(SIGINT),
-    # TODO don't print when loading validators
+      sigtermHandleFut: waitSignal(SIGTERM),
-    let feeRecipient = default(Eth1Address)
+      keystoreCache: KeystoreCacheRef.init()
-    case keystore.kind
+    )
-    of KeystoreKind.Local:
+  else:
-      discard sn.attachedValidators.addValidator(keystore,
+    SigningNodeRef(
-                                                 feeRecipient,
+      config: config,
-                                                 defaultGasLimit)
+      sigintHandleFut: newFuture[void]("sigint_placeholder"),
-      publicKeyIdents.add("\"0x" & keystore.pubkey.toHex() & "\"")
+      sigtermHandleFut: newFuture[void]("sigterm_placeholder"),
-    of KeystoreKind.Remote:
+      keystoreCache: KeystoreCacheRef.init()
-      error "Signing node do not support remote validators",
+    )
-            validator_pubkey = keystore.pubkey
-      return false
-  sn.keysList = "[" & publicKeyIdents.join(", ") & "]"
-  true
-
-proc init(t: typedesc[SigningNode], config: SigningNodeConf): SigningNode =
-  var sn = SigningNode(
-    config: config,
-    keystoreCache: KeystoreCacheRef.init()
-  )
-
-  if not(initValidators(sn)):
-    fatal "Could not find/initialize local validators"
-    quit 1
-
-  asyncSpawn runKeystoreCachePruningLoop(sn.keystoreCache)
-
-  let
-    address = initTAddress(config.bindAddress, config.bindPort)
-    serverFlags = {HttpServerFlags.QueryCommaSeparatedArray,
-                   HttpServerFlags.NotifyDisconnect}
-    timeout =
-      if config.requestTimeout < 0:
-        warn "Negative value of request timeout, using default instead"
-        seconds(defaultSigningNodeRequestTimeout)
-      else:
-        seconds(config.requestTimeout)
-    serverIdent =
-      if config.serverIdent.isSome():
-        config.serverIdent.get()
-      else:
-        NimbusSigningNodeIdent
-
-  sn.signingServer =
-    if config.tlsEnabled:
-      if config.tlsCertificate.isNone():
-        fatal "TLS certificate path is missing, please use --tls-cert option"
-        quit 1
-
-      if config.tlsPrivateKey.isNone():
-        fatal "TLS private key path is missing, please use --tls-key option"
-        quit 1
-
-      let cert =
-        block:
-          let res = loadTLSCert(config.tlsCertificate.get())
-          if res.isErr():
-            fatal "Could not initialize SSL certificate", reason = $res.error()
-            quit 1
-          res.get()
-      let key =
-        block:
-          let res = loadTLSKey(config.tlsPrivateKey.get())
-          if res.isErr():
-            fatal "Could not initialize SSL private key", reason = $res.error()
-            quit 1
-          res.get()
-      let res = SecureRestServerRef.new(getRouter(), address, key, cert,
-                                        serverFlags = serverFlags,
-                                        httpHeadersTimeout = timeout,
-                                        serverIdent = serverIdent)
-      if res.isErr():
-        fatal "HTTPS(REST) server could not be started", address = $address,
-              reason = $res.error()
-        quit 1
-      SigningNodeServer(kind: SigningNodeKind.Secure, sserver: res.get())
-    else:
-      let res = RestServerRef.new(getRouter(), address,
-                                  serverFlags = serverFlags,
-                                  httpHeadersTimeout = timeout,
-                                  serverIdent = serverIdent)
-      if res.isErr():
-        fatal "HTTP(REST) server could not be started", address = $address,
-               reason = $res.error()
-        quit 1
-      SigningNodeServer(kind: SigningNodeKind.NonSecure, nserver: res.get())
-  sn
 
 template errorResponse(code: HttpCode, message: string): RestApiResponse =
   RestApiResponse.response("{\"error\": \"" & message & "\"}", code)
 
 template signatureResponse(code: HttpCode, signature: string): RestApiResponse =
-  RestApiResponse.response("{\"signature\": \"0x" & signature & "\"}", code, "application/json")
+  RestApiResponse.response("{\"signature\": \"0x" & signature & "\"}",
+                           code, "application/json")
 
-proc installApiHandlers*(node: SigningNode) =
+proc loadKeystores*(node: SigningNodeRef) =
+  var keysList: seq[string]
+  for keystore in listLoadableKeystores(node.config, node.keystoreCache):
+    # Not relevant in signing node
+    # TODO don't print when loading validators
+    let feeRecipient = default(Eth1Address)
+    case keystore.kind
+    of KeystoreKind.Local:
+      discard node.attachedValidators.addValidator(keystore,
+                                                   feeRecipient,
+                                                   defaultGasLimit)
+      keysList.add("\"0x" & keystore.pubkey.toHex() & "\"")
+    of KeystoreKind.Remote:
+      warn "Signing node do not support remote validators",
+           path = node.config.validatorsDir(),
+           validator_pubkey = keystore.pubkey
+
+  node.keysList = "[" & keysList.join(", ") & "]"
+
+proc installApiHandlers*(node: SigningNodeRef) =
   var router = node.router()
 
   router.api(MethodGet, "/api/v1/eth2/publicKeys") do () -> RestApiResponse:
@@ -205,6 +163,11 @@ proc installApiHandlers*(node: SigningNode) =
     return RestApiResponse.response("{\"status\": \"OK\"}", Http200,
                                     "application/json")
 
+  router.api(MethodPost, "/reload") do () -> RestApiResponse:
+    node.attachedValidators.close()
+    node.loadKeystores()
+    return RestApiResponse.response(Http200)
+
   router.api(MethodPost, "/api/v1/eth2/sign/{validator_key}") do (
     validator_key: ValidatorPubKey,
     contentBody: Option[ContentBody]) -> RestApiResponse:
@@ -231,108 +194,137 @@ proc installApiHandlers*(node: SigningNode) =
       of Web3SignerRequestKind.AggregationSlot:
         let
           forkInfo = request.forkInfo.get()
-          cooked = get_slot_signature(forkInfo.fork,
+          signature = get_slot_signature(forkInfo.fork,
             forkInfo.genesis_validators_root,
-            request.aggregationSlot.slot, validator.data.privateKey)
+            request.aggregationSlot.slot,
-          signature = cooked.toValidatorSig().toHex()
+            validator.data.privateKey).toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.AggregateAndProof:
         let
           forkInfo = request.forkInfo.get()
-          cooked = get_aggregate_and_proof_signature(forkInfo.fork,
+          signature = get_aggregate_and_proof_signature(forkInfo.fork,
             forkInfo.genesis_validators_root, request.aggregateAndProof,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.Attestation:
         let
           forkInfo = request.forkInfo.get()
-          cooked = get_attestation_signature(forkInfo.fork,
+          signature = get_attestation_signature(forkInfo.fork,
             forkInfo.genesis_validators_root, request.attestation,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.Block:
         let
           forkInfo = request.forkInfo.get()
           blck = request.blck
-          blockRoot = hash_tree_root(blck)
+          signature = get_block_signature(forkInfo.fork,
-          cooked = get_block_signature(forkInfo.fork,
+            forkInfo.genesis_validators_root, blck.slot, hash_tree_root(blck),
-            forkInfo.genesis_validators_root, blck.slot, blockRoot,
+            validator.data.privateKey).toValidatorSig().toHex()
-            validator.data.privateKey)
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.BlockV2:
-        let
+        if node.config.expectedFeeRecipient.isNone():
-          forkInfo = request.forkInfo.get()
+          let
-          forked = request.beaconBlock
+            forkInfo = request.forkInfo.get()
-          blockRoot = hash_tree_root(forked)
+            blockRoot = hash_tree_root(request.beaconBlock)
-          cooked =
+            signature = withBlck(request.beaconBlock):
-            withBlck(forked):
               get_block_signature(forkInfo.fork,
                 forkInfo.genesis_validators_root, blck.slot, blockRoot,
-                validator.data.privateKey)
+                validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
+          return signatureResponse(Http200, signature)
+
+        let (feeRecipientIndex, blockHeader) =
+          case request.beaconBlock.kind
+          of ConsensusFork.Phase0, ConsensusFork.Altair:
+            # `phase0` and `altair` blocks do not have `fee_recipient`, so
+            # we return an error.
+            return errorResponse(Http400, BlockIncorrectFork)
+          of ConsensusFork.Bellatrix:
+            (GeneralizedIndex(401), request.beaconBlock.bellatrixData)
+          of ConsensusFork.Capella:
+            (GeneralizedIndex(401), request.beaconBlock.capellaData)
+          of ConsensusFork.Deneb:
+            (GeneralizedIndex(401), request.beaconBlock.denebData)
+
+        if request.proofs.isNone() or len(request.proofs.get()) == 0:
+          return errorResponse(Http400, MissingMerkleProofError)
+
+        let proof = request.proofs.get()[0]
+
+        if proof.index != feeRecipientIndex:
+          return errorResponse(Http400, InvalidMerkleProofIndexError)
+
+        let feeRecipientRoot = hash_tree_root(distinctBase(
+          node.config.expectedFeeRecipient.get()))
+
+        if not(is_valid_merkle_branch(feeRecipientRoot, proof.merkleProofs,
+                                      log2trunc(proof.index),
+                                      get_subtree_index(proof.index),
+                                      blockHeader.body_root)):
+          return errorResponse(Http400, InvalidMerkleProofError)
+
+        let
+          forkInfo = request.forkInfo.get()
+          blockRoot = hash_tree_root(request.beaconBlock)
+          signature = withBlck(request.beaconBlock):
+            get_block_signature(forkInfo.fork,
+              forkInfo.genesis_validators_root, blck.slot, blockRoot,
+              validator.data.privateKey).toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.Deposit:
         let
           data = DepositMessage(pubkey: request.deposit.pubkey,
             withdrawal_credentials: request.deposit.withdrawalCredentials,
             amount: request.deposit.amount)
-          cooked = get_deposit_signature(data,
+          signature = get_deposit_signature(data,
-            request.deposit.genesisForkVersion, validator.data.privateKey)
+            request.deposit.genesisForkVersion,
-          signature = cooked.toValidatorSig().toHex()
+            validator.data.privateKey).toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.RandaoReveal:
         let
           forkInfo = request.forkInfo.get()
-          cooked = get_epoch_signature(forkInfo.fork,
+          signature = get_epoch_signature(forkInfo.fork,
             forkInfo.genesis_validators_root, request.randaoReveal.epoch,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.VoluntaryExit:
         let
           forkInfo = request.forkInfo.get()
-          cooked = get_voluntary_exit_signature(forkInfo.fork,
+          signature = get_voluntary_exit_signature(forkInfo.fork,
             forkInfo.genesis_validators_root, request.voluntaryExit,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.SyncCommitteeMessage:
         let
           forkInfo = request.forkInfo.get()
           msg = request.syncCommitteeMessage
-          cooked = get_sync_committee_message_signature(forkInfo.fork,
+          signature = get_sync_committee_message_signature(forkInfo.fork,
             forkInfo.genesis_validators_root, msg.slot, msg.beaconBlockRoot,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.SyncCommitteeSelectionProof:
         let
           forkInfo = request.forkInfo.get()
           msg = request.syncAggregatorSelectionData
-          subcommittee = SyncSubcommitteeIndex.init(msg.subcommittee_index).valueOr:
+          subcommittee =
-            return errorResponse(Http400, InvalidSubCommitteeIndexValueError)
+            SyncSubcommitteeIndex.init(msg.subcommittee_index).valueOr:
-          cooked = get_sync_committee_selection_proof(forkInfo.fork,
+              return errorResponse(Http400, InvalidSubCommitteeIndexValueError)
+          signature = get_sync_committee_selection_proof(forkInfo.fork,
             forkInfo.genesis_validators_root, msg.slot, subcommittee,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.SyncCommitteeContributionAndProof:
         let
           forkInfo = request.forkInfo.get()
           msg = request.syncCommitteeContributionAndProof
-          cooked = get_contribution_and_proof_signature(
+          signature = get_contribution_and_proof_signature(
             forkInfo.fork, forkInfo.genesis_validators_root, msg,
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
       of Web3SignerRequestKind.ValidatorRegistration:
         let
           forkInfo = request.forkInfo.get()
-          cooked = get_builder_signature(
+          signature = get_builder_signature(forkInfo.fork,
-            forkInfo.fork, ValidatorRegistrationV1(
+            ValidatorRegistrationV1(
               fee_recipient:
                 ExecutionAddress(data: distinctBase(Eth1Address.fromHex(
                   request.validatorRegistration.feeRecipient))),
@@ -340,34 +332,142 @@ proc installApiHandlers*(node: SigningNode) =
               timestamp: request.validatorRegistration.timestamp,
               pubkey: request.validatorRegistration.pubkey,
             ),
-            validator.data.privateKey)
+            validator.data.privateKey).toValidatorSig().toHex()
-          signature = cooked.toValidatorSig().toHex()
         signatureResponse(Http200, signature)
 
-proc validate(key: string, value: string): int =
+proc asyncInit(sn: SigningNodeRef) {.async.} =
-  case key
-  of "{validator_key}":
-    0
-  else:
-    1
-
-proc getRouter*(): RestRouter =
-  RestRouter.init(validate)
-
-programMain:
-  let config = makeBannerAndConfig("Nimbus signing node " & fullVersionStr,
-                                   SigningNodeConf)
-  setupLogging(config.logLevel, config.logStdout, config.logFile)
-
-  var sn = SigningNode.init(config)
   notice "Launching signing node", version = fullVersionStr,
-         cmdParams = commandLineParams(), config,
+         cmdParams = commandLineParams(), config = sn.config
-         validators_count = sn.attachedValidators.count()
+
+  info "Initializaing validators", path = sn.config.validatorsDir()
+  sn.loadKeystores()
+
+  if sn.attachedValidators.count() == 0:
+    fatal "Could not find/initialize local validators"
+    raise newException(SigningNodeError, "")
+
+  let
+    address = initTAddress(sn.config.bindAddress, sn.config.bindPort)
+    serverFlags = {HttpServerFlags.QueryCommaSeparatedArray,
+                   HttpServerFlags.NotifyDisconnect}
+    timeout =
+      if sn.config.requestTimeout < 0:
+        warn "Negative value of request timeout, using default instead"
+        seconds(defaultSigningNodeRequestTimeout)
+      else:
+        seconds(sn.config.requestTimeout)
+    serverIdent =
+      if sn.config.serverIdent.isSome():
+        sn.config.serverIdent.get()
+      else:
+        NimbusSigningNodeIdent
+
+  sn.signingServer =
+    if sn.config.tlsEnabled:
+      if sn.config.tlsCertificate.isNone():
+        fatal "TLS certificate path is missing, please use --tls-cert option"
+        raise newException(SigningNodeError, "")
+
+      if sn.config.tlsPrivateKey.isNone():
+        fatal "TLS private key path is missing, please use --tls-key option"
+        raise newException(SigningNodeError, "")
+
+      let cert =
+        block:
+          let res = loadTLSCert(sn.config.tlsCertificate.get())
+          if res.isErr():
+            fatal "Could not initialize SSL certificate",
+                  reason = $res.error()
+            raise newException(SigningNodeError, "")
+          res.get()
+      let key =
+        block:
+          let res = loadTLSKey(sn.config.tlsPrivateKey.get())
+          if res.isErr():
+            fatal "Could not initialize SSL private key",
+                  reason = $res.error()
+            raise newException(SigningNodeError, "")
+          res.get()
+      let res = SecureRestServerRef.new(getRouter(), address, key, cert,
+                                        serverFlags = serverFlags,
+                                        httpHeadersTimeout = timeout,
+                                        serverIdent = serverIdent)
+      if res.isErr():
+        fatal "HTTPS(REST) server could not be started", address = $address,
+              reason = $res.error()
+        raise newException(SigningNodeError, "")
+      SigningNodeServer(kind: SigningNodeKind.Secure, sserver: res.get())
+    else:
+      let res = RestServerRef.new(getRouter(), address,
+                                  serverFlags = serverFlags,
+                                  httpHeadersTimeout = timeout,
+                                  serverIdent = serverIdent)
+      if res.isErr():
+        fatal "HTTP(REST) server could not be started", address = $address,
+               reason = $res.error()
+        raise newException(SigningNodeError, "")
+      SigningNodeServer(kind: SigningNodeKind.NonSecure, nserver: res.get())
+
+proc asyncRun*(sn: SigningNodeRef) {.async.} =
+  sn.runKeystoreCachePruningLoopFut =
+    runKeystorecachePruningLoop(sn.keystoreCache)
   sn.installApiHandlers()
   sn.start()
+
+  var future = newFuture[void]("signing-node-mainLoop")
   try:
-    runForever()
+    await future
-  finally:
+  except CancelledError:
-    waitFor sn.stop()
+    debug "Main loop interrupted"
-    waitFor sn.close()
+  except CatchableError as exc:
-  discard sn.stop()
+    warn "Main loop failed with unexpected error", err_name = $exc.name,
+         reason = $exc.msg
+
+  debug "Stopping main processing loop"
+  var pending: seq[Future[void]]
+  if not(sn.runKeystoreCachePruningLoopFut.finished()):
+    pending.add(cancelAndWait(sn.runKeystoreCachePruningLoopFut))
+  pending.add(sn.stop())
+  pending.add(sn.close())
+  await allFutures(pending)
+
+template runWithSignals(sn: SigningNodeRef, body: untyped): bool =
+  let future = body
+  discard await race(future, sn.sigintHandleFut, sn.sigtermHandleFut)
+  if future.finished():
+    if future.failed() or future.cancelled():
+      let exc = future.readError()
+      debug "Signing node initialization failed"
+      var pending: seq[Future[void]]
+      if not(sn.sigintHandleFut.finished()):
+        pending.add(cancelAndWait(sn.sigintHandleFut))
+      if not(sn.sigtermHandleFut.finished()):
+        pending.add(cancelAndWait(sn.sigtermHandleFut))
+      await allFutures(pending)
+      false
+    else:
+      true
+  else:
+    let signal = if sn.sigintHandleFut.finished(): "SIGINT" else: "SIGTERM"
+    info "Got interrupt, trying to shutdown gracefully", signal = signal
+    var pending = @[cancelAndWait(future)]
+    if not(sn.sigintHandleFut.finished()):
+      pending.add(cancelAndWait(sn.sigintHandleFut))
+    if not(sn.sigtermHandleFut.finished()):
+      pending.add(cancelAndWait(sn.sigtermHandleFut))
+    await allFutures(pending)
+    false
+
+proc runSigningNode(config: SigningNodeConf) {.async.} =
+  let sn = SigningNodeRef.new(config)
+  if not sn.runWithSignals(asyncInit sn):
+    return
+  if not sn.runWithSignals(asyncRun sn):
+    return
+
+programMain:
+  let config =
+    makeBannerAndConfig("Nimbus signing node " & fullVersionStr,
+                        SigningNodeConf)
+  setupLogging(config.logLevel, config.logStdout, config.logFile)
+  waitFor runSigningNode(config)

beacon_chain/rpc/rest_constants.nim

@@ -228,3 +228,9 @@ const
     "BLS to execution change was broadcast"
   AggregationSelectionNotImplemented* =
     "Attestation and sync committee aggreggation selection are not implemented"
+  MissingMerkleProofError* =
+    "Required merkle proof is missing"
+  InvalidMerkleProofError* =
+    "The given merkle proof is invalid"
+  InvalidMerkleProofIndexError* =
+    "The given merkle proof index is invalid"

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -1901,6 +1901,8 @@ proc writeValue*(writer: var JsonWriter[RestJson],
     # https://github.com/ConsenSys/web3signer/blob/41c0cbfabcb1fca9587b59e058b7eb29f152c60c/core/src/main/resources/openapi-specs/eth2/signing/schemas.yaml#L418-L497
     writer.writeField("beacon_block", value.beaconBlock)
 
+    if isSome(value.proofs):
+      writer.writeField("proofs", value.proofs.get())
   of Web3SignerRequestKind.Deposit:
     writer.writeField("type", "DEPOSIT")
     if isSome(value.signingRoot):
@@ -1967,6 +1969,7 @@ proc readValue*(reader: var JsonReader[RestJson],
     forkInfo: Option[Web3SignerForkInfo]
     signingRoot: Option[Eth2Digest]
     data: Option[JsonString]
+    proofs: seq[Web3SignerMerkleProof]
     dataName: string
 
   for fieldName in readObjectFields(reader):
@@ -2015,14 +2018,19 @@ proc readValue*(reader: var JsonReader[RestJson],
         reader.raiseUnexpectedField("Multiple `signingRoot` fields found",
                                     "Web3SignerRequest")
       signingRoot = some(reader.readValue(Eth2Digest))
+    of "proofs":
+      let newProofs = reader.readValue(seq[Web3SignerMerkleProof])
+      proofs.add(newProofs)
     of "aggregation_slot", "aggregate_and_proof", "block", "beacon_block",
        "randao_reveal", "voluntary_exit", "sync_committee_message",
-       "sync_aggregator_selection_data", "contribution_and_proof", "attestation":
+       "sync_aggregator_selection_data", "contribution_and_proof",
+       "attestation", "deposit", "validator_registration":
       if data.isSome():
         reader.raiseUnexpectedField("Multiple data fields found",
                                     "Web3SignerRequest")
       dataName = fieldName
       data = some(reader.readValue(JsonString))
+
     else:
       unrecognizedFieldWarning()
 
@@ -2108,10 +2116,17 @@ proc readValue*(reader: var JsonReader[RestJson],
             reader.raiseUnexpectedValue(
               "Incorrect field `beacon_block` format")
           res.get()
-      Web3SignerRequest(
+      if len(proofs) > 0:
-        kind: Web3SignerRequestKind.BlockV2,
+        Web3SignerRequest(
-        forkInfo: forkInfo, signingRoot: signingRoot, beaconBlock: data
+          kind: Web3SignerRequestKind.BlockV2,
-      )
+          forkInfo: forkInfo, signingRoot: signingRoot, beaconBlock: data,
+          proofs: Opt.some(proofs)
+        )
+      else:
+        Web3SignerRequest(
+          kind: Web3SignerRequestKind.BlockV2,
+          forkInfo: forkInfo, signingRoot: signingRoot, beaconBlock: data
+        )
     of Web3SignerRequestKind.Deposit:
       if dataName != "deposit":
         reader.raiseUnexpectedValue("Field `deposit` is missing")

beacon_chain/spec/eth2_apis/rest_remote_signer_calls.nim

@@ -10,14 +10,23 @@ import
   chronicles, metrics,
   chronos, chronos/apps/http/httpclient, presto, presto/client,
   serialization, json_serialization,
-  json_serialization/std/[options, net, sets],
+  json_serialization/std/[net, sets],
   stew/[results, base10, byteutils],
   "."/[rest_types, eth2_rest_serialization]
 
 export chronos, httpclient, client, rest_types, eth2_rest_serialization, results
 
 type
-  Web3SignerResult*[T] = Result[T, string]
+  Web3SignerErrorKind* {.pure.} = enum
+    Error400, Error404, Error412, Error500, CommError, UnexpectedError,
+    UknownStatus, InvalidContentType, InvalidPlain, InvalidContent,
+    InvalidSignature, TimeoutError
+
+  Web3SignerError* = object
+    kind*: Web3SignerErrorKind
+    message*: string
+
+  Web3SignerResult*[T] = Result[T, Web3SignerError]
   Web3SignerDataResponse* = Web3SignerResult[CookedSig]
 
 declareCounter nbc_remote_signer_requests,
@@ -50,7 +59,7 @@ declareCounter nbc_remote_signer_unknown_responses,
 declareCounter nbc_remote_signer_communication_errors,
   "Number of communication errors"
 
-declareHistogram nbc_remote_signer_time,
+declareHistogram nbc_remote_signer_duration,
   "Time(s) used to generate signature usign remote signer",
    buckets = [0.050, 0.100, 0.500, 1.0, 5.0, 10.0]
 
@@ -59,6 +68,11 @@ proc getUpcheck*(): RestResponse[Web3SignerStatusResponse] {.
      meth: MethodGet, accept: "application/json" .}
   ## https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Server-Status
 
+proc reload*(): RestPlainResponse {.
+     rest, endpoint: "/reload",
+     meth: MethodPost, accept: "application/json" .}
+  ## https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Reload-Signer-Keys/operation/RELOAD
+
 proc getKeys*(): RestResponse[Web3SignerKeysResponse] {.
      rest, endpoint: "/api/v1/eth2/publicKeys",
      meth: MethodGet, accept: "application/json" .}
@@ -70,132 +84,225 @@ proc signDataPlain*(identifier: ValidatorPubKey,
      meth: MethodPost, accept: "application/json" .}
   # https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Signing
 
+proc init(t: typedesc[Web3SignerError], kind: Web3SignerErrorKind,
+          message: string): Web3SignerError =
+  Web3SignerError(kind: kind, message: message)
+
 proc signData*(client: RestClientRef, identifier: ValidatorPubKey,
                body: Web3SignerRequest
               ): Future[Web3SignerDataResponse] {.async.} =
-  let startSignTick = Moment.now()
   inc(nbc_remote_signer_requests)
-  let response =
-    try:
-      await client.signDataPlain(identifier, body,
-                                 restAcceptType = "application/json")
-    except RestError as exc:
-      let msg = "[" & $exc.name & "] " & $exc.msg
-      debug "Error occured while generating signature",
-            validator = shortLog(identifier),
-            remote_signer = $client.address.getUri(),
-            error_name = $exc.name, error_msg = $exc.msg,
-            signDur = Moment.now() - startSignTick
-      inc(nbc_remote_signer_communication_errors)
-      return Web3SignerDataResponse.err(msg)
-    except CatchableError as exc:
-      let msg = "[" & $exc.name & "] " & $exc.msg
-      debug "Unexpected error occured while generating signature",
-            validator = shortLog(identifier),
-            remote_signer = $client.address.getUri(),
-            error_name = $exc.name, error_msg = $exc.msg,
-            signDur = Moment.now() - startSignTick
-      inc(nbc_remote_signer_communication_errors)
-      return Web3SignerDataResponse.err(msg)
 
-  let res =
+  let
+    startSignMoment = Moment.now()
+    response =
+      try:
+        let
+          res = await client.signDataPlain(identifier, body,
+                                           restAcceptType = "application/json")
+          duration = Moment.now() - startSignMoment
+        nbc_remote_signer_duration.observe(
+          float(milliseconds(duration)) / 1000.0)
+        res
+      except RestError as exc:
+        return Web3SignerDataResponse.err(
+          Web3SignerError.init(Web3SignerErrorKind.CommError, $exc.msg))
+      except CancelledError as exc:
+        raise exc
+      except CatchableError as exc:
+        return Web3SignerDataResponse.err(
+          Web3SignerError.init(Web3SignerErrorKind.UnexpectedError, $exc.msg))
+
+  return
     case response.status
     of 200:
       inc(nbc_remote_signer_200_responses)
       let sig =
         if response.contentType.isNone() or
            isWildCard(response.contentType.get().mediaType):
+          inc(nbc_remote_signer_failures)
           return Web3SignerDataResponse.err(
-            "Unable to decode signature from missing or incorrect content")
+            Web3SignerError.init(
+              Web3SignerErrorKind.InvalidContentType,
+              "Unable to decode signature from missing or incorrect content"
+            )
+          )
         else:
           let mediaType = response.contentType.get().mediaType
           if mediaType == TextPlainMediaType:
-            let asStr = fromBytes(string, response.data)
+            let
-            let sigFromText = fromHex(ValidatorSig, asStr)
+              asStr = fromBytes(string, response.data)
-            if sigFromText.isErr:
+              sigFromText = fromHex(ValidatorSig, asStr).valueOr:
-              return Web3SignerDataResponse.err(
+                inc(nbc_remote_signer_failures)
-                "Unable to decode signature from plain text")
+                return Web3SignerDataResponse.err(
-            sigFromText.get.load
+                  Web3SignerError.init(
+                    Web3SignerErrorKind.InvalidPlain,
+                    "Unable to decode signature from plain text"
+                  )
+                )
+            sigFromText.load()
           else:
             let res = decodeBytes(Web3SignerSignatureResponse, response.data,
-                                  response.contentType)
+                                  response.contentType).valueOr:
-            if res.isErr:
-              let msg = "Unable to decode remote signer response [" &
-                        $res.error() & "]"
               inc(nbc_remote_signer_failures)
-              return Web3SignerDataResponse.err(msg)
+              return Web3SignerDataResponse.err(
-            res.get.signature.load
+                Web3SignerError.init(
+                  Web3SignerErrorKind.InvalidContent,
+                  "Unable to decode remote signer response [" & $error & "]"
+                )
+              )
+            res.signature.load()
 
-      if sig.isNone:
+      if sig.isNone():
-        let msg = "Remote signer returns invalid signature"
         inc(nbc_remote_signer_failures)
-        return Web3SignerDataResponse.err(msg)
+        return Web3SignerDataResponse.err(
+          Web3SignerError.init(
+            Web3SignerErrorKind.InvalidSignature,
+            "Remote signer returns invalid signature"
+          )
+        )
 
-      Web3SignerDataResponse.ok(sig.get)
+      inc(nbc_remote_signer_signatures)
+      Web3SignerDataResponse.ok(sig.get())
     of 400:
       inc(nbc_remote_signer_400_responses)
-      let res = decodeBytes(Web3SignerErrorResponse, response.data,
+      let message =
-                            response.contentType)
+        block:
-      let msg =
+          let res = decodeBytes(Web3SignerErrorResponse, response.data,
-        if res.isErr():
+                                response.contentType)
-          "Remote signer returns 400 Bad Request Format Error"
+          if res.isErr():
-        else:
+            "Remote signer returns 400 Bad Request Format Error"
-          "Remote signer returns 400 Bad Request Format Error [" &
+          else:
-          res.get().error & "]"
+            res.get().error
-      Web3SignerDataResponse.err(msg)
+      Web3SignerDataResponse.err(
+        Web3SignerError.init(Web3SignerErrorKind.Error400, message))
     of 404:
-      let res = decodeBytes(Web3SignerErrorResponse, response.data,
-                            response.contentType)
-      let msg =
-        if res.isErr():
-          "Remote signer returns 404 Validator's Key Not Found Error"
-        else:
-          "Remote signer returns 404 Validator's Key Not Found Error [" &
-          res.get().error & "]"
       inc(nbc_remote_signer_404_responses)
-      Web3SignerDataResponse.err(msg)
+      let message =
+        block:
+          let res = decodeBytes(Web3SignerErrorResponse, response.data,
+                                response.contentType)
+          if res.isErr():
+            "Remote signer returns 404 Validator's Key Not Found Error"
+          else:
+            res.get().error
+      Web3SignerDataResponse.err(
+        Web3SignerError.init(Web3SignerErrorKind.Error404, message))
     of 412:
-      let res = decodeBytes(Web3SignerErrorResponse, response.data,
-                            response.contentType)
-      let msg =
-        if res.isErr():
-          "Remote signer returns 412 Slashing Protection Error"
-        else:
-          "Remote signer returns 412 Slashing Protection Error [" &
-          res.get().error & "]"
       inc(nbc_remote_signer_412_responses)
-      Web3SignerDataResponse.err(msg)
+      let message =
+        block:
+          let res = decodeBytes(Web3SignerErrorResponse, response.data,
+                                response.contentType)
+          if res.isErr():
+            "Remote signer returns 412 Slashing Protection Error"
+          else:
+            res.get().error
+      Web3SignerDataResponse.err(
+        Web3SignerError.init(Web3SignerErrorKind.Error412, message))
     of 500:
-      let res = decodeBytes(Web3SignerErrorResponse, response.data,
-                            response.contentType)
-      let msg =
-        if res.isErr():
-          "Remote signer returns 500 Internal Server Error"
-        else:
-          "Remote signer returns 500 Internal Server Error [" &
-          res.get().error & "]"
       inc(nbc_remote_signer_500_responses)
-      Web3SignerDataResponse.err(msg)
+      let message =
+        block:
+          let res = decodeBytes(Web3SignerErrorResponse, response.data,
+                                response.contentType)
+          if res.isErr():
+            "Remote signer returns 500 Internal Server Error"
+          else:
+            res.get().error
+      Web3SignerDataResponse.err(
+        Web3SignerError.init(Web3SignerErrorKind.Error500, message))
     else:
-      let msg = "Remote signer returns unexpected status code " &
-                Base10.toString(uint64(response.status))
       inc(nbc_remote_signer_unknown_responses)
-      Web3SignerDataResponse.err(msg)
+      let message =
+        block:
+          let res = decodeBytes(Web3SignerErrorResponse, response.data,
+                                response.contentType)
+          if res.isErr():
+            "Remote signer returns unexpected status code " &
+              Base10.toString(uint64(response.status))
+          else:
+            res.get().error
+      Web3SignerDataResponse.err(
+        Web3SignerError.init(Web3SignerErrorKind.UknownStatus, message))
 
-  if res.isOk():
+proc signData*(
-    let delay = Moment.now() - startSignTick
+       client: RestClientRef,
-    inc(nbc_remote_signer_signatures)
+       identifier: ValidatorPubKey,
-    nbc_remote_signer_time.observe(float(milliseconds(delay)) / 1000.0)
+       timerFut: Future[void],
-    debug "Signature was successfully generated",
+       attemptsCount: int,
-          validator = shortLog(identifier),
+       body: Web3SignerRequest
-          remote_signer = $client.address.getUri(),
+     ): Future[Web3SignerDataResponse] {.async.} =
-          signDur = delay
+  doAssert(attemptsCount >= 1)
-  else:
-    inc(nbc_remote_signer_failures)
-    debug "Signature generation was failed",
-          validator = shortLog(identifier),
-          remote_signer = $client.address.getUri(),
-          error_msg = res.error(),
-          signDur = Moment.now() - startSignTick
 
-  return res
+  const BackoffTimeouts = [
+    10.milliseconds, 100.milliseconds, 1.seconds, 2.seconds, 5.seconds
+  ]
+
+  var
+    attempt = 0
+    currentTimeout = 0
+
+  while true:
+    var
+      operationFut: Future[Web3SignerDataResponse]
+      lastError: Opt[Web3SignerError]
+    try:
+      operationFut = signData(client, identifier, body)
+      if isNil(timerFut):
+        await allFutures(operationFut)
+      else:
+        discard await race(timerFut, operationFut)
+    except CancelledError as exc:
+      if not(operationFut.finished()):
+        await operationFut.cancelAndWait()
+      raise exc
+
+    if not(operationFut.finished()):
+      await operationFut.cancelAndWait()
+      if lastError.isSome():
+        # We return last know error instead of timeout error.
+        return Web3SignerDataResponse.err(lastError.get())
+      else:
+        return Web3SignerDataResponse.err(
+          Web3SignerError.init(
+            Web3SignerErrorKind.TimeoutError,
+            "Operation timed out"
+          )
+        )
+    else:
+      let resp = operationFut.read()
+      if resp.isOk():
+        return resp
+
+      case resp.error.kind
+      of Web3SignerErrorKind.Error404,
+         Web3SignerErrorKind.Error412,
+         Web3SignerErrorKind.Error500,
+         Web3SignerErrorKind.CommError,
+         Web3SignerErrorKind.UnexpectedError:
+        ## Non-critical errors
+        if attempt == attemptsCount:
+          # Number of attempts exceeded, so we return result we have.
+          return resp
+        else:
+          # We have some attempts left, so we show debug log about current
+          # attempt
+          debug "Unable to get signature using remote signer",
+                kind = resp.error.kind, reason = resp.error.message,
+                attempts_count = attemptsCount, attempt = attempt
+          lastError = Opt.some(resp.error)
+          inc(attempt)
+          await sleepAsync(BackoffTimeouts[currentTimeout])
+          if currentTimeout < len(BackoffTimeouts) - 1:
+            inc currentTimeout
+      of Web3SignerErrorKind.Error400,
+         Web3SignerErrorKind.UknownStatus,
+         Web3SignerErrorKind.InvalidContentType,
+         Web3SignerErrorKind.InvalidPlain,
+         Web3SignerErrorKind.InvalidContent,
+         Web3SignerErrorKind.InvalidSignature:
+        # Critical errors
+        return resp
+      of Web3SignerErrorKind.TimeoutError:
+        raiseAssert "Timeout error should not be happened"

beacon_chain/spec/eth2_apis/rest_types.nim

@@ -520,8 +520,7 @@ type
     signature*: ValidatorSig
     slot*: Slot
 
-  Web3SignerKeysResponse* = object
+  Web3SignerKeysResponse* = seq[ValidatorPubKey]
-    keys*: seq[ValidatorPubKey]
 
   Web3SignerStatusResponse* = object
     status*: string
@@ -564,6 +563,10 @@ type
     timestamp*: uint64
     pubkey*: ValidatorPubKey
 
+  Web3SignerMerkleProof* = object
+    index*: GeneralizedIndex
+    merkleProofs* {.serializedFieldName: "merkle_proofs".}: seq[Eth2Digest]
+
   Web3SignerRequestKind* {.pure.} = enum
     AggregationSlot, AggregateAndProof, Attestation, Block, BlockV2,
     Deposit, RandaoReveal, VoluntaryExit, SyncCommitteeMessage,
@@ -588,6 +591,7 @@ type
     of Web3SignerRequestKind.BlockV2:
       beaconBlock* {.
         serializedFieldName: "beacon_block".}: Web3SignerForkedBeaconBlock
+      proofs*: Opt[seq[Web3SignerMerkleProof]]
     of Web3SignerRequestKind.Deposit:
       deposit*: Web3SignerDepositData
     of Web3SignerRequestKind.RandaoReveal:
@@ -759,7 +763,8 @@ func init*(t: typedesc[Web3SignerRequest], fork: Fork,
   )
 
 func init*(t: typedesc[Web3SignerRequest], fork: Fork,
-           genesis_validators_root: Eth2Digest, data: Web3SignerForkedBeaconBlock,
+           genesis_validators_root: Eth2Digest,
+           data: Web3SignerForkedBeaconBlock,
            signingRoot: Option[Eth2Digest] = none[Eth2Digest]()
           ): Web3SignerRequest =
   Web3SignerRequest(
@@ -771,6 +776,22 @@ func init*(t: typedesc[Web3SignerRequest], fork: Fork,
     beaconBlock: data
   )
 
+func init*(t: typedesc[Web3SignerRequest], fork: Fork,
+           genesis_validators_root: Eth2Digest,
+           data: Web3SignerForkedBeaconBlock,
+           proofs: openArray[Web3SignerMerkleProof],
+           signingRoot: Option[Eth2Digest] = none[Eth2Digest]()
+          ): Web3SignerRequest =
+  Web3SignerRequest(
+    kind: Web3SignerRequestKind.BlockV2,
+    forkInfo: some(Web3SignerForkInfo(
+      fork: fork, genesis_validators_root: genesis_validators_root
+    )),
+    signingRoot: signingRoot,
+    proofs: Opt.some(@proofs),
+    beaconBlock: data
+  )
+
 func init*(t: typedesc[Web3SignerRequest], genesisForkVersion: Version,
            data: DepositMessage,
            signingRoot: Option[Eth2Digest] = none[Eth2Digest]()

beacon_chain/spec/keystore.nim

@@ -157,6 +157,7 @@ type
       flags*: set[RemoteKeystoreFlag]
       remotes*: seq[RemoteSignerInfo]
       threshold*: uint32
+      remoteType*: RemoteSignerType
 
   NetKeystore* = object
     crypto*: Crypto
@@ -166,7 +167,7 @@ type
     version*: int
 
   RemoteSignerType* {.pure.} = enum
-    Web3Signer
+    Web3Signer, Web3SignerDiva
 
   RemoteKeystore* = object
     version*: uint64
@@ -598,6 +599,8 @@ proc writeValue*(writer: var JsonWriter, value: RemoteKeystore)
   case value.remoteType
   of RemoteSignerType.Web3Signer:
     writer.writeField("type", "web3signer")
+  of RemoteSignerType.Web3SignerDiva:
+    writer.writeField("type", "web3signer-diva")
   if value.description.isSome():
     writer.writeField("description", value.description.get())
   if RemoteKeystoreFlag.IgnoreSSLVerification in value.flags:
@@ -704,6 +707,8 @@ proc readValue*(reader: var JsonReader, value: var RemoteKeystore)
       case res.toLowerAscii()
       of "web3signer":
         RemoteSignerType.Web3Signer
+      of "web3signer-diva":
+        RemoteSignerType.Web3SignerDiva
       else:
         reader.raiseUnexpectedValue("Unsupported remote signer `type` value")
     else:

beacon_chain/validators/keystore_management.nim

@@ -140,7 +140,8 @@ func init*(T: type KeystoreData, keystore: RemoteKeystore,
     description: keystore.description,
     version: keystore.version,
     remotes: keystore.remotes,
-    threshold: keystore.threshold
+    threshold: keystore.threshold,
+    remoteType: keystore.remoteType
   ))
 
 func init*(T: type KeystoreData, cookedKey: CookedPubKey,

beacon_chain/validators/validator_pool.nim

@@ -9,7 +9,7 @@
 
 import
   std/[tables, json, streams, sequtils, uri],
-  chronos, chronicles, metrics, eth/async_utils,
+  chronos, chronicles, metrics,
   json_serialization/std/net,
   presto, presto/client,
 
@@ -268,6 +268,7 @@ proc close*(pool: var ValidatorPool) =
     if res.isErr():
       notice "Could not unlock validator's keystore file",
              pubkey = validator.pubkey, validator = shortLog(validator)
+  pool.validators.clear()
 
 iterator publicKeys*(pool: ValidatorPool): ValidatorPubKey =
   for item in pool.validators.keys():
@@ -388,10 +389,13 @@ proc signWithDistributedKey(v: AttachedValidator,
   doAssert v.data.threshold <= uint32(v.clients.len)
 
   let
-    signatureReqs = mapIt(v.clients, it[0].signData(it[1].pubkey, request))
     deadline = sleepAsync(WEB3_SIGNER_DELAY_TOLERANCE)
+    signatureReqs = mapIt(v.clients, it[0].signData(it[1].pubkey, deadline,
+                                                    2, request))
 
-  await allFutures(signatureReqs) or deadline
+  await allFutures(signatureReqs)
+
+  if not(deadline.finished()): await cancelAndWait(deadline)
 
   var shares: seq[SignatureShare]
   var neededShares = v.data.threshold
@@ -404,7 +408,9 @@ proc signWithDistributedKey(v: AttachedValidator,
     else:
       warn "Failed to obtain signature from remote signer",
            pubkey = shareInfo.pubkey,
-           signerUrl = $(v.clients[i][0].address)
+           signerUrl = $(v.clients[i][0].address),
+           reason = req.read.error.message,
+           kind = req.read.error.kind
 
     if neededShares == 0:
       let recovered = shares.recoverSignature()
@@ -413,17 +419,19 @@ proc signWithDistributedKey(v: AttachedValidator,
   return SignatureResult.err "Not enough shares to recover the signature"
 
 proc signWithSingleKey(v: AttachedValidator,
-                       request: Web3SignerRequest): Future[SignatureResult]
+                       request: Web3SignerRequest): Future[SignatureResult] {.
-                      {.async.} =
+     async.} =
   doAssert v.clients.len == 1
-  let (client, info) = v.clients[0]
+  let
-  let res = awaitWithTimeout(client.signData(info.pubkey, request),
+    deadline = sleepAsync(WEB3_SIGNER_DELAY_TOLERANCE)
-                             WEB3_SIGNER_DELAY_TOLERANCE):
+    (client, info) = v.clients[0]
-    return SignatureResult.err "Timeout"
+    res = await client.signData(info.pubkey, deadline, 2, request)
-  if res.isErr:
+
-    return SignatureResult.err res.error
+  if not(deadline.finished()): await cancelAndWait(deadline)
+  if res.isErr():
+    return SignatureResult.err(res.error.message)
   else:
-    return SignatureResult.ok res.get.toValidatorSig
+    return SignatureResult.ok(res.get().toValidatorSig())
 
 proc signData(v: AttachedValidator,
               request: Web3SignerRequest): Future[SignatureResult] =
@@ -435,6 +443,55 @@ proc signData(v: AttachedValidator,
   else:
     v.signWithDistributedKey(request)
 
+proc getFeeRecipientProof(blck: ForkedBeaconBlock | ForkedBlindedBeaconBlock |
+                                bellatrix_mev.BlindedBeaconBlock |
+                                capella_mev.BlindedBeaconBlock
+                         ): Result[Web3SignerMerkleProof, string] =
+  when blck is ForkedBlindedBeaconBlock:
+    case blck.kind
+    of ConsensusFork.Phase0:
+      err("Invalid block fork: phase0")
+    of ConsensusFork.Altair:
+      err("Invalid block fork: altair")
+    of ConsensusFork.Bellatrix:
+      const FeeRecipientIndex = GeneralizedIndex(401)
+      let res = ? build_proof(blck.bellatrixData.body, FeeRecipientIndex)
+      ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+    of ConsensusFork.Capella:
+      const FeeRecipientIndex = GeneralizedIndex(401)
+      let res = ? build_proof(blck.capellaData.body, FeeRecipientIndex)
+      ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+    of ConsensusFork.Deneb:
+      const FeeRecipientIndex = GeneralizedIndex(401)
+      let res = ? build_proof(blck.denebData.body, FeeRecipientIndex)
+      ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+  elif blck is bellatrix_mev.BlindedBeaconBlock:
+    const FeeRecipientIndex = GeneralizedIndex(401)
+    let res = ? build_proof(blck.body, FeeRecipientIndex)
+    ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+  elif blck is capella_mev.BlindedBeaconBlock:
+    const FeeRecipientIndex = GeneralizedIndex(401)
+    let res = ? build_proof(blck.body, FeeRecipientIndex)
+    ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+  else:
+    case blck.kind
+    of ConsensusFork.Phase0:
+      err("Invalid block fork: phase0")
+    of ConsensusFork.Altair:
+      err("Invalid block fork: altair")
+    of ConsensusFork.Bellatrix:
+      const FeeRecipientIndex = GeneralizedIndex(401)
+      let res = ? build_proof(blck.bellatrixData.body, FeeRecipientIndex)
+      ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+    of ConsensusFork.Capella:
+      const FeeRecipientIndex = GeneralizedIndex(401)
+      let res = ? build_proof(blck.capellaData.body, FeeRecipientIndex)
+      ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+    of ConsensusFork.Deneb:
+      const FeeRecipientIndex = GeneralizedIndex(401)
+      let res = ? build_proof(blck.denebData.body, FeeRecipientIndex)
+      ok(Web3SignerMerkleProof(index: FeeRecipientIndex, merkleProofs: @res))
+
 # https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.5/specs/phase0/validator.md#signature
 proc getBlockSignature*(v: AttachedValidator, fork: Fork,
                         genesis_validators_root: Eth2Digest, slot: Slot,
@@ -451,76 +508,151 @@ proc getBlockSignature*(v: AttachedValidator, fork: Fork,
           fork, genesis_validators_root, slot, block_root,
           v.data.privateKey).toValidatorSig())
     of ValidatorKind.Remote:
-      when blck is ForkedBlindedBeaconBlock:
+      let web3SignerRequest =
-        let
+        when blck is ForkedBlindedBeaconBlock:
-          web3SignerBlock =
+          case blck.kind
-            case blck.kind
+          of ConsensusFork.Phase0:
-            of ConsensusFork.Phase0:
+            case v.data.remoteType
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3Signer:
-                kind: ConsensusFork.Phase0,
+              Web3SignerRequest.init(fork, genesis_validators_root,
-                phase0Data: blck.phase0Data)
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Phase0,
-            of ConsensusFork.Altair:
+                  phase0Data: blck.phase0Data))
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3SignerDiva:
-                kind: ConsensusFork.Altair,
+              return SignatureResult.err("Invalid beacon block fork version")
-                altairData: blck.altairData)
+          of ConsensusFork.Altair:
-            of ConsensusFork.Bellatrix:
+            case v.data.remoteType
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3Signer:
-                kind: ConsensusFork.Bellatrix,
+              Web3SignerRequest.init(fork, genesis_validators_root,
-                bellatrixData: blck.bellatrixData.toBeaconBlockHeader)
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Altair,
-            of ConsensusFork.Capella:
+                  altairData: blck.altairData))
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3SignerDiva:
-                kind: ConsensusFork.Capella,
+              return SignatureResult.err("Invalid beacon block fork version")
-                capellaData: blck.capellaData.toBeaconBlockHeader)
+          of ConsensusFork.Bellatrix:
-            of ConsensusFork.Deneb:
+            case v.data.remoteType
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3Signer:
-                kind: ConsensusFork.Deneb,
+              Web3SignerRequest.init(fork, genesis_validators_root,
-                denebData: blck.denebData.toBeaconBlockHeader)
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Bellatrix,
-
+                  bellatrixData: blck.bellatrixData.toBeaconBlockHeader))
-          request = Web3SignerRequest.init(
+            of RemoteSignerType.Web3SignerDiva:
-            fork, genesis_validators_root, web3SignerBlock)
+              let res = getFeeRecipientProof(blck)
-        await v.signData(request)
+              if res.isErr(): return SignatureResult.err(res.error)
-      elif blck is bellatrix_mev.BlindedBeaconBlock:
+              Web3SignerRequest.init(fork, genesis_validators_root,
-        let request = Web3SignerRequest.init(
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Bellatrix,
-          fork, genesis_validators_root,
+                  bellatrixData: blck.bellatrixData.toBeaconBlockHeader),
-          Web3SignerForkedBeaconBlock(
+                [res.get()])
-            kind: ConsensusFork.Bellatrix,
+          of ConsensusFork.Capella:
-            bellatrixData: blck.toBeaconBlockHeader))
+            case v.data.remoteType
-        await v.signData(request)
+            of RemoteSignerType.Web3Signer:
-      elif blck is capella_mev.BlindedBeaconBlock:
+              Web3SignerRequest.init(fork, genesis_validators_root,
-        let request = Web3SignerRequest.init(
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Capella,
-          fork, genesis_validators_root,
+                  capellaData: blck.capellaData.toBeaconBlockHeader))
-          Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3SignerDiva:
-            kind: ConsensusFork.Capella,
+              let res = getFeeRecipientProof(blck)
-            capellaData: blck.toBeaconBlockHeader))
+              if res.isErr(): return SignatureResult.err(res.error)
-        await v.signData(request)
+              Web3SignerRequest.init(fork, genesis_validators_root,
-      else:
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Capella,
-        let
+                  capellaData: blck.capellaData.toBeaconBlockHeader),
-          web3SignerBlock =
+                [res.get()])
-            case blck.kind
+          of ConsensusFork.Deneb:
-            of ConsensusFork.Phase0:
+            case v.data.remoteType
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3Signer:
-                kind: ConsensusFork.Phase0,
+              Web3SignerRequest.init(fork, genesis_validators_root,
-                phase0Data: blck.phase0Data)
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Deneb,
-            of ConsensusFork.Altair:
+                  denebData: blck.denebData.toBeaconBlockHeader))
-              Web3SignerForkedBeaconBlock(
+            of RemoteSignerType.Web3SignerDiva:
-                kind: ConsensusFork.Altair,
+              let res = getFeeRecipientProof(blck)
-                altairData: blck.altairData)
+              if res.isErr(): return SignatureResult.err(res.error)
-            of ConsensusFork.Bellatrix:
+              Web3SignerRequest.init(fork, genesis_validators_root,
-              Web3SignerForkedBeaconBlock(
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Deneb,
-                kind: ConsensusFork.Bellatrix,
+                  denebData: blck.denebData.toBeaconBlockHeader),
-                bellatrixData: blck.bellatrixData.toBeaconBlockHeader)
+                [res.get()])
-            of ConsensusFork.Capella:
+        elif blck is bellatrix_mev.BlindedBeaconBlock:
-              Web3SignerForkedBeaconBlock(
+          case v.data.remoteType
-                kind: ConsensusFork.Capella,
+          of RemoteSignerType.Web3Signer:
-                capellaData: blck.capellaData.toBeaconBlockHeader)
+            Web3SignerRequest.init(fork, genesis_validators_root,
-            of ConsensusFork.Deneb:
+              Web3SignerForkedBeaconBlock(kind: ConsensusFork.Bellatrix,
-              Web3SignerForkedBeaconBlock(
+                bellatrixData: blck.toBeaconBlockHeader)
-                kind: ConsensusFork.Deneb,
+            )
-                denebData: blck.denebData.toBeaconBlockHeader)
+          of RemoteSignerType.Web3SignerDiva:
-
+            let res = getFeeRecipientProof(blck)
-          request = Web3SignerRequest.init(
+            if res.isErr(): return SignatureResult.err(res.error)
-            fork, genesis_validators_root, web3SignerBlock)
+            Web3SignerRequest.init(fork, genesis_validators_root,
-        await v.signData(request)
+              Web3SignerForkedBeaconBlock(kind: ConsensusFork.Bellatrix,
+                bellatrixData: blck.toBeaconBlockHeader),
+              [res.get()])
+        elif blck is capella_mev.BlindedBeaconBlock:
+          case v.data.remoteType
+          of RemoteSignerType.Web3Signer:
+            Web3SignerRequest.init(fork, genesis_validators_root,
+              Web3SignerForkedBeaconBlock(kind: ConsensusFork.Capella,
+                capellaData: blck.toBeaconBlockHeader))
+          of RemoteSignerType.Web3SignerDiva:
+            let res = getFeeRecipientProof(blck)
+            if res.isErr(): return SignatureResult.err(res.error)
+            Web3SignerRequest.init(fork, genesis_validators_root,
+              Web3SignerForkedBeaconBlock(kind: ConsensusFork.Capella,
+                capellaData: blck.toBeaconBlockHeader),
+              [res.get()])
+        else:
+          case blck.kind
+          of ConsensusFork.Phase0:
+            # In case of `phase0` block we did not send merkle proof.
+            case v.data.remoteType
+            of RemoteSignerType.Web3Signer:
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Phase0,
+                                            phase0Data: blck.phase0Data))
+            of RemoteSignerType.Web3SignerDiva:
+              return SignatureResult.err("Invalid beacon block fork version")
+          of ConsensusFork.Altair:
+            # In case of `altair` block we did not send merkle proof.
+            case v.data.remoteType
+            of RemoteSignerType.Web3Signer:
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Altair,
+                                            altairData: blck.altairData))
+            of RemoteSignerType.Web3SignerDiva:
+              return SignatureResult.err("Invalid beacon block fork version")
+          of ConsensusFork.Bellatrix:
+            case v.data.remoteType
+            of RemoteSignerType.Web3Signer:
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Bellatrix,
+                  bellatrixData: blck.bellatrixData.toBeaconBlockHeader))
+            of RemoteSignerType.Web3SignerDiva:
+              let res = getFeeRecipientProof(blck)
+              if res.isErr(): return SignatureResult.err(res.error)
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Bellatrix,
+                  bellatrixData: blck.bellatrixData.toBeaconBlockHeader),
+                [res.get()])
+          of ConsensusFork.Capella:
+            case v.data.remoteType
+            of RemoteSignerType.Web3Signer:
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Capella,
+                  capellaData: blck.capellaData.toBeaconBlockHeader))
+            of RemoteSignerType.Web3SignerDiva:
+              let res = getFeeRecipientProof(blck)
+              if res.isErr(): return SignatureResult.err(res.error)
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Capella,
+                  capellaData: blck.capellaData.toBeaconBlockHeader),
+                [res.get()])
+          of ConsensusFork.Deneb:
+            case v.data.remoteType
+            of RemoteSignerType.Web3Signer:
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Deneb,
+                  denebData: blck.denebData.toBeaconBlockHeader))
+            of RemoteSignerType.Web3SignerDiva:
+              let res = getFeeRecipientProof(blck)
+              if res.isErr(): return SignatureResult.err(res.error)
+              Web3SignerRequest.init(fork, genesis_validators_root,
+                Web3SignerForkedBeaconBlock(kind: ConsensusFork.Deneb,
+                  denebData: blck.denebData.toBeaconBlockHeader),
+                [res.get()])
+      await v.signData(web3SignerRequest)
 
 # https://github.com/ethereum/consensus-specs/blob/v1.3.0-rc.5/specs/phase0/validator.md#aggregate-signature
 proc getAttestationSignature*(v: AttachedValidator, fork: Fork,
@@ -662,6 +794,34 @@ proc getSlotSignature*(v: AttachedValidator, fork: Fork,
   v.slotSignature = Opt.some((slot, signature.get))
   return signature
 
+proc getValidatorExitSignature*(v: AttachedValidator, fork: Fork,
+                                genesis_validators_root: Eth2Digest,
+                                voluntary_exit: VoluntaryExit
+                               ): Future[SignatureResult] {.async.} =
+  return
+    case v.kind
+    of ValidatorKind.Local:
+      SignatureResult.ok(get_voluntary_exit_signature(
+        fork, genesis_validators_root, voluntary_exit,
+        v.data.privateKey).toValidatorSig())
+    of ValidatorKind.Remote:
+      let request = Web3SignerRequest.init(fork, genesis_validators_root,
+                                           voluntary_exit)
+      await v.signData(request)
+
+proc getDepositMessageSignature*(v: AttachedValidator, version: Version,
+                                 deposit_message: DepositMessage
+                                ): Future[SignatureResult] {.async.} =
+  return
+    case v.kind
+    of ValidatorKind.Local:
+      SignatureResult.ok(get_deposit_signature(
+        deposit_message, version,
+        v.data.privateKey).toValidatorSig())
+    of ValidatorKind.Remote:
+      let request = Web3SignerRequest.init(version, deposit_message)
+      await v.signData(request)
+
 # https://github.com/ethereum/builder-specs/blob/v0.3.0/specs/bellatrix/builder.md#signing
 proc getBuilderSignature*(v: AttachedValidator, fork: Fork,
     validatorRegistration: ValidatorRegistrationV1):

tests/all_tests.nim

@@ -45,6 +45,7 @@ import # Unit test
   ./test_sync_manager,
   ./test_validator_pool,
   ./test_zero_signature,
+  ./test_signing_node,
   ./fork_choice/tests_fork_choice,
   ./consensus_spec/all_tests as consensus_all_tests,
   ./slashing_protection/test_fixtures,

vendor/nim-presto

@@ -1 +1 @@
-Subproject commit ba20cebf0f729b52d90487f34efb5923dfc7b2c7
+Subproject commit 2b440a443f3fc29197f267879e16bb8057ccc0ed