VC: Use SSZ encoding while processing blocks data (#4999)

* Refactor api.nim to provide more informative failure reasons.
Distinct between unexpected data and unexpected code.
Deprecate Option[T] usage.

* Fix 400 for produceBlindedBlock().
Get proper string conversion for strategy.

* Fix SSZ encoded versions of ProduceBlockResponseV2, ProduceBlockResponseV2 can be received and decoded.
Fix done() warnings.
Bump presto.

* Fix compilation error with new presto.
Use TcpNoDelay option for Web3Signer.

* Fix produceBlockV2() should provide SSZ responses too.

* Address block encoding issue.

* Fix signing test.

* Bump presto.

* Address review comments.

beacon_chain/rpc/rest_validator_api.nim

@@ -325,6 +325,9 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) =
       slot: Slot, randao_reveal: Option[ValidatorSig],
       graffiti: Option[GraffitiBytes],
       skip_randao_verification: Option[string]) -> RestApiResponse:
+    let
+      contentType = preferredContentType(jsonMediaType, sszMediaType).valueOr:
+        return RestApiResponse.jsonError(Http406, ContentNotAcceptableError)
     let message =
       block:
         let qslot = block:
@@ -348,8 +351,8 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) =
           else:
             let res = skip_randao_verification.get()
             if res.isErr() or res.get() != "":
-              return RestApiResponse.jsonError(Http400,
+              return RestApiResponse.jsonError(
-                                               InvalidSkipRandaoVerificationValue)
+                Http400, InvalidSkipRandaoVerificationValue)
             true
         let qrandao =
           if randao_reveal.isNone():
@@ -416,7 +419,16 @@ proc installValidatorApiHandlers*(router: var RestRouter, node: BeaconNode) =
         if res.isErr():
           return RestApiResponse.jsonError(Http400, res.error())
         res.get.blck
-    return RestApiResponse.jsonResponsePlain(message)
+    return
+      if contentType == sszMediaType:
+        let headers = [("eth-consensus-version", message.kind.toString())]
+        withBlck(message):
+          RestApiResponse.sszResponse(blck, headers)
+      elif contentType == jsonMediaType:
+        withBlck(message):
+          RestApiResponse.jsonResponseWVersion(blck, message.kind)
+      else:
+        raiseAssert "preferredContentType() returns invalid content type"
 
   # https://ethereum.github.io/beacon-APIs/#/Validator/produceBlindedBlock
   # https://github.com/ethereum/beacon-APIs/blob/v2.4.0/apis/validator/blinded_block.yaml

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -140,8 +140,6 @@ type
     KeystoresAndSlashingProtection |
     ListFeeRecipientResponse |
     PrepareBeaconProposer |
-    ProduceBlockResponseV2 |
-    ProduceBlindedBlockResponse |
     RestIndexedErrorMessage |
     RestErrorMessage |
     RestValidator |
@@ -154,11 +152,19 @@ type
     SomeForkedLightClientObject |
     seq[SomeForkedLightClientObject]
 
+  DecodeConsensysTypes* =
+    ProduceBlockResponseV2 | ProduceBlindedBlockResponse
+
   RestVersioned*[T] = object
     data*: T
     jsonVersion*: ConsensusFork
     sszContext*: ForkDigest
 
+  RestBlockTypes* = phase0.BeaconBlock | altair.BeaconBlock |
+                    bellatrix.BeaconBlock | capella.BeaconBlock |
+                    deneb.BeaconBlock | bellatrix_mev.BlindedBeaconBlock |
+                    capella_mev.BlindedBeaconBlock
+
 {.push raises: [].}
 
 proc prepareJsonResponse*(t: typedesc[RestApiResponse], d: auto): seq[byte] =
@@ -2990,6 +2996,87 @@ proc encodeBytes*[T: EncodeOctetTypes](
   else:
     err("Content-Type not supported")
 
+func readSszResBytes(T: typedesc[RestBlockTypes],
+                     data: openArray[byte]): RestResult[T] =
+  var res: T
+  try:
+    readSszBytes(data, res)
+    ok(res)
+  except MalformedSszError as exc:
+    err("Invalid SSZ object")
+  except SszSizeMismatchError:
+    err("Incorrect SSZ object's size")
+
+proc decodeBytes*[T: DecodeConsensysTypes](
+       t: typedesc[T],
+       value: openArray[byte],
+       contentType: Opt[ContentTypeData],
+       consensusVersion: string
+     ): RestResult[T] =
+  let mediaType =
+    if contentType.isNone() or
+       isWildCard(contentType.get().mediaType):
+      return err("Invalid/missing Content-Type value")
+    else:
+      contentType.get().mediaType
+
+  if mediaType == ApplicationJsonMediaType:
+    try:
+      ok(RestJson.decode(value, T,
+                         requireAllFields = true,
+                         allowUnknownFields = true))
+    except SerializationError as exc:
+      debug "Failed to deserialize REST JSON data",
+            err = exc.formatMsg("<data>"),
+            data = string.fromBytes(value)
+      return err("Serialization error")
+  elif mediaType == OctetStreamMediaType:
+    when t is ProduceBlockResponseV2:
+      let fork = decodeEthConsensusVersion(consensusVersion).valueOr:
+        return err("Invalid or Unsupported consensus version")
+      case fork
+      of ConsensusFork.Deneb:
+        let blck = ? readSszResBytes(deneb.BeaconBlock, value)
+        ok(ProduceBlockResponseV2(ForkedBeaconBlock.init(blck)))
+      of ConsensusFork.Capella:
+        let blck = ? readSszResBytes(capella.BeaconBlock, value)
+        ok(ProduceBlockResponseV2(ForkedBeaconBlock.init(blck)))
+      of ConsensusFork.Bellatrix:
+        let blck = ? readSszResBytes(bellatrix.BeaconBlock, value)
+        ok(ProduceBlockResponseV2(ForkedBeaconBlock.init(blck)))
+      of ConsensusFork.Altair:
+        let blck = ? readSszResBytes(altair.BeaconBlock, value)
+        ok(ProduceBlockResponseV2(ForkedBeaconBlock.init(blck)))
+      of ConsensusFork.Phase0:
+        let blck = ? readSszResBytes(phase0.BeaconBlock, value)
+        ok(ProduceBlockResponseV2(ForkedBeaconBlock.init(blck)))
+    elif t is ProduceBlindedBlockResponse:
+      let fork = decodeEthConsensusVersion(consensusVersion).valueOr:
+        return err("Invalid or Unsupported consensus version")
+      case fork
+      of ConsensusFork.Deneb:
+        let
+          blck = ? readSszResBytes(capella_mev.BlindedBeaconBlock, value)
+          forked = ForkedBlindedBeaconBlock(
+            kind: ConsensusFork.Deneb, denebData: blck)
+        ok(ProduceBlindedBlockResponse(forked))
+      of ConsensusFork.Capella:
+        let
+          blck = ? readSszResBytes(capella_mev.BlindedBeaconBlock, value)
+          forked = ForkedBlindedBeaconBlock(
+            kind: ConsensusFork.Capella, capellaData: blck)
+        ok(ProduceBlindedBlockResponse(forked))
+      of ConsensusFork.Bellatrix:
+        let
+          blck = ? readSszResBytes(bellatrix_mev.BlindedBeaconBlock, value)
+          forked = ForkedBlindedBeaconBlock(
+            kind: ConsensusFork.Bellatrix, bellatrixData: blck)
+        ok(ProduceBlindedBlockResponse(forked))
+      of ConsensusFork.Altair, ConsensusFork.Phase0:
+        err("Unable to decode blinded block for Altair and Phase0 fork")
+  else:
+    err("Unsupported Content-Type")
+
 proc decodeBytes*[T: DecodeTypes](
        t: typedesc[T],
        value: openArray[byte],

beacon_chain/spec/eth2_apis/rest_validator_calls.nim

@@ -58,40 +58,22 @@ proc getSyncCommitteeDutiesPlain*(
      meth: MethodPost.}
   ## https://ethereum.github.io/beacon-APIs/#/Validator/getSyncCommitteeDuties
 
-proc produceBlockV2*(
-       slot: Slot,
-       randao_reveal: ValidatorSig,
-       graffiti: GraffitiBytes
-     ): RestResponse[ProduceBlockResponseV2] {.
-     rest, endpoint: "/eth/v2/validator/blocks/{slot}",
-     meth: MethodGet.}
-  ## https://ethereum.github.io/beacon-APIs/#/Validator/produceBlockV2
-
 proc produceBlockV2Plain*(
        slot: Slot,
        randao_reveal: ValidatorSig,
        graffiti: GraffitiBytes
      ): RestPlainResponse {.
      rest, endpoint: "/eth/v2/validator/blocks/{slot}",
-     meth: MethodGet.}
+     accept: preferSSZ, meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Validator/produceBlockV2
 
-proc produceBlindedBlock*(
-       slot: Slot,
-       randao_reveal: ValidatorSig,
-       graffiti: GraffitiBytes
-     ): RestResponse[ProduceBlindedBlockResponse] {.
-     rest, endpoint: "/eth/v1/validator/blinded_blocks/{slot}",
-     meth: MethodGet.}
-  ## https://ethereum.github.io/beacon-APIs/#/Validator/produceBlindedBlock
-
 proc produceBlindedBlockPlain*(
        slot: Slot,
        randao_reveal: ValidatorSig,
        graffiti: GraffitiBytes
      ): RestPlainResponse {.
      rest, endpoint: "/eth/v1/validator/blinded_blocks/{slot}",
-     meth: MethodGet.}
+     accept: preferSSZ, meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Validator/produceBlindedBlock
 
 proc produceAttestationData*(

beacon_chain/validator_client/api.nim

@@ -1777,8 +1777,10 @@ proc produceBlockV2*(
         let response = apiResponse.get()
         case response.status:
         of 200:
-          let res = decodeBytes(ProduceBlockResponseV2, response.data,
+          let
-                                response.contentType)
+            version = response.headers.getString("eth-consensus-version")
+            res = decodeBytes(ProduceBlockResponseV2, response.data,
+                              response.contentType, version)
           if res.isErr():
             handleUnexpectedData()
             ApiResponse[ProduceBlockResponseV2].err($res.error)
@@ -1815,8 +1817,10 @@ proc produceBlockV2*(
         let response = apiResponse.get()
         case response.status:
         of 200:
-          let res = decodeBytes(ProduceBlockResponseV2, response.data,
+          let
-                                response.contentType)
+            version = response.headers.getString("eth-consensus-version")
+            res = decodeBytes(ProduceBlockResponseV2, response.data,
+                              response.contentType, version)
           if res.isOk(): return res.get()
           handleUnexpectedData()
           false
@@ -1976,8 +1980,10 @@ proc produceBlindedBlock*(
         let response = apiResponse.get()
         case response.status:
         of 200:
-          let res = decodeBytes(ProduceBlindedBlockResponse, response.data,
+          let
-                                response.contentType)
+            version = response.headers.getString("eth-consensus-version")
+            res = decodeBytes(ProduceBlindedBlockResponse, response.data,
+                              response.contentType, version)
           if res.isErr():
             handleUnexpectedData()
             ApiResponse[ProduceBlindedBlockResponse].err($res.error)
@@ -2019,8 +2025,10 @@ proc produceBlindedBlock*(
         let response = apiResponse.get()
         case response.status:
         of 200:
-          let res = decodeBytes(ProduceBlindedBlockResponse, response.data,
+          let
-                                response.contentType)
+            version = response.headers.getString("eth-consensus-version")
+            res = decodeBytes(ProduceBlindedBlockResponse, response.data,
+                              response.contentType, version)
           if res.isOk(): return res.get()
           handleUnexpectedData()
           false

beacon_chain/validator_client/sync_committee_service.nim

@@ -246,7 +246,7 @@ proc produceAndPublishContributions(service: SyncCommitteeServiceRef,
   let validatorContributions = block:
     var res: seq[ContributionItem]
     for idx, fut in slotSignatureReqs:
-      if fut.completed:
+      if fut.completed():
         let
           sigRes = fut.read
           validator = validators[idx][0]

beacon_chain/validators/validator_pool.nim

@@ -27,12 +27,6 @@ export
 
 const
   WEB3_SIGNER_DELAY_TOLERANCE = 3.seconds
-  WEB3_SIGNER_DEFAULT_TIMEOUT = (int64(SECONDS_PER_SLOT) + 1).seconds
-    # This timeout value should not be greater than default value specified at:
-    # https://docs.web3signer.consensys.net/Reference/CLI/CLI-Syntax#idle-connection-timeout-seconds
-  WEB3_SIGNER_CHECKING_PERIOD = ((int64(SECONDS_PER_SLOT) + 1) div 3).seconds
-    # Host often connection pool will collect/destroy connections which are
-    # expired.
 
 declareGauge validators,
   "Number of validators attached to the beacon node"
@@ -174,15 +168,13 @@ proc addRemoteValidator(pool: var ValidatorPool,
       else:
         {}
     prestoFlags = {RestClientFlag.CommaSeparatedArray}
+    socketFlags = {SocketFlags.TcpNoDelay}
     clients =
       block:
         var res: seq[(RestClientRef, RemoteSignerInfo)]
         for remote in keystore.remotes:
           let client = RestClientRef.new(
-            $remote.url, prestoFlags, httpFlags,
+            $remote.url, prestoFlags, httpFlags, socketFlags = socketFlags)
-            idleTimeout = WEB3_SIGNER_DEFAULT_TIMEOUT,
-            idlePeriod = WEB3_SIGNER_CHECKING_PERIOD,
-          )
           if client.isErr():
             # TODO keep trying in case of temporary network failure
             warn "Unable to resolve distributed signer address",
@@ -396,7 +388,7 @@ proc signWithDistributedKey(v: AttachedValidator,
 
   for i, req in signatureReqs:
     template shareInfo: untyped = v.clients[i][1]
-    if req.completed and req.read.isOk:
+    if req.completed() and req.read.isOk:
       shares.add req.read.get.toSignatureShare(shareInfo.id)
       neededShares = neededShares - 1
     else:

tests/test_signing_node.nim

@@ -94,7 +94,8 @@ proc getBlock(fork: ConsensusFork,
 
   decodeBytes(ProduceBlockResponseV2,
               blckData.toOpenArrayByte(0, len(blckData) - 1),
-              Opt.some(contentType)).tryGet()
+              Opt.some(contentType),
+              $fork).tryGet()
 
 proc init(t: typedesc[Web3SignerForkedBeaconBlock],
           forked: ForkedBeaconBlock): Web3SignerForkedBeaconBlock =

vendor/nim-presto

@@ -1 +1 @@
-Subproject commit 2b440a443f3fc29197f267879e16bb8057ccc0ed
+Subproject commit 35652ed19ccbbf042e95941bc2f8bab39e3f6030