Client SSZ API revisited. (#2907)

* Initial commit.

* Add SSZ getBlock().

* Automatic types detection for SSZ encoded objects.

* Change SSZ.decode() to readSszBytes().

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -9,13 +9,14 @@ import stew/[results, base10, byteutils, endians2], presto/common,
        libp2p/peerid, serialization,
        json_serialization, json_serialization/std/[options, net],
        nimcrypto/utils as ncrutils
-import ".."/forks, ".."/datatypes/[phase0, altair, merge],
-       ".."/".."/ssz/ssz_serialization,
+import ".."/[forks, ssz_codec], ".."/datatypes/[phase0, altair, merge],
+       ".."/eth2_ssz_serialization,
+       ".."/".."/ssz/[ssz_serialization, codec, types],
        "."/rest_types
 
 export
   results, peerid, common, serialization, json_serialization, options, net,
-  rest_types
+  rest_types, ssz_codec, ssz_serialization, codec, types
 
 Json.createFlavor RestJson
 
@@ -59,12 +60,21 @@ type
 
   DecodeTypes* =
     DataEnclosedObject |
-    GetBlockV2Response |
     ProduceBlockResponseV2 |
     DataMetaEnclosedObject |
     DataRootEnclosedObject |
     RestAttestationError |
-    RestGenericError
+    RestGenericError |
+    GetBlockV2Response |
+    GetStateV2Response
+
+  SszDecodeTypes* =
+    GetPhase0StateSszResponse |
+    GetAltairStateSszResponse |
+    GetPhase0BlockSszResponse |
+    GetAltairBlockSszResponse |
+    GetBlockV2Header |
+    GetStateV2Header
 
 {.push raises: [Defect].}
 
@@ -858,16 +868,6 @@ proc writeValue*(writer: var JsonWriter[RestJson], value: ForkedBeaconState) {.
       writer.writeField("data", value.bsMerge)
   writer.endRecord()
 
-template toSszType*(v: BeaconBlockFork): auto =
-  case v
-  of BeaconBlockFork.Phase0: Phase0Version
-  of BeaconBlockFork.Altair: AltairVersion
-
-template toSszType*(v: BeaconStateFork): auto =
-  case v
-  of BeaconStateFork.forkPhase0: Phase0Version
-  of BeaconStateFork.forkAltair: AltairVersion
-
 # SyncCommitteeIndex
 proc writeValue*(writer: var JsonWriter[RestJson],
                  value: SyncCommitteeIndex) {.
@@ -965,6 +965,19 @@ proc decodeBytes*[T: DecodeTypes](t: typedesc[T], value: openarray[byte],
   else:
     err("Content-Type not supported")
 
+proc decodeBytes*[T: SszDecodeTypes](t: typedesc[T], value: openarray[byte],
+                                     contentType: string): RestResult[T] =
+  case contentType
+  of "application/octet-stream":
+    try:
+      var v: T
+      readSszBytes(value, v)
+      ok(v)
+    except SerializationError as exc:
+      err("Serialization error")
+  else:
+    err("Content-Type not supported")
+
 proc encodeString*(value: string): RestResult[string] =
   ok(value)
 

beacon_chain/spec/eth2_apis/rest_beacon_calls.nim

@@ -7,8 +7,9 @@
 {.push raises: [Defect].}
 
 import
-  chronos, presto/client,
-  ../datatypes/[phase0, altair, merge],
+  chronos, presto/client, chronicles,
+  ".."/[helpers, forks], ".."/datatypes/[phase0, altair, merge],
+  ".."/eth2_ssz_serialization,
   "."/[rest_types, eth2_rest_serialization]
 
 export chronos, client, rest_types, eth2_rest_serialization
@@ -88,16 +89,129 @@ proc publishBlock*(body: altair.SignedBeaconBlock): RestPlainResponse {.
      meth: MethodPost.}
   ## https://ethereum.github.io/beacon-APIs/#/Beacon/publishBlock
 
-proc getBlock*(block_id: BlockIdent): RestResponse[GetBlockResponse] {.
+proc getBlockPlain*(block_id: BlockIdent): RestPlainResponse {.
      rest, endpoint: "/api/eth/v1/beacon/blocks/{block_id}",
+     accept: "application/octet-stream,application-json;q=0.9",
      meth: MethodGet.}
-  ## https://ethereum.github.io/beacon-APIs/#/Beacon/getBlock
 
-proc getBlockV2*(block_id: BlockIdent): RestResponse[GetBlockV2Response] {.
+proc getBlock*(client: RestClientRef, block_id: BlockIdent,
+               restAccept = ""): Future[ForkedSignedBeaconBlock] {.async.} =
+  let resp =
+    if len(restAccept) > 0:
+      await client.getBlockPlain(block_id, restAcceptType = restAccept)
+    else:
+      await client.getBlockPlain(block_id)
+  let data =
+    case resp.status
+    of 200:
+      case resp.contentType
+      of "application/json":
+        let blck =
+          block:
+            let res = decodeBytes(GetBlockResponse, resp.data,
+                                  resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        ForkedSignedBeaconBlock.init(blck.data)
+      of "application/octet-stream":
+        let blck =
+          block:
+            let res = decodeBytes(GetPhase0BlockSszResponse, resp.data,
+                                  resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        ForkedSignedBeaconBlock.init(blck)
+      else:
+        raise newException(RestError, "Unsupported content-type")
+    of 400, 404, 500:
+      let error =
+        block:
+          let res = decodeBytes(RestGenericError, resp.data, resp.contentType)
+          if res.isErr():
+            let msg = "Incorrect response error format (" & $resp.status &
+                      ") [" & $res.error() & "]"
+            raise newException(RestError, msg)
+          res.get()
+      let msg = "Error response (" & $resp.status & ") [" & error.message & "]"
+      raise newException(RestError, msg)
+    else:
+      let msg = "Unknown response status error (" & $resp.status & ")"
+      raise newException(RestError, msg)
+  return data
+
+proc getBlockV2Plain*(block_id: BlockIdent): RestPlainResponse {.
      rest, endpoint: "/api/eth/v2/beacon/blocks/{block_id}",
+     accept: "application/octet-stream,application-json;q=0.9",
      meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Beacon/getBlockV2
 
+proc getBlockV2*(client: RestClientRef, block_id: BlockIdent,
+                 forks: array[2, Fork],
+                 restAccept = ""): Future[ForkedSignedBeaconBlock] {.
+     async.} =
+  let resp =
+    if len(restAccept) > 0:
+      await client.getBlockV2Plain(block_id, restAcceptType = restAccept)
+    else:
+      await client.getBlockV2Plain(block_id)
+  let data =
+    case resp.status
+    of 200:
+      case resp.contentType
+      of "application/json":
+        let blck =
+          block:
+            let res = decodeBytes(GetBlockV2Response, resp.data,
+                                  resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        blck
+      of "application/octet-stream":
+        let header =
+          block:
+            let res = decodeBytes(GetBlockV2Header, resp.data, resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        if header.slot.epoch() < forks[1].epoch:
+          let blck =
+            block:
+              let res = decodeBytes(GetPhase0BlockSszResponse, resp.data,
+                                    resp.contentType)
+              if res.isErr():
+                raise newException(RestError, $res.error())
+              res.get()
+          ForkedSignedBeaconBlock.init(blck)
+        else:
+          let blck =
+            block:
+              let res = decodeBytes(GetAltairBlockSszResponse, resp.data,
+                                    resp.contentType)
+              if res.isErr():
+                raise newException(RestError, $res.error())
+              res.get()
+          ForkedSignedBeaconBlock.init(blck)
+      else:
+        raise newException(RestError, "Unsupported content-type")
+    of 400, 404, 500:
+      let error =
+        block:
+          let res = decodeBytes(RestGenericError, resp.data, resp.contentType)
+          if res.isErr():
+            let msg = "Incorrect response error format (" & $resp.status &
+                      ") [" & $res.error() & "]"
+            raise newException(RestError, msg)
+          res.get()
+      let msg = "Error response (" & $resp.status & ") [" & error.message & "]"
+      raise newException(RestError, msg)
+    else:
+      let msg = "Unknown response status error (" & $resp.status & ")"
+      raise newException(RestError, msg)
+  return data
+
 proc getBlockRoot*(block_id: BlockIdent): RestResponse[GetBlockRootResponse] {.
      rest, endpoint: "/eth/v1/beacon/blocks/{block_id}/root",
      meth: MethodGet.}

beacon_chain/spec/eth2_apis/rest_debug_calls.nim

@@ -8,22 +8,135 @@
 
 import
   chronos, presto/client,
+  ".."/[helpers, forks], ".."/datatypes/[phase0, altair, merge],
   "."/[rest_types, eth2_rest_serialization]
 
 export chronos, client, rest_types, eth2_rest_serialization
 
-proc getState*(state_id: StateIdent): RestResponse[GetStateResponse] {.
+proc getStatePlain*(state_id: StateIdent): RestPlainResponse {.
      rest, endpoint: "/eth/v1/debug/beacon/states/{state_id}",
+     accept: "application/octet-stream,application-json;q=0.9",
      meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Beacon/getState
 
-# TODO altair
-# proc getStateV2*(state_id: StateIdent): RestResponse[GetStateV2Response] {.
-#      rest, endpoint: "/eth/v2/debug/beacon/states/{state_id}",
-#      meth: MethodGet.}
-#   ## https://ethereum.github.io/beacon-APIs/#/Beacon/getState
+proc getState*(client: RestClientRef, state_id: StateIdent,
+               restAccept = ""): Future[ForkedBeaconState] {.async.} =
+  let resp =
+    if len(restAccept) > 0:
+      await client.getStatePlain(state_id, restAcceptType = restAccept)
+    else:
+      await client.getStatePlain(state_id)
+  let data =
+    case resp.status
+    of 200:
+      case resp.contentType
+      of "application/json":
+        let state =
+          block:
+            let res = decodeBytes(GetStateResponse, resp.data,
+                                  resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        ForkedBeaconState.init(state.data)
+      of "application/octet-stream":
+        let state =
+          block:
+            let res = decodeBytes(GetPhase0StateSszResponse, resp.data,
+                                  resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        ForkedBeaconState.init(state)
+      else:
+        raise newException(RestError, "Unsupported content-type")
+    of 400, 404, 500:
+      let error =
+        block:
+          let res = decodeBytes(RestGenericError, resp.data, resp.contentType)
+          if res.isErr():
+            let msg = "Incorrect response error format (" & $resp.status &
+                      ") [" & $res.error() & "]"
+            raise newException(RestError, msg)
+          res.get()
+      let msg = "Error response (" & $resp.status & ") [" & error.message & "]"
+      raise newException(RestError, msg)
+    else:
+      let msg = "Unknown response status error (" & $resp.status & ")"
+      raise newException(RestError, msg)
+  return data
 
 proc getDebugChainHeads*(): RestResponse[GetDebugChainHeadsResponse] {.
      rest, endpoint: "/eth/v1/debug/beacon/heads",
      meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Beacon/getDebugChainHeads
+
+proc getStateV2Plain*(state_id: StateIdent): RestPlainResponse {.
+     rest, endpoint: "/eth/v2/debug/beacon/states/{state_id}",
+     accept: "application/octet-stream,application-json;q=0.9",
+     meth: MethodGet.}
+  ## https://ethereum.github.io/beacon-APIs/#/Debug/getStateV2
+
+proc getStateV2*(client: RestClientRef, state_id: StateIdent,
+                 forks: array[2, Fork],
+                 restAccept = ""): Future[ForkedBeaconState] {.async.} =
+  let resp =
+    if len(restAccept) > 0:
+      await client.getStateV2Plain(state_id, restAcceptType = restAccept)
+    else:
+      await client.getStateV2Plain(state_id)
+  let data =
+    case resp.status
+    of 200:
+      case resp.contentType
+      of "application/json":
+        let state =
+          block:
+            let res = decodeBytes(GetStateV2Response, resp.data,
+                                  resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        state
+      of "application/octet-stream":
+        let header =
+          block:
+            let res = decodeBytes(GetStateV2Header, resp.data, resp.contentType)
+            if res.isErr():
+              raise newException(RestError, $res.error())
+            res.get()
+        if header.slot.epoch() < forks[1].epoch:
+          let state =
+            block:
+              let res = decodeBytes(GetPhase0StateSszResponse, resp.data,
+                                    resp.contentType)
+              if res.isErr():
+                raise newException(RestError, $res.error())
+              res.get()
+          ForkedBeaconState.init(state)
+        else:
+          let blck =
+            block:
+              let res = decodeBytes(GetAltairStateSszResponse, resp.data,
+                                    resp.contentType)
+              if res.isErr():
+                raise newException(RestError, $res.error())
+              res.get()
+          ForkedBeaconState.init(blck)
+      else:
+        raise newException(RestError, "Unsupported content-type")
+    of 400, 404, 500:
+      let error =
+        block:
+          let res = decodeBytes(RestGenericError, resp.data, resp.contentType)
+          if res.isErr():
+            let msg = "Incorrect response error format (" & $resp.status &
+                      ") [" & $res.error() & "]"
+            raise newException(RestError, msg)
+          res.get()
+      let msg = "Error response (" & $resp.status & ") [" & error.message & "]"
+      raise newException(RestError, msg)
+    else:
+      let msg = "Unknown response status error (" & $resp.status & ")"
+      raise newException(RestError, msg)
+  return data

beacon_chain/spec/eth2_apis/rest_types.nim

@@ -281,14 +281,31 @@ type
     dependent_root*: Eth2Digest
     data*: T
 
+  ForkedSignedBlockHeader* = object
+    slot*: Slot
+
+  ForkedBeaconStateHeader* = object
+    genesis_time*: uint64
+    genesis_validators_root*: Eth2Digest
+    slot*: Slot
+
+  GetBlockResponse* = DataEnclosedObject[phase0.SignedBeaconBlock]
+  GetStateResponse* = DataEnclosedObject[phase0.BeaconState]
+  GetBlockV2Response* = ForkedSignedBeaconBlock
+  GetBlockV2Header* = ForkedSignedBlockHeader
+  GetStateV2Response* = ForkedBeaconState
+  GetStateV2Header* = ForkedBeaconStateHeader
+  GetPhase0StateSszResponse* = phase0.BeaconState
+  GetAltairStateSszResponse* = altair.BeaconState
+  GetPhase0BlockSszResponse* = phase0.SignedBeaconBlock
+  GetAltairBlockSszResponse* = altair.SignedBeaconBlock
+
   # Types based on the OAPI yaml file - used in responses to requests
   GetAggregatedAttestationResponse* = DataEnclosedObject[Attestation]
   GetAttesterDutiesResponse* = DataRootEnclosedObject[seq[RestAttesterDuty]]
   GetBlockAttestationsResponse* = DataEnclosedObject[seq[Attestation]]
   GetBlockHeaderResponse* = DataEnclosedObject[RestBlockHeaderInfo]
   GetBlockHeadersResponse* = DataEnclosedObject[seq[RestBlockHeaderInfo]]
-  GetBlockResponse* = DataEnclosedObject[phase0.SignedBeaconBlock]
-  GetBlockV2Response* = ForkedSignedBeaconBlock
   GetBlockRootResponse* = DataEnclosedObject[Eth2Digest]
   GetDebugChainHeadsResponse* = DataEnclosedObject[seq[RestChainHead]]
   GetDepositContractResponse* = DataEnclosedObject[RestDepositContract]
@@ -307,7 +324,6 @@ type
   GetSpecResponse* = DataEnclosedObject[RestSpec]
   GetStateFinalityCheckpointsResponse* = DataEnclosedObject[RestBeaconStatesFinalityCheckpoints]
   GetStateForkResponse* = DataEnclosedObject[Fork]
-  GetStateResponse* = DataEnclosedObject[phase0.BeaconState]
   GetStateRootResponse* = DataEnclosedObject[Eth2Digest]
   GetStateValidatorBalancesResponse* = DataEnclosedObject[seq[RestValidatorBalance]]
   GetStateValidatorResponse* = DataEnclosedObject[RestValidator]