REST: Add implementation of postStateValidators and postStateValidatorBalances. (#5632)

* Add implementation POST versions of /eth/v1/beacon/states/{state_id}/validators and /eth/v1/beacon/states/{state_id}/validator_balances.
Add tests.

* Address review comments.
Fix toList() issue.

* Fix tests.

* Address review comments 2.

* Address review comments 3.
Fix unique check for validator identifiers.

* Address review comments.

* Fix constant value.

beacon_chain/rpc/rest_beacon_api.nim

@@ -236,6 +236,136 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
       )
     return RestApiResponse.jsonError(Http404, StateNotFoundError)
 
+  proc getIndices(
+         node: BeaconNode,
+         validatorIds: openArray[ValidatorIdent],
+         state: ForkedHashedBeaconState
+       ): Result[seq[ValidatorIndex], RestErrorMessage] =
+    var
+      keyset: HashSet[ValidatorPubKey]
+      indexset: HashSet[ValidatorIndex]
+
+    let validatorsCount = lenu64(getStateField(state, validators))
+
+    for item in validatorIds:
+      case item.kind
+      of ValidatorQueryKind.Key:
+        # Test for uniqueness of value.
+        if keyset.containsOrIncl(item.key):
+          return err(RestErrorMessage.init(
+            Http400, NonUniqueValidatorIdError, $item.key))
+      of ValidatorQueryKind.Index:
+        let vindex = item.index.toValidatorIndex().valueOr:
+          case error
+          of ValidatorIndexError.TooHighValue:
+            return err(RestErrorMessage.init(
+              Http400, TooHighValidatorIndexValueError))
+          of ValidatorIndexError.UnsupportedValue:
+            return err(RestErrorMessage.init(
+              Http500, UnsupportedValidatorIndexValueError))
+        if uint64(vindex) < validatorsCount:
+          # We're only adding validator indices which are present in
+          # validators list at this moment.
+          if indexset.containsOrIncl(vindex):
+            return err(RestErrorMessage.init(
+              Http400, NonUniqueValidatorIdError,
+              Base10.toString(uint64(vindex))))
+
+    if len(keyset) > 0:
+      let optIndices = keysToIndices(node.restKeysCache, state, keyset.toSeq())
+      # Remove all the duplicates.
+      for item in optIndices:
+        # We ignore missing keys.
+        if item.isSome():
+          indexset.incl(item.get())
+    ok(indexset.toSeq())
+
+  proc getValidators(
+         node: BeaconNode,
+         bslot: BlockSlotId,
+         validatorsMask: ValidatorFilter,
+         validatorIds: openArray[ValidatorIdent]
+       ): RestApiResponse =
+    node.withStateForBlockSlotId(bslot):
+      let
+        stateEpoch = getStateField(state, slot).epoch()
+        validatorsCount = lenu64(getStateField(state, validators))
+        indices = node.getIndices(validatorIds, state).valueOr:
+          return RestApiResponse.jsonError(error)
+        response =
+          block:
+            var res: seq[RestValidator]
+            if len(indices) == 0:
+              # Case when `len(indices) == 0 and len(validatorIds) != 0` means
+              # that we can't find validator identifiers in state, so we should
+              # return empty response.
+              if len(validatorIds) == 0:
+                # There are no indices, so we're going to filter all the
+                # validators.
+                for index, validator in getStateField(state, validators):
+                  let
+                    balance = getStateField(state, balances).item(index)
+                    status = validator.getStatus(stateEpoch).valueOr:
+                      return RestApiResponse.jsonError(
+                        Http400, ValidatorStatusNotFoundError, $error)
+                  if status in validatorsMask:
+                    res.add(RestValidator.init(ValidatorIndex(index), balance,
+                                               toString(status), validator))
+            else:
+              for index in indices:
+                let
+                  validator = getStateField(state, validators).item(index)
+                  balance = getStateField(state, balances).item(index)
+                  status = validator.getStatus(stateEpoch).valueOr:
+                    return RestApiResponse.jsonError(
+                      Http400, ValidatorStatusNotFoundError, $error)
+                if status in validatorsMask:
+                  res.add(RestValidator.init(index, balance, toString(status),
+                                             validator))
+            res
+      return RestApiResponse.jsonResponseFinalized(
+        response,
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
+      )
+    RestApiResponse.jsonError(Http404, StateNotFoundError)
+
+  proc getBalances(
+         node: BeaconNode,
+         bslot: BlockSlotId,
+         validatorIds: openArray[ValidatorIdent]
+       ): RestApiResponse =
+    node.withStateForBlockSlotId(bslot):
+      let
+        validatorsCount = lenu64(getStateField(state, validators))
+        indices = node.getIndices(validatorIds, state).valueOr:
+          return RestApiResponse.jsonError(error)
+        response =
+          block:
+            var res: seq[RestValidatorBalance]
+            if len(indices) == 0:
+              # Case when `len(indices) == 0 and len(validatorIds) != 0` means
+              # that we can't find validator identifiers in state, so we should
+              # return empty response.
+              if len(validatorIds) == 0:
+                # There are no indices, so we're going to return balances of all
+                # known validators.
+                for index, balance in getStateField(state, balances):
+                  res.add(RestValidatorBalance.init(ValidatorIndex(index),
+                                                    balance))
+            else:
+              for index in indices:
+                let balance = getStateField(state, balances).item(index)
+                res.add(RestValidatorBalance.init(index, balance))
+            res
+
+      return RestApiResponse.jsonResponseFinalized(
+        response,
+        node.getStateOptimistic(state),
+        node.dag.isFinalized(bslot.bid)
+      )
+    RestApiResponse.jsonError(Http404, StateNotFoundError)
+
   # https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidators
   router.api(MethodGet, "/eth/v1/beacon/states/{state_id}/validators") do (
     state_id: StateIdent, id: seq[ValidatorIdent],
@@ -249,119 +379,54 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
           # TODO (cheatfate): Its impossible to retrieve state by `state_root`
           # in current version of database.
           return RestApiResponse.jsonError(Http500, NoImplementationError)
-        return RestApiResponse.jsonError(Http404, StateNotFoundError,
-                                          $error)
-    let validatorIds =
+        return RestApiResponse.jsonError(
+          Http404, StateNotFoundError, $error)
+      validatorIds =
         block:
           if id.isErr():
-          return RestApiResponse.jsonError(Http400,
-                                           InvalidValidatorIdValueError)
+            return RestApiResponse.jsonError(
+              Http400, InvalidValidatorIdValueError)
           let ires = id.get()
           if len(ires) > ServerMaximumValidatorIds:
-          return RestApiResponse.jsonError(Http414,
-                                           MaximumNumberOfValidatorIdsError)
+            return RestApiResponse.jsonError(
+              Http414, MaximumNumberOfValidatorIdsError)
           ires
-
-    let validatorsMask =
+      validatorsMask =
         block:
           if status.isErr():
             return RestApiResponse.jsonError(Http400,
                                              InvalidValidatorStatusValueError)
-        let res = validateFilter(status.get())
-        if res.isErr():
-          return RestApiResponse.jsonError(Http400,
-                                           InvalidValidatorStatusValueError,
-                                           $res.error())
-        res.get()
+          validateFilter(status.get()).valueOr:
+            return RestApiResponse.jsonError(
+              Http400, InvalidValidatorStatusValueError, $error)
+    getValidators(node, bslot, validatorsMask, validatorIds)
 
-    node.withStateForBlockSlotId(bslot):
+  # https://ethereum.github.io/beacon-APIs/#/Beacon/postStateValidators
+  router.api(MethodPost, "/eth/v1/beacon/states/{state_id}/validators") do (
+    state_id: StateIdent, contentBody: Option[ContentBody]) -> RestApiResponse:
     let
-        current_epoch = getStateField(state, slot).epoch()
-        validatorsCount = lenu64(getStateField(state, validators))
-
-      let indices =
+      (validatorIds, validatorsMask) =
         block:
-          var keyset: HashSet[ValidatorPubKey]
-          var indexset: HashSet[ValidatorIndex]
-          for item in validatorIds:
-            case item.kind
-            of ValidatorQueryKind.Key:
-              keyset.incl(item.key)
-            of ValidatorQueryKind.Index:
-              let vindex =
-                block:
-                  let vres = item.index.toValidatorIndex()
-                  if vres.isErr():
-                    case vres.error()
-                    of ValidatorIndexError.TooHighValue:
-                      return RestApiResponse.jsonError(Http400,
-                                                TooHighValidatorIndexValueError)
-                    of ValidatorIndexError.UnsupportedValue:
-                      return RestApiResponse.jsonError(Http500,
-                                            UnsupportedValidatorIndexValueError)
-                  let index = vres.get()
-                  index
-              if uint64(vindex) < validatorsCount:
-                # We only adding validator indices which are present in
-                # validators list at this moment.
-                indexset.incl(vindex)
-
-          if len(keyset) > 0:
-            let optIndices = keysToIndices(node.restKeysCache, state,
-                                           keyset.toSeq())
-            # Remove all the duplicates.
-            for item in optIndices:
-              # We ignore missing keys.
-              if item.isSome():
-                indexset.incl(item.get())
-          indexset.toSeq()
-
-      let response =
-        block:
-          var res: seq[RestValidator]
-          if len(indices) == 0:
-            # Case when `len(indices) == 0 and len(validatorIds) != 0` means
-            # that we can't find validator identifiers in state, so we should
-            # return empty response.
-            if len(validatorIds) == 0:
-              # There is no indices, so we going to filter all the validators.
-              for index, validator in getStateField(state, validators):
+          if contentBody.isNone():
+            return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
+          let request =
+            decodeBody(RestValidatorRequest, contentBody.get()).valueOr:
+              return RestApiResponse.jsonError(
+                Http400, InvalidRequestBodyError, $error)
           let
-                  balance = getStateField(state, balances).item(index)
-                  status =
-                    block:
-                      let sres = validator.getStatus(current_epoch)
-                      if sres.isErr():
-                        return RestApiResponse.jsonError(Http400,
-                                                     ValidatorStatusNotFoundError,
-                                                     $sres.get())
-                      sres.get()
-                if status in validatorsMask:
-                  res.add(RestValidator.init(ValidatorIndex(index), balance,
-                                             toString(status), validator))
-          else:
-            for index in indices:
-              let
-                validator = getStateField(state, validators).item(index)
-                balance = getStateField(state, balances).item(index)
-                status =
-                  block:
-                    let sres = validator.getStatus(current_epoch)
-                    if sres.isErr():
-                      return RestApiResponse.jsonError(Http400,
-                                                   ValidatorStatusNotFoundError,
-                                                   $sres.get())
-                    sres.get()
-              if status in validatorsMask:
-                res.add(RestValidator.init(index, balance, toString(status),
-                                           validator))
-          res
-      return RestApiResponse.jsonResponseFinalized(
-        response,
-        node.getStateOptimistic(state),
-        node.dag.isFinalized(bslot.bid)
-      )
-    return RestApiResponse.jsonError(Http404, StateNotFoundError)
+            ids = request.ids.valueOr: @[]
+            filter = request.status.valueOr: {}
+          (ids, filter)
+      sid = state_id.valueOr:
+        return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
+                                         $error)
+      bslot = node.getBlockSlotId(sid).valueOr:
+        if sid.kind == StateQueryKind.Root:
+          # TODO (cheatfate): Its impossible to retrieve state by `state_root`
+          # in current version of database.
+          return RestApiResponse.jsonError(Http500, NoImplementationError)
+        return RestApiResponse.jsonError(Http404, StateNotFoundError, $error)
+    getValidators(node, bslot, validatorsMask, validatorIds)
 
   # https://ethereum.github.io/beacon-APIs/#/Beacon/getStateValidator
   router.api(MethodGet,
@@ -441,84 +506,42 @@ proc installBeaconApiHandlers*(router: var RestRouter, node: BeaconNode) =
           # TODO (cheatfate): Its impossible to retrieve state by `state_root`
           # in current version of database.
           return RestApiResponse.jsonError(Http500, NoImplementationError)
-        return RestApiResponse.jsonError(Http404, StateNotFoundError,
-                                          $error)
-
-    let validatorIds =
+        return RestApiResponse.jsonError(Http404, StateNotFoundError, $error)
+      validatorIds =
         block:
           if id.isErr():
-          return RestApiResponse.jsonError(Http400,
-                                           InvalidValidatorIdValueError)
+            return RestApiResponse.jsonError(
+              Http400, InvalidValidatorIdValueError)
           let ires = id.get()
           if len(ires) > ServerMaximumValidatorIds:
-          return RestApiResponse.jsonError(Http400,
-                                           MaximumNumberOfValidatorIdsError)
+            return RestApiResponse.jsonError(
+              Http400, MaximumNumberOfValidatorIdsError)
           ires
+    getBalances(node, bslot, validatorIds)
 
-    node.withStateForBlockSlotId(bslot):
-      let validatorsCount = lenu64(getStateField(state, validators))
-
-      let indices =
+  # https://ethereum.github.io/beacon-APIs/#/Beacon/postStateValidatorBalances
+  router.api(MethodPost,
+             "/eth/v1/beacon/states/{state_id}/validator_balances") do (
+    state_id: StateIdent, contentBody: Option[ContentBody]) -> RestApiResponse:
+    let
+      validatorIds =
         block:
-          var keyset: HashSet[ValidatorPubKey]
-          var indexset: HashSet[ValidatorIndex]
-          for item in validatorIds:
-            case item.kind
-            of ValidatorQueryKind.Key:
-              keyset.incl(item.key)
-            of ValidatorQueryKind.Index:
-              let vindex =
-                block:
-                  let vres = item.index.toValidatorIndex()
-                  if vres.isErr():
-                    case vres.error()
-                    of ValidatorIndexError.TooHighValue:
-                      return RestApiResponse.jsonError(Http400,
-                                                TooHighValidatorIndexValueError)
-                    of ValidatorIndexError.UnsupportedValue:
-                      return RestApiResponse.jsonError(Http500,
-                                            UnsupportedValidatorIndexValueError)
-                  vres.get()
-              # We only adding validator indices which are present in
-              # validators list at this moment.
-              if uint64(vindex) < validatorsCount:
-                indexset.incl(vindex)
-
-          if len(keyset) > 0:
-            let optIndices = keysToIndices(node.restKeysCache, state,
-                                           keyset.toSeq())
-            # Remove all the duplicates.
-            for item in optIndices:
-              # We ignore missing keys.
-              if item.isSome():
-                indexset.incl(item.get())
-          indexset.toSeq()
-
-      let response =
-        block:
-          var res: seq[RestValidatorBalance]
-          if len(indices) == 0:
-            # Case when `len(indices) == 0 and len(validatorIds) != 0` means
-            # that we can't find validator identifiers in state, so we should
-            # return empty response.
-            if len(validatorIds) == 0:
-              # There is no indices, so we going to return balances of all
-              # known validators.
-              for index, balance in getStateField(state, balances):
-                res.add(RestValidatorBalance.init(ValidatorIndex(index),
-                                                  balance))
-          else:
-            for index in indices:
-              let balance = getStateField(state, balances).item(index)
-              res.add(RestValidatorBalance.init(index, balance))
-          res
-      return RestApiResponse.jsonResponseFinalized(
-        response,
-        node.getStateOptimistic(state),
-        node.dag.isFinalized(bslot.bid)
-      )
-
-    return RestApiResponse.jsonError(Http404, StateNotFoundError)
+          if contentBody.isNone():
+            return RestApiResponse.jsonError(Http400, EmptyRequestBodyError)
+          let body = contentBody.get()
+          decodeBody(seq[ValidatorIdent], body).valueOr:
+            return RestApiResponse.jsonError(
+              Http400, InvalidValidatorIdValueError, $error)
+      sid = state_id.valueOr:
+        return RestApiResponse.jsonError(Http400, InvalidStateIdValueError,
+                                         $error)
+      bslot = node.getBlockSlotId(sid).valueOr:
+        if sid.kind == StateQueryKind.Root:
+          # TODO (cheatfate): Its impossible to retrieve state by `state_root`
+          # in current version of database.
+          return RestApiResponse.jsonError(Http500, NoImplementationError)
+        return RestApiResponse.jsonError(Http404, StateNotFoundError, $error)
+    getBalances(node, bslot, validatorIds)
 
   # https://ethereum.github.io/beacon-APIs/#/Beacon/getEpochCommittees
   router.api(MethodGet,

beacon_chain/rpc/rest_constants.nim

@@ -25,7 +25,9 @@ const
   BlockNotFoundError* =
     "Block header/data has not been found"
   EmptyRequestBodyError* =
-    "Empty request's body"
+    "Empty request body"
+  InvalidRequestBodyError* =
+    "Invalid request body"
   InvalidBlockObjectError* =
     "Unable to decode block object(s)"
   InvalidAttestationObjectError* =
@@ -100,7 +102,9 @@ const
   InvalidBlockIdValueError* =
     "Invalid block identifier value"
   InvalidValidatorIdValueError* =
-    "Invalid validator's identifier value(s)"
+    "Invalid validator identifier value(s)"
+  NonUniqueValidatorIdError* =
+    "Non-unique validator identifier value(s)"
   MaximumNumberOfValidatorIdsError* =
     "Maximum number of validator identifier values exceeded"
   InvalidValidatorStatusValueError* =

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -3991,6 +3991,45 @@ proc encodeString*(value: set[EventTopic]): Result[string, cstring] =
   res.setLen(len(res) - 1)
   ok(res)
 
+proc toList*(value: set[ValidatorFilterKind]): seq[string] =
+  const
+    pendingSet = {ValidatorFilterKind.PendingInitialized,
+                  ValidatorFilterKind.PendingQueued}
+    activeSet = {ValidatorFilterKind.ActiveOngoing,
+                 ValidatorFilterKind.ActiveExiting,
+                 ValidatorFilterKind.ActiveSlashed}
+    exitedSet = {ValidatorFilterKind.ExitedUnslashed,
+                 ValidatorFilterKind.ExitedSlashed}
+    withdrawSet = {ValidatorFilterKind.WithdrawalPossible,
+                   ValidatorFilterKind.WithdrawalDone}
+  var
+    res: seq[string]
+    v = value
+
+  template processSet(argSet, argName: untyped): untyped =
+    if argSet * v == argSet:
+      res.add(argName)
+      v.excl(argSet)
+
+  template processSingle(argSingle, argName): untyped =
+    if argSingle in v:
+      res.add(argName)
+
+  processSet(pendingSet, "pending")
+  processSet(activeSet, "active")
+  processSet(exitedSet, "exited")
+  processSet(withdrawSet, "withdrawal")
+  processSingle(ValidatorFilterKind.PendingInitialized, "pending_initialized")
+  processSingle(ValidatorFilterKind.PendingQueued, "pending_queued")
+  processSingle(ValidatorFilterKind.ActiveOngoing, "active_ongoing")
+  processSingle(ValidatorFilterKind.ActiveExiting, "active_exiting")
+  processSingle(ValidatorFilterKind.ActiveSlashed, "active_slashed")
+  processSingle(ValidatorFilterKind.ExitedUnslashed, "exited_unslashed")
+  processSingle(ValidatorFilterKind.ExitedSlashed, "exited_slashed")
+  processSingle(ValidatorFilterKind.WithdrawalPossible, "withdrawal_possible")
+  processSingle(ValidatorFilterKind.WithdrawalDone, "withdrawal_done")
+  res
+
 proc decodeString*(t: typedesc[ValidatorSig],
                    value: string): Result[ValidatorSig, cstring] =
   if len(value) != ValidatorSigSize + 2:
@@ -4215,3 +4254,82 @@ proc decodeString*(t: typedesc[EventBeaconBlockObject],
                        allowUnknownFields = true))
   except SerializationError as exc:
     err(exc.formatMsg("<data>"))
+
+## ValidatorIdent
+proc writeValue*(w: var JsonWriter[RestJson],
+                 value: ValidatorIdent) {.raises: [IOError].} =
+  writeValue(w, value.encodeString().get())
+
+proc readValue*(reader: var JsonReader[RestJson],
+                value: var ValidatorIdent) {.
+     raises: [IOError, SerializationError].} =
+  value = decodeString(ValidatorIdent, reader.readValue(string)).valueOr:
+    raise newException(SerializationError, $error)
+
+## RestValidatorRequest
+proc readValue*(reader: var JsonReader[RestJson],
+                value: var RestValidatorRequest) {.
+     raises: [IOError, SerializationError].} =
+  var
+    statuses: Opt[seq[string]]
+    ids: Opt[seq[string]]
+
+  for fieldName in readObjectFields(reader):
+    case fieldName
+    of "ids":
+      if ids.isSome():
+        reader.raiseUnexpectedField("Multiple `ids` fields found",
+                                    "RestValidatorRequest")
+      ids = Opt.some(reader.readValue(seq[string]))
+    of "statuses":
+      if statuses.isSome():
+        reader.raiseUnexpectedField("Multiple `statuses` fields found",
+                                    "RestValidatorRequest")
+      statuses = Opt.some(reader.readValue(seq[string]))
+    else:
+      unrecognizedFieldWarning()
+
+  let
+    validatorIds =
+      block:
+        # Test for uniqueness of value will be happened on higher layer.
+        if ids.isSome():
+          var res: seq[ValidatorIdent]
+          for item in ids.get():
+            let value = decodeString(ValidatorIdent, item).valueOr:
+              reader.raiseUnexpectedValue($error)
+            res.add(value)
+          Opt.some(res)
+        else:
+          Opt.none(seq[ValidatorIdent])
+    filter =
+      block:
+        if statuses.isSome():
+          var res: ValidatorFilter
+          for item in statuses.get():
+            let value = decodeString(ValidatorFilter, item).valueOr:
+              reader.raiseUnexpectedValue($error)
+            # Test for uniqueness of value.
+            if value * res != {}:
+              reader.raiseUnexpectedValue(
+                "The `statuses` array should consist of only unique values")
+            res.incl(value)
+          Opt.some(res)
+        else:
+          Opt.none(ValidatorFilter)
+
+  value = RestValidatorRequest(ids: validatorIds, status: filter)
+
+proc writeValue*(writer: var JsonWriter[RestJson],
+                 value: RestValidatorRequest) {.raises: [IOError].} =
+  writer.beginRecord()
+  if value.ids.isSome():
+    var res: seq[string]
+    for item in value.ids.get():
+      res.add(item.encodeString().get())
+    writer.writeField("ids", res)
+  if value.status.isSome():
+    let res = value.status.get().toList()
+    if len(res) > 0:
+      writer.writeField("statuses", res)
+  writer.endRecord()

beacon_chain/spec/eth2_apis/rest_types.nim

@@ -121,6 +121,10 @@ type
 
   RestNumeric* = distinct int
 
+  RestValidatorRequest* = object
+    ids*: Opt[seq[ValidatorIdent]]
+    status*: Opt[ValidatorFilter]
+
   RestAttesterDuty* = object
     pubkey*: ValidatorPubKey
     validator_index*: ValidatorIndex