diff --git a/beacon_chain/nimbus_validator_client.nim b/beacon_chain/nimbus_validator_client.nim index 042615faa..e474d79eb 100644 --- a/beacon_chain/nimbus_validator_client.nim +++ b/beacon_chain/nimbus_validator_client.nim @@ -129,7 +129,7 @@ proc initMetrics(vc: ValidatorClientRef): Future[bool] {.async.} = error_msg = res.error() return false res.get() - vc.metricsServer = some(server) + vc.metricsServer = Opt.some(server) try: await server.start() except MetricsError as exc: diff --git a/beacon_chain/validator_client/api.nim b/beacon_chain/validator_client/api.nim index 87777da05..6c5cf8520 100644 --- a/beacon_chain/validator_client/api.nim +++ b/beacon_chain/validator_client/api.nim @@ -27,9 +27,6 @@ type ApiOperation = enum Success, Timeout, Failure, Interrupt - ApiStrategyKind* {.pure.} = enum - Priority, Best, First - ApiNodeResponse*[T] = object node*: BeaconNodeServerRef data*: ApiResponse[T] @@ -44,33 +41,29 @@ const RestBeaconNodeStatus.OptSynced, RestBeaconNodeStatus.Synced} -proc `$`*(strategy: ApiStrategyKind): string = - case strategy - of ApiStrategyKind.First: - "first" - of ApiStrategyKind.Best: - "best" - of ApiStrategyKind.Priority: - "priority" - -proc lazyWaiter(node: BeaconNodeServerRef, request: FutureBase) {.async.} = +proc lazyWaiter(node: BeaconNodeServerRef, request: FutureBase, + requestName: string, strategy: ApiStrategyKind) {.async.} = try: await allFutures(request) if request.failed(): - node.updateStatus(RestBeaconNodeStatus.Offline) + let failure = ApiNodeFailure.init( + ApiFailure.Communication, requestName, strategy, node, + $request.error.msg) + node.updateStatus(RestBeaconNodeStatus.Offline, failure) except CancelledError as exc: - node.updateStatus(RestBeaconNodeStatus.Offline) await cancelAndWait(request) proc lazyWait(nodes: seq[BeaconNodeServerRef], requests: seq[FutureBase], - timerFut: Future[void]) {.async.} = + timerFut: Future[void], requestName: string, + strategy: ApiStrategyKind) {.async.} = doAssert(len(nodes) == len(requests)) if len(nodes) == 0: return var futures: seq[Future[void]] for index in 0 ..< len(requests): - futures.add(lazyWaiter(nodes[index], requests[index])) + futures.add(lazyWaiter(nodes[index], requests[index], requestName, + strategy)) if not(isNil(timerFut)): await allFutures(futures) or timerFut @@ -132,18 +125,20 @@ template firstSuccessParallel*( retRes = ApiResponse[handlerType].err("No online beacon node(s)") resultReady = true else: - var (pendingRequests, pendingNodes) = - block: - var requests: seq[FutureBase] - var nodes: seq[BeaconNodeServerRef] - for node {.inject.} in onlineNodes: - it = node.client - let fut = FutureBase(body1) - requests.add(fut) - nodes.add(node) - (requests, nodes) + var + (pendingRequests, pendingNodes) = + block: + var requests: seq[FutureBase] + var nodes: seq[BeaconNodeServerRef] + for node {.inject.} in onlineNodes: + it = node.client + let fut = FutureBase(body1) + requests.add(fut) + nodes.add(node) + (requests, nodes) + raceFut: Future[FutureBase] + requestsCancelled = false - var raceFut: Future[FutureBase] while true: try: if len(pendingRequests) == 0: @@ -152,6 +147,7 @@ template firstSuccessParallel*( retRes = ApiResponse[handlerType].err( "Beacon node(s) unable to satisfy request") resultReady = true + break else: raceFut = race(pendingRequests) @@ -160,54 +156,57 @@ template firstSuccessParallel*( else: await allFutures(raceFut) - if raceFut.finished(): - # One of the requests in the race completed. - let index = pendingRequests.find(raceFut.read()) - doAssert(index >= 0) + let + index = + if not(isNil(timerFut)) and timerFut.finished(): + # Timeout exceeded first. + if not(requestsCancelled): + var pending: seq[Future[void]] + pending.add(raceFut.cancelAndWait()) + for future in pendingRequests.items(): + if not(future.finished()): + pending.add(future.cancelAndWait()) + await allFutures(pending) + requestsCancelled = true + 0 + else: + let res = pendingRequests.find(raceFut.read()) + doAssert(res >= 0) + res + requestFut = pendingRequests[index] + beaconNode = pendingNodes[index] - let - requestFut = pendingRequests[index] - beaconNode = pendingNodes[index] + # Remove completed future from pending list. + pendingRequests.del(index) + pendingNodes.del(index) - # Remove completed future from pending list. - pendingRequests.del(index) - pendingNodes.del(index) - - let - node {.inject.} = beaconNode - apiResponse {.inject.} = + let + node {.inject.} = beaconNode + apiResponse {.inject.} = + if timerFut.finished(): + ApiResponse[responseType].err( + "Timeout exceeded while awaiting for the response") + else: if requestFut.failed(): - let exc = Future[responseType](requestFut).readError() - ApiResponse[responseType].err("[" & $exc.name & "] " & - $exc.msg) + ApiResponse[responseType].err($requestFut.error.msg) else: ApiResponse[responseType].ok( Future[responseType](requestFut).read()) - handlerResponse = - try: - body2 - except CancelledError as exc: - raise exc - except CatchableError: - raiseAssert("Response handler must not raise exceptions") + handlerResponse = + try: + body2 + except CancelledError as exc: + raise exc + except CatchableError: + raiseAssert("Response handler must not raise exceptions") - if apiResponse.isOk() and handlerResponse.isOk(): - retRes = handlerResponse - resultReady = true - asyncSpawn lazyWait(pendingNodes, pendingRequests, timerFut) - break - else: - # Timeout exceeded first. - var pendingCancel: seq[Future[void]] - pendingCancel.add(raceFut.cancelAndWait()) - for index, future in pendingRequests.pairs(): - if not(future.finished()): - pendingNodes[index].updateStatus(RestBeaconNodeStatus.Offline) - pendingCancel.add(future.cancelAndWait()) - await allFutures(pendingCancel) - retRes = ApiResponse[handlerType].err( - "Beacon nodes unable to satisfy request in time") + if apiResponse.isOk() and handlerResponse.isOk(): + retRes = handlerResponse resultReady = true + asyncSpawn lazyWait(pendingNodes, pendingRequests, timerFut, + RequestName, strategy) + break + except CancelledError as exc: var pendingCancel: seq[Future[void]] if not(isNil(raceFut)) and not(raceFut.finished()): @@ -216,7 +215,6 @@ template firstSuccessParallel*( pendingCancel.add(timerFut.cancelAndWait()) for index, future in pendingRequests.pairs(): if not(future.finished()): - pendingNodes[index].updateStatus(RestBeaconNodeStatus.Offline) pendingCancel.add(future.cancelAndWait()) await allFutures(pendingCancel) raise exc @@ -636,16 +634,61 @@ proc getErrorMessage(response: RestPlainResponse): string = else: "Unable to decode error response: [" & $res.error & "]" +template handleCommunicationError(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.Communication, RequestName, + strategy, node, apiResponse.error) + node.updateStatus(RestBeaconNodeStatus.Offline, failure) + failures.add(failure) + +template handleUnexpectedCode(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.UnexpectedCode, RequestName, + strategy, node, response.status, response.getErrorMessage()) + node.updateStatus(RestBeaconNodeStatus.UnexpectedCode, failure) + failures.add(failure) + +template handleUnexpectedData(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.UnexpectedResponse, RequestName, + strategy, node, response.status, $res.error) + node.updateStatus(RestBeaconNodeStatus.UnexpectedResponse, failure) + failures.add(failure) + +template handleOptimistic(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.OptSynced, RequestName, + strategy, node, response.status, + "Response was sent by optimistically synced node") + node.updateStatus(RestBeaconNodeStatus.OptSynced, failure) + +template handle400(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.Invalid, RequestName, + strategy, node, response.status, response.getErrorMessage()) + node.updateStatus(RestBeaconNodeStatus.Incompatible, failure) + failures.add(failure) + +template handle404(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.NotFound, RequestName, + strategy, node, response.status, response.getErrorMessage()) + node.updateStatus(RestBeaconNodeStatus.Incompatible, failure) + failures.add(failure) + +template handle500(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.Internal, RequestName, + strategy, node, response.status, response.getErrorMessage()) + node.updateStatus(RestBeaconNodeStatus.InternalError, failure) + failures.add(failure) + +template handle503(): untyped {.dirty.} = + let failure = ApiNodeFailure.init(ApiFailure.NotSynced, RequestName, + strategy, node, response.status, response.getErrorMessage()) + node.updateStatus(RestBeaconNodeStatus.NotSynced, failure) + failures.add(failure) + proc getProposerDuties*( vc: ValidatorClientRef, epoch: Epoch, strategy: ApiStrategyKind ): Future[GetProposerDutiesResponse] {.async.} = - logScope: - request = "getProposerDuties" - strategy = $strategy + const RequestName = "getProposerDuties" - const ErrorMessage = "Unable to retrieve proposer duties" var failures: seq[ApiNodeFailure] case strategy @@ -657,9 +700,7 @@ proc getProposerDuties*( {BeaconNodeRole.Duties}, getProposerDutiesPlain(it, epoch)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetProposerDutiesResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -668,36 +709,24 @@ proc getProposerDuties*( let res = decodeBytes(GetProposerDutiesResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetProposerDutiesResponse].err($res.error) else: let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() ApiResponse[GetProposerDutiesResponse].ok(data) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[GetProposerDutiesResponse].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetProposerDutiesResponse].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[GetProposerDutiesResponse].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetProposerDutiesResponse].err(ResponseUnexpectedError) if res.isErr(): @@ -711,9 +740,7 @@ proc getProposerDuties*( {BeaconNodeRole.Duties}, getProposerDutiesPlain(it, epoch)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -724,40 +751,25 @@ proc getProposerDuties*( if res.isOk(): let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() return data - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get proposer duties", data: failures) proc getAttesterDuties*( vc: ValidatorClientRef, @@ -765,11 +777,8 @@ proc getAttesterDuties*( validators: seq[ValidatorIndex], strategy: ApiStrategyKind ): Future[GetAttesterDutiesResponse] {.async.} = - logScope: - request = "getAttesterDuties" - strategy = $strategy + const RequestName = "getAttesterDuties" - const ErrorMessage = "Unable to retrieve attester duties" var failures: seq[ApiNodeFailure] case strategy @@ -782,9 +791,7 @@ proc getAttesterDuties*( getAttesterDutiesPlain(it, epoch, validators)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetAttesterDutiesResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -793,36 +800,24 @@ proc getAttesterDuties*( let res = decodeBytes(GetAttesterDutiesResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetAttesterDutiesResponse].err($res.error) else: let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() ApiResponse[GetAttesterDutiesResponse].ok(data) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[GetAttesterDutiesResponse].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetAttesterDutiesResponse].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[GetAttesterDutiesResponse].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetAttesterDutiesResponse].err(ResponseUnexpectedError) if res.isErr(): @@ -836,9 +831,7 @@ proc getAttesterDuties*( {BeaconNodeRole.Duties}, getAttesterDutiesPlain(it, epoch, validators)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -846,39 +839,28 @@ proc getAttesterDuties*( of 200: let res = decodeBytes(GetAttesterDutiesResponse, response.data, response.contentType) - if res.isOk(): return res.get() - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + if res.isOk(): + let data = res.get() + if data.execution_optimistic.get(false): + handleOptimistic() + return data + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get attester duties", data: failures) proc getSyncCommitteeDuties*( vc: ValidatorClientRef, @@ -886,11 +868,8 @@ proc getSyncCommitteeDuties*( validators: seq[ValidatorIndex], strategy: ApiStrategyKind ): Future[GetSyncCommitteeDutiesResponse] {.async.} = - logScope: - request = "getSyncCommitteeDuties" - strategy = $strategy + const RequestName = "getSyncCommitteeDuties" - const ErrorMessage = "Unable to retrieve sync committee duties" var failures: seq[ApiNodeFailure] case strategy @@ -903,9 +882,7 @@ proc getSyncCommitteeDuties*( {BeaconNodeRole.Duties}, getSyncCommitteeDutiesPlain(it, epoch, validators)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetSyncCommitteeDutiesResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -914,36 +891,24 @@ proc getSyncCommitteeDuties*( let res = decodeBytes(GetSyncCommitteeDutiesResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetSyncCommitteeDutiesResponse].err($res.error) else: let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() ApiResponse[GetSyncCommitteeDutiesResponse].ok(data) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[GetSyncCommitteeDutiesResponse].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetSyncCommitteeDutiesResponse].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[GetSyncCommitteeDutiesResponse].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetSyncCommitteeDutiesResponse].err( ResponseUnexpectedError) @@ -959,9 +924,7 @@ proc getSyncCommitteeDuties*( {BeaconNodeRole.Duties}, getSyncCommitteeDutiesPlain(it, epoch, validators)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -972,50 +935,32 @@ proc getSyncCommitteeDuties*( if res.isOk(): let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() return data - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get sync committee duties", data: failures) proc getForkSchedule*( vc: ValidatorClientRef, strategy: ApiStrategyKind ): Future[seq[Fork]] {.async.} = - logScope: - request = "getForkSchedule" - strategy = $strategy + const RequestName = "getForkSchedule" - const ErrorMessage = "Unable to retrieve fork schedule" var failures: seq[ApiNodeFailure] case strategy @@ -1027,9 +972,7 @@ proc getForkSchedule*( {BeaconNodeRole.Duties}, getForkSchedulePlain(it)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetForkScheduleResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -1038,21 +981,15 @@ proc getForkSchedule*( let res = decodeBytes(GetForkScheduleResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetForkScheduleResponse].err($res.error) else: ApiResponse[GetForkScheduleResponse].ok(res.get()) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetForkScheduleResponse].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetForkScheduleResponse].err(ResponseUnexpectedError) if res.isErr(): @@ -1066,9 +1003,7 @@ proc getForkSchedule*( {BeaconNodeRole.Duties}, getForkSchedulePlain(it)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1078,41 +1013,28 @@ proc getForkSchedule*( response.contentType) if res.isOk(): return res.get().data - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 500: - debug ResponseInternalError, - response_code = response.status, endpoint = node, - reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, - response_code = response.status, endpoint = node, - reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get fork schedule", data: failures) proc getHeadBlockRoot*( vc: ValidatorClientRef, strategy: ApiStrategyKind ): Future[DataOptimisticObject[RestRoot]] {.async.} = - logScope: - request = "getHeadBlockRoot" - strategy = $strategy + const RequestName = "getHeadBlockRoot" + + var failures: seq[ApiNodeFailure] let blockIdent = BlockIdent.init(BlockIdentType.Head) - const ErrorMessage = "Unable to retrieve head block's root" - var failures: seq[ApiNodeFailure] - case strategy of ApiStrategyKind.First, ApiStrategyKind.Best: let res = vc.firstSuccessParallel(RestPlainResponse, @@ -1122,9 +1044,7 @@ proc getHeadBlockRoot*( {BeaconNodeRole.SyncCommitteeData}, getBlockRootPlain(it, blockIdent)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetBlockRootResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -1133,38 +1053,27 @@ proc getHeadBlockRoot*( let res = decodeBytes(GetBlockRootResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetBlockRootResponse].err($res.error) else: let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() + failures.add(failure) ApiResponse[GetBlockRootResponse].err(ResponseECNotInSyncError) else: ApiResponse[GetBlockRootResponse].ok(data) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[GetBlockRootResponse].err(ResponseInvalidError) of 404: - debug ResponseNotFoundError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotFound)) + handle404() ApiResponse[GetBlockRootResponse].err(ResponseNotFoundError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetBlockRootResponse].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetBlockRootResponse].err(ResponseUnexpectedError) if res.isErr(): @@ -1178,9 +1087,7 @@ proc getHeadBlockRoot*( {BeaconNodeRole.SyncCommitteeData}, getBlockRootPlain(it, blockIdent)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1188,59 +1095,42 @@ proc getHeadBlockRoot*( of 200: let res = decodeBytes(GetBlockRootResponse, response.data, response.contentType) - if res.isOk(): + if res.isErr(): + handleUnexpectedData() + false + else: let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleOptimistic() + failures.add(failure) false else: return data - else: - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) - false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 404: - debug ResponseNotFoundError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotFound)) + handle404() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get head block root", data: failures) proc getValidators*( vc: ValidatorClientRef, id: seq[ValidatorIdent], strategy: ApiStrategyKind ): Future[seq[RestValidator]] {.async.} = - logScope: - request = "getStateValidators" - strategy = $strategy + const RequestName = "getStateValidators" let stateIdent = StateIdent.init(StateIdentType.Head) - const ErrorMessage = "Unable to retrieve head state's validator information" var failures: seq[ApiNodeFailure] case strategy @@ -1253,9 +1143,7 @@ proc getValidators*( {BeaconNodeRole.Duties}, getStateValidatorsPlain(it, stateIdent, id)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetStateValidatorsResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -1264,36 +1152,24 @@ proc getValidators*( let res = decodeBytes(GetStateValidatorsResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetStateValidatorsResponse].err($res.error) else: let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() ApiResponse[GetStateValidatorsResponse].ok(data) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[GetStateValidatorsResponse].err(ResponseInvalidError) of 404: - debug ResponseNotFoundError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotFound)) + handle404() ApiResponse[GetStateValidatorsResponse].err(ResponseNotFoundError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetStateValidatorsResponse].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetStateValidatorsResponse].err(ResponseUnexpectedError) if res.isErr(): @@ -1307,9 +1183,7 @@ proc getValidators*( {BeaconNodeRole.Duties}, getStateValidatorsPlain(it, stateIdent, id)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1320,40 +1194,25 @@ proc getValidators*( if res.isOk(): let data = res.get() if data.execution_optimistic.get(false): - node.updateStatus(RestBeaconNodeStatus.OptSynced) + handleOptimistic() return data.data - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 404: - debug ResponseNotFoundError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotFound)) + handle404() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get state's validators", data: failures) proc produceAttestationData*( vc: ValidatorClientRef, @@ -1361,11 +1220,8 @@ proc produceAttestationData*( committee_index: CommitteeIndex, strategy: ApiStrategyKind ): Future[AttestationData] {.async.} = - logScope: - request = "produceAttestationData" - strategy = $strategy + const RequestName = "produceAttestationData" - const ErrorMessage = "Unable to retrieve attestation data" var failures: seq[ApiNodeFailure] case strategy @@ -1378,9 +1234,7 @@ proc produceAttestationData*( {BeaconNodeRole.AttestationData}, produceAttestationDataPlain(it, slot, committee_index)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[ProduceAttestationDataResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -1389,34 +1243,22 @@ proc produceAttestationData*( let res = decodeBytes(ProduceAttestationDataResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[ProduceAttestationDataResponse].err($res.error) else: ApiResponse[ProduceAttestationDataResponse].ok(res.get()) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[ProduceAttestationDataResponse].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[ProduceAttestationDataResponse].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[ProduceAttestationDataResponse].err( ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[ProduceAttestationDataResponse].err( ResponseUnexpectedError) @@ -1433,9 +1275,7 @@ proc produceAttestationData*( produceAttestationDataPlain(it, slot, committee_index)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1445,50 +1285,32 @@ proc produceAttestationData*( response.contentType) if res.isOk(): return res.get().data - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to produce attestation data", data: failures) proc submitPoolAttestations*( vc: ValidatorClientRef, data: seq[Attestation], strategy: ApiStrategyKind ): Future[bool] {.async.} = - logScope: - request = "submitPoolAttestations" - strategy = $strategy - const - ErrorMessage = "Unable to submit attestation" - NoErrorMessage = "Attestation was sucessfully published" + RequestName = "submitPoolAttestations" + var failures: seq[ApiNodeFailure] case strategy @@ -1500,9 +1322,7 @@ proc submitPoolAttestations*( {BeaconNodeRole.AttestationPublish}, submitPoolAttestations(it, data)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[bool].err(apiResponse.error) else: let response = apiResponse.get() @@ -1510,22 +1330,13 @@ proc submitPoolAttestations*( of 200: ApiResponse[bool].ok(true) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[bool].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[bool].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[bool].err(ResponseUnexpectedError) if res.isErr(): @@ -1539,9 +1350,7 @@ proc submitPoolAttestations*( {BeaconNodeRole.AttestationPublish}, submitPoolAttestations(it, data)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1549,34 +1358,25 @@ proc submitPoolAttestations*( of 200: return true of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to submit attestations", data: failures) proc submitPoolSyncCommitteeSignature*( vc: ValidatorClientRef, data: SyncCommitteeMessage, strategy: ApiStrategyKind ): Future[bool] {.async.} = - logScope: - request = "submitPoolSyncCommitteeSignatures" - strategy = $strategy + const + RequestName = "submitPoolSyncCommitteeSignatures" let restData = RestSyncCommitteeMessage.init( data.slot, @@ -1585,9 +1385,6 @@ proc submitPoolSyncCommitteeSignature*( data.signature ) - const - ErrorMessage = "Unable to submit sync committee message" - NoErrorMessage = "Sync committee message was successfully published" var failures: seq[ApiNodeFailure] case strategy @@ -1600,9 +1397,7 @@ proc submitPoolSyncCommitteeSignature*( {BeaconNodeRole.SyncCommitteePublish}, submitPoolSyncCommitteeSignatures(it, @[restData])): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[bool].err(apiResponse.error) else: let response = apiResponse.get() @@ -1610,22 +1405,13 @@ proc submitPoolSyncCommitteeSignature*( of 200: ApiResponse[bool].ok(true) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[bool].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[bool].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[bool].err(ResponseUnexpectedError) if res.isErr(): @@ -1640,9 +1426,7 @@ proc submitPoolSyncCommitteeSignature*( {BeaconNodeRole.SyncCommitteePublish}, submitPoolSyncCommitteeSignatures(it, @[restData])): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1650,25 +1434,17 @@ proc submitPoolSyncCommitteeSignature*( of 200: return true of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getIndexedErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to submit sync committee message", data: failures) proc getAggregatedAttestation*( vc: ValidatorClientRef, @@ -1676,11 +1452,9 @@ proc getAggregatedAttestation*( root: Eth2Digest, strategy: ApiStrategyKind ): Future[Attestation] {.async.} = - logScope: - request = "getAggregatedAttestation" - strategy = $strategy + const + RequestName = "getAggregatedAttestation" - const ErrorMessage = "Unable to retrieve aggregated attestation data" var failures: seq[ApiNodeFailure] case strategy @@ -1693,9 +1467,7 @@ proc getAggregatedAttestation*( {BeaconNodeRole.AggregatedData}, getAggregatedAttestationPlain(it, root, slot)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[GetAggregatedAttestationResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -1704,29 +1476,20 @@ proc getAggregatedAttestation*( let res = decodeBytes(GetAggregatedAttestationResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[GetAggregatedAttestationResponse].err($res.error) else: ApiResponse[GetAggregatedAttestationResponse].ok(res.get()) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[GetAggregatedAttestationResponse].err( ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[GetAggregatedAttestationResponse].err( ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[GetAggregatedAttestationResponse].err( ResponseUnexpectedError) @@ -1742,9 +1505,7 @@ proc getAggregatedAttestation*( {BeaconNodeRole.AggregatedData}, getAggregatedAttestationPlain(it, root, slot)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1753,32 +1514,20 @@ proc getAggregatedAttestation*( let res = decodeBytes(GetAggregatedAttestationResponse, response.data, response.contentType) if res.isOk(): return res.get().data - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to get aggregated attestation", data: failures) proc produceSyncCommitteeContribution*( vc: ValidatorClientRef, @@ -1787,11 +1536,9 @@ proc produceSyncCommitteeContribution*( root: Eth2Digest, strategy: ApiStrategyKind ): Future[SyncCommitteeContribution] {.async.} = - logScope: - request = "produceSyncCommitteeContribution" - strategy = $strategy + const + RequestName = "produceSyncCommitteeContribution" - const ErrorMessage = "Unable to retrieve sync committee contribution data" var failures: seq[ApiNodeFailure] case strategy @@ -1804,9 +1551,7 @@ proc produceSyncCommitteeContribution*( {BeaconNodeRole.SyncCommitteeData}, produceSyncCommitteeContributionPlain(it, slot, subcommitteeIndex, root)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[ProduceSyncCommitteeContributionResponse].err( apiResponse.error) else: @@ -1816,30 +1561,21 @@ proc produceSyncCommitteeContribution*( let res = decodeBytes(ProduceSyncCommitteeContributionResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[ProduceSyncCommitteeContributionResponse].err( $res.error) else: ApiResponse[ProduceSyncCommitteeContributionResponse].ok(res.get()) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[ProduceSyncCommitteeContributionResponse].err( ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[ProduceSyncCommitteeContributionResponse].err( ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[ProduceSyncCommitteeContributionResponse].err( ResponseUnexpectedError) @@ -1855,9 +1591,7 @@ proc produceSyncCommitteeContribution*( {BeaconNodeRole.SyncCommitteeData}, produceSyncCommitteeContributionPlain(it, slot, subcommitteeIndex, root)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1866,45 +1600,29 @@ proc produceSyncCommitteeContribution*( let res = decodeBytes(ProduceSyncCommitteeContributionResponse, response.data, response.contentType) if res.isOk(): return res.get().data - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to produce sync committee contribution", data: failures) proc publishAggregateAndProofs*( vc: ValidatorClientRef, data: seq[SignedAggregateAndProof], strategy: ApiStrategyKind ): Future[bool] {.async.} = - logScope: - request = "publishAggregateAndProofs" - strategy = $strategy - const - ErrorMessage = "Unable to publish aggregate and proofs" - NoErrorMessage = "Aggregate and proofs was sucessfully published" + RequestName = "publishAggregateAndProofs" + var failures: seq[ApiNodeFailure] case strategy @@ -1916,9 +1634,7 @@ proc publishAggregateAndProofs*( {BeaconNodeRole.AggregatedPublish}, publishAggregateAndProofs(it, data)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[bool].err(apiResponse.error) else: let response = apiResponse.get() @@ -1926,22 +1642,13 @@ proc publishAggregateAndProofs*( of 200: ApiResponse[bool].ok(true) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[bool].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[bool].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[bool].err(ResponseUnexpectedError) if res.isErr(): @@ -1955,9 +1662,7 @@ proc publishAggregateAndProofs*( {BeaconNodeRole.AggregatedPublish}, publishAggregateAndProofs(it, data)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -1965,38 +1670,26 @@ proc publishAggregateAndProofs*( of 200: return true of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to publish aggregated attestation", data: failures) proc publishContributionAndProofs*( vc: ValidatorClientRef, data: seq[RestSignedContributionAndProof], strategy: ApiStrategyKind ): Future[bool] {.async.} = - logScope: - request = "publishContributionAndProofs" - strategy = $strategy - const - ErrorMessage = "Unable to publish contribution and proofs" - NoErrorMessage = "Contribution and proofs were successfully published" + RequestName = "publishContributionAndProofs" + var failures: seq[ApiNodeFailure] case strategy @@ -2008,9 +1701,7 @@ proc publishContributionAndProofs*( {BeaconNodeRole.SyncCommitteePublish}, publishContributionAndProofs(it, data)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[bool].err(apiResponse.error) else: let response = apiResponse.get() @@ -2018,22 +1709,13 @@ proc publishContributionAndProofs*( of 200: ApiResponse[bool].ok(true) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[bool].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[bool].err(ResponseInternalError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[bool].err(ResponseUnexpectedError) if res.isErr(): @@ -2047,36 +1729,25 @@ proc publishContributionAndProofs*( {BeaconNodeRole.SyncCommitteePublish}, publishContributionAndProofs(it, data)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() case response.status: of 200: - trace NoErrorMessage, endpoint = node return true of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to publish sync committee contribution", data: failures) proc produceBlockV2*( vc: ValidatorClientRef, @@ -2085,11 +1756,9 @@ proc produceBlockV2*( graffiti: GraffitiBytes, strategy: ApiStrategyKind ): Future[ProduceBlockResponseV2] {.async.} = - logScope: - request = "produceBlockV2" - strategy = $strategy + const + RequestName = "produceBlockV2" - const ErrorMessage = "Unable to retrieve block data" var failures: seq[ApiNodeFailure] case strategy @@ -2102,9 +1771,7 @@ proc produceBlockV2*( {BeaconNodeRole.BlockProposalData}, produceBlockV2Plain(it, slot, randao_reveal, graffiti)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[ProduceBlockResponseV2].err(apiResponse.error) else: let response = apiResponse.get() @@ -2113,33 +1780,21 @@ proc produceBlockV2*( let res = decodeBytes(ProduceBlockResponseV2, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[ProduceBlockResponseV2].err($res.error) else: ApiResponse[ProduceBlockResponseV2].ok(res.get()) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[ProduceBlockResponseV2].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[ProduceBlockResponseV2].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[ProduceBlockResponseV2].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[ProduceBlockResponseV2].err(ResponseUnexpectedError) if res.isErr(): @@ -2154,9 +1809,7 @@ proc produceBlockV2*( {BeaconNodeRole.BlockProposalData}, produceBlockV2Plain(it, slot, randao_reveal, graffiti)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -2165,52 +1818,33 @@ proc produceBlockV2*( let res = decodeBytes(ProduceBlockResponseV2, response.data, response.contentType) if res.isOk(): return res.get() - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to produce block", data: failures) proc publishBlock*( vc: ValidatorClientRef, data: ForkedSignedBeaconBlock, strategy: ApiStrategyKind ): Future[bool] {.async.} = - logScope: - request = "publishBlock" - strategy = $strategy - const - BlockPublished = "Block was successfully published" + RequestName = "publishBlock" BlockBroadcasted = "Block not passed validation, but still published" - ErrorMessage = "Unable to publish block" + var failures: seq[ApiNodeFailure] case strategy @@ -2239,9 +1873,7 @@ proc publishBlock*( do: if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[bool].err(apiResponse.error) else: let response = apiResponse.get() @@ -2249,31 +1881,19 @@ proc publishBlock*( of 200: ApiResponse[bool].ok(true) of 202: - debug BlockBroadcasted, endpoint = node + debug BlockBroadcasted, node = node, blck = shortLog(data) ApiResponse[bool].ok(true) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[bool].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[bool].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[bool].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[bool].err(ResponseUnexpectedError) if res.isErr(): @@ -2302,9 +1922,7 @@ proc publishBlock*( f do: if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -2312,34 +1930,23 @@ proc publishBlock*( of 200: return true of 202: - debug BlockBroadcasted, endpoint = node + debug BlockBroadcasted, node = node, blck = shortLog(data) return true of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to publish block", data: failures) proc produceBlindedBlock*( vc: ValidatorClientRef, @@ -2348,11 +1955,9 @@ proc produceBlindedBlock*( graffiti: GraffitiBytes, strategy: ApiStrategyKind ): Future[ProduceBlindedBlockResponse] {.async.} = - logScope: - request = "produceBlindedBlock" - strategy = $strategy + const + RequestName = "produceBlindedBlock" - const ErrorMessage = "Unable to retrieve block data" var failures: seq[ApiNodeFailure] case strategy @@ -2365,9 +1970,7 @@ proc produceBlindedBlock*( {BeaconNodeRole.BlockProposalData}, produceBlindedBlockPlain(it, slot, randao_reveal, graffiti)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[ProduceBlindedBlockResponse].err(apiResponse.error) else: let response = apiResponse.get() @@ -2376,33 +1979,26 @@ proc produceBlindedBlock*( let res = decodeBytes(ProduceBlindedBlockResponse, response.data, response.contentType) if res.isErr(): - node.updateStatus(RestBeaconNodeStatus.Unexpected) + handleUnexpectedData() ApiResponse[ProduceBlindedBlockResponse].err($res.error) else: ApiResponse[ProduceBlindedBlockResponse].ok(res.get()) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + # TODO(cheatfate): We not going to update BN status for this handler, + # because BN reports 400 for any type of error that does not mean + # that BN is incompatible. + let failure = ApiNodeFailure.init(ApiFailure.Invalid, RequestName, + strategy, node, response.status, response.getErrorMessage()) + failures.add(failure) ApiResponse[ProduceBlindedBlockResponse].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[ProduceBlindedBlockResponse].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[ProduceBlindedBlockResponse].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[ProduceBlindedBlockResponse].err(ResponseUnexpectedError) if res.isErr(): @@ -2417,9 +2013,7 @@ proc produceBlindedBlock*( {BeaconNodeRole.BlockProposalData}, produceBlindedBlockPlain(it, slot, randao_reveal, graffiti)): if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -2428,52 +2022,33 @@ proc produceBlindedBlock*( let res = decodeBytes(ProduceBlindedBlockResponse, response.data, response.contentType) if res.isOk(): return res.get() - - debug ResponseDecodeError, response_code = response.status, - endpoint = node, reason = res.error - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handleUnexpectedData() false of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to produce blinded block", data: failures) proc publishBlindedBlock*( vc: ValidatorClientRef, data: ForkedSignedBlindedBeaconBlock, strategy: ApiStrategyKind ): Future[bool] {.async.} = - logScope: - request = "publishBlindedBlock" - strategy = $strategy - const - BlockPublished = "Block was successfully published" + RequestName = "publishBlindedBlock" BlockBroadcasted = "Block not passed validation, but still published" - ErrorMessage = "Unable to publish block" + var failures: seq[ApiNodeFailure] case strategy @@ -2501,9 +2076,7 @@ proc publishBlindedBlock*( f do: if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() ApiResponse[bool].err(apiResponse.error) else: let response = apiResponse.get() @@ -2511,31 +2084,19 @@ proc publishBlindedBlock*( of 200: ApiResponse[bool].ok(true) of 202: - debug BlockBroadcasted, endpoint = node + debug BlockBroadcasted, node = node, blck = shortLog(data) ApiResponse[bool].ok(true) of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() ApiResponse[bool].err(ResponseInvalidError) of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() ApiResponse[bool].err(ResponseInternalError) of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() ApiResponse[bool].err(ResponseNoSyncError) else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() ApiResponse[bool].err(ResponseUnexpectedError) if res.isErr(): @@ -2564,9 +2125,7 @@ proc publishBlindedBlock*( f do: if apiResponse.isErr(): - debug ErrorMessage, endpoint = node, error = apiResponse.error - node.updateStatus(RestBeaconNodeStatus.Offline) - failures.add(ApiNodeFailure.init(node, ApiFailure.Communication)) + handleCommunicationError() false else: let response = apiResponse.get() @@ -2574,34 +2133,23 @@ proc publishBlindedBlock*( of 200: return true of 202: - debug BlockBroadcasted, endpoint = node + debug BlockBroadcasted, node = node, blck = shortLog(data) return true of 400: - debug ResponseInvalidError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Incompatible) - failures.add(ApiNodeFailure.init(node, ApiFailure.Invalid)) + handle400() false of 500: - debug ResponseInternalError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.InternalError) - failures.add(ApiNodeFailure.init(node, ApiFailure.Internal)) + handle500() false of 503: - debug ResponseNoSyncError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.NotSynced) - failures.add(ApiNodeFailure.init(node, ApiFailure.NotSynced)) + handle503() false else: - debug ResponseUnexpectedError, response_code = response.status, - endpoint = node, reason = response.getErrorMessage() - node.updateStatus(RestBeaconNodeStatus.Unexpected) - failures.add(ApiNodeFailure.init(node, ApiFailure.Unexpected)) + handleUnexpectedCode() false - raise (ref ValidatorApiError)(msg: ErrorMessage, data: failures) + raise (ref ValidatorApiError)( + msg: "Failed to publish blinded block", data: failures) proc prepareBeaconCommitteeSubnet*( vc: ValidatorClientRef, @@ -2690,112 +2238,6 @@ proc prepareSyncCommitteeSubnets*( message = response.getErrorMessage() return count -proc getValidatorsActivity*( - vc: ValidatorClientRef, epoch: Epoch, - validators: seq[ValidatorIndex] - ): Future[GetValidatorsActivityResponse] {.async.} = - logScope: request = "getValidatorsActivity" - let resp = vc.onceToAll(RestPlainResponse, - SlotDuration, - ViableNodeStatus, - {BeaconNodeRole.Duties}, - getValidatorsActivity(it, epoch, validators)) - case resp.status - of ApiOperation.Timeout: - debug "Unable to perform validator's activity request in time", - timeout = SlotDuration - return GetValidatorsActivityResponse() - of ApiOperation.Interrupt: - debug "Validator's activity request was interrupted" - return GetValidatorsActivityResponse() - of ApiOperation.Failure: - debug "Unexpected error happened while receiving validator's activity" - return GetValidatorsActivityResponse() - of ApiOperation.Success: - var activities: seq[RestActivityItem] - for apiResponse in resp.data: - if apiResponse.data.isErr(): - debug "Unable to retrieve validators activity data", - endpoint = apiResponse.node, error = apiResponse.data.error - else: - let - response = apiResponse.data.get() - activity = - block: - var default: seq[RestActivityItem] - case response.status - of 200: - let res = decodeBytes(GetValidatorsActivityResponse, - response.data, response.contentType) - if res.isOk(): - let list = res.get().data - if len(list) != len(validators): - debug "Received incomplete validators activity response", - endpoint = apiResponse.node, - validators_count = len(validators), - activities_count = len(list) - default - else: - let isOrdered = - block: - var res = true - for index in 0 ..< len(validators): - if list[index].index != validators[index]: - res = false - break - res - if not(isOrdered): - debug "Received unordered validators activity response", - endpoint = apiResponse.node, - validators_count = len(validators), - activities_count = len(list) - default - else: - debug "Received validators activity response", - endpoint = apiResponse.node, - validators_count = len(validators), - activities_count = len(list) - list - else: - debug "Received invalid/incomplete response", - endpoint = apiResponse.node, error_message = res.error - apiResponse.node.updateStatus( - RestBeaconNodeStatus.Unexpected) - default - of 400: - debug "Server reports invalid request", - response_code = response.status, - endpoint = apiResponse.node, - reason = response.getErrorMessage() - apiResponse.node.updateStatus(RestBeaconNodeStatus.Incompatible) - default - of 500: - debug "Server reports internal error", - response_code = response.status, - endpoint = apiResponse.node, - reason = response.getErrorMessage() - apiResponse.node.updateStatus( - RestBeaconNodeStatus.InternalError) - default - else: - debug "Server reports unexpected error code", - response_code = response.status, - endpoint = apiResponse.node, - reason = response.getErrorMessage() - apiResponse.node.updateStatus(RestBeaconNodeStatus.Unexpected) - default - - if len(activity) > 0: - if len(activities) == 0: - activities = activity - else: - # If single node returns `active` it means that validator's - # activity was seen by this node, so result would be `active`. - for index in 0 ..< len(activities): - if activity[index].active: - activities[index].active = true - return GetValidatorsActivityResponse(data: activities) - proc prepareBeaconProposer*( vc: ValidatorClientRef, data: seq[PrepareBeaconProposer] @@ -2861,7 +2303,7 @@ proc registerValidator*( return 0 of ApiOperation.Interrupt: debug "Validator registration was interrupted" - return 00 + return 0 of ApiOperation.Failure: debug "Unexpected error happened while registering validators" return 0 @@ -2885,7 +2327,8 @@ proc getValidatorsLiveness*( vc: ValidatorClientRef, epoch: Epoch, validators: seq[ValidatorIndex] ): Future[GetValidatorsLivenessResponse] {.async.} = - logScope: request = "getValidatorsActivity" + const + RequestName = "getValidatorsActivity" let resp = vc.onceToAll(RestPlainResponse, SlotDuration, ViableNodeStatus, @@ -2939,33 +2382,39 @@ proc getValidatorsLiveness*( activities_count = len(list), updated_count = updated else: - debug "Received invalid/incomplete response", - endpoint = apiResponse.node, error_message = res.error - apiResponse.node.updateStatus(RestBeaconNodeStatus.Unexpected) + let failure = ApiNodeFailure.init( + ApiFailure.UnexpectedResponse, RequestName, + apiResponse.node, response.status, $res.error) + apiResponse.node.updateStatus( + RestBeaconNodeStatus.UnexpectedResponse, failure) continue of 400: - debug "Server reports invalid request", - response_code = response.status, - endpoint = apiResponse.node, reason = response.getErrorMessage() - apiResponse.node.updateStatus(RestBeaconNodeStatus.Incompatible) + let failure = ApiNodeFailure.init( + ApiFailure.Invalid, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + apiResponse.node.updateStatus( + RestBeaconNodeStatus.Incompatible, failure) continue of 500: - debug "Server reports internal error", - response_code = response.status, - endpoint = apiResponse.node, reason = response.getErrorMessage() - apiResponse.node.updateStatus(RestBeaconNodeStatus.InternalError) + let failure = ApiNodeFailure.init( + ApiFailure.Internal, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + apiResponse.node.updateStatus( + RestBeaconNodeStatus.InternalError, failure) continue of 503: - debug "Server reports that it not in sync", - response_code = response.status, - endpoint = apiResponse.node, reason = response.getErrorMessage() - apiResponse.node.updateStatus(RestBeaconNodeStatus.NotSynced) + let failure = ApiNodeFailure.init( + ApiFailure.NotSynced, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + apiResponse.node.updateStatus( + RestBeaconNodeStatus.NotSynced, failure) continue else: - debug "Server reports unexpected error code", - response_code = response.status, - endpoint = apiResponse.node, reason = response.getErrorMessage() - apiResponse.node.updateStatus(RestBeaconNodeStatus.Unexpected) + let failure = ApiNodeFailure.init( + ApiFailure.UnexpectedCode, RequestName, + apiResponse.node, response.status, response.getErrorMessage()) + apiResponse.node.updateStatus( + RestBeaconNodeStatus.UnexpectedCode, failure) continue var response = diff --git a/beacon_chain/validator_client/attestation_service.nim b/beacon_chain/validator_client/attestation_service.nim index 8433bb3fa..98fec5c85 100644 --- a/beacon_chain/validator_client/attestation_service.nim +++ b/beacon_chain/validator_client/attestation_service.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2021-2022 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). diff --git a/beacon_chain/validator_client/block_service.nim b/beacon_chain/validator_client/block_service.nim index e76be0c3c..9cc46d166 100644 --- a/beacon_chain/validator_client/block_service.nim +++ b/beacon_chain/validator_client/block_service.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2021-2022 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). diff --git a/beacon_chain/validator_client/common.nim b/beacon_chain/validator_client/common.nim index b53b1fa01..380d039a7 100644 --- a/beacon_chain/validator_client/common.nim +++ b/beacon_chain/validator_client/common.nim @@ -9,7 +9,7 @@ import std/[tables, os, sets, sequtils, strutils, uri], stew/[base10, results, byteutils], bearssl/rand, chronos, presto, presto/client as presto_client, - chronicles, confutils, json_serialization/std/[options, net], + chronicles, confutils, metrics, metrics/chronos_httpserver, ".."/spec/datatypes/[phase0, altair], ".."/spec/[eth2_merkleization, helpers, signatures, validator], @@ -22,7 +22,7 @@ from std/times import Time, toUnix, fromUnix, getTime export os, sets, sequtils, chronos, presto, chronicles, confutils, - nimbus_binary_common, version, conf, options, tables, results, base10, + nimbus_binary_common, version, conf, tables, results, base10, byteutils, presto_client, eth2_rest_serialization, rest_beacon_client, phase0, altair, helpers, signatures, validator, eth2_merkleization, beacon_clock, keystore_management, slashing_protection, validator_pool, @@ -46,8 +46,8 @@ type Initialized, Running, Error, Closing, Closed RegistrationKind* {.pure.} = enum - Cached, IncorrectTime, MissingIndex, MissingFee, MissingGasLimit - ErrorSignature, NoSignature + Cached, IncorrectTime, MissingIndex, MissingFee, MissingGasLimit, + ErrorSignature, NoSignature PendingValidatorRegistration* = object registration*: SignedValidatorRegistrationV1 @@ -79,7 +79,7 @@ type epoch*: Epoch dependentRoot*: Eth2Digest data*: RestAttesterDuty - slotSig*: Option[ValidatorSig] + slotSig*: Opt[ValidatorSig] SyncCommitteeDuty* = object pubkey*: ValidatorPubKey @@ -109,10 +109,10 @@ type BeaconNodeServer* = object client*: RestClientRef endpoint*: string - config*: Option[RestSpecVC] - ident*: Option[string] - genesis*: Option[RestGenesis] - syncInfo*: Option[RestSyncInfo] + config*: Opt[RestSpecVC] + ident*: Opt[string] + genesis*: Opt[RestGenesis] + syncInfo*: Opt[RestSyncInfo] status*: RestBeaconNodeStatus roles*: set[BeaconNodeRole] logIdent*: string @@ -125,15 +125,16 @@ type duties*: Table[Epoch, SyncCommitteeDuty] RestBeaconNodeStatus* {.pure.} = enum - Offline, ## BN is offline. - Online, ## BN is online, passed checkOnline() check. - Incompatible, ## BN configuration is NOT compatible with VC configuration. - Compatible, ## BN configuration is compatible with VC configuration. - NotSynced, ## BN is not in sync. - OptSynced, ## BN is optimistically synced (EL is not in sync). - Synced, ## BN and EL are synced. - Unexpected, ## BN sends unexpected/incorrect response. - InternalError ## BN reports internal error. + Offline, ## BN is offline. + Online, ## BN is online, passed checkOnline() check. + Incompatible, ## BN configuration is NOT compatible with VC. + Compatible, ## BN configuration is compatible with VC configuration. + NotSynced, ## BN is not in sync. + OptSynced, ## BN is optimistically synced (EL is not in sync). + Synced, ## BN and EL are synced. + UnexpectedCode, ## BN sends unexpected/incorrect HTTP status code . + UnexpectedResponse, ## BN sends unexpected/incorrect response. + InternalError ## BN reports internal error. BeaconNodesCounters* = object data*: array[int(high(RestBeaconNodeStatus)) + 1, int] @@ -152,7 +153,7 @@ type ValidatorClient* = object config*: ValidatorClientConf - metricsServer*: Option[MetricsHttpServerRef] + metricsServer*: Opt[MetricsHttpServerRef] graffitiBytes*: GraffitiBytes beaconNodes*: seq[BeaconNodeServerRef] fallbackService*: FallbackServiceRef @@ -187,12 +188,20 @@ type validatorsRegCache*: Table[ValidatorPubKey, SignedValidatorRegistrationV1] rng*: ref HmacDrbgContext + ApiStrategyKind* {.pure.} = enum + Priority, Best, First + ApiFailure* {.pure.} = enum - Communication, Invalid, NotFound, NotSynced, Internal, Unexpected + Communication, Invalid, NotFound, OptSynced, NotSynced, Internal, + UnexpectedCode, UnexpectedResponse, NoError ApiNodeFailure* = object node*: BeaconNodeServerRef + request*: string + strategy*: Opt[ApiStrategyKind] failure*: ApiFailure + status*: Opt[int] + reason*: string ValidatorClientRef* = ref ValidatorClient @@ -253,23 +262,52 @@ proc `$`*(status: RestBeaconNodeStatus): string = of RestBeaconNodeStatus.NotSynced: "bn-unsynced" of RestBeaconNodeStatus.OptSynced: "el-unsynced" of RestBeaconNodeStatus.Synced: "synced" - of RestBeaconNodeStatus.Unexpected: "unexpected data" + of RestBeaconNodeStatus.UnexpectedCode: "unexpected code" + of RestBeaconNodeStatus.UnexpectedResponse: "unexpected data" of RestBeaconNodeStatus.InternalError: "internal error" proc `$`*(failure: ApiFailure): string = case failure - of ApiFailure.Communication: "Connection with beacon node has been lost" - of ApiFailure.Invalid: "Invalid response received from beacon node" - of ApiFailure.NotFound: "Beacon node did not found requested entity" - of ApiFailure.NotSynced: "Beacon node not in sync with network" - of ApiFailure.Internal: "Beacon node reports internal failure" - of ApiFailure.Unexpected: "Beacon node reports unexpected status" + of ApiFailure.Communication: "communication" + of ApiFailure.Invalid: "invalid-request" + of ApiFailure.NotFound: "not-found" + of ApiFailure.NotSynced: "not-synced" + of ApiFailure.OptSynced: "opt-synced" + of ApiFailure.Internal: "internal-issue" + of ApiFailure.UnexpectedCode: "unexpected-code" + of ApiFailure.UnexpectedResponse: "unexpected-data" + of ApiFailure.NoError: "status-update" proc getNodeCounts*(vc: ValidatorClientRef): BeaconNodesCounters = var res = BeaconNodesCounters() for node in vc.beaconNodes: inc(res.data[int(node.status)]) res +proc hash*(f: ApiNodeFailure): Hash = + hash(f.failure) + +proc toString*(strategy: ApiStrategyKind): string = + case strategy + of ApiStrategyKind.First: + "first" + of ApiStrategyKind.Best: + "best" + of ApiStrategyKind.Priority: + "priority" + +func getFailureReason*(failure: ApiNodeFailure): string = + let status = + if failure.status.isSome(): + Base10.toString(uint32(failure.status.get())) + else: + "n/a" + let request = + if failure.strategy.isSome(): + failure.request & "(" & failure.strategy.get().toString() & ")" + else: + failure.request & "()" + [failure.reason, status, request, $failure.failure].join(";") + proc getFailureReason*(exc: ref ValidatorApiError): string = var counts: array[int(high(ApiFailure)) + 1, int] let @@ -277,19 +315,20 @@ proc getFailureReason*(exc: ref ValidatorApiError): string = errorsCount = len(errors) if errorsCount > 1: - var maxFailure = + let distinctErrors = block: - var maxCount = -1 - var res = ApiFailure.Unexpected - for item in errors: - inc(counts[int(item.failure)]) - if counts[int(item.failure)] > maxCount: - maxCount = counts[int(item.failure)] - res = item.failure + var res: seq[ApiNodeFailure] + for item in errors.toHashSet().items(): + res.add(item) res - $maxFailure + if len(distinctErrors) > 1: + # If we have many unique errors, we going to report only failures, + # full reasons could be obtained via previosly made log statements. + "[" & distinctErrors.mapIt($it.failure).join(",") & "]" + else: + getFailureReason(distinctErrors[0]) elif errorsCount == 1: - $errors[0].failure + getFailureReason(errors[0]) else: exc.msg @@ -381,13 +420,17 @@ proc checkConfig*(info: RestSpecVC): bool = info.DOMAIN_SELECTION_PROOF == DOMAIN_SELECTION_PROOF and info.DOMAIN_AGGREGATE_AND_PROOF == DOMAIN_AGGREGATE_AND_PROOF -proc updateStatus*(node: BeaconNodeServerRef, status: RestBeaconNodeStatus) = +proc updateStatus*(node: BeaconNodeServerRef, + status: RestBeaconNodeStatus, + failure: ApiNodeFailure) = logScope: - endpoint = node + node = node + case status of RestBeaconNodeStatus.Offline: if node.status != status: - warn "Beacon node down" + warn "Beacon node down", + reason = failure.getFailureReason() node.status = status of RestBeaconNodeStatus.Online: if node.status != status: @@ -396,7 +439,8 @@ proc updateStatus*(node: BeaconNodeServerRef, status: RestBeaconNodeStatus) = node.status = status of RestBeaconNodeStatus.Incompatible: if node.status != status: - warn "Beacon node has incompatible configuration" + warn "Beacon node has incompatible configuration", + reason = failure.getFailureReason() node.status = status of RestBeaconNodeStatus.Compatible: if node.status != status: @@ -407,7 +451,7 @@ proc updateStatus*(node: BeaconNodeServerRef, status: RestBeaconNodeStatus) = RestBeaconNodeStatus.OptSynced}: doAssert(node.syncInfo.isSome()) let si = node.syncInfo.get() - warn "Beacon node not in sync", + warn "Beacon node not in sync", reason = failure.getFailureReason(), last_head_slot = si.head_slot, last_sync_distance = si.sync_distance, last_optimistic = si.is_optimistic.get(false) @@ -416,7 +460,8 @@ proc updateStatus*(node: BeaconNodeServerRef, status: RestBeaconNodeStatus) = if node.status != status: doAssert(node.syncInfo.isSome()) let si = node.syncInfo.get() - notice "Execution client not in sync (beacon node optimistically synced)", + notice "Beacon node optimistically synced (Execution client not in sync)", + reason = failure.getFailureReason(), last_head_slot = si.head_slot, last_sync_distance = si.sync_distance, last_optimistic = si.is_optimistic.get(false) @@ -426,16 +471,24 @@ proc updateStatus*(node: BeaconNodeServerRef, status: RestBeaconNodeStatus) = doAssert(node.syncInfo.isSome()) let si = node.syncInfo.get() notice "Beacon node is in sync", - head_slot = si.head_slot, sync_distance = si.sync_distance, + head_slot = si.head_slot, + sync_distance = si.sync_distance, is_optimistic = si.is_optimistic.get(false) node.status = status - of RestBeaconNodeStatus.Unexpected: + of RestBeaconNodeStatus.UnexpectedResponse: if node.status != status: - error "Beacon node provides unexpected response" + error "Beacon node provides unexpected response", + reason = failure.getFailureReason() + node.status = status + of RestBeaconNodeStatus.UnexpectedCode: + if node.status != status: + error "Beacon node provides unexpected status code", + reason = failure.getFailureReason() node.status = status of RestBeaconNodeStatus.InternalError: if node.status != status: - warn "Beacon node reports internal error" + warn "Beacon node reports internal error", + reason = failure.getFailureReason() node.status = status proc stop*(csr: ClientServiceRef) {.async.} = @@ -565,7 +618,7 @@ proc getMissingRoles*(n: openArray[BeaconNodeServerRef]): set[BeaconNodeRole] = proc init*(t: typedesc[DutyAndProof], epoch: Epoch, dependentRoot: Eth2Digest, duty: RestAttesterDuty, - slotSig: Option[ValidatorSig]): DutyAndProof = + slotSig: Opt[ValidatorSig]): DutyAndProof = DutyAndProof(epoch: epoch, dependentRoot: dependentRoot, data: duty, slotSig: slotSig) @@ -930,9 +983,35 @@ proc prepareRegistrationList*( return registrations -proc init*(t: typedesc[ApiNodeFailure], node: BeaconNodeServerRef, - failure: ApiFailure): ApiNodeFailure = - ApiNodeFailure(node: node, failure: failure) +func init*(t: typedesc[ApiNodeFailure], failure: ApiFailure, + request: string, strategy: ApiStrategyKind, + node: BeaconNodeServerRef): ApiNodeFailure = + ApiNodeFailure(node: node, request: request, strategy: Opt.some(strategy), + failure: failure) + +func init*(t: typedesc[ApiNodeFailure], failure: ApiFailure, + request: string, strategy: ApiStrategyKind, + node: BeaconNodeServerRef, reason: string): ApiNodeFailure = + ApiNodeFailure(node: node, request: request, strategy: Opt.some(strategy), + failure: failure, reason: reason) + +func init*(t: typedesc[ApiNodeFailure], failure: ApiFailure, + request: string, strategy: ApiStrategyKind, + node: BeaconNodeServerRef, status: int, + reason: string): ApiNodeFailure = + ApiNodeFailure(node: node, request: request, strategy: Opt.some(strategy), + failure: failure, status: Opt.some(status), reason: reason) + +func init*(t: typedesc[ApiNodeFailure], failure: ApiFailure, + request: string, node: BeaconNodeServerRef, status: int, + reason: string): ApiNodeFailure = + ApiNodeFailure(node: node, request: request, + failure: failure, status: Opt.some(status), reason: reason) + +func init*(t: typedesc[ApiNodeFailure], failure: ApiFailure, + request: string, node: BeaconNodeServerRef, + reason: string): ApiNodeFailure = + ApiNodeFailure(node: node, request: request, failure: failure, reason: reason) proc checkedWaitForSlot*(vc: ValidatorClientRef, destinationSlot: Slot, offset: TimeDiff, diff --git a/beacon_chain/validator_client/doppelganger_service.nim b/beacon_chain/validator_client/doppelganger_service.nim index 9cef4488b..656c22f0a 100644 --- a/beacon_chain/validator_client/doppelganger_service.nim +++ b/beacon_chain/validator_client/doppelganger_service.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). diff --git a/beacon_chain/validator_client/duties_service.nim b/beacon_chain/validator_client/duties_service.nim index dc67a3804..9cc879886 100644 --- a/beacon_chain/validator_client/duties_service.nim +++ b/beacon_chain/validator_client/duties_service.nim @@ -227,13 +227,13 @@ proc pollForAttesterDuties*(service: DutiesServiceRef, validator = shortLog(validators[index]), error_msg = sigRes.error() DutyAndProof.init(item.epoch, currentRoot.get(), item.duty, - none[ValidatorSig]()) + Opt.none(ValidatorSig)) else: DutyAndProof.init(item.epoch, currentRoot.get(), item.duty, - some(sigRes.get())) + Opt.some(sigRes.get())) else: DutyAndProof.init(item.epoch, currentRoot.get(), item.duty, - none[ValidatorSig]()) + Opt.none(ValidatorSig)) var validatorDuties = vc.attesters.getOrDefault(item.duty.pubkey) validatorDuties.duties[item.epoch] = dap diff --git a/beacon_chain/validator_client/fallback_service.nim b/beacon_chain/validator_client/fallback_service.nim index b64de0ab5..a6c53e155 100644 --- a/beacon_chain/validator_client/fallback_service.nim +++ b/beacon_chain/validator_client/fallback_service.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2021-2022 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -78,6 +78,7 @@ proc checkCompatible( vc: ValidatorClientRef, node: BeaconNodeServerRef ): Future[RestBeaconNodeStatus] {.async.} = + ## Could return only {Offline, Incompatible, Compatible} logScope: endpoint = node let info = try: @@ -121,8 +122,8 @@ proc checkCompatible( genesisFlag = (genesis != vc.beaconGenesis) configFlag = not(checkConfig(info)) - node.config = some(info) - node.genesis = some(genesis) + node.config = Opt.some(info) + node.genesis = Opt.some(genesis) let res = if configFlag or genesisFlag: if node.status != RestBeaconNodeStatus.Incompatible: @@ -137,6 +138,7 @@ proc checkSync( vc: ValidatorClientRef, node: BeaconNodeServerRef ): Future[RestBeaconNodeStatus] {.async.} = + ## Could return only {Offline, NotSynced, Synced, OptSynced} logScope: endpoint = node let syncInfo = try: @@ -156,7 +158,7 @@ proc checkSync( error "Unexpected exception", error_name = exc.name, error_message = exc.msg return RestBeaconNodeStatus.Offline - node.syncInfo = some(syncInfo) + node.syncInfo = Opt.some(syncInfo) let res = block: let optimistic = @@ -177,6 +179,7 @@ proc checkSync( proc checkOnline( node: BeaconNodeServerRef ): Future[RestBeaconNodeStatus] {.async.} = + ## Could return only {Offline, Online}. logScope: endpoint = node debug "Checking beacon node status" let agent = @@ -194,42 +197,63 @@ proc checkOnline( error "Unexpected exception", error_name = exc.name, error_message = exc.msg return RestBeaconNodeStatus.Offline - node.ident = some(agent.version) + node.ident = Opt.some(agent.version) return RestBeaconNodeStatus.Online +func getReason(status: RestBeaconNodeStatus): string = + case status + of RestBeaconNodeStatus.Offline: + "Connection with node has been lost" + of RestBeaconNodeStatus.Online: + "Connection with node has been established" + else: + "Beacon node reports" + proc checkNode(vc: ValidatorClientRef, node: BeaconNodeServerRef): Future[bool] {.async.} = let nstatus = node.status debug "Checking beacon node", endpoint = node, status = node.status if nstatus in {RestBeaconNodeStatus.Offline, - RestBeaconNodeStatus.Unexpected, + RestBeaconNodeStatus.UnexpectedCode, + RestBeaconNodeStatus.UnexpectedResponse, RestBeaconNodeStatus.InternalError}: - let status = await node.checkOnline() - node.updateStatus(status) + let + status = await node.checkOnline() + failure = ApiNodeFailure.init(ApiFailure.NoError, "checkOnline", + node, status.getReason()) + node.updateStatus(status, failure) if status != RestBeaconNodeStatus.Online: return nstatus != status if nstatus in {RestBeaconNodeStatus.Offline, - RestBeaconNodeStatus.Unexpected, + RestBeaconNodeStatus.UnexpectedCode, + RestBeaconNodeStatus.UnexpectedResponse, RestBeaconNodeStatus.InternalError, RestBeaconNodeStatus.Online, RestBeaconNodeStatus.Incompatible}: - let status = await vc.checkCompatible(node) - node.updateStatus(status) + let + status = await vc.checkCompatible(node) + failure = ApiNodeFailure.init(ApiFailure.NoError, "checkCompatible", + node, status.getReason()) + node.updateStatus(status, failure) if status != RestBeaconNodeStatus.Compatible: return nstatus != status if nstatus in {RestBeaconNodeStatus.Offline, - RestBeaconNodeStatus.Unexpected, + RestBeaconNodeStatus.UnexpectedCode, + RestBeaconNodeStatus.UnexpectedResponse, RestBeaconNodeStatus.InternalError, RestBeaconNodeStatus.Online, RestBeaconNodeStatus.Incompatible, RestBeaconNodeStatus.Compatible, RestBeaconNodeStatus.OptSynced, RestBeaconNodeStatus.NotSynced}: - let status = await vc.checkSync(node) - node.updateStatus(status) + let + status = await vc.checkSync(node) + failure = ApiNodeFailure.init(ApiFailure.NoError, "checkSync", + node, status.getReason()) + node.updateStatus(status, failure) return nstatus != status proc checkNodes*(service: FallbackServiceRef): Future[bool] {.async.} = diff --git a/beacon_chain/validator_client/fork_service.nim b/beacon_chain/validator_client/fork_service.nim index dbd1cb53e..3fb1ca254 100644 --- a/beacon_chain/validator_client/fork_service.nim +++ b/beacon_chain/validator_client/fork_service.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2021-2022 Status Research & Development GmbH +# Copyright (c) 2021-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).