diff --git a/beacon_chain/validator_client/duties_service.nim b/beacon_chain/validator_client/duties_service.nim index 7cb2e1d4b..c4b5b043a 100644 --- a/beacon_chain/validator_client/duties_service.nim +++ b/beacon_chain/validator_client/duties_service.nim @@ -40,6 +40,10 @@ type validatorSyncCommitteeIndex: IndexInSyncCommittee validatorSubCommitteeIndex: SyncSubcommitteeIndex + FillSignaturesResult = object + signaturesRequested: int + signaturesReceived: int + chronicles.formatIt(DutiesServiceLoop): case it of AttesterLoop: "attester_loop" @@ -142,7 +146,7 @@ proc pollForValidatorIndices*(service: DutiesServiceRef) {.async.} = proc fillAttestationSlotSignatures*( service: DutiesServiceRef, epochPeriods: seq[Epoch] - ) {.async.} = + ): Future[FillSignaturesResult] {.async.} = let vc = service.client genesisRoot = vc.beaconGenesis.genesis_validators_root @@ -171,10 +175,13 @@ proc fillAttestationSlotSignatures*( # of validators. In this case tasks that run concurrently will be able to use # signatures for slots at the beginning of the epoch even before this # processing will be completed. + var sigres = FillSignaturesResult() for chunk in requests.chunks(ATTESTATION_SIGNING_CHUNK_SIZE): let pendingRequests = chunk.mapIt( getSlotSignature(it.validator, it.fork, genesisRoot, it.slot)) + inc(sigres.signaturesRequested, len(chunk)) + try: await allFutures(pendingRequests) except CancelledError as exc: @@ -196,6 +203,7 @@ proc fillAttestationSlotSignatures*( slot = request.slot Opt.none(ValidatorSig) else: + inc(sigres.signaturesReceived) Opt.some(sres.get()) else: Opt.none(ValidatorSig) @@ -282,6 +290,8 @@ proc fillAttestationSlotSignatures*( map[].duties.withValue(selection.slot.epoch(), dap): dap[].slotSig = Opt.some(selectionProof.toValidatorSig()) + return sigres + proc pollForAttesterDuties*(service: DutiesServiceRef, epoch: Epoch): Future[int] {.async.} = let vc = service.client @@ -392,7 +402,7 @@ proc pruneSyncCommitteeDuties*(service: DutiesServiceRef, slot: Slot) = proc fillSyncSlotSignatures*( service: DutiesServiceRef, epochPeriods: seq[Epoch] - ) {.async.} = + ): Future[FillSignaturesResult] {.async.} = let vc = service.client genesisRoot = vc.beaconGenesis.genesis_validators_root @@ -428,6 +438,7 @@ proc fillSyncSlotSignatures*( validatorSubCommitteeIndex: subCommitteeIndex)) res + var sigres = FillSignaturesResult() # We creating signatures in chunks to make VC more responsive for big number # of validators. In this case tasks that run concurrently will be able to use # signatures for slots at the beginning of the epoch even before this @@ -438,6 +449,8 @@ proc fillSyncSlotSignatures*( it.validator, it.fork, genesisRoot, it.slot, it.validatorSubCommitteeIndex)) + inc(sigres.signaturesRequested, len(chunk)) + try: await allFutures(pendingRequests) except CancelledError as exc: @@ -462,6 +475,7 @@ proc fillSyncSlotSignatures*( validator = shortLog(request.validator) Opt.none(ValidatorSig) else: + inc(sigres.signaturesReceived) Opt.some(sres.get()) else: Opt.none(ValidatorSig) @@ -563,6 +577,8 @@ proc fillSyncSlotSignatures*( sdap[].slotSigs.withValue(subCommitteeIndex, proofs): proofs[][slotIndex] = Opt.some(selectionProof.toValidatorSig()) + return sigres + proc pollForSyncCommitteeDuties*(service: DutiesServiceRef, epoch: Epoch): Future[int] {.async.} = let vc = service.client @@ -672,9 +688,14 @@ proc pollForAttesterDuties*(service: DutiesServiceRef) {.async.} = debug "No new attester's duties received", slot = currentSlot block: - let moment = Moment.now() - await service.fillAttestationSlotSignatures(@[currentEpoch, nextEpoch]) - debug "Slot signatures has been obtained", time = (Moment.now() - moment) + let + moment = Moment.now() + sigres = await service.fillAttestationSlotSignatures( + @[currentEpoch, nextEpoch]) + debug "Slot signatures has been received", + signatures_requested = sigres.signaturesRequested, + signatures_received = sigres.signaturesReceived, + time = (Moment.now() - moment) let subscriptions = block: @@ -754,10 +775,13 @@ proc pollForSyncCommitteeDuties*(service: DutiesServiceRef) {.async.} = slot = currentSlot block: - let moment = Moment.now() - await service.fillSyncSlotSignatures(epochs) + let + moment = Moment.now() + sigres = await service.fillSyncSlotSignatures(epochs) debug "Sync selection proofs has been obtained", - time = (Moment.now() - moment) + signatures_requested = sigres.signaturesRequested, + signatures_received = sigres.signaturesReceived, + time = (Moment.now() - moment) let subscriptions = block: diff --git a/tests/all_tests.nim b/tests/all_tests.nim index e49a51e77..35b82cc82 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -47,6 +47,7 @@ import # Unit test ./test_validator_pool, ./test_zero_signature, ./test_signing_node, + ./test_validator_client, ./consensus_spec/all_tests as consensus_all_tests, ./slashing_protection/test_fixtures, ./slashing_protection/test_slashing_protection_db, diff --git a/tests/test_validator_client.nim b/tests/test_validator_client.nim index e798dc896..df5f03ecf 100644 --- a/tests/test_validator_client.nim +++ b/tests/test_validator_client.nim @@ -143,9 +143,384 @@ const ("", "err(Missing hostname)") ] + ObolBeaconRequestTestVector = """ +[ + { + "validator_index": "1", + "slot": "1", + "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "slot": "2", + "validator_index": "2", + "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "3", + "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + "slot": "3" + }, + { + "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + "validator_index": "4", + "slot": "4" + } +]""" + ObolBeaconResponseTestVector = """ +{ + "data": [ + { + "validator_index": "1", + "slot": "1", + "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "2", + "slot": "2", + "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "3", + "slot": "3", + "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "4", + "slot": "4", + "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + } + ] +}""" + ObolBeaconResponseTestVectorObject = [ + ( + validator_index: RestValidatorIndex(1), + slot: Slot(1), + selection_proof: "1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ), + ( + validator_index: RestValidatorIndex(2), + slot: Slot(2), + selection_proof: "2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ), + ( + validator_index: RestValidatorIndex(3), + slot: Slot(3), + selection_proof: "3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ), + ( + validator_index: RestValidatorIndex(4), + slot: Slot(4), + selection_proof: "4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ) + ] + ObolSyncRequestTestVector = """ +[ + { + "validator_index": "1", + "slot": "1", + "subcommittee_index": "1", + "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "2", + "subcommittee_index": "2", + "slot": "2", + "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "subcommittee_index": "3", + "validator_index": "3", + "slot": "3", + "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "4", + "slot": "4", + "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + "subcommittee_index": "4" + } +]""" + ObolSyncResponseTestVector = """ +{ + "data": [ + { + "validator_index": "1", + "slot": "1", + "subcommittee_index": "1", + "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "2", + "subcommittee_index": "2", + "slot": "2", + "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "subcommittee_index": "3", + "validator_index": "3", + "slot": "3", + "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + }, + { + "validator_index": "4", + "slot": "4", + "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + "subcommittee_index": "4" + } + ] +}""" + ObolSyncResponseTestVectorObject = [ + ( + validator_index: RestValidatorIndex(1), + slot: Slot(1), + subcommittee_index: 1'u64, + selection_proof: "1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ), + ( + validator_index: RestValidatorIndex(2), + slot: Slot(2), + subcommittee_index: 2'u64, + selection_proof: "2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ), + ( + validator_index: RestValidatorIndex(3), + slot: Slot(3), + subcommittee_index: 3'u64, + selection_proof: "3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ), + ( + validator_index: RestValidatorIndex(4), + slot: Slot(4), + subcommittee_index: 4'u64, + selection_proof: "4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + ) + ] + +type + TestDecodeTypes = seq[RestBeaconCommitteeSelection] | + seq[RestSyncCommitteeSelection] + suite "Validator Client test suite": + proc decodeBytes[T: TestDecodeTypes]( + t: typedesc[T], + value: openArray[byte], + contentType: Opt[ContentTypeData] = Opt.none(ContentTypeData) + ): RestResult[T] = + + let mediaType = + if contentType.isNone(): + ApplicationJsonMediaType + else: + if isWildCard(contentType.get().mediaType): + return err("Incorrect Content-Type") + contentType.get().mediaType + + if mediaType == ApplicationJsonMediaType: + try: + ok RestJson.decode(value, T, + requireAllFields = true, + allowUnknownFields = true) + except SerializationError as exc: + err("Serialization error") + else: + err("Content-Type not supported") + + proc submitBeaconCommitteeSelectionsPlain*( + body: seq[RestBeaconCommitteeSelection] + ): RestPlainResponse {. + rest, endpoint: "/eth/v1/validator/beacon_committee_selections", + meth: MethodPost.} + ## https://ethereum.github.io/beacon-APIs/#/Validator/submitBeaconCommitteeSelections + + proc submitSyncCommitteeSelectionsPlain*( + body: seq[RestSyncCommitteeSelection] + ): RestPlainResponse {. + rest, endpoint: "/eth/v1/validator/sync_committee_selections", + meth: MethodPost.} + ## https://ethereum.github.io/beacon-APIs/#/Validator/submitSyncCommitteeSelections + + proc createServer(address: TransportAddress, + process: HttpProcessCallback, secure: bool): HttpServerRef = + let + socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} + res = HttpServerRef.new(address, process, socketFlags = socketFlags) + res.get() + test "normalizeUri() test vectors": for hostname in HostNames: for vector in GoodTestVectors: let expect = vector[1] % (hostname) check $normalizeUri(parseUri(vector[0] % (hostname))) == expect + + asyncTest "/eth/v1/validator/sync_committee_selections " & + "serialization/deserialization test": + var clientRequest: seq[byte] + proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + if r.isOk(): + let request = r.get() + case request.uri.path + of "/eth/v1/validator/beacon_committee_selections": + clientRequest = await request.getBody() + let headers = HttpTable.init([("Content-Type", "application/json")]) + return await request.respond(Http200, ObolBeaconResponseTestVector, + headers) + else: + return await request.respond(Http404, "Page not found") + else: + return dumbResponse() + + let server = createServer(initTAddress("127.0.0.1:0"), process, false) + server.start() + defer: + await server.stop() + await server.closeWait() + + let + serverAddress = server.instance.localAddress + flags = {RestClientFlag.CommaSeparatedArray} + remoteUri = "http://" & $serverAddress + client = + block: + let res = RestClientRef.new(remoteUri, flags = flags) + check res.isOk() + res.get() + selections = + block: + let res = decodeBytes( + seq[RestBeaconCommitteeSelection], + ObolBeaconRequestTestVector.toOpenArrayByte( + 0, len(ObolBeaconRequestTestVector) - 1)) + check res.isOk() + res.get() + + defer: + await client.closeWait() + + let resp = await client.submitBeaconCommitteeSelectionsPlain(selections) + check: + resp.status == 200 + resp.contentType == MediaType.init("application/json") + + let request = + block: + let res = decodeBytes( + seq[RestBeaconCommitteeSelection], + clientRequest) + check res.isOk() + res.get() + + let response = block: + let res = decodeBytes(SubmitBeaconCommitteeSelectionsResponse, + resp.data, resp.contentType) + check res.isOk() + res.get() + + check: + len(request) == len(selections) + len(response.data) == len(ObolBeaconResponseTestVectorObject) + + # Checking response + for index, item in response.data.pairs(): + check: + item.validator_index == + ObolBeaconResponseTestVectorObject[index].validator_index + item.slot == + ObolBeaconResponseTestVectorObject[index].slot + item.selection_proof.toHex() == + ObolBeaconResponseTestVectorObject[index].selection_proof + + # Checking request + for index, item in selections.pairs(): + check: + item.validator_index == request[index].validator_index + item.slot == request[index].slot + item.selection_proof.toHex() == request[index].selection_proof.toHex() + + + + asyncTest "/eth/v1/validator/sync_committee_selections " & + "serialization/deserialization test": + + var clientRequest: seq[byte] + proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + if r.isOk(): + let request = r.get() + case request.uri.path + of "/eth/v1/validator/sync_committee_selections": + clientRequest = await request.getBody() + let headers = HttpTable.init([("Content-Type", "application/json")]) + return await request.respond(Http200, ObolSyncResponseTestVector, + headers) + else: + return await request.respond(Http404, "Page not found") + else: + return dumbResponse() + + let server = createServer(initTAddress("127.0.0.1:0"), process, false) + server.start() + defer: + await server.stop() + await server.closeWait() + + let + serverAddress = server.instance.localAddress + flags = {RestClientFlag.CommaSeparatedArray} + remoteUri = "http://" & $serverAddress + client = + block: + let res = RestClientRef.new(remoteUri, flags = flags) + check res.isOk() + res.get() + selections = + block: + let res = decodeBytes( + seq[RestSyncCommitteeSelection], + ObolSyncRequestTestVector.toOpenArrayByte( + 0, len(ObolSyncRequestTestVector) - 1)) + check res.isOk() + res.get() + + defer: + await client.closeWait() + + let resp = await client.submitSyncCommitteeSelectionsPlain(selections) + check: + resp.status == 200 + resp.contentType == MediaType.init("application/json") + + let request = + block: + let res = decodeBytes( + seq[RestSyncCommitteeSelection], + clientRequest) + check res.isOk() + res.get() + + let response = block: + let res = decodeBytes(SubmitSyncCommitteeSelectionsResponse, + resp.data, resp.contentType) + check res.isOk() + res.get() + + check: + len(request) == len(selections) + len(response.data) == len(ObolSyncResponseTestVectorObject) + + # Checking response + for index, item in response.data.pairs(): + check: + item.validator_index == + ObolSyncResponseTestVectorObject[index].validator_index + item.slot == + ObolSyncResponseTestVectorObject[index].slot + item.selection_proof.toHex() == + ObolSyncResponseTestVectorObject[index].selection_proof + item.subcommittee_index == request[index].subcommittee_index + + # Checking request + for index, item in selections.pairs(): + check: + item.validator_index == request[index].validator_index + item.slot == request[index].slot + item.subcommittee_index == request[index].subcommittee_index + item.selection_proof.toHex() == request[index].selection_proof.toHex()