Add more debug timing logs.

Adding tests.
Update AllTests.

beacon_chain/validator_client/duties_service.nim

@@ -40,6 +40,10 @@ type
     validatorSyncCommitteeIndex: IndexInSyncCommittee
     validatorSubCommitteeIndex: SyncSubcommitteeIndex
 
+  FillSignaturesResult = object
+    signaturesRequested: int
+    signaturesReceived: int
+
 chronicles.formatIt(DutiesServiceLoop):
   case it
   of AttesterLoop: "attester_loop"
@@ -142,7 +146,7 @@ proc pollForValidatorIndices*(service: DutiesServiceRef) {.async.} =
 proc fillAttestationSlotSignatures*(
        service: DutiesServiceRef,
        epochPeriods: seq[Epoch]
-     ) {.async.} =
+     ): Future[FillSignaturesResult] {.async.} =
   let
     vc = service.client
     genesisRoot = vc.beaconGenesis.genesis_validators_root
@@ -171,10 +175,13 @@ proc fillAttestationSlotSignatures*(
   # of validators. In this case tasks that run concurrently will be able to use
   # signatures for slots at the beginning of the epoch even before this
   # processing will be completed.
+  var sigres = FillSignaturesResult()
   for chunk in requests.chunks(ATTESTATION_SIGNING_CHUNK_SIZE):
     let pendingRequests = chunk.mapIt(
       getSlotSignature(it.validator, it.fork, genesisRoot, it.slot))
 
+    inc(sigres.signaturesRequested, len(chunk))
+
     try:
       await allFutures(pendingRequests)
     except CancelledError as exc:
@@ -196,6 +203,7 @@ proc fillAttestationSlotSignatures*(
                    slot = request.slot
               Opt.none(ValidatorSig)
             else:
+              inc(sigres.signaturesReceived)
               Opt.some(sres.get())
           else:
             Opt.none(ValidatorSig)
@@ -282,6 +290,8 @@ proc fillAttestationSlotSignatures*(
         map[].duties.withValue(selection.slot.epoch(), dap):
           dap[].slotSig = Opt.some(selectionProof.toValidatorSig())
 
+  return sigres
+
 proc pollForAttesterDuties*(service: DutiesServiceRef,
                             epoch: Epoch): Future[int] {.async.} =
   let vc = service.client
@@ -392,7 +402,7 @@ proc pruneSyncCommitteeDuties*(service: DutiesServiceRef, slot: Slot) =
 proc fillSyncSlotSignatures*(
        service: DutiesServiceRef,
        epochPeriods: seq[Epoch]
-     ) {.async.} =
+     ): Future[FillSignaturesResult] {.async.} =
   let
     vc = service.client
     genesisRoot = vc.beaconGenesis.genesis_validators_root
@@ -428,6 +438,7 @@ proc fillSyncSlotSignatures*(
                   validatorSubCommitteeIndex: subCommitteeIndex))
         res
 
+  var sigres = FillSignaturesResult()
   # We creating signatures in chunks to make VC more responsive for big number
   # of validators. In this case tasks that run concurrently will be able to use
   # signatures for slots at the beginning of the epoch even before this
@@ -438,6 +449,8 @@ proc fillSyncSlotSignatures*(
         it.validator, it.fork, genesisRoot, it.slot,
         it.validatorSubCommitteeIndex))
 
+    inc(sigres.signaturesRequested, len(chunk))
+
     try:
       await allFutures(pendingRequests)
     except CancelledError as exc:
@@ -462,6 +475,7 @@ proc fillSyncSlotSignatures*(
                    validator = shortLog(request.validator)
               Opt.none(ValidatorSig)
             else:
+              inc(sigres.signaturesReceived)
               Opt.some(sres.get())
           else:
             Opt.none(ValidatorSig)
@@ -563,6 +577,8 @@ proc fillSyncSlotSignatures*(
           sdap[].slotSigs.withValue(subCommitteeIndex, proofs):
             proofs[][slotIndex] = Opt.some(selectionProof.toValidatorSig())
 
+  return sigres
+
 proc pollForSyncCommitteeDuties*(service: DutiesServiceRef,
                                  epoch: Epoch): Future[int] {.async.} =
   let vc = service.client
@@ -672,9 +688,14 @@ proc pollForAttesterDuties*(service: DutiesServiceRef) {.async.} =
       debug "No new attester's duties received", slot = currentSlot
 
     block:
-      let moment = Moment.now()
+      let
-      await service.fillAttestationSlotSignatures(@[currentEpoch, nextEpoch])
+        moment = Moment.now()
-      debug "Slot signatures has been obtained", time = (Moment.now() - moment)
+        sigres = await service.fillAttestationSlotSignatures(
+          @[currentEpoch, nextEpoch])
+      debug "Slot signatures has been received",
+            signatures_requested = sigres.signaturesRequested,
+            signatures_received = sigres.signaturesReceived,
+            time = (Moment.now() - moment)
 
     let subscriptions =
       block:
@@ -754,10 +775,13 @@ proc pollForSyncCommitteeDuties*(service: DutiesServiceRef) {.async.} =
             slot = currentSlot
 
     block:
-      let moment = Moment.now()
+      let
-      await service.fillSyncSlotSignatures(epochs)
+        moment = Moment.now()
+        sigres = await service.fillSyncSlotSignatures(epochs)
       debug "Sync selection proofs has been obtained",
-             time = (Moment.now() - moment)
+            signatures_requested = sigres.signaturesRequested,
+            signatures_received = sigres.signaturesReceived,
+            time = (Moment.now() - moment)
 
     let subscriptions =
       block:

tests/all_tests.nim

@@ -47,6 +47,7 @@ import # Unit test
   ./test_validator_pool,
   ./test_zero_signature,
   ./test_signing_node,
+  ./test_validator_client,
   ./consensus_spec/all_tests as consensus_all_tests,
   ./slashing_protection/test_fixtures,
   ./slashing_protection/test_slashing_protection_db,

tests/test_validator_client.nim

@@ -143,9 +143,384 @@ const
     ("", "err(Missing hostname)")
   ]
 
+  ObolBeaconRequestTestVector = """
+[
+  {
+    "validator_index": "1",
+    "slot": "1",
+    "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  },
+  {
+    "slot": "2",
+    "validator_index": "2",
+    "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  },
+  {
+    "validator_index": "3",
+    "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505",
+    "slot": "3"
+  },
+  {
+    "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505",
+    "validator_index": "4",
+    "slot": "4"
+  }
+]"""
+  ObolBeaconResponseTestVector = """
+{
+  "data": [
+    {
+      "validator_index": "1",
+      "slot": "1",
+      "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    },
+    {
+      "validator_index": "2",
+      "slot": "2",
+      "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    },
+    {
+      "validator_index": "3",
+      "slot": "3",
+      "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    },
+    {
+      "validator_index": "4",
+      "slot": "4",
+      "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    }
+  ]
+}"""
+  ObolBeaconResponseTestVectorObject = [
+    (
+      validator_index: RestValidatorIndex(1),
+      slot: Slot(1),
+      selection_proof: "1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    ),
+    (
+      validator_index: RestValidatorIndex(2),
+      slot: Slot(2),
+      selection_proof: "2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    ),
+    (
+      validator_index: RestValidatorIndex(3),
+      slot: Slot(3),
+      selection_proof: "3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    ),
+    (
+      validator_index: RestValidatorIndex(4),
+      slot: Slot(4),
+      selection_proof: "4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    )
+  ]
+  ObolSyncRequestTestVector = """
+[
+  {
+    "validator_index": "1",
+    "slot": "1",
+    "subcommittee_index": "1",
+    "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  },
+  {
+    "validator_index": "2",
+    "subcommittee_index": "2",
+    "slot": "2",
+    "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  },
+  {
+    "subcommittee_index": "3",
+    "validator_index": "3",
+    "slot": "3",
+    "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  },
+  {
+    "validator_index": "4",
+    "slot": "4",
+    "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505",
+    "subcommittee_index": "4"
+  }
+]"""
+  ObolSyncResponseTestVector = """
+{
+  "data": [
+    {
+      "validator_index": "1",
+      "slot": "1",
+      "subcommittee_index": "1",
+      "selection_proof": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    },
+    {
+      "validator_index": "2",
+      "subcommittee_index": "2",
+      "slot": "2",
+      "selection_proof": "0x2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    },
+    {
+      "subcommittee_index": "3",
+      "validator_index": "3",
+      "slot": "3",
+      "selection_proof": "0x3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    },
+    {
+      "validator_index": "4",
+      "slot": "4",
+      "selection_proof": "0x4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505",
+      "subcommittee_index": "4"
+    }
+  ]
+}"""
+  ObolSyncResponseTestVectorObject = [
+    (
+      validator_index: RestValidatorIndex(1),
+      slot: Slot(1),
+      subcommittee_index: 1'u64,
+      selection_proof: "1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    ),
+    (
+      validator_index: RestValidatorIndex(2),
+      slot: Slot(2),
+      subcommittee_index: 2'u64,
+      selection_proof: "2b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    ),
+    (
+      validator_index: RestValidatorIndex(3),
+      slot: Slot(3),
+      subcommittee_index: 3'u64,
+      selection_proof: "3b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    ),
+    (
+      validator_index: RestValidatorIndex(4),
+      slot: Slot(4),
+      subcommittee_index: 4'u64,
+      selection_proof: "4b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+    )
+  ]
+
+type
+  TestDecodeTypes = seq[RestBeaconCommitteeSelection] |
+                    seq[RestSyncCommitteeSelection]
+
 suite "Validator Client test suite":
+  proc decodeBytes[T: TestDecodeTypes](
+         t: typedesc[T],
+         value: openArray[byte],
+         contentType: Opt[ContentTypeData] = Opt.none(ContentTypeData)
+       ): RestResult[T] =
+
+    let mediaType =
+      if contentType.isNone():
+        ApplicationJsonMediaType
+      else:
+        if isWildCard(contentType.get().mediaType):
+          return err("Incorrect Content-Type")
+        contentType.get().mediaType
+
+    if mediaType == ApplicationJsonMediaType:
+      try:
+        ok RestJson.decode(value, T,
+                           requireAllFields = true,
+                           allowUnknownFields = true)
+      except SerializationError as exc:
+        err("Serialization error")
+    else:
+      err("Content-Type not supported")
+
+  proc submitBeaconCommitteeSelectionsPlain*(
+         body: seq[RestBeaconCommitteeSelection]
+       ): RestPlainResponse {.
+       rest, endpoint: "/eth/v1/validator/beacon_committee_selections",
+       meth: MethodPost.}
+    ## https://ethereum.github.io/beacon-APIs/#/Validator/submitBeaconCommitteeSelections
+
+  proc submitSyncCommitteeSelectionsPlain*(
+         body: seq[RestSyncCommitteeSelection]
+       ): RestPlainResponse {.
+       rest, endpoint: "/eth/v1/validator/sync_committee_selections",
+       meth: MethodPost.}
+    ## https://ethereum.github.io/beacon-APIs/#/Validator/submitSyncCommitteeSelections
+
+  proc createServer(address: TransportAddress,
+                    process: HttpProcessCallback, secure: bool): HttpServerRef =
+    let
+      socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr}
+      res = HttpServerRef.new(address, process, socketFlags = socketFlags)
+    res.get()
+
   test "normalizeUri() test vectors":
     for hostname in HostNames:
       for vector in GoodTestVectors:
         let expect = vector[1] % (hostname)
         check $normalizeUri(parseUri(vector[0] % (hostname))) == expect
+
+  asyncTest "/eth/v1/validator/sync_committee_selections " &
+            "serialization/deserialization test":
+    var clientRequest: seq[byte]
+    proc process(r: RequestFence): Future[HttpResponseRef] {.async.} =
+      if r.isOk():
+        let request = r.get()
+        case request.uri.path
+        of "/eth/v1/validator/beacon_committee_selections":
+          clientRequest = await request.getBody()
+          let headers = HttpTable.init([("Content-Type", "application/json")])
+          return await request.respond(Http200, ObolBeaconResponseTestVector,
+                                       headers)
+        else:
+          return await request.respond(Http404, "Page not found")
+      else:
+        return dumbResponse()
+
+    let  server = createServer(initTAddress("127.0.0.1:0"), process, false)
+    server.start()
+    defer:
+      await server.stop()
+      await server.closeWait()
+
+    let
+      serverAddress = server.instance.localAddress
+      flags = {RestClientFlag.CommaSeparatedArray}
+      remoteUri = "http://" & $serverAddress
+      client =
+        block:
+          let res = RestClientRef.new(remoteUri, flags = flags)
+          check res.isOk()
+          res.get()
+      selections =
+        block:
+          let res = decodeBytes(
+            seq[RestBeaconCommitteeSelection],
+            ObolBeaconRequestTestVector.toOpenArrayByte(
+              0, len(ObolBeaconRequestTestVector) - 1))
+          check res.isOk()
+          res.get()
+
+    defer:
+      await client.closeWait()
+
+    let resp = await client.submitBeaconCommitteeSelectionsPlain(selections)
+    check:
+      resp.status == 200
+      resp.contentType == MediaType.init("application/json")
+
+    let request =
+      block:
+        let res = decodeBytes(
+          seq[RestBeaconCommitteeSelection],
+          clientRequest)
+        check res.isOk()
+        res.get()
+
+    let response = block:
+      let res = decodeBytes(SubmitBeaconCommitteeSelectionsResponse,
+                            resp.data, resp.contentType)
+      check res.isOk()
+      res.get()
+
+    check:
+      len(request) == len(selections)
+      len(response.data) == len(ObolBeaconResponseTestVectorObject)
+
+    # Checking response
+    for index, item in response.data.pairs():
+      check:
+        item.validator_index ==
+          ObolBeaconResponseTestVectorObject[index].validator_index
+        item.slot ==
+          ObolBeaconResponseTestVectorObject[index].slot
+        item.selection_proof.toHex() ==
+          ObolBeaconResponseTestVectorObject[index].selection_proof
+
+    # Checking request
+    for index, item in selections.pairs():
+      check:
+        item.validator_index == request[index].validator_index
+        item.slot == request[index].slot
+        item.selection_proof.toHex() == request[index].selection_proof.toHex()
+
+
+
+  asyncTest "/eth/v1/validator/sync_committee_selections " &
+            "serialization/deserialization test":
+
+    var clientRequest: seq[byte]
+    proc process(r: RequestFence): Future[HttpResponseRef] {.async.} =
+      if r.isOk():
+        let request = r.get()
+        case request.uri.path
+        of "/eth/v1/validator/sync_committee_selections":
+          clientRequest = await request.getBody()
+          let headers = HttpTable.init([("Content-Type", "application/json")])
+          return await request.respond(Http200, ObolSyncResponseTestVector,
+                                       headers)
+        else:
+          return await request.respond(Http404, "Page not found")
+      else:
+        return dumbResponse()
+
+    let  server = createServer(initTAddress("127.0.0.1:0"), process, false)
+    server.start()
+    defer:
+      await server.stop()
+      await server.closeWait()
+
+    let
+      serverAddress = server.instance.localAddress
+      flags = {RestClientFlag.CommaSeparatedArray}
+      remoteUri = "http://" & $serverAddress
+      client =
+        block:
+          let res = RestClientRef.new(remoteUri, flags = flags)
+          check res.isOk()
+          res.get()
+      selections =
+        block:
+          let res = decodeBytes(
+            seq[RestSyncCommitteeSelection],
+            ObolSyncRequestTestVector.toOpenArrayByte(
+              0, len(ObolSyncRequestTestVector) - 1))
+          check res.isOk()
+          res.get()
+
+    defer:
+      await client.closeWait()
+
+    let resp = await client.submitSyncCommitteeSelectionsPlain(selections)
+    check:
+      resp.status == 200
+      resp.contentType == MediaType.init("application/json")
+
+    let request =
+      block:
+        let res = decodeBytes(
+          seq[RestSyncCommitteeSelection],
+          clientRequest)
+        check res.isOk()
+        res.get()
+
+    let response = block:
+      let res = decodeBytes(SubmitSyncCommitteeSelectionsResponse,
+                            resp.data, resp.contentType)
+      check res.isOk()
+      res.get()
+
+    check:
+      len(request) == len(selections)
+      len(response.data) == len(ObolSyncResponseTestVectorObject)
+
+    # Checking response
+    for index, item in response.data.pairs():
+      check:
+        item.validator_index ==
+          ObolSyncResponseTestVectorObject[index].validator_index
+        item.slot ==
+          ObolSyncResponseTestVectorObject[index].slot
+        item.selection_proof.toHex() ==
+          ObolSyncResponseTestVectorObject[index].selection_proof
+        item.subcommittee_index == request[index].subcommittee_index
+
+    # Checking request
+    for index, item in selections.pairs():
+      check:
+        item.validator_index == request[index].validator_index
+        item.slot == request[index].slot
+        item.subcommittee_index == request[index].subcommittee_index
+        item.selection_proof.toHex() == request[index].selection_proof.toHex()