VC: Refactor some timing code around sync committee processing (#6073)

* Add some duration metering.
Refactor some log statements.
Rework sync contribution deadline waiting.
Add some cancellation reporting handlers.

* Make all validator's shortLog to become validatorLog.
Optimize some logs with logScope.

* Add `raises`.

* More log statements polishing.
This commit is contained in:
Eugene Kabanov 2024-03-22 04:37:44 +02:00 committed by GitHub
parent 9d5643240b
commit a6e9e0774c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 147 additions and 116 deletions

View File

@ -5,6 +5,8 @@
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import import
std/sets, std/sets,
chronicles, chronicles,
@ -167,7 +169,7 @@ proc produceAndPublishAttestations*(service: AttestationServiceRef,
if (duty.data.slot != data.slot) or if (duty.data.slot != data.slot) or
(uint64(duty.data.committee_index) != data.index): (uint64(duty.data.committee_index) != data.index):
warn "Inconsistent validator duties during attestation signing", warn "Inconsistent validator duties during attestation signing",
validator = shortLog(duty.data.pubkey), pubkey = shortLog(duty.data.pubkey),
duty_slot = duty.data.slot, duty_slot = duty.data.slot,
duty_index = duty.data.committee_index, duty_index = duty.data.committee_index,
attestation_slot = data.slot, attestation_index = data.index attestation_slot = data.slot, attestation_index = data.index

View File

@ -59,7 +59,7 @@ proc produceBlock(
logScope: logScope:
slot = slot slot = slot
wall_slot = currentSlot wall_slot = currentSlot
validator = shortLog(validator) validator = validatorLog(validator)
let let
produceBlockResponse = produceBlockResponse =
try: try:
@ -120,7 +120,7 @@ proc produceBlindedBlock(
logScope: logScope:
slot = slot slot = slot
wall_slot = currentSlot wall_slot = currentSlot
validator = shortLog(validator) validator = validatorLog(validator)
let let
beaconBlock = beaconBlock =
try: try:
@ -178,17 +178,17 @@ proc prepareRandao(vc: ValidatorClientRef, slot: Slot,
timeElapsed = Moment.now() - start timeElapsed = Moment.now() - start
if rsig.isErr(): if rsig.isErr():
debug "Unable to prepare RANDAO signature", epoch = epoch, debug "Unable to prepare RANDAO signature", epoch = epoch,
validator = shortLog(validator), elapsed_time = timeElapsed, validator = validatorLog(validator), elapsed_time = timeElapsed,
current_slot = currentSlot, destination_slot = destSlot, current_slot = currentSlot, destination_slot = destSlot,
delay = vc.getDelay(deadline) delay = vc.getDelay(deadline)
else: else:
debug "RANDAO signature has been prepared", epoch = epoch, debug "RANDAO signature has been prepared", epoch = epoch,
validator = shortLog(validator), elapsed_time = timeElapsed, validator = validatorLog(validator), elapsed_time = timeElapsed,
current_slot = currentSlot, destination_slot = destSlot, current_slot = currentSlot, destination_slot = destSlot,
delay = vc.getDelay(deadline) delay = vc.getDelay(deadline)
else: else:
debug "RANDAO signature preparation timed out", epoch = epoch, debug "RANDAO signature preparation timed out", epoch = epoch,
validator = shortLog(validator), validator = validatorLog(validator),
current_slot = currentSlot, destination_slot = destSlot, current_slot = currentSlot, destination_slot = destSlot,
delay = vc.getDelay(deadline) delay = vc.getDelay(deadline)
@ -225,7 +225,7 @@ proc publishBlockV3(vc: ValidatorClientRef, currentSlot, slot: Slot,
vindex = validator.index.get() vindex = validator.index.get()
logScope: logScope:
validator = shortLog(validator) validator = validatorLog(validator)
validator_index = vindex validator_index = vindex
slot = slot slot = slot
wall_slot = currentSlot wall_slot = currentSlot
@ -415,7 +415,7 @@ proc publishBlockV2(vc: ValidatorClientRef, currentSlot, slot: Slot,
vindex = validator.index.get() vindex = validator.index.get()
logScope: logScope:
validator = shortLog(validator) validator = validatorLog(validator)
validator_index = vindex validator_index = vindex
slot = slot slot = slot
wall_slot = currentSlot wall_slot = currentSlot
@ -634,7 +634,7 @@ proc publishBlock(vc: ValidatorClientRef, currentSlot, slot: Slot,
vindex = validator.index.get() vindex = validator.index.get()
logScope: logScope:
validator = shortLog(validator) validator = validatorLog(validator)
validator_index = vindex validator_index = vindex
slot = slot slot = slot
wall_slot = currentSlot wall_slot = currentSlot
@ -690,11 +690,11 @@ proc proposeBlock(vc: ValidatorClientRef, slot: Slot,
await vc.publishBlock(currentSlot, slot, validator) await vc.publishBlock(currentSlot, slot, validator)
except CancelledError as exc: except CancelledError as exc:
debug "Block proposing process was interrupted", debug "Block proposing process was interrupted",
slot = slot, validator = shortLog(proposerKey) slot = slot, validator = validatorLog(validator)
raise exc raise exc
except CatchableError: except CatchableError:
error "Unexpected error encountered while proposing block", error "Unexpected error encountered while proposing block",
slot = slot, validator = shortLog(validator) slot = slot, validator = validatorLog(validator)
proc contains(data: openArray[RestProposerDuty], task: ProposerTask): bool = proc contains(data: openArray[RestProposerDuty], task: ProposerTask): bool =
for item in data: for item in data:
@ -715,12 +715,12 @@ proc checkDuty(duty: RestProposerDuty, epoch: Epoch, slot: Slot): bool =
true true
else: else:
warn "Block proposal duty is in the far future, ignoring", warn "Block proposal duty is in the far future, ignoring",
duty_slot = duty.slot, validator = shortLog(duty.pubkey), duty_slot = duty.slot, pubkey = shortLog(duty.pubkey),
wall_slot = slot, last_slot_in_epoch = (lastSlot - 1'u64) wall_slot = slot, last_slot_in_epoch = (lastSlot - 1'u64)
false false
else: else:
warn "Block proposal duty is in the past, ignoring", duty_slot = duty.slot, warn "Block proposal duty is in the past, ignoring", duty_slot = duty.slot,
validator = shortLog(duty.pubkey), wall_slot = slot pubkey = shortLog(duty.pubkey), wall_slot = slot
false false
proc addOrReplaceProposers*(vc: ValidatorClientRef, epoch: Epoch, proc addOrReplaceProposers*(vc: ValidatorClientRef, epoch: Epoch,
@ -747,20 +747,20 @@ proc addOrReplaceProposers*(vc: ValidatorClientRef, epoch: Epoch,
# Task is no more relevant, so cancel it. # Task is no more relevant, so cancel it.
debug "Cancelling running proposal duty tasks", debug "Cancelling running proposal duty tasks",
slot = task.duty.slot, slot = task.duty.slot,
validator = shortLog(task.duty.pubkey) pubkey = shortLog(task.duty.pubkey)
task.proposeFut.cancelSoon() task.proposeFut.cancelSoon()
task.randaoFut.cancelSoon() task.randaoFut.cancelSoon()
else: else:
# If task is already running for proper slot, we keep it alive. # If task is already running for proper slot, we keep it alive.
debug "Keep running previous proposal duty tasks", debug "Keep running previous proposal duty tasks",
slot = task.duty.slot, slot = task.duty.slot,
validator = shortLog(task.duty.pubkey) pubkey = shortLog(task.duty.pubkey)
res.add(task) res.add(task)
for duty in duties: for duty in duties:
if duty notin res: if duty notin res:
debug "New proposal duty received", slot = duty.slot, info "Received new proposer duty", slot = duty.slot,
validator = shortLog(duty.pubkey) pubkey = shortLog(duty.pubkey)
if checkDuty(duty, epoch, currentSlot): if checkDuty(duty, epoch, currentSlot):
let task = vc.spawnProposalTask(duty) let task = vc.spawnProposalTask(duty)
if duty.slot in hashset: if duty.slot in hashset:
@ -781,8 +781,8 @@ proc addOrReplaceProposers*(vc: ValidatorClientRef, epoch: Epoch,
var hashset = initHashSet[Slot]() var hashset = initHashSet[Slot]()
var res: seq[ProposerTask] var res: seq[ProposerTask]
for duty in duties: for duty in duties:
debug "New proposal duty received", slot = duty.slot, info "Received new proposer duty", slot = duty.slot,
validator = shortLog(duty.pubkey) pubkey = shortLog(duty.pubkey)
if checkDuty(duty, epoch, currentSlot): if checkDuty(duty, epoch, currentSlot):
let task = vc.spawnProposalTask(duty) let task = vc.spawnProposalTask(duty)
if duty.slot in hashset: if duty.slot in hashset:

View File

@ -1003,7 +1003,7 @@ proc getValidatorRegistration(
): Result[PendingValidatorRegistration, RegistrationKind] = ): Result[PendingValidatorRegistration, RegistrationKind] =
if validator.index.isNone(): if validator.index.isNone():
debug "Validator registration missing validator index", debug "Validator registration missing validator index",
validator = shortLog(validator) validator = validatorLog(validator)
return err(RegistrationKind.MissingIndex) return err(RegistrationKind.MissingIndex)
let let
@ -1039,13 +1039,13 @@ proc getValidatorRegistration(
if not(sigfut.completed()): if not(sigfut.completed()):
let exc = sigfut.error() let exc = sigfut.error()
debug "Got unexpected exception while signing validator registration", debug "Got unexpected exception while signing validator registration",
validator = shortLog(validator), error_name = $exc.name, validator = validatorLog(validator), error = exc.name,
error_msg = $exc.msg reason = exc.msg
return err(RegistrationKind.ErrorSignature) return err(RegistrationKind.ErrorSignature)
let sigres = sigfut.value() let sigres = sigfut.value()
if sigres.isErr(): if sigres.isErr():
debug "Failed to get signature for validator registration", debug "Failed to get signature for validator registration",
validator = shortLog(validator), error = sigres.error() validator = validatorLog(validator), reason = sigres.error()
return err(RegistrationKind.NoSignature) return err(RegistrationKind.NoSignature)
registration.signature = sigres.get() registration.signature = sigres.get()
# Updating cache table with new signed registration data # Updating cache table with new signed registration data

View File

@ -5,6 +5,8 @@
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import chronicles import chronicles
import "."/[common, api] import "."/[common, api]
@ -35,7 +37,7 @@ proc processActivities(service: DoppelgangerServiceRef, epoch: Epoch,
if item.is_live and validator.triggersDoppelganger(epoch): if item.is_live and validator.triggersDoppelganger(epoch):
warn "Doppelganger detection triggered", warn "Doppelganger detection triggered",
validator = shortLog(validator), epoch validator = validatorLog(validator), epoch
vc.doppelExit.fire() vc.doppelExit.fire()
return return

View File

@ -271,8 +271,8 @@ proc pollForSyncCommitteeDuties*(
res res
for item in addOrReplaceItems: for item in addOrReplaceItems:
vc.syncCommitteeDuties.mgetOrPut(item.duty.pubkey, vc.syncCommitteeDuties.mgetOrPut(
default(SyncPeriodDuties)).duties[item.period] = item.duty.pubkey, default(SyncPeriodDuties)).duties[item.period] =
item.duty item.duty
len(addOrReplaceItems) len(addOrReplaceItems)
@ -285,8 +285,8 @@ proc pruneAttesterDuties(service: DutiesServiceRef, epoch: Epoch) =
if (epochKey + HISTORICAL_DUTIES_EPOCHS) >= epoch: if (epochKey + HISTORICAL_DUTIES_EPOCHS) >= epoch:
v.duties[epochKey] = epochDuty v.duties[epochKey] = epochDuty
else: else:
debug "Attester duties for the epoch has been pruned", validator = key, debug "Attester duties for the epoch has been pruned",
epoch = epochKey, loop = AttesterLoop pubkey = shortLog(key), epoch = epochKey, loop = AttesterLoop
attesters[key] = v attesters[key] = v
vc.attesters = attesters vc.attesters = attesters

View File

@ -217,7 +217,7 @@ proc fillAttestationSelectionProofs*(
notice "Found missing validator while processing " & notice "Found missing validator while processing " &
"beacon committee selections", validator_index = vindex, "beacon committee selections", validator_index = vindex,
slot = selection.slot, slot = selection.slot,
validator = shortLog(key.get()), pubkey = shortLog(key.get()),
selection_proof = shortLog(selection.selection_proof) selection_proof = shortLog(selection.selection_proof)
continue continue
@ -483,7 +483,7 @@ proc fillSyncCommitteeSelectionProofs*(
notice "Found missing validator while processing " & notice "Found missing validator while processing " &
"sync committee selections", validator_index = vindex, "sync committee selections", validator_index = vindex,
slot = slot, slot = slot,
validator = shortLog(key.get()), pubkey = shortLog(key.get()),
selection_proof = shortLog(selection.selection_proof) selection_proof = shortLog(selection.selection_proof)
continue continue
request = request =
@ -494,7 +494,7 @@ proc fillSyncCommitteeSelectionProofs*(
warn "Found sync committee selection proof which was not " & warn "Found sync committee selection proof which was not " &
"requested", "requested",
slot = slot, subcommittee_index = subcommittee_index, slot = slot, subcommittee_index = subcommittee_index,
validator = shortLog(validator), validator = validatorLog(validator),
selection_proof = shortLog(selection.selection_proof) selection_proof = shortLog(selection.selection_proof)
continue continue
res.get() res.get()

View File

@ -5,6 +5,8 @@
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import import
std/sets, std/sets,
metrics, chronicles, metrics, chronicles,
@ -31,25 +33,33 @@ proc serveSyncCommitteeMessage*(service: SyncCommitteeServiceRef,
async.} = async.} =
let let
vc = service.client vc = service.client
startTime = Moment.now()
fork = vc.forkAtEpoch(slot.epoch) fork = vc.forkAtEpoch(slot.epoch)
genesisValidatorsRoot = vc.beaconGenesis.genesis_validators_root genesisValidatorsRoot = vc.beaconGenesis.genesis_validators_root
vindex = duty.validator_index vindex = duty.validator_index
validator = vc.getValidatorForDuties( validator = vc.getValidatorForDuties(
duty.pubkey, slot, slashingSafe = true).valueOr: return false duty.pubkey, slot, slashingSafe = true).valueOr: return false
logScope:
validator = validatorLog(validator)
block_root = shortLog(beaconBlockRoot)
slot = slot
let
message = message =
block: block:
let res = await getSyncCommitteeMessage(validator, fork, let res = await getSyncCommitteeMessage(validator, fork,
genesisValidatorsRoot, genesisValidatorsRoot,
slot, beaconBlockRoot) slot, beaconBlockRoot)
if res.isErr(): if res.isErr():
warn "Unable to sign committee message using remote signer", warn "Unable to sign committee message using remote signer"
validator = shortLog(validator), slot = slot,
block_root = shortLog(beaconBlockRoot)
return return
res.get() res.get()
debug "Sending sync committee message", message = shortLog(message), logScope:
validator = shortLog(validator), validator_index = vindex, message = shortLog(message)
debug "Sending sync committee message",
delay = vc.getDelay(message.slot.sync_committee_message_deadline()) delay = vc.getDelay(message.slot.sync_committee_message_deadline())
let res = let res =
@ -57,9 +67,6 @@ proc serveSyncCommitteeMessage*(service: SyncCommitteeServiceRef,
await vc.submitPoolSyncCommitteeSignature(message, ApiStrategyKind.First) await vc.submitPoolSyncCommitteeSignature(message, ApiStrategyKind.First)
except ValidatorApiError as exc: except ValidatorApiError as exc:
warn "Unable to publish sync committee message", warn "Unable to publish sync committee message",
message = shortLog(message),
validator = shortLog(validator),
validator_index = vindex,
reason = exc.getFailureReason() reason = exc.getFailureReason()
return false return false
except CancelledError: except CancelledError:
@ -67,35 +74,31 @@ proc serveSyncCommitteeMessage*(service: SyncCommitteeServiceRef,
return false return false
except CatchableError as exc: except CatchableError as exc:
error "Unexpected error occurred while publishing sync committee message", error "Unexpected error occurred while publishing sync committee message",
message = shortLog(message), error = exc.name, reason = exc.msg
validator = shortLog(validator),
validator_index = vindex,
err_name = exc.name, err_msg = exc.msg
return false return false
let delay = vc.getDelay(message.slot.sync_committee_message_deadline()) let
delay = vc.getDelay(message.slot.sync_committee_message_deadline())
dur = Moment.now() - startTime
if res: if res:
beacon_sync_committee_messages_sent.inc() beacon_sync_committee_messages_sent.inc()
beacon_sync_committee_message_sent_delay.observe(delay.toFloatSeconds()) beacon_sync_committee_message_sent_delay.observe(delay.toFloatSeconds())
notice "Sync committee message published", notice "Sync committee message published",
message = shortLog(message), validator_index = vindex, delay = delay, duration = dur
validator = shortLog(validator),
validator_index = vindex,
delay = delay
else: else:
warn "Sync committee message was not accepted by beacon node", warn "Sync committee message was not accepted by beacon node",
message = shortLog(message), validator_index = vindex, delay = delay, duration = dur
validator = shortLog(validator), res
validator_index = vindex, delay = delay
return res
proc produceAndPublishSyncCommitteeMessages(service: SyncCommitteeServiceRef, proc produceAndPublishSyncCommitteeMessages(service: SyncCommitteeServiceRef,
slot: Slot, slot: Slot,
beaconBlockRoot: Eth2Digest, beaconBlockRoot: Eth2Digest,
duties: seq[SyncCommitteeDuty]) duties: seq[SyncCommitteeDuty])
{.async.} = {.async.} =
let vc = service.client let
vc = service.client
startTime = Moment.now()
let pendingSyncCommitteeMessages = let pendingSyncCommitteeMessages =
block: block:
@ -128,38 +131,54 @@ proc produceAndPublishSyncCommitteeMessages(service: SyncCommitteeServiceRef,
inc(errored) inc(errored)
(succeed, errored, failed) (succeed, errored, failed)
let delay = vc.getDelay(slot.attestation_deadline()) let
delay = vc.getDelay(slot.attestation_deadline())
dur = Moment.now() - startTime
debug "Sync committee message statistics", debug "Sync committee message statistics",
total = len(pendingSyncCommitteeMessages), total = len(pendingSyncCommitteeMessages),
succeed = statistics[0], failed_to_deliver = statistics[1], succeed = statistics[0], failed_to_deliver = statistics[1],
not_accepted = statistics[2], delay = delay, slot = slot, not_accepted = statistics[2], delay = delay, duration = dur,
duties_count = len(duties) slot = slot, duties_count = len(duties)
proc serveContributionAndProof*(service: SyncCommitteeServiceRef, proc serveContributionAndProof*(service: SyncCommitteeServiceRef,
proof: ContributionAndProof, proof: ContributionAndProof,
validator: AttachedValidator): Future[bool] {. validator: AttachedValidator): Future[bool] {.
async.} = async.} =
## Signs ConributionAndProof object and sends it to BN.
let let
vc = service.client vc = service.client
startTime = Moment.now()
slot = proof.contribution.slot slot = proof.contribution.slot
validatorIdx = validator.index.get() validatorIdx = validator.index.get()
genesisRoot = vc.beaconGenesis.genesis_validators_root genesisRoot = vc.beaconGenesis.genesis_validators_root
fork = vc.forkAtEpoch(slot.epoch) fork = vc.forkAtEpoch(slot.epoch)
logScope:
validator = validatorLog(validator)
contribution = shortLog(proof.contribution)
let signature = let signature =
block: block:
let res = await validator.getContributionAndProofSignature( let res =
try:
await validator.getContributionAndProofSignature(
fork, genesisRoot, proof) fork, genesisRoot, proof)
except CancelledError:
debug "Sync contribution signing process was interrupted"
return false
except CatchableError as exc:
error "Unexpected error occurred while signing sync contribution",
error = exc.name, reason = exc.msg
return false
if res.isErr(): if res.isErr():
warn "Unable to sign sync committee contribution using remote signer", warn "Unable to sign sync committee contribution using remote signer",
validator = shortLog(validator), reason = res.error()
contribution = shortLog(proof.contribution),
error_msg = res.error()
return false return false
res.get() res.get()
debug "Sending sync contribution", debug "Sending sync contribution",
contribution = shortLog(proof.contribution),
validator = shortLog(validator), validator_index = validatorIdx,
delay = vc.getDelay(slot.sync_contribution_deadline()) delay = vc.getDelay(slot.sync_contribution_deadline())
let restSignedProof = RestSignedContributionAndProof.init( let restSignedProof = RestSignedContributionAndProof.init(
@ -171,39 +190,35 @@ proc serveContributionAndProof*(service: SyncCommitteeServiceRef,
ApiStrategyKind.First) ApiStrategyKind.First)
except ValidatorApiError as exc: except ValidatorApiError as exc:
warn "Unable to publish sync contribution", warn "Unable to publish sync contribution",
contribution = shortLog(proof.contribution),
validator = shortLog(validator),
validator_index = validatorIdx,
err_msg = exc.msg,
reason = exc.getFailureReason() reason = exc.getFailureReason()
false false
except CancelledError: except CancelledError:
debug "Publish sync contribution request was interrupted" debug "Publication process of sync contribution was interrupted"
return false return false
except CatchableError as err: except CatchableError as err:
error "Unexpected error occurred while publishing sync contribution", error "Unexpected error occurred while publishing sync contribution",
contribution = shortLog(proof.contribution), error = err.name, reason = err.msg
validator = shortLog(validator),
err_name = err.name, err_msg = err.msg
false false
let dur = Moment.now() - startTime
if res: if res:
beacon_sync_committee_contributions_sent.inc() beacon_sync_committee_contributions_sent.inc()
notice "Sync contribution published", notice "Sync contribution published", duration = dur
validator = shortLog(validator),
validator_index = validatorIdx
else: else:
warn "Sync contribution was not accepted by beacon node", warn "Sync contribution was not accepted by beacon node", duration = dur
contribution = shortLog(proof.contribution), res
validator = shortLog(validator),
validator_index = validatorIdx
return res
proc produceAndPublishContributions(service: SyncCommitteeServiceRef, proc produceAndPublishContributions(service: SyncCommitteeServiceRef,
slot: Slot, slot: Slot,
beaconBlockRoot: Eth2Digest, beaconBlockRoot: Eth2Digest,
duties: seq[SyncCommitteeDuty]) {.async.} = duties: seq[SyncCommitteeDuty]) {.async.} =
let vc = service.client let
vc = service.client
startTime = Moment.now()
logScope:
slot = slot
block_root = shortLog(beaconBlockRoot)
var (contributions, pendingFutures, contributionsMap) = var (contributions, pendingFutures, contributionsMap) =
block: block:
@ -263,12 +278,9 @@ proc produceAndPublishContributions(service: SyncCommitteeServiceRef,
if index notin completed: completed.add(index) if index notin completed: completed.add(index)
let aggContribution = let aggContribution =
try: try:
let res = future.read() Opt.some(future.read())
Opt.some(res)
except ValidatorApiError as exc: except ValidatorApiError as exc:
warn "Unable to get sync message contribution data", warn "Unable to get sync message contribution data",
slot = slot,
beacon_block_root = shortLog(beaconBlockRoot),
reason = exc.getFailureReason() reason = exc.getFailureReason()
Opt.none(SyncCommitteeContribution) Opt.none(SyncCommitteeContribution)
except CancelledError as exc: except CancelledError as exc:
@ -277,9 +289,8 @@ proc produceAndPublishContributions(service: SyncCommitteeServiceRef,
raise exc raise exc
except CatchableError as exc: except CatchableError as exc:
error "Unexpected error occurred while getting sync " & error "Unexpected error occurred while getting sync " &
"message contribution", slot = slot, "message contribution",
beacon_block_root = shortLog(beaconBlockRoot), error = exc.name, reason = exc.msg
err_name = exc.name, err_msg = exc.msg
Opt.none(SyncCommitteeContribution) Opt.none(SyncCommitteeContribution)
if aggContribution.isSome(): if aggContribution.isSome():
@ -319,17 +330,20 @@ proc produceAndPublishContributions(service: SyncCommitteeServiceRef,
inc(errored) inc(errored)
(succeed, errored, failed) (succeed, errored, failed)
let delay = vc.getDelay(slot.aggregate_deadline()) let
delay = vc.getDelay(slot.aggregate_deadline())
dur = Moment.now() - startTime
debug "Sync message contribution statistics", debug "Sync message contribution statistics",
total = len(contributions), total = len(contributions),
succeed = statistics[0], succeed = statistics[0],
failed_to_create = len(pendingAggregates) - len(contributions), failed_to_create = len(pendingAggregates) - len(contributions),
failed_to_deliver = statistics[1], failed_to_deliver = statistics[1],
not_accepted = statistics[2], not_accepted = statistics[2],
delay = delay, slot = slot delay = delay, duration = dur
else: else:
debug "No contribution and proofs scheduled for the slot", slot = slot debug "No contribution and proofs scheduled for the slot"
proc publishSyncMessagesAndContributions(service: SyncCommitteeServiceRef, proc publishSyncMessagesAndContributions(service: SyncCommitteeServiceRef,
slot: Slot, slot: Slot,
@ -339,9 +353,12 @@ proc publishSyncMessagesAndContributions(service: SyncCommitteeServiceRef,
await vc.waitForBlock(slot, syncCommitteeMessageSlotOffset) await vc.waitForBlock(slot, syncCommitteeMessageSlotOffset)
logScope:
slot = slot
block: block:
let delay = vc.getDelay(slot.sync_committee_message_deadline()) let delay = vc.getDelay(slot.sync_committee_message_deadline())
debug "Producing sync committee messages", delay = delay, slot = slot, debug "Producing sync committee messages", delay = delay,
duties_count = len(duties) duties_count = len(duties)
let beaconBlockRoot = let beaconBlockRoot =
@ -357,7 +374,7 @@ proc publishSyncMessagesAndContributions(service: SyncCommitteeServiceRef,
res.data.root res.data.root
else: else:
if res.execution_optimistic.get(): if res.execution_optimistic.get():
notice "Execution client not in sync", slot = slot notice "Execution client not in sync"
return return
res.data.root res.data.root
except ValidatorApiError as exc: except ValidatorApiError as exc:
@ -369,38 +386,45 @@ proc publishSyncMessagesAndContributions(service: SyncCommitteeServiceRef,
return return
except CatchableError as exc: except CatchableError as exc:
error "Unexpected error while requesting sync message block root", error "Unexpected error while requesting sync message block root",
err_name = exc.name, err_msg = exc.msg, slot = slot error = exc.name, reason = exc.msg
return return
try: try:
await service.produceAndPublishSyncCommitteeMessages(slot, await service.produceAndPublishSyncCommitteeMessages(
beaconBlockRoot, slot, beaconBlockRoot, duties)
duties)
except ValidatorApiError as exc: except ValidatorApiError as exc:
warn "Unable to proceed sync committee messages", slot = slot, warn "Unable to proceed sync committee messages",
duties_count = len(duties), reason = exc.getFailureReason() duties_count = len(duties), reason = exc.getFailureReason()
return return
except CancelledError: except CancelledError:
debug "Sync committee producing process was interrupted" debug "Sync committee messages production was interrupted"
return return
except CatchableError as exc: except CatchableError as exc:
error "Unexpected error while producing sync committee messages", error "Unexpected error while producing sync committee messages",
slot = slot, duties_count = len(duties), error = exc.name, reason = exc.msg
duties_count = len(duties),
err_name = exc.name, err_msg = exc.msg
return return
let contributionTime = let currentTime = vc.beaconClock.now()
# chronos.Duration subtraction cannot return a negative value; in such if slot.sync_contribution_deadline() > currentTime:
# case it will return `ZeroDuration`. let waitDur =
vc.beaconClock.durationToNextSlot() - OneThirdDuration nanoseconds((slot.sync_contribution_deadline() - currentTime).nanoseconds)
if contributionTime != ZeroDuration: # Sleeping until `sync_contribution_deadline`.
await sleepAsync(contributionTime) debug "Waiting for sync contribution deadline", wait_time = waitDur
await sleepAsync(waitDur)
block: block:
let delay = vc.getDelay(slot.sync_contribution_deadline()) let delay = vc.getDelay(slot.sync_contribution_deadline())
debug "Producing contribution and proofs", delay = delay debug "Producing contribution and proofs", delay = delay
try:
await service.produceAndPublishContributions(slot, beaconBlockRoot, duties) await service.produceAndPublishContributions(slot, beaconBlockRoot, duties)
except CancelledError:
debug "Sync committee contributions production was interrupted"
return
except CatchableError as exc:
error "Unexpected error while producing sync committee contributions",
duties_count = len(duties), error = exc.name, reason = exc.msg
return
proc processSyncCommitteeTasks(service: SyncCommitteeServiceRef, proc processSyncCommitteeTasks(service: SyncCommitteeServiceRef,
slot: Slot) {.async.} = slot: Slot) {.async.} =
@ -409,12 +433,15 @@ proc processSyncCommitteeTasks(service: SyncCommitteeServiceRef,
duties = vc.getSyncCommitteeDutiesForSlot(slot + 1) duties = vc.getSyncCommitteeDutiesForSlot(slot + 1)
timeout = vc.beaconClock.durationToNextSlot() timeout = vc.beaconClock.durationToNextSlot()
logScope:
slot = slot
try: try:
await service.publishSyncMessagesAndContributions(slot, await service.publishSyncMessagesAndContributions(slot,
duties).wait(timeout) duties).wait(timeout)
except AsyncTimeoutError: except AsyncTimeoutError:
warn "Unable to publish sync committee messages and contributions in time", warn "Unable to publish sync committee messages and contributions in time",
slot = slot, timeout = timeout timeout = timeout
except CancelledError as exc: except CancelledError as exc:
debug "Sync committee publish task has been interrupted" debug "Sync committee publish task has been interrupted"
raise exc raise exc
@ -439,8 +466,8 @@ proc mainLoop(service: SyncCommitteeServiceRef) {.async.} =
debug "Service interrupted" debug "Service interrupted"
return return
except CatchableError as exc: except CatchableError as exc:
warn "Service crashed with unexpected error", err_name = exc.name, warn "Service crashed with unexpected error", error = exc.name,
err_msg = exc.msg reason = exc.msg
return return
doAssert(len(vc.forks) > 0, "Fork schedule must not be empty at this point") doAssert(len(vc.forks) > 0, "Fork schedule must not be empty at this point")
@ -468,8 +495,8 @@ proc mainLoop(service: SyncCommitteeServiceRef) {.async.} =
debug "Service interrupted" debug "Service interrupted"
true true
except CatchableError as exc: except CatchableError as exc:
warn "Service crashed with unexpected error", err_name = exc.name, warn "Service crashed with unexpected error", error = exc.name,
err_msg = exc.msg reason = exc.msg
true true
if breakLoop: if breakLoop:
@ -481,7 +508,7 @@ proc init*(t: typedesc[SyncCommitteeServiceRef],
let res = SyncCommitteeServiceRef(name: ServiceName, client: vc, let res = SyncCommitteeServiceRef(name: ServiceName, client: vc,
state: ServiceState.Initialized) state: ServiceState.Initialized)
debug "Initializing service" debug "Initializing service"
return res res
proc start*(service: SyncCommitteeServiceRef) = proc start*(service: SyncCommitteeServiceRef) =
service.lifeFut = mainLoop(service) service.lifeFut = mainLoop(service)