delay `onSlotEnd` if there are duties (#5196)

We currently call `onSlotEnd` whenever all in-BN validator duties are
completed. VC validator duties are not awaited. When `onSlotEnd` is
processed close to the slot start, a VC may therefore miss duties.
Adding a delay before `onSlotEnd` improves this situation.
The logic can be optimized further if `ActionTracker` would track
`knownValidators` from REST separately from in-process ones.

beacon_chain/nimbus_beacon_node.nim

@@ -1144,6 +1144,15 @@ proc onSlotEnd(node: BeaconNode, slot: Slot) {.async.} =
   # Things we do when slot processing has ended and we're about to wait for the
   # next slot
 
+  # By waiting until close before slot end, ensure that preparation for next
+  # slot does not interfere with propagation of messages and with VC duties.
+  const endOffset = aggregateSlotOffset + nanos(
+    (NANOSECONDS_PER_SLOT - aggregateSlotOffset.nanoseconds.uint64).int64 div 2)
+  let endCutoff = node.beaconClock.fromNow(slot.start_beacon_time + endOffset)
+  if endCutoff.inFuture:
+    debug "Waiting for slot end", slot, endCutoff = shortLog(endCutoff.offset)
+    await sleepAsync(endCutoff.offset)
+
   if node.dag.needStateCachesAndForkChoicePruning():
     if node.attachedValidators[].validators.len > 0:
       node.attachedValidators[]

beacon_chain/spec/beacon_time.nim

@@ -48,7 +48,7 @@ const
 
   FAR_FUTURE_BEACON_TIME* = BeaconTime(ns_since_genesis: int64.high())
 
-  NANOSECONDS_PER_SLOT = SECONDS_PER_SLOT * 1_000_000_000'u64
+  NANOSECONDS_PER_SLOT* = SECONDS_PER_SLOT * 1_000_000_000'u64
 
 template ethTimeUnit*(typ: type) {.dirty.} =
   func `+`*(x: typ, y: uint64): typ {.borrow.}