EIP4844 sync (#4581)
* EIP4844 Sync * Pass eip4844 fork epoch rather than cfg to syncmanager * Fix sync * Update test * map->mapIt
This commit is contained in:
parent
db91710ee2
commit
59e41dc65d
|
@ -180,7 +180,7 @@ proc storeBackfillBlock(
|
||||||
# writing the block in case of blob error.
|
# writing the block in case of blob error.
|
||||||
let blobsOk =
|
let blobsOk =
|
||||||
when typeof(signedBlock).toFork() >= ConsensusFork.EIP4844:
|
when typeof(signedBlock).toFork() >= ConsensusFork.EIP4844:
|
||||||
blobs.isSome() and not
|
blobs.isNone or
|
||||||
validate_blobs_sidecar(signedBlock.message.slot,
|
validate_blobs_sidecar(signedBlock.message.slot,
|
||||||
signedBlock.root,
|
signedBlock.root,
|
||||||
signedBlock.message
|
signedBlock.message
|
||||||
|
|
|
@ -800,7 +800,8 @@ func chunkMaxSize[T](): uint32 =
|
||||||
# compiler error on (T: type) syntax...
|
# compiler error on (T: type) syntax...
|
||||||
when T is ForkySignedBeaconBlock:
|
when T is ForkySignedBeaconBlock:
|
||||||
when T is phase0.SignedBeaconBlock or T is altair.SignedBeaconBlock or
|
when T is phase0.SignedBeaconBlock or T is altair.SignedBeaconBlock or
|
||||||
T is bellatrix.SignedBeaconBlock or T is capella.SignedBeaconBlock:
|
T is bellatrix.SignedBeaconBlock or T is capella.SignedBeaconBlock or
|
||||||
|
T is eip4844.SignedBeaconBlock:
|
||||||
MAX_CHUNK_SIZE_BELLATRIX
|
MAX_CHUNK_SIZE_BELLATRIX
|
||||||
else:
|
else:
|
||||||
{.fatal: "what's the chunk size here?".}
|
{.fatal: "what's the chunk size here?".}
|
||||||
|
|
|
@ -330,10 +330,13 @@ proc initFullNode(
|
||||||
# that should probably be reimagined more holistically in the future.
|
# that should probably be reimagined more holistically in the future.
|
||||||
let resfut = newFuture[Result[void, VerifierError]]("blockVerifier")
|
let resfut = newFuture[Result[void, VerifierError]]("blockVerifier")
|
||||||
blockProcessor[].addBlock(MsgSource.gossip, signedBlock,
|
blockProcessor[].addBlock(MsgSource.gossip, signedBlock,
|
||||||
Opt.none(eip4844.BlobsSidecar), resfut,
|
Opt.none(eip4844.BlobsSidecar),
|
||||||
|
resfut,
|
||||||
maybeFinalized = maybeFinalized)
|
maybeFinalized = maybeFinalized)
|
||||||
resfut
|
resfut
|
||||||
blockBlobsVerifier = proc(signedBlock: ForkedSignedBeaconBlock, blobs: eip4844.BlobsSidecar):
|
blockBlobsVerifier = proc(signedBlock: ForkedSignedBeaconBlock,
|
||||||
|
blobs: eip4844.BlobsSidecar,
|
||||||
|
maybeFinalized: bool):
|
||||||
Future[Result[void, VerifierError]] =
|
Future[Result[void, VerifierError]] =
|
||||||
# The design with a callback for block verification is unusual compared
|
# The design with a callback for block verification is unusual compared
|
||||||
# to the rest of the application, but fits with the general approach
|
# to the rest of the application, but fits with the general approach
|
||||||
|
@ -341,7 +344,7 @@ proc initFullNode(
|
||||||
# that should probably be reimagined more holistically in the future.
|
# that should probably be reimagined more holistically in the future.
|
||||||
let resfut = newFuture[Result[void, VerifierError]]("blockVerifier")
|
let resfut = newFuture[Result[void, VerifierError]]("blockVerifier")
|
||||||
blockProcessor[].addBlock(MsgSource.gossip, signedBlock,
|
blockProcessor[].addBlock(MsgSource.gossip, signedBlock,
|
||||||
Opt.some(blobs), resfut)
|
Opt.some(blobs), resfut, maybeFinalized = maybeFinalized)
|
||||||
resfut
|
resfut
|
||||||
processor = Eth2Processor.new(
|
processor = Eth2Processor.new(
|
||||||
config.doppelgangerDetection,
|
config.doppelgangerDetection,
|
||||||
|
@ -349,13 +352,14 @@ proc initFullNode(
|
||||||
validatorChangePool, node.attachedValidators, syncCommitteeMsgPool,
|
validatorChangePool, node.attachedValidators, syncCommitteeMsgPool,
|
||||||
lightClientPool, quarantine, rng, getBeaconTime, taskpool)
|
lightClientPool, quarantine, rng, getBeaconTime, taskpool)
|
||||||
syncManager = newSyncManager[Peer, PeerId](
|
syncManager = newSyncManager[Peer, PeerId](
|
||||||
node.network.peerPool, SyncQueueKind.Forward, getLocalHeadSlot,
|
node.network.peerPool, dag.cfg.EIP4844_FORK_EPOCH, SyncQueueKind.Forward, getLocalHeadSlot,
|
||||||
getLocalWallSlot, getFirstSlotAtFinalizedEpoch, getBackfillSlot,
|
getLocalWallSlot, getFirstSlotAtFinalizedEpoch, getBackfillSlot,
|
||||||
getFrontfillSlot, dag.tail.slot, blockVerifier)
|
getFrontfillSlot, dag.tail.slot, blockVerifier, blockBlobsVerifier)
|
||||||
backfiller = newSyncManager[Peer, PeerId](
|
backfiller = newSyncManager[Peer, PeerId](
|
||||||
node.network.peerPool, SyncQueueKind.Backward, getLocalHeadSlot,
|
node.network.peerPool, dag.cfg.EIP4844_FORK_EPOCH, SyncQueueKind.Backward, getLocalHeadSlot,
|
||||||
getLocalWallSlot, getFirstSlotAtFinalizedEpoch, getBackfillSlot,
|
getLocalWallSlot, getFirstSlotAtFinalizedEpoch, getBackfillSlot,
|
||||||
getFrontfillSlot, dag.backfill.slot, blockVerifier, maxHeadAge = 0)
|
getFrontfillSlot, dag.backfill.slot, blockVerifier, blockBlobsVerifier,
|
||||||
|
maxHeadAge = 0)
|
||||||
router = (ref MessageRouter)(
|
router = (ref MessageRouter)(
|
||||||
processor: processor,
|
processor: processor,
|
||||||
network: node.network)
|
network: node.network)
|
||||||
|
|
|
@ -33,7 +33,8 @@ type
|
||||||
proc(signedBlock: ForkedSignedBeaconBlock, maybeFinalized: bool):
|
proc(signedBlock: ForkedSignedBeaconBlock, maybeFinalized: bool):
|
||||||
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
||||||
BlockBlobsVerifier* =
|
BlockBlobsVerifier* =
|
||||||
proc(signedBlock: ForkedSignedBeaconBlock, blobs: eip4844.BlobsSidecar):
|
proc(signedBlock: ForkedSignedBeaconBlock, blobs: eip4844.BlobsSidecar,
|
||||||
|
maybeFinalized: bool):
|
||||||
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
||||||
|
|
||||||
RequestManager* = object
|
RequestManager* = object
|
||||||
|
@ -173,7 +174,8 @@ proc fetchAncestorBlocksAndBlobsFromNetwork(rman: RequestManager,
|
||||||
debug "Requesting blocks and sidecars by root",
|
debug "Requesting blocks and sidecars by root",
|
||||||
peer = peer, blocks = shortLog(items), peer_score = peer.getScore()
|
peer = peer, blocks = shortLog(items), peer_score = peer.getScore()
|
||||||
|
|
||||||
let blocks = (await beaconBlockAndBlobsSidecarByRoot_v1(peer, BlockRootsList items))
|
let blocks = await beaconBlockAndBlobsSidecarByRoot_v1(peer,
|
||||||
|
BlockRootsList items)
|
||||||
|
|
||||||
if blocks.isOk:
|
if blocks.isOk:
|
||||||
let ublocks = blocks.get()
|
let ublocks = blocks.get()
|
||||||
|
@ -183,7 +185,9 @@ proc fetchAncestorBlocksAndBlobsFromNetwork(rman: RequestManager,
|
||||||
gotUnviableBlock = false
|
gotUnviableBlock = false
|
||||||
|
|
||||||
for b in ublocks:
|
for b in ublocks:
|
||||||
let ver = await rman.blockBlobsVerifier(ForkedSignedBeaconBlock.init(b[].beacon_block), b[].blobs_sidecar)
|
let ver = await rman.blockBlobsVerifier(
|
||||||
|
ForkedSignedBeaconBlock.init(b[].beacon_block),
|
||||||
|
b[].blobs_sidecar, false)
|
||||||
if ver.isErr():
|
if ver.isErr():
|
||||||
case ver.error()
|
case ver.error()
|
||||||
of VerifierError.MissingParent:
|
of VerifierError.MissingParent:
|
||||||
|
|
|
@ -47,6 +47,7 @@ type
|
||||||
|
|
||||||
SyncManager*[A, B] = ref object
|
SyncManager*[A, B] = ref object
|
||||||
pool: PeerPool[A, B]
|
pool: PeerPool[A, B]
|
||||||
|
EIP4844_FORK_EPOCH: Epoch
|
||||||
responseTimeout: chronos.Duration
|
responseTimeout: chronos.Duration
|
||||||
maxHeadAge: uint64
|
maxHeadAge: uint64
|
||||||
getLocalHeadSlot: GetSlotCallback
|
getLocalHeadSlot: GetSlotCallback
|
||||||
|
@ -62,6 +63,7 @@ type
|
||||||
queue: SyncQueue[A]
|
queue: SyncQueue[A]
|
||||||
syncFut: Future[void]
|
syncFut: Future[void]
|
||||||
blockVerifier: BlockVerifier
|
blockVerifier: BlockVerifier
|
||||||
|
blockBlobsVerifier: BlockBlobsVerifier
|
||||||
inProgress*: bool
|
inProgress*: bool
|
||||||
insSyncSpeed*: float
|
insSyncSpeed*: float
|
||||||
avgSyncSpeed*: float
|
avgSyncSpeed*: float
|
||||||
|
@ -75,6 +77,7 @@ type
|
||||||
slots*: uint64
|
slots*: uint64
|
||||||
|
|
||||||
BeaconBlocksRes = NetRes[List[ref ForkedSignedBeaconBlock, MAX_REQUEST_BLOCKS]]
|
BeaconBlocksRes = NetRes[List[ref ForkedSignedBeaconBlock, MAX_REQUEST_BLOCKS]]
|
||||||
|
BlobsSidecarRes = NetRes[List[ref BlobsSidecar, MAX_REQUEST_BLOBS_SIDECARS]]
|
||||||
|
|
||||||
proc now*(sm: typedesc[SyncMoment], slots: uint64): SyncMoment {.inline.} =
|
proc now*(sm: typedesc[SyncMoment], slots: uint64): SyncMoment {.inline.} =
|
||||||
SyncMoment(stamp: now(chronos.Moment), slots: slots)
|
SyncMoment(stamp: now(chronos.Moment), slots: slots)
|
||||||
|
@ -94,7 +97,8 @@ proc initQueue[A, B](man: SyncManager[A, B]) =
|
||||||
of SyncQueueKind.Forward:
|
of SyncQueueKind.Forward:
|
||||||
man.queue = SyncQueue.init(A, man.direction, man.getFirstSlot(),
|
man.queue = SyncQueue.init(A, man.direction, man.getFirstSlot(),
|
||||||
man.getLastSlot(), man.chunkSize,
|
man.getLastSlot(), man.chunkSize,
|
||||||
man.getSafeSlot, man.blockVerifier, 1,
|
man.getSafeSlot, man.blockVerifier,
|
||||||
|
man.blockBlobsVerifier, 1,
|
||||||
man.ident)
|
man.ident)
|
||||||
of SyncQueueKind.Backward:
|
of SyncQueueKind.Backward:
|
||||||
let
|
let
|
||||||
|
@ -108,10 +112,11 @@ proc initQueue[A, B](man: SyncManager[A, B]) =
|
||||||
Slot(firstSlot - 1'u64)
|
Slot(firstSlot - 1'u64)
|
||||||
man.queue = SyncQueue.init(A, man.direction, startSlot, lastSlot,
|
man.queue = SyncQueue.init(A, man.direction, startSlot, lastSlot,
|
||||||
man.chunkSize, man.getSafeSlot,
|
man.chunkSize, man.getSafeSlot,
|
||||||
man.blockVerifier, 1,
|
man.blockVerifier, man.blockBlobsVerifier, 1,
|
||||||
man.ident)
|
man.ident)
|
||||||
|
|
||||||
proc newSyncManager*[A, B](pool: PeerPool[A, B],
|
proc newSyncManager*[A, B](pool: PeerPool[A, B],
|
||||||
|
eip4844Epoch: Epoch,
|
||||||
direction: SyncQueueKind,
|
direction: SyncQueueKind,
|
||||||
getLocalHeadSlotCb: GetSlotCallback,
|
getLocalHeadSlotCb: GetSlotCallback,
|
||||||
getLocalWallSlotCb: GetSlotCallback,
|
getLocalWallSlotCb: GetSlotCallback,
|
||||||
|
@ -120,6 +125,7 @@ proc newSyncManager*[A, B](pool: PeerPool[A, B],
|
||||||
getFrontfillSlotCb: GetSlotCallback,
|
getFrontfillSlotCb: GetSlotCallback,
|
||||||
progressPivot: Slot,
|
progressPivot: Slot,
|
||||||
blockVerifier: BlockVerifier,
|
blockVerifier: BlockVerifier,
|
||||||
|
blockBlobsVerifier: BlockBlobsVerifier,
|
||||||
maxHeadAge = uint64(SLOTS_PER_EPOCH * 1),
|
maxHeadAge = uint64(SLOTS_PER_EPOCH * 1),
|
||||||
chunkSize = uint64(SLOTS_PER_EPOCH),
|
chunkSize = uint64(SLOTS_PER_EPOCH),
|
||||||
flags: set[SyncManagerFlag] = {},
|
flags: set[SyncManagerFlag] = {},
|
||||||
|
@ -133,6 +139,7 @@ proc newSyncManager*[A, B](pool: PeerPool[A, B],
|
||||||
|
|
||||||
var res = SyncManager[A, B](
|
var res = SyncManager[A, B](
|
||||||
pool: pool,
|
pool: pool,
|
||||||
|
EIP4844_FORK_EPOCH: eip4844Epoch,
|
||||||
getLocalHeadSlot: getLocalHeadSlotCb,
|
getLocalHeadSlot: getLocalHeadSlotCb,
|
||||||
getLocalWallSlot: getLocalWallSlotCb,
|
getLocalWallSlot: getLocalWallSlotCb,
|
||||||
getSafeSlot: getSafeSlot,
|
getSafeSlot: getSafeSlot,
|
||||||
|
@ -142,6 +149,7 @@ proc newSyncManager*[A, B](pool: PeerPool[A, B],
|
||||||
maxHeadAge: maxHeadAge,
|
maxHeadAge: maxHeadAge,
|
||||||
chunkSize: chunkSize,
|
chunkSize: chunkSize,
|
||||||
blockVerifier: blockVerifier,
|
blockVerifier: blockVerifier,
|
||||||
|
blockBlobsVerifier: blockBlobsVerifier,
|
||||||
notInSyncEvent: newAsyncEvent(),
|
notInSyncEvent: newAsyncEvent(),
|
||||||
direction: direction,
|
direction: direction,
|
||||||
ident: ident,
|
ident: ident,
|
||||||
|
@ -179,6 +187,41 @@ proc getBlocks*[A, B](man: SyncManager[A, B], peer: A,
|
||||||
errName = exc.name, errMsg = exc.msg
|
errName = exc.name, errMsg = exc.msg
|
||||||
return
|
return
|
||||||
|
|
||||||
|
proc shouldGetBlobs[A, B](man: SyncManager[A, B], e: Epoch): bool =
|
||||||
|
let wallEpoch = man.getLocalWallSlot().epoch
|
||||||
|
e >= man.EIP4844_FORK_EPOCH and
|
||||||
|
(wallEpoch < MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS or
|
||||||
|
e >= wallEpoch - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS)
|
||||||
|
|
||||||
|
proc getBlobsSidecars*[A, B](man: SyncManager[A, B], peer: A,
|
||||||
|
req: SyncRequest): Future[BlobsSidecarRes] {.async.} =
|
||||||
|
mixin getScore, `==`
|
||||||
|
|
||||||
|
logScope:
|
||||||
|
peer_score = peer.getScore()
|
||||||
|
peer_speed = peer.netKbps()
|
||||||
|
sync_ident = man.ident
|
||||||
|
direction = man.direction
|
||||||
|
topics = "syncman"
|
||||||
|
|
||||||
|
doAssert(not(req.isEmpty()), "Request must not be empty!")
|
||||||
|
debug "Requesting blobs sidecars from peer", request = req
|
||||||
|
try:
|
||||||
|
let res = await blobsSidecarsByRange(peer, req.slot, req.count)
|
||||||
|
|
||||||
|
if res.isErr():
|
||||||
|
debug "Error, while reading blobsSidecarsByRange response", request = req,
|
||||||
|
error = $res.error()
|
||||||
|
return
|
||||||
|
return res
|
||||||
|
except CancelledError:
|
||||||
|
debug "Interrupt, while waiting blobsSidecarsByRange response", request = req
|
||||||
|
return
|
||||||
|
except CatchableError as exc:
|
||||||
|
debug "Error, while waiting blobsSidecarsByRange response", request = req,
|
||||||
|
errName = exc.name, errMsg = exc.msg
|
||||||
|
return
|
||||||
|
|
||||||
proc remainingSlots(man: SyncManager): uint64 =
|
proc remainingSlots(man: SyncManager): uint64 =
|
||||||
let
|
let
|
||||||
first = man.getFirstSlot()
|
first = man.getFirstSlot()
|
||||||
|
@ -333,20 +376,62 @@ proc syncStep[A, B](man: SyncManager[A, B], index: int, peer: A) {.async.} =
|
||||||
|
|
||||||
try:
|
try:
|
||||||
let blocks = await man.getBlocks(peer, req)
|
let blocks = await man.getBlocks(peer, req)
|
||||||
if blocks.isOk():
|
if blocks.isErr():
|
||||||
let data = blocks.get().asSeq()
|
peer.updateScore(PeerScoreNoValues)
|
||||||
let smap = getShortMap(req, data)
|
man.queue.push(req)
|
||||||
debug "Received blocks on request", blocks_count = len(data),
|
debug "Failed to receive blocks on request", request = req
|
||||||
blocks_map = smap, request = req
|
return
|
||||||
|
let blockData = blocks.get().asSeq()
|
||||||
|
let blockSmap = getShortMap(req, blockData)
|
||||||
|
debug "Received blocks on request", blocks_count = len(blockData),
|
||||||
|
blocks_map = blockSmap, request = req
|
||||||
|
|
||||||
if not(checkResponse(req, data)):
|
let slots = mapIt(blockData, it[].slot)
|
||||||
|
if not(checkResponse(req, slots)):
|
||||||
peer.updateScore(PeerScoreBadResponse)
|
peer.updateScore(PeerScoreBadResponse)
|
||||||
warn "Received blocks sequence is not in requested range",
|
warn "Received blocks sequence is not in requested range",
|
||||||
blocks_count = len(data), blocks_map = smap,
|
blocks_count = len(blockData), blocks_map = blockSmap,
|
||||||
request = req
|
request = req
|
||||||
return
|
return
|
||||||
|
|
||||||
if len(data) == 0 and man.direction == SyncQueueKind.Backward and
|
let blobData =
|
||||||
|
if man.shouldGetBlobs(req.slot.epoch):
|
||||||
|
let blobs = await man.getBlobsSidecars(peer, req)
|
||||||
|
if blobs.isErr():
|
||||||
|
peer.updateScore(PeerScoreNoValues)
|
||||||
|
man.queue.push(req)
|
||||||
|
debug "Failed to receive blobs on request", request = req
|
||||||
|
return
|
||||||
|
let blobData = blobs.get().asSeq()
|
||||||
|
let slots = mapIt(blobData, it[].beacon_block_slot)
|
||||||
|
if not(checkResponse(req, slots)):
|
||||||
|
peer.updateScore(PeerScoreBadResponse)
|
||||||
|
warn "Received blobs sequence is not in requested range",
|
||||||
|
blobs_count = len(blobData), blobs_map = getShortMap(req, blobData),
|
||||||
|
request = req
|
||||||
|
return
|
||||||
|
Opt.some(blobData)
|
||||||
|
else:
|
||||||
|
Opt.none(seq[ref BlobsSidecar])
|
||||||
|
|
||||||
|
if blobData.isSome:
|
||||||
|
let blobSmap = getShortMap(req, blobData.get(@[]))
|
||||||
|
debug "Received blobs on request", blobs_count = len(blobData.get(@[])),
|
||||||
|
blobs_map = blobSmap, request = req
|
||||||
|
let blobs = blobData.get()
|
||||||
|
if len(blobs) != len(blockData):
|
||||||
|
info "block and blobs have different lengths", blobs=len(blobs), blocks=len(blockData)
|
||||||
|
peer.updateScore(PeerScoreNoValues)
|
||||||
|
man.queue.push(req)
|
||||||
|
return
|
||||||
|
for i, blk in blockData:
|
||||||
|
if blk[].slot != blobs[i].beacon_block_slot:
|
||||||
|
peer.updateScore(PeerScoreNoValues)
|
||||||
|
man.queue.push(req)
|
||||||
|
debug "block and blobs data have inconsistent slots"
|
||||||
|
return
|
||||||
|
|
||||||
|
if len(blockData) == 0 and man.direction == SyncQueueKind.Backward and
|
||||||
req.contains(man.getSafeSlot()):
|
req.contains(man.getSafeSlot()):
|
||||||
# The sync protocol does not distinguish between:
|
# The sync protocol does not distinguish between:
|
||||||
# - All requested slots are empty
|
# - All requested slots are empty
|
||||||
|
@ -370,13 +455,8 @@ proc syncStep[A, B](man: SyncManager[A, B], index: int, peer: A) {.async.} =
|
||||||
# TODO descore peers that lie
|
# TODO descore peers that lie
|
||||||
maybeFinalized = lastSlot < peerFinalized
|
maybeFinalized = lastSlot < peerFinalized
|
||||||
|
|
||||||
await man.queue.push(req, data, maybeFinalized, proc() =
|
await man.queue.push(req, blockData, blobData, maybeFinalized, proc() =
|
||||||
man.workers[index].status = SyncWorkerStatus.Processing)
|
man.workers[index].status = SyncWorkerStatus.Processing)
|
||||||
else:
|
|
||||||
peer.updateScore(PeerScoreNoValues)
|
|
||||||
man.queue.push(req)
|
|
||||||
debug "Failed to receive blocks on request", request = req
|
|
||||||
return
|
|
||||||
|
|
||||||
except CatchableError as exc:
|
except CatchableError as exc:
|
||||||
debug "Unexpected exception while receiving blocks", request = req,
|
debug "Unexpected exception while receiving blocks", request = req,
|
||||||
|
|
|
@ -23,7 +23,7 @@ logScope:
|
||||||
|
|
||||||
const
|
const
|
||||||
MAX_REQUEST_BLOCKS* = 1024
|
MAX_REQUEST_BLOCKS* = 1024
|
||||||
MAX_REQUEST_BLOBS_SIDECARS = 128
|
MAX_REQUEST_BLOBS_SIDECARS* = 128
|
||||||
|
|
||||||
blockResponseCost = allowedOpsPerSecondCost(64) # Allow syncing ~64 blocks/sec (minus request costs)
|
blockResponseCost = allowedOpsPerSecondCost(64) # Allow syncing ~64 blocks/sec (minus request costs)
|
||||||
|
|
||||||
|
@ -110,6 +110,12 @@ proc readChunkPayload*(
|
||||||
return ok newClone(ForkedSignedBeaconBlock.init(res.get))
|
return ok newClone(ForkedSignedBeaconBlock.init(res.get))
|
||||||
else:
|
else:
|
||||||
return err(res.error)
|
return err(res.error)
|
||||||
|
elif contextBytes == peer.network.forkDigests.eip4844:
|
||||||
|
let res = await readChunkPayload(conn, peer, eip4844.SignedBeaconBlock)
|
||||||
|
if res.isOk:
|
||||||
|
return ok newClone(ForkedSignedBeaconBlock.init(res.get))
|
||||||
|
else:
|
||||||
|
return err(res.error)
|
||||||
else:
|
else:
|
||||||
return neterr InvalidContextBytes
|
return neterr InvalidContextBytes
|
||||||
|
|
||||||
|
@ -131,6 +137,24 @@ proc readChunkPayload*(
|
||||||
else:
|
else:
|
||||||
return neterr InvalidContextBytes
|
return neterr InvalidContextBytes
|
||||||
|
|
||||||
|
proc readChunkPayload*(
|
||||||
|
conn: Connection, peer: Peer, MsgType: type (ref BlobsSidecar)):
|
||||||
|
Future[NetRes[MsgType]] {.async.} =
|
||||||
|
var contextBytes: ForkDigest
|
||||||
|
try:
|
||||||
|
await conn.readExactly(addr contextBytes, sizeof contextBytes)
|
||||||
|
except CatchableError:
|
||||||
|
return neterr UnexpectedEOF
|
||||||
|
|
||||||
|
if contextBytes == peer.network.forkDigests.eip4844:
|
||||||
|
let res = await readChunkPayload(conn, peer, BlobsSidecar)
|
||||||
|
if res.isOk:
|
||||||
|
return ok newClone(res.get)
|
||||||
|
else:
|
||||||
|
return err(res.error)
|
||||||
|
else:
|
||||||
|
return neterr InvalidContextBytes
|
||||||
|
|
||||||
proc readChunkPayload*(
|
proc readChunkPayload*(
|
||||||
conn: Connection, peer: Peer, MsgType: type SomeForkedLightClientObject):
|
conn: Connection, peer: Peer, MsgType: type SomeForkedLightClientObject):
|
||||||
Future[NetRes[MsgType]] {.async.} =
|
Future[NetRes[MsgType]] {.async.} =
|
||||||
|
@ -509,7 +533,7 @@ p2pProtocol BeaconSync(version = 1,
|
||||||
startSlot: Slot,
|
startSlot: Slot,
|
||||||
reqCount: uint64,
|
reqCount: uint64,
|
||||||
response: MultipleChunksResponse[
|
response: MultipleChunksResponse[
|
||||||
BlobsSidecar, MAX_REQUEST_BLOBS_SIDECARS])
|
ref BlobsSidecar, MAX_REQUEST_BLOBS_SIDECARS])
|
||||||
{.async, libp2pProtocol("blobs_sidecars_by_range", 1).} =
|
{.async, libp2pProtocol("blobs_sidecars_by_range", 1).} =
|
||||||
# TODO This code is more complicated than it needs to be, since the type
|
# TODO This code is more complicated than it needs to be, since the type
|
||||||
# of the multiple chunks response is not actually used in this server
|
# of the multiple chunks response is not actually used in this server
|
||||||
|
|
|
@ -28,6 +28,10 @@ type
|
||||||
BlockVerifier* =
|
BlockVerifier* =
|
||||||
proc(signedBlock: ForkedSignedBeaconBlock, maybeFinalized: bool):
|
proc(signedBlock: ForkedSignedBeaconBlock, maybeFinalized: bool):
|
||||||
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
||||||
|
BlockBlobsVerifier* =
|
||||||
|
proc(signedBlock: ForkedSignedBeaconBlock, blobs: eip4844.BlobsSidecar,
|
||||||
|
maybeFinalized: bool):
|
||||||
|
Future[Result[void, VerifierError]] {.gcsafe, raises: [Defect].}
|
||||||
|
|
||||||
SyncQueueKind* {.pure.} = enum
|
SyncQueueKind* {.pure.} = enum
|
||||||
Forward, Backward
|
Forward, Backward
|
||||||
|
@ -42,6 +46,7 @@ type
|
||||||
SyncResult*[T] = object
|
SyncResult*[T] = object
|
||||||
request*: SyncRequest[T]
|
request*: SyncRequest[T]
|
||||||
data*: seq[ref ForkedSignedBeaconBlock]
|
data*: seq[ref ForkedSignedBeaconBlock]
|
||||||
|
blobs*: Opt[seq[ref BlobsSidecar]]
|
||||||
|
|
||||||
GapItem*[T] = object
|
GapItem*[T] = object
|
||||||
start*: Slot
|
start*: Slot
|
||||||
|
@ -74,6 +79,7 @@ type
|
||||||
readyQueue: HeapQueue[SyncResult[T]]
|
readyQueue: HeapQueue[SyncResult[T]]
|
||||||
rewind: Option[RewindPoint]
|
rewind: Option[RewindPoint]
|
||||||
blockVerifier: BlockVerifier
|
blockVerifier: BlockVerifier
|
||||||
|
blockBlobsVerifier: BlockBlobsVerifier
|
||||||
ident*: string
|
ident*: string
|
||||||
|
|
||||||
chronicles.formatIt SyncQueueKind: toLowerAscii($it)
|
chronicles.formatIt SyncQueueKind: toLowerAscii($it)
|
||||||
|
@ -109,6 +115,27 @@ proc getShortMap*[T](req: SyncRequest[T],
|
||||||
slider = slider + 1
|
slider = slider + 1
|
||||||
res
|
res
|
||||||
|
|
||||||
|
proc getShortMap*[T](req: SyncRequest[T],
|
||||||
|
data: openArray[ref BlobsSidecar]): string =
|
||||||
|
## Returns all slot numbers in ``data`` as placement map.
|
||||||
|
var res = newStringOfCap(req.count)
|
||||||
|
var slider = req.slot
|
||||||
|
var last = 0
|
||||||
|
for i in 0 ..< req.count:
|
||||||
|
if last < len(data):
|
||||||
|
for k in last ..< len(data):
|
||||||
|
if slider == data[k].beacon_block_slot:
|
||||||
|
res.add('x')
|
||||||
|
last = k + 1
|
||||||
|
break
|
||||||
|
elif slider < data[k].beacon_block_slot:
|
||||||
|
res.add('.')
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
res.add('.')
|
||||||
|
slider = slider + 1
|
||||||
|
res
|
||||||
|
|
||||||
proc contains*[T](req: SyncRequest[T], slot: Slot): bool {.inline.} =
|
proc contains*[T](req: SyncRequest[T], slot: Slot): bool {.inline.} =
|
||||||
slot >= req.slot and slot < req.slot + req.count
|
slot >= req.slot and slot < req.slot + req.count
|
||||||
|
|
||||||
|
@ -116,7 +143,7 @@ proc cmp*[T](a, b: SyncRequest[T]): int =
|
||||||
cmp(uint64(a.slot), uint64(b.slot))
|
cmp(uint64(a.slot), uint64(b.slot))
|
||||||
|
|
||||||
proc checkResponse*[T](req: SyncRequest[T],
|
proc checkResponse*[T](req: SyncRequest[T],
|
||||||
data: openArray[ref ForkedSignedBeaconBlock]): bool =
|
data: openArray[Slot]): bool =
|
||||||
if len(data) == 0:
|
if len(data) == 0:
|
||||||
# Impossible to verify empty response.
|
# Impossible to verify empty response.
|
||||||
return true
|
return true
|
||||||
|
@ -131,9 +158,9 @@ proc checkResponse*[T](req: SyncRequest[T],
|
||||||
var dindex = 0
|
var dindex = 0
|
||||||
|
|
||||||
while (rindex < req.count) and (dindex < len(data)):
|
while (rindex < req.count) and (dindex < len(data)):
|
||||||
if slot < data[dindex][].slot:
|
if slot < data[dindex]:
|
||||||
discard
|
discard
|
||||||
elif slot == data[dindex][].slot:
|
elif slot == data[dindex]:
|
||||||
inc(dindex)
|
inc(dindex)
|
||||||
else:
|
else:
|
||||||
return false
|
return false
|
||||||
|
@ -174,6 +201,7 @@ proc init*[T](t1: typedesc[SyncQueue], t2: typedesc[T],
|
||||||
start, final: Slot, chunkSize: uint64,
|
start, final: Slot, chunkSize: uint64,
|
||||||
getSafeSlotCb: GetSlotCallback,
|
getSafeSlotCb: GetSlotCallback,
|
||||||
blockVerifier: BlockVerifier,
|
blockVerifier: BlockVerifier,
|
||||||
|
blockBlobsVerifier: BlockBlobsVerifier,
|
||||||
syncQueueSize: int = -1,
|
syncQueueSize: int = -1,
|
||||||
ident: string = "main"): SyncQueue[T] =
|
ident: string = "main"): SyncQueue[T] =
|
||||||
## Create new synchronization queue with parameters
|
## Create new synchronization queue with parameters
|
||||||
|
@ -245,6 +273,7 @@ proc init*[T](t1: typedesc[SyncQueue], t2: typedesc[T],
|
||||||
inpSlot: start,
|
inpSlot: start,
|
||||||
outSlot: start,
|
outSlot: start,
|
||||||
blockVerifier: blockVerifier,
|
blockVerifier: blockVerifier,
|
||||||
|
blockBlobsVerifier: blockBlobsVerifier,
|
||||||
ident: ident
|
ident: ident
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -578,6 +607,7 @@ func numAlreadyKnownSlots[T](sq: SyncQueue[T], sr: SyncRequest[T]): uint64 =
|
||||||
|
|
||||||
proc push*[T](sq: SyncQueue[T], sr: SyncRequest[T],
|
proc push*[T](sq: SyncQueue[T], sr: SyncRequest[T],
|
||||||
data: seq[ref ForkedSignedBeaconBlock],
|
data: seq[ref ForkedSignedBeaconBlock],
|
||||||
|
blobs: Opt[seq[ref BlobsSidecar]],
|
||||||
maybeFinalized: bool = false,
|
maybeFinalized: bool = false,
|
||||||
processingCb: ProcessingCallback = nil) {.async.} =
|
processingCb: ProcessingCallback = nil) {.async.} =
|
||||||
logScope:
|
logScope:
|
||||||
|
@ -605,7 +635,7 @@ proc push*[T](sq: SyncQueue[T], sr: SyncRequest[T],
|
||||||
# SyncQueue reset happens. We are exiting to wake up sync-worker.
|
# SyncQueue reset happens. We are exiting to wake up sync-worker.
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
let syncres = SyncResult[T](request: sr, data: data)
|
let syncres = SyncResult[T](request: sr, data: data, blobs: blobs)
|
||||||
sq.readyQueue.push(syncres)
|
sq.readyQueue.push(syncres)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
@ -654,8 +684,14 @@ proc push*[T](sq: SyncQueue[T], sr: SyncRequest[T],
|
||||||
# Nim versions, remove workaround and move `res` into for loop
|
# Nim versions, remove workaround and move `res` into for loop
|
||||||
res: Result[void, VerifierError]
|
res: Result[void, VerifierError]
|
||||||
|
|
||||||
|
var i=0
|
||||||
for blk in sq.blocks(item):
|
for blk in sq.blocks(item):
|
||||||
|
if reqres.get().blobs.isNone():
|
||||||
res = await sq.blockVerifier(blk[], maybeFinalized)
|
res = await sq.blockVerifier(blk[], maybeFinalized)
|
||||||
|
else:
|
||||||
|
res = await sq.blockBlobsVerifier(blk[], reqres.get().blobs.get()[i][], maybeFinalized)
|
||||||
|
inc(i)
|
||||||
|
|
||||||
if res.isOk():
|
if res.isOk():
|
||||||
goodBlock = some(blk[].slot)
|
goodBlock = some(blk[].slot)
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
|
|
||||||
{.used.}
|
{.used.}
|
||||||
|
|
||||||
import std/strutils
|
import std/[strutils, sequtils]
|
||||||
import unittest2
|
import unittest2
|
||||||
import chronos
|
import chronos
|
||||||
import ../beacon_chain/gossip_processing/block_processor,
|
import ../beacon_chain/gossip_processing/block_processor,
|
||||||
|
@ -57,6 +57,7 @@ proc collector(queue: AsyncQueue[BlockEntry]): BlockVerifier =
|
||||||
return fut
|
return fut
|
||||||
|
|
||||||
return verify
|
return verify
|
||||||
|
|
||||||
suite "SyncManager test suite":
|
suite "SyncManager test suite":
|
||||||
proc createChain(start, finish: Slot): seq[ref ForkedSignedBeaconBlock] =
|
proc createChain(start, finish: Slot): seq[ref ForkedSignedBeaconBlock] =
|
||||||
doAssert(start <= finish)
|
doAssert(start <= finish)
|
||||||
|
@ -85,7 +86,8 @@ suite "SyncManager test suite":
|
||||||
|
|
||||||
var queue = SyncQueue.init(SomeTPeer, kind,
|
var queue = SyncQueue.init(SomeTPeer, kind,
|
||||||
Slot(0), Slot(0), 1'u64,
|
Slot(0), Slot(0), 1'u64,
|
||||||
getStaticSlotCb(Slot(0)), collector(aq))
|
getStaticSlotCb(Slot(0)),
|
||||||
|
collector(aq), nil)
|
||||||
check:
|
check:
|
||||||
len(queue) == 1
|
len(queue) == 1
|
||||||
pendingLen(queue) == 0
|
pendingLen(queue) == 0
|
||||||
|
@ -180,7 +182,8 @@ suite "SyncManager test suite":
|
||||||
let aq = newAsyncQueue[BlockEntry]()
|
let aq = newAsyncQueue[BlockEntry]()
|
||||||
var queue = SyncQueue.init(SomeTPeer, kind,
|
var queue = SyncQueue.init(SomeTPeer, kind,
|
||||||
item[0], item[1], item[2],
|
item[0], item[1], item[2],
|
||||||
getStaticSlotCb(item[0]), collector(aq))
|
getStaticSlotCb(item[0]),
|
||||||
|
collector(aq), nil)
|
||||||
check:
|
check:
|
||||||
len(queue) == item[4]
|
len(queue) == item[4]
|
||||||
pendingLen(queue) == item[5]
|
pendingLen(queue) == item[5]
|
||||||
|
@ -204,11 +207,11 @@ suite "SyncManager test suite":
|
||||||
of SyncQueueKind.Forward:
|
of SyncQueueKind.Forward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
Slot(0), Slot(1), 1'u64,
|
Slot(0), Slot(1), 1'u64,
|
||||||
getStaticSlotCb(Slot(0)), collector(aq))
|
getStaticSlotCb(Slot(0)), collector(aq), nil)
|
||||||
of SyncQueueKind.Backward:
|
of SyncQueueKind.Backward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
Slot(1), Slot(0), 1'u64,
|
Slot(1), Slot(0), 1'u64,
|
||||||
getStaticSlotCb(Slot(1)), collector(aq))
|
getStaticSlotCb(Slot(1)), collector(aq), nil)
|
||||||
|
|
||||||
let p1 = SomeTPeer()
|
let p1 = SomeTPeer()
|
||||||
let p2 = SomeTPeer()
|
let p2 = SomeTPeer()
|
||||||
|
@ -302,11 +305,11 @@ suite "SyncManager test suite":
|
||||||
of SyncQueueKind.Forward:
|
of SyncQueueKind.Forward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
start, finish, chunkSize,
|
start, finish, chunkSize,
|
||||||
getStaticSlotCb(start), collector(aq))
|
getStaticSlotCb(start), collector(aq), nil)
|
||||||
of SyncQueueKind.Backward:
|
of SyncQueueKind.Backward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
finish, start, chunkSize,
|
finish, start, chunkSize,
|
||||||
getStaticSlotCb(finish), collector(aq))
|
getStaticSlotCb(finish), collector(aq), nil)
|
||||||
chain = createChain(start, finish)
|
chain = createChain(start, finish)
|
||||||
validatorFut =
|
validatorFut =
|
||||||
case kkind
|
case kkind
|
||||||
|
@ -322,7 +325,8 @@ suite "SyncManager test suite":
|
||||||
var request = queue.pop(finish, p1)
|
var request = queue.pop(finish, p1)
|
||||||
if request.isEmpty():
|
if request.isEmpty():
|
||||||
break
|
break
|
||||||
await queue.push(request, getSlice(chain, start, request))
|
await queue.push(request, getSlice(chain, start, request),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await validatorFut.cancelAndWait()
|
await validatorFut.cancelAndWait()
|
||||||
|
|
||||||
waitFor runSmokeTest()
|
waitFor runSmokeTest()
|
||||||
|
@ -372,12 +376,12 @@ suite "SyncManager test suite":
|
||||||
of SyncQueueKind.Forward:
|
of SyncQueueKind.Forward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
startSlot, finishSlot, chunkSize,
|
startSlot, finishSlot, chunkSize,
|
||||||
getStaticSlotCb(startSlot), collector(aq),
|
getStaticSlotCb(startSlot), collector(aq), nil,
|
||||||
queueSize)
|
queueSize)
|
||||||
of SyncQueueKind.Backward:
|
of SyncQueueKind.Backward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
finishSlot, startSlot, chunkSize,
|
finishSlot, startSlot, chunkSize,
|
||||||
getStaticSlotCb(finishSlot), collector(aq),
|
getStaticSlotCb(finishSlot), collector(aq), nil,
|
||||||
queueSize)
|
queueSize)
|
||||||
validatorFut =
|
validatorFut =
|
||||||
case kkind
|
case kkind
|
||||||
|
@ -396,7 +400,8 @@ suite "SyncManager test suite":
|
||||||
var r12 = queue.pop(finishSlot, p2)
|
var r12 = queue.pop(finishSlot, p2)
|
||||||
var r13 = queue.pop(finishSlot, p3)
|
var r13 = queue.pop(finishSlot, p3)
|
||||||
|
|
||||||
var f13 = queue.push(r13, chain.getSlice(startSlot, r13))
|
var f13 = queue.push(r13, chain.getSlice(startSlot, r13),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check:
|
check:
|
||||||
f13.finished == false
|
f13.finished == false
|
||||||
|
@ -404,7 +409,8 @@ suite "SyncManager test suite":
|
||||||
of SyncQueueKind.Forward: counter == int(startSlot)
|
of SyncQueueKind.Forward: counter == int(startSlot)
|
||||||
of SyncQueueKind.Backward: counter == int(finishSlot)
|
of SyncQueueKind.Backward: counter == int(finishSlot)
|
||||||
|
|
||||||
var f11 = queue.push(r11, chain.getSlice(startSlot, r11))
|
var f11 = queue.push(r11, chain.getSlice(startSlot, r11),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check:
|
check:
|
||||||
case kkind
|
case kkind
|
||||||
|
@ -413,7 +419,8 @@ suite "SyncManager test suite":
|
||||||
f11.finished == true and f11.failed == false
|
f11.finished == true and f11.failed == false
|
||||||
f13.finished == false
|
f13.finished == false
|
||||||
|
|
||||||
var f12 = queue.push(r12, chain.getSlice(startSlot, r12))
|
var f12 = queue.push(r12, chain.getSlice(startSlot, r12),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f11, f12, f13)
|
await allFutures(f11, f12, f13)
|
||||||
check:
|
check:
|
||||||
f12.finished == true and f12.failed == false
|
f12.finished == true and f12.failed == false
|
||||||
|
@ -477,11 +484,11 @@ suite "SyncManager test suite":
|
||||||
of SyncQueueKind.Forward:
|
of SyncQueueKind.Forward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
start, finish, chunkSize,
|
start, finish, chunkSize,
|
||||||
getFowardSafeSlotCb, collector(aq))
|
getFowardSafeSlotCb, collector(aq), nil)
|
||||||
of SyncQueueKind.Backward:
|
of SyncQueueKind.Backward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
finish, start, chunkSize,
|
finish, start, chunkSize,
|
||||||
getBackwardSafeSlotCb, collector(aq))
|
getBackwardSafeSlotCb, collector(aq), nil)
|
||||||
chain = createChain(start, finish)
|
chain = createChain(start, finish)
|
||||||
validatorFut =
|
validatorFut =
|
||||||
case kkind
|
case kkind
|
||||||
|
@ -516,7 +523,7 @@ suite "SyncManager test suite":
|
||||||
check response[0][].slot >= getFowardSafeSlotCb()
|
check response[0][].slot >= getFowardSafeSlotCb()
|
||||||
else:
|
else:
|
||||||
check response[^1][].slot <= getBackwardSafeSlotCb()
|
check response[^1][].slot <= getBackwardSafeSlotCb()
|
||||||
await queue.push(request, response)
|
await queue.push(request, response, Opt.none(seq[ref BlobsSidecar]))
|
||||||
await validatorFut.cancelAndWait()
|
await validatorFut.cancelAndWait()
|
||||||
|
|
||||||
waitFor runTest()
|
waitFor runTest()
|
||||||
|
@ -570,11 +577,11 @@ suite "SyncManager test suite":
|
||||||
of SyncQueueKind.Forward:
|
of SyncQueueKind.Forward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
start, finish, chunkSize,
|
start, finish, chunkSize,
|
||||||
getFowardSafeSlotCb, collector(aq))
|
getFowardSafeSlotCb, collector(aq), nil)
|
||||||
of SyncQueueKind.Backward:
|
of SyncQueueKind.Backward:
|
||||||
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
finish, start, chunkSize,
|
finish, start, chunkSize,
|
||||||
getBackwardSafeSlotCb, collector(aq))
|
getBackwardSafeSlotCb, collector(aq), nil)
|
||||||
chain = createChain(start, finish)
|
chain = createChain(start, finish)
|
||||||
validatorFut = failingValidator(aq)
|
validatorFut = failingValidator(aq)
|
||||||
|
|
||||||
|
@ -599,7 +606,7 @@ suite "SyncManager test suite":
|
||||||
|
|
||||||
# Handle request 1. Should be re-enqueued as it simulates `Invalid`.
|
# Handle request 1. Should be re-enqueued as it simulates `Invalid`.
|
||||||
let response1 = getSlice(chain, start, request1)
|
let response1 = getSlice(chain, start, request1)
|
||||||
await queue.push(request1, response1)
|
await queue.push(request1, response1, Opt.none(seq[ref BlobsSidecar]))
|
||||||
check debtLen(queue) == request2.count + request1.count
|
check debtLen(queue) == request2.count + request1.count
|
||||||
|
|
||||||
# Request 1 should be discarded as it is no longer relevant.
|
# Request 1 should be discarded as it is no longer relevant.
|
||||||
|
@ -611,7 +618,7 @@ suite "SyncManager test suite":
|
||||||
|
|
||||||
# Handle request 3. Should be re-enqueued as it simulates `Invalid`.
|
# Handle request 3. Should be re-enqueued as it simulates `Invalid`.
|
||||||
let response3 = getSlice(chain, start, request3)
|
let response3 = getSlice(chain, start, request3)
|
||||||
await queue.push(request3, response3)
|
await queue.push(request3, response3, Opt.none(seq[ref BlobsSidecar]))
|
||||||
check debtLen(queue) == request3.count
|
check debtLen(queue) == request3.count
|
||||||
|
|
||||||
# Request 2 should be re-issued.
|
# Request 2 should be re-issued.
|
||||||
|
@ -625,7 +632,7 @@ suite "SyncManager test suite":
|
||||||
|
|
||||||
# Handle request 4. Should be re-enqueued as it simulates `Invalid`.
|
# Handle request 4. Should be re-enqueued as it simulates `Invalid`.
|
||||||
let response4 = getSlice(chain, start, request4)
|
let response4 = getSlice(chain, start, request4)
|
||||||
await queue.push(request4, response4)
|
await queue.push(request4, response4, Opt.none(seq[ref BlobsSidecar]))
|
||||||
check debtLen(queue) == request4.count
|
check debtLen(queue) == request4.count
|
||||||
|
|
||||||
# Advance `safeSlot` out of band.
|
# Advance `safeSlot` out of band.
|
||||||
|
@ -721,7 +728,7 @@ suite "SyncManager test suite":
|
||||||
chain = createChain(startSlot, finishSlot)
|
chain = createChain(startSlot, finishSlot)
|
||||||
queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
startSlot, finishSlot, chunkSize,
|
startSlot, finishSlot, chunkSize,
|
||||||
getStaticSlotCb(startSlot), collector(aq),
|
getStaticSlotCb(startSlot), collector(aq), nil,
|
||||||
queueSize)
|
queueSize)
|
||||||
validatorFut = forwardValidator(aq)
|
validatorFut = forwardValidator(aq)
|
||||||
|
|
||||||
|
@ -741,20 +748,23 @@ suite "SyncManager test suite":
|
||||||
var r13 = queue.pop(finishSlot, p3)
|
var r13 = queue.pop(finishSlot, p3)
|
||||||
var r14 = queue.pop(finishSlot, p4)
|
var r14 = queue.pop(finishSlot, p4)
|
||||||
|
|
||||||
var f14 = queue.push(r14, chain.getSlice(startSlot, r14))
|
var f14 = queue.push(r14, chain.getSlice(startSlot, r14),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check:
|
check:
|
||||||
f14.finished == false
|
f14.finished == false
|
||||||
counter == int(startSlot)
|
counter == int(startSlot)
|
||||||
|
|
||||||
var f12 = queue.push(r12, chain.getSlice(startSlot, r12))
|
var f12 = queue.push(r12, chain.getSlice(startSlot, r12),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check:
|
check:
|
||||||
counter == int(startSlot)
|
counter == int(startSlot)
|
||||||
f12.finished == false
|
f12.finished == false
|
||||||
f14.finished == false
|
f14.finished == false
|
||||||
|
|
||||||
var f11 = queue.push(r11, chain.getSlice(startSlot, r11))
|
var f11 = queue.push(r11, chain.getSlice(startSlot, r11),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f11, f12)
|
await allFutures(f11, f12)
|
||||||
check:
|
check:
|
||||||
counter == int(startSlot + chunkSize + chunkSize)
|
counter == int(startSlot + chunkSize + chunkSize)
|
||||||
|
@ -765,7 +775,8 @@ suite "SyncManager test suite":
|
||||||
var missingSlice = chain.getSlice(startSlot, r13)
|
var missingSlice = chain.getSlice(startSlot, r13)
|
||||||
withBlck(missingSlice[0][]):
|
withBlck(missingSlice[0][]):
|
||||||
blck.message.proposer_index = 0xDEADBEAF'u64
|
blck.message.proposer_index = 0xDEADBEAF'u64
|
||||||
var f13 = queue.push(r13, missingSlice)
|
var f13 = queue.push(r13, missingSlice,
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f13, f14)
|
await allFutures(f13, f14)
|
||||||
check:
|
check:
|
||||||
f11.finished == true and f11.failed == false
|
f11.finished == true and f11.failed == false
|
||||||
|
@ -786,15 +797,18 @@ suite "SyncManager test suite":
|
||||||
|
|
||||||
check r18.isEmpty() == true
|
check r18.isEmpty() == true
|
||||||
|
|
||||||
var f17 = queue.push(r17, chain.getSlice(startSlot, r17))
|
var f17 = queue.push(r17, chain.getSlice(startSlot, r17),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check f17.finished == false
|
check f17.finished == false
|
||||||
|
|
||||||
var f16 = queue.push(r16, chain.getSlice(startSlot, r16))
|
var f16 = queue.push(r16, chain.getSlice(startSlot, r16),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check f16.finished == false
|
check f16.finished == false
|
||||||
|
|
||||||
var f15 = queue.push(r15, chain.getSlice(startSlot, r15))
|
var f15 = queue.push(r15, chain.getSlice(startSlot, r15),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f15, f16, f17)
|
await allFutures(f15, f16, f17)
|
||||||
check:
|
check:
|
||||||
f15.finished == true and f15.failed == false
|
f15.finished == true and f15.failed == false
|
||||||
|
@ -829,7 +843,7 @@ suite "SyncManager test suite":
|
||||||
chain = createChain(startSlot, finishSlot)
|
chain = createChain(startSlot, finishSlot)
|
||||||
queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
startSlot, finishSlot, chunkSize,
|
startSlot, finishSlot, chunkSize,
|
||||||
getStaticSlotCb(startSlot), collector(aq),
|
getStaticSlotCb(startSlot), collector(aq), nil,
|
||||||
queueSize)
|
queueSize)
|
||||||
validatorFut = forwardValidator(aq)
|
validatorFut = forwardValidator(aq)
|
||||||
|
|
||||||
|
@ -840,7 +854,8 @@ suite "SyncManager test suite":
|
||||||
var r11 = queue.pop(finishSlot, p1)
|
var r11 = queue.pop(finishSlot, p1)
|
||||||
|
|
||||||
# Push a single request that will fail with all blocks being unviable
|
# Push a single request that will fail with all blocks being unviable
|
||||||
var f11 = queue.push(r11, chain.getSlice(startSlot, r11))
|
var f11 = queue.push(r11, chain.getSlice(startSlot, r11),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
discard await f11.withTimeout(100.milliseconds)
|
discard await f11.withTimeout(100.milliseconds)
|
||||||
|
|
||||||
check:
|
check:
|
||||||
|
@ -887,7 +902,7 @@ suite "SyncManager test suite":
|
||||||
chain = createChain(startSlot, finishSlot)
|
chain = createChain(startSlot, finishSlot)
|
||||||
queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
finishSlot, startSlot, chunkSize,
|
finishSlot, startSlot, chunkSize,
|
||||||
getSafeSlot, collector(aq), queueSize)
|
getSafeSlot, collector(aq), nil, queueSize)
|
||||||
validatorFut = backwardValidator(aq)
|
validatorFut = backwardValidator(aq)
|
||||||
|
|
||||||
let
|
let
|
||||||
|
@ -905,20 +920,23 @@ suite "SyncManager test suite":
|
||||||
var r13 = queue.pop(finishSlot, p3)
|
var r13 = queue.pop(finishSlot, p3)
|
||||||
var r14 = queue.pop(finishSlot, p4)
|
var r14 = queue.pop(finishSlot, p4)
|
||||||
|
|
||||||
var f14 = queue.push(r14, chain.getSlice(startSlot, r14))
|
var f14 = queue.push(r14, chain.getSlice(startSlot, r14),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check:
|
check:
|
||||||
f14.finished == false
|
f14.finished == false
|
||||||
counter == int(finishSlot)
|
counter == int(finishSlot)
|
||||||
|
|
||||||
var f12 = queue.push(r12, chain.getSlice(startSlot, r12))
|
var f12 = queue.push(r12, chain.getSlice(startSlot, r12),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check:
|
check:
|
||||||
counter == int(finishSlot)
|
counter == int(finishSlot)
|
||||||
f12.finished == false
|
f12.finished == false
|
||||||
f14.finished == false
|
f14.finished == false
|
||||||
|
|
||||||
var f11 = queue.push(r11, chain.getSlice(startSlot, r11))
|
var f11 = queue.push(r11, chain.getSlice(startSlot, r11),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f11, f12)
|
await allFutures(f11, f12)
|
||||||
check:
|
check:
|
||||||
counter == int(finishSlot - chunkSize - chunkSize)
|
counter == int(finishSlot - chunkSize - chunkSize)
|
||||||
|
@ -929,7 +947,7 @@ suite "SyncManager test suite":
|
||||||
var missingSlice = chain.getSlice(startSlot, r13)
|
var missingSlice = chain.getSlice(startSlot, r13)
|
||||||
withBlck(missingSlice[0][]):
|
withBlck(missingSlice[0][]):
|
||||||
blck.message.proposer_index = 0xDEADBEAF'u64
|
blck.message.proposer_index = 0xDEADBEAF'u64
|
||||||
var f13 = queue.push(r13, missingSlice)
|
var f13 = queue.push(r13, missingSlice, Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f13, f14)
|
await allFutures(f13, f14)
|
||||||
check:
|
check:
|
||||||
f11.finished == true and f11.failed == false
|
f11.finished == true and f11.failed == false
|
||||||
|
@ -946,11 +964,13 @@ suite "SyncManager test suite":
|
||||||
|
|
||||||
check r17.isEmpty() == true
|
check r17.isEmpty() == true
|
||||||
|
|
||||||
var f16 = queue.push(r16, chain.getSlice(startSlot, r16))
|
var f16 = queue.push(r16, chain.getSlice(startSlot, r16),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await sleepAsync(100.milliseconds)
|
await sleepAsync(100.milliseconds)
|
||||||
check f16.finished == false
|
check f16.finished == false
|
||||||
|
|
||||||
var f15 = queue.push(r15, chain.getSlice(startSlot, r15))
|
var f15 = queue.push(r15, chain.getSlice(startSlot, r15),
|
||||||
|
Opt.none(seq[ref BlobsSidecar]))
|
||||||
await allFutures(f15, f16)
|
await allFutures(f15, f16)
|
||||||
check:
|
check:
|
||||||
f15.finished == true and f15.failed == false
|
f15.finished == true and f15.failed == false
|
||||||
|
@ -1014,25 +1034,26 @@ suite "SyncManager test suite":
|
||||||
let chain = createChain(Slot(10), Slot(20))
|
let chain = createChain(Slot(10), Slot(20))
|
||||||
let r1 = SyncRequest[SomeTPeer](slot: Slot(11), count: 1'u64)
|
let r1 = SyncRequest[SomeTPeer](slot: Slot(11), count: 1'u64)
|
||||||
let r21 = SyncRequest[SomeTPeer](slot: Slot(11), count: 2'u64)
|
let r21 = SyncRequest[SomeTPeer](slot: Slot(11), count: 2'u64)
|
||||||
|
let slots = mapIt(chain, it[].slot)
|
||||||
|
|
||||||
check:
|
check:
|
||||||
checkResponse(r1, @[chain[1]]) == true
|
checkResponse(r1, @[slots[1]]) == true
|
||||||
checkResponse(r1, @[]) == true
|
checkResponse(r1, @[]) == true
|
||||||
checkResponse(r1, @[chain[1], chain[1]]) == false
|
checkResponse(r1, @[slots[1], slots[1]]) == false
|
||||||
checkResponse(r1, @[chain[0]]) == false
|
checkResponse(r1, @[slots[0]]) == false
|
||||||
checkResponse(r1, @[chain[2]]) == false
|
checkResponse(r1, @[slots[2]]) == false
|
||||||
|
|
||||||
checkResponse(r21, @[chain[1]]) == true
|
checkResponse(r21, @[slots[1]]) == true
|
||||||
checkResponse(r21, @[]) == true
|
checkResponse(r21, @[]) == true
|
||||||
checkResponse(r21, @[chain[1], chain[2]]) == true
|
checkResponse(r21, @[slots[1], slots[2]]) == true
|
||||||
checkResponse(r21, @[chain[2]]) == true
|
checkResponse(r21, @[slots[2]]) == true
|
||||||
checkResponse(r21, @[chain[1], chain[2], chain[3]]) == false
|
checkResponse(r21, @[slots[1], slots[2], slots[3]]) == false
|
||||||
checkResponse(r21, @[chain[0], chain[1]]) == false
|
checkResponse(r21, @[slots[0], slots[1]]) == false
|
||||||
checkResponse(r21, @[chain[0]]) == false
|
checkResponse(r21, @[slots[0]]) == false
|
||||||
checkResponse(r21, @[chain[2], chain[1]]) == false
|
checkResponse(r21, @[slots[2], slots[1]]) == false
|
||||||
checkResponse(r21, @[chain[2], chain[1]]) == false
|
checkResponse(r21, @[slots[2], slots[1]]) == false
|
||||||
checkResponse(r21, @[chain[2], chain[3]]) == false
|
checkResponse(r21, @[slots[2], slots[3]]) == false
|
||||||
checkResponse(r21, @[chain[3]]) == false
|
checkResponse(r21, @[slots[3]]) == false
|
||||||
|
|
||||||
test "[SyncQueue#Forward] getRewindPoint() test":
|
test "[SyncQueue#Forward] getRewindPoint() test":
|
||||||
let aq = newAsyncQueue[BlockEntry]()
|
let aq = newAsyncQueue[BlockEntry]()
|
||||||
|
@ -1040,7 +1061,7 @@ suite "SyncManager test suite":
|
||||||
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
||||||
1'u64, getStaticSlotCb(Slot(0)),
|
1'u64, getStaticSlotCb(Slot(0)),
|
||||||
collector(aq), 2)
|
collector(aq), nil, 2)
|
||||||
let finalizedSlot = start_slot(Epoch(0'u64))
|
let finalizedSlot = start_slot(Epoch(0'u64))
|
||||||
let startSlot = start_slot(Epoch(0'u64)) + 1'u64
|
let startSlot = start_slot(Epoch(0'u64)) + 1'u64
|
||||||
let finishSlot = start_slot(Epoch(2'u64))
|
let finishSlot = start_slot(Epoch(2'u64))
|
||||||
|
@ -1052,7 +1073,7 @@ suite "SyncManager test suite":
|
||||||
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
||||||
1'u64, getStaticSlotCb(Slot(0)),
|
1'u64, getStaticSlotCb(Slot(0)),
|
||||||
collector(aq), 2)
|
collector(aq), nil, 2)
|
||||||
let finalizedSlot = start_slot(Epoch(1'u64))
|
let finalizedSlot = start_slot(Epoch(1'u64))
|
||||||
let startSlot = start_slot(Epoch(1'u64)) + 1'u64
|
let startSlot = start_slot(Epoch(1'u64)) + 1'u64
|
||||||
let finishSlot = start_slot(Epoch(3'u64))
|
let finishSlot = start_slot(Epoch(3'u64))
|
||||||
|
@ -1064,7 +1085,7 @@ suite "SyncManager test suite":
|
||||||
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
||||||
1'u64, getStaticSlotCb(Slot(0)),
|
1'u64, getStaticSlotCb(Slot(0)),
|
||||||
collector(aq), 2)
|
collector(aq), nil, 2)
|
||||||
let finalizedSlot = start_slot(Epoch(0'u64))
|
let finalizedSlot = start_slot(Epoch(0'u64))
|
||||||
let failSlot = Slot(0xFFFF_FFFF_FFFF_FFFFF'u64)
|
let failSlot = Slot(0xFFFF_FFFF_FFFF_FFFFF'u64)
|
||||||
let failEpoch = epoch(failSlot)
|
let failEpoch = epoch(failSlot)
|
||||||
|
@ -1082,7 +1103,7 @@ suite "SyncManager test suite":
|
||||||
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Forward,
|
||||||
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
Slot(0), Slot(0xFFFF_FFFF_FFFF_FFFFF'u64),
|
||||||
1'u64, getStaticSlotCb(Slot(0)),
|
1'u64, getStaticSlotCb(Slot(0)),
|
||||||
collector(aq), 2)
|
collector(aq), nil, 2)
|
||||||
let finalizedSlot = start_slot(Epoch(1'u64))
|
let finalizedSlot = start_slot(Epoch(1'u64))
|
||||||
let failSlot = Slot(0xFFFF_FFFF_FFFF_FFFFF'u64)
|
let failSlot = Slot(0xFFFF_FFFF_FFFF_FFFFF'u64)
|
||||||
let failEpoch = epoch(failSlot)
|
let failEpoch = epoch(failSlot)
|
||||||
|
@ -1101,7 +1122,7 @@ suite "SyncManager test suite":
|
||||||
let getSafeSlot = getStaticSlotCb(Slot(1024))
|
let getSafeSlot = getStaticSlotCb(Slot(1024))
|
||||||
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
var queue = SyncQueue.init(SomeTPeer, SyncQueueKind.Backward,
|
||||||
Slot(1024), Slot(0),
|
Slot(1024), Slot(0),
|
||||||
1'u64, getSafeSlot, collector(aq), 2)
|
1'u64, getSafeSlot, collector(aq), nil, 2)
|
||||||
let safeSlot = getSafeSlot()
|
let safeSlot = getSafeSlot()
|
||||||
for i in countdown(1023, 0):
|
for i in countdown(1023, 0):
|
||||||
check queue.getRewindPoint(Slot(i), safeSlot) == safeSlot
|
check queue.getRewindPoint(Slot(i), safeSlot) == safeSlot
|
||||||
|
|
Loading…
Reference in New Issue