Advertise manifests and trees (#797)

* wip update advertise loop to announce only manifests and trees

* fixes e2e tests

* fixes tests for discovery engine

---------

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
This commit is contained in:
Ben Bierens 2024-05-09 10:03:35 +02:00 committed by GitHub
parent 1a0d2d424e
commit 9379c7c662
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 102 additions and 95 deletions

View File

@ -11,6 +11,7 @@ import std/sequtils
import pkg/chronos import pkg/chronos
import pkg/libp2p/cid import pkg/libp2p/cid
import pkg/libp2p/multicodec
import pkg/metrics import pkg/metrics
import pkg/questionable import pkg/questionable
import pkg/questionable/results import pkg/questionable/results
@ -25,6 +26,7 @@ import ../../utils
import ../../discovery import ../../discovery
import ../../stores/blockstore import ../../stores/blockstore
import ../../logutils import ../../logutils
import ../../manifest
logScope: logScope:
topics = "codex discoveryengine" topics = "codex discoveryengine"
@ -76,14 +78,32 @@ proc discoveryQueueLoop(b: DiscoveryEngine) {.async.} =
await sleepAsync(b.discoveryLoopSleep) await sleepAsync(b.discoveryLoopSleep)
proc advertiseBlock(b: DiscoveryEngine, cid: Cid) {.async.} =
without isM =? cid.isManifest, err:
warn "Unable to determine if cid is manifest"
return
if isM:
without blk =? await b.localStore.getBlock(cid), err:
error "Error retrieving manifest block", cid, err = err.msg
return
without manifest =? Manifest.decode(blk), err:
error "Unable to decode as manifest", err = err.msg
return
# announce manifest cid and tree cid
await b.advertiseQueue.put(cid)
await b.advertiseQueue.put(manifest.treeCid)
proc advertiseQueueLoop(b: DiscoveryEngine) {.async.} = proc advertiseQueueLoop(b: DiscoveryEngine) {.async.} =
while b.discEngineRunning: while b.discEngineRunning:
if cids =? await b.localStore.listBlocks(blockType = b.advertiseType): if cids =? await b.localStore.listBlocks(blockType = b.advertiseType):
trace "Begin iterating blocks..." trace "Begin iterating blocks..."
for c in cids: for c in cids:
if cid =? await c: if cid =? await c:
await b.advertiseQueue.put(cid) b.advertiseBlock(cid)
await sleepAsync(50.millis) await sleepAsync(100.millis)
trace "Iterating blocks finished." trace "Iterating blocks finished."
await sleepAsync(b.advertiseLoopSleep) await sleepAsync(b.advertiseLoopSleep)
@ -248,7 +268,7 @@ proc new*(
discoveryLoopSleep = DefaultDiscoveryLoopSleep, discoveryLoopSleep = DefaultDiscoveryLoopSleep,
advertiseLoopSleep = DefaultAdvertiseLoopSleep, advertiseLoopSleep = DefaultAdvertiseLoopSleep,
minPeersPerBlock = DefaultMinPeersPerBlock, minPeersPerBlock = DefaultMinPeersPerBlock,
advertiseType = BlockType.Both advertiseType = BlockType.Manifest
): DiscoveryEngine = ): DiscoveryEngine =
## Create a discovery engine instance for advertising services ## Create a discovery engine instance for advertising services
## ##

View File

@ -50,7 +50,7 @@ asyncchecksuite "Block Advertising and Discovery":
blockDiscovery = MockDiscovery.new() blockDiscovery = MockDiscovery.new()
wallet = WalletRef.example wallet = WalletRef.example
network = BlockExcNetwork.new(switch) network = BlockExcNetwork.new(switch)
localStore = CacheStore.new(blocks.mapIt( it )) localStore = CacheStore.new(blocks.mapIt(it))
peerStore = PeerCtxStore.new() peerStore = PeerCtxStore.new()
pendingBlocks = PendingBlocksManager.new() pendingBlocks = PendingBlocksManager.new()
@ -92,57 +92,40 @@ asyncchecksuite "Block Advertising and Discovery":
blockDiscovery.findBlockProvidersHandler = blockDiscovery.findBlockProvidersHandler =
proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async.} = proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async.} =
await engine.resolveBlocks(blocks.filterIt( it.cid == cid )) await engine.resolveBlocks(blocks.filterIt(it.cid == cid))
await allFuturesThrowing( await allFuturesThrowing(
allFinished(pendingBlocks)) allFinished(pendingBlocks))
await engine.stop() await engine.stop()
test "Should advertise both manifests and blocks": test "Should advertise both manifests and trees":
let let
cids = @[manifest.cid.tryGet, manifest.treeCid]
advertised = initTable.collect: advertised = initTable.collect:
for b in (blocks & manifestBlock): {b.cid: newFuture[void]()} for cid in cids: {cid: newFuture[void]()}
blockDiscovery blockDiscovery
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} = .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} =
if cid in advertised and not advertised[cid].finished(): if cid in advertised and not advertised[cid].finished():
advertised[cid].complete() advertised[cid].complete()
discovery.advertiseType = BlockType.Both await engine.start()
await engine.start() # fire up advertise loop
await allFuturesThrowing( await allFuturesThrowing(
allFinished(toSeq(advertised.values))) allFinished(toSeq(advertised.values)))
await engine.stop() await engine.stop()
test "Should advertise local manifests": test "Should not advertise local blocks":
let let
advertised = newFuture[Cid]() blockCids = blocks.mapIt(it.cid)
blockDiscovery blockDiscovery
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} = .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} =
check manifestBlock.cid == cid check:
advertised.complete(cid) cid notin blockCids
discovery.advertiseType = BlockType.Manifest await engine.start()
await engine.start() # fire up advertise loop await sleepAsync(3.seconds)
check (await advertised.wait(10.millis)) == manifestBlock.cid
await engine.stop()
test "Should advertise local blocks":
let
advertised = initTable.collect:
for b in blocks: {b.cid: newFuture[void]()}
blockDiscovery
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} =
if cid in advertised and not advertised[cid].finished():
advertised[cid].complete()
discovery.advertiseType = BlockType.Block
await engine.start() # fire up advertise loop
await allFuturesThrowing(
allFinished(toSeq(advertised.values)))
await engine.stop() await engine.stop()
test "Should not launch discovery if remote peer has block": test "Should not launch discovery if remote peer has block":
@ -165,7 +148,7 @@ asyncchecksuite "Block Advertising and Discovery":
proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] = proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] =
check false check false
await engine.start() # fire up discovery loop await engine.start()
engine.pendingBlocks.resolve(blocks.mapIt(BlockDelivery(blk: it, address: it.address))) engine.pendingBlocks.resolve(blocks.mapIt(BlockDelivery(blk: it, address: it.address)))
await allFuturesThrowing( await allFuturesThrowing(
@ -173,23 +156,33 @@ asyncchecksuite "Block Advertising and Discovery":
await engine.stop() await engine.stop()
asyncchecksuite "E2E - Multiple Nodes Discovery": proc asBlock(m: Manifest): bt.Block =
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256) let mdata = m.encode().tryGet()
bt.Block.new(data = mdata, codec = ManifestCodec).tryGet()
asyncchecksuite "E2E - Multiple Nodes Discovery":
var var
switch: seq[Switch] switch: seq[Switch]
blockexc: seq[NetworkStore] blockexc: seq[NetworkStore]
blocks: seq[bt.Block] manifests: seq[Manifest]
mBlocks: seq[bt.Block]
trees: seq[CodexTree]
setup: setup:
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
blocks.add(bt.Block.new(chunk).tryGet())
for _ in 0..<4: for _ in 0..<4:
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256)
var blocks = newSeq[bt.Block]()
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
blocks.add(bt.Block.new(chunk).tryGet())
let (manifest, tree) = makeManifestAndTree(blocks).tryGet()
manifests.add(manifest)
mBlocks.add(manifest.asBlock())
trees.add(tree)
let let
s = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) s = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr})
blockDiscovery = MockDiscovery.new() blockDiscovery = MockDiscovery.new()
@ -223,9 +216,12 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
teardown: teardown:
switch = @[] switch = @[]
blockexc = @[] blockexc = @[]
manifests = @[]
mBlocks = @[]
trees = @[]
test "E2E - Should advertise and discover blocks": test "E2E - Should advertise and discover blocks":
# Distribute the blocks amongst 1..3 # Distribute the manifests and trees amongst 1..3
# Ask 0 to download everything without connecting him beforehand # Ask 0 to download everything without connecting him beforehand
var advertised: Table[Cid, SignedPeerRecord] var advertised: Table[Cid, SignedPeerRecord]
@ -242,14 +238,14 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} =
advertised[cid] = switch[3].peerInfo.signedPeerRecord advertised[cid] = switch[3].peerInfo.signedPeerRecord
discard blocks[0..5].mapIt(blockexc[1].engine.pendingBlocks.getWantHandle(it.address)) discard blockexc[1].engine.pendingBlocks.getWantHandle(mBlocks[0].cid)
await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[0..5].mapIt(BlockDelivery(blk: it, address: it.address))) await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[0], address: BlockAddress(leaf: false, cid: mBlocks[0].cid))])
discard blocks[4..10].mapIt(blockexc[2].engine.pendingBlocks.getWantHandle(it.address)) discard blockexc[2].engine.pendingBlocks.getWantHandle(mBlocks[1].cid)
await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[4..10].mapIt(BlockDelivery(blk: it, address: it.address))) await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[1], address: BlockAddress(leaf: false, cid: mBlocks[1].cid))])
discard blocks[10..15].mapIt(blockexc[3].engine.pendingBlocks.getWantHandle(it.address)) discard blockexc[3].engine.pendingBlocks.getWantHandle(mBlocks[2].cid)
await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[10..15].mapIt(BlockDelivery(blk: it, address: it.address))) await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[2], address: BlockAddress(leaf: false, cid: mBlocks[2].cid))])
MockDiscovery(blockexc[0].engine.discovery.discovery) MockDiscovery(blockexc[0].engine.discovery.discovery)
.findBlockProvidersHandler = proc(d: MockDiscovery, cid: Cid): .findBlockProvidersHandler = proc(d: MockDiscovery, cid: Cid):
@ -258,22 +254,22 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
result.add(advertised[cid]) result.add(advertised[cid])
let futs = collect(newSeq): let futs = collect(newSeq):
for b in blocks: for m in mBlocks[0..2]:
blockexc[0].engine.requestBlock(b.cid) blockexc[0].engine.requestBlock(m.cid)
await allFuturesThrowing( await allFuturesThrowing(
switch.mapIt( it.start() ) & switch.mapIt(it.start()) &
blockexc.mapIt( it.engine.start() )).wait(10.seconds) blockexc.mapIt(it.engine.start())).wait(10.seconds)
await allFutures(futs).wait(10.seconds) await allFutures(futs).wait(10.seconds)
await allFuturesThrowing( await allFuturesThrowing(
blockexc.mapIt( it.engine.stop() ) & blockexc.mapIt(it.engine.stop()) &
switch.mapIt( it.stop() )).wait(10.seconds) switch.mapIt(it.stop())).wait(10.seconds)
test "E2E - Should advertise and discover blocks with peers already connected": test "E2E - Should advertise and discover blocks with peers already connected":
# Distribute the blocks amongst 1..3 # Distribute the blocks amongst 1..3
# Ask 0 to download everything without connecting him beforehand # Ask 0 to download everything *WITH* connecting him beforehand
var advertised: Table[Cid, SignedPeerRecord] var advertised: Table[Cid, SignedPeerRecord]
@ -289,14 +285,14 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} = .publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} =
advertised[cid] = switch[3].peerInfo.signedPeerRecord advertised[cid] = switch[3].peerInfo.signedPeerRecord
discard blocks[0..5].mapIt(blockexc[1].engine.pendingBlocks.getWantHandle(it.address)) discard blockexc[1].engine.pendingBlocks.getWantHandle(mBlocks[0].cid)
await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[0..5].mapIt(BlockDelivery(blk: it, address: it.address))) await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[0], address: BlockAddress(leaf: false, cid: mBlocks[0].cid))])
discard blocks[4..10].mapIt(blockexc[2].engine.pendingBlocks.getWantHandle(it.address)) discard blockexc[2].engine.pendingBlocks.getWantHandle(mBlocks[1].cid)
await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[4..10].mapIt(BlockDelivery(blk: it, address: it.address))) await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[1], address: BlockAddress(leaf: false, cid: mBlocks[1].cid))])
discard blocks[10..15].mapIt(blockexc[3].engine.pendingBlocks.getWantHandle(it.address)) discard blockexc[3].engine.pendingBlocks.getWantHandle(mBlocks[2].cid)
await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[10..15].mapIt(BlockDelivery(blk: it, address: it.address))) await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[2], address: BlockAddress(leaf: false, cid: mBlocks[2].cid))])
MockDiscovery(blockexc[0].engine.discovery.discovery) MockDiscovery(blockexc[0].engine.discovery.discovery)
.findBlockProvidersHandler = proc(d: MockDiscovery, cid: Cid): .findBlockProvidersHandler = proc(d: MockDiscovery, cid: Cid):
@ -305,14 +301,14 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
return @[advertised[cid]] return @[advertised[cid]]
let let
futs = blocks.mapIt( blockexc[0].engine.requestBlock( it.cid ) ) futs = mBlocks[0..2].mapIt(blockexc[0].engine.requestBlock(it.cid))
await allFuturesThrowing( await allFuturesThrowing(
switch.mapIt( it.start() ) & switch.mapIt(it.start()) &
blockexc.mapIt( it.engine.start() )).wait(10.seconds) blockexc.mapIt(it.engine.start())).wait(10.seconds)
await allFutures(futs).wait(10.seconds) await allFutures(futs).wait(10.seconds)
await allFuturesThrowing( await allFuturesThrowing(
blockexc.mapIt( it.engine.stop() ) & blockexc.mapIt(it.engine.stop()) &
switch.mapIt( it.stop() )).wait(10.seconds) switch.mapIt(it.stop())).wait(10.seconds)

View File

@ -10,17 +10,26 @@ import pkg/codex/blockexchange
import pkg/codex/chunker import pkg/codex/chunker
import pkg/codex/blocktype as bt import pkg/codex/blocktype as bt
import pkg/codex/blockexchange/engine import pkg/codex/blockexchange/engine
import pkg/codex/manifest
import pkg/codex/merkletree
import ../../../asynctest import ../../../asynctest
import ../../helpers import ../../helpers
import ../../helpers/mockdiscovery import ../../helpers/mockdiscovery
import ../../examples import ../../examples
proc asBlock(m: Manifest): bt.Block =
let mdata = m.encode().tryGet()
bt.Block.new(data = mdata, codec = ManifestCodec).tryGet()
asyncchecksuite "Test Discovery Engine": asyncchecksuite "Test Discovery Engine":
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256) let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256)
var var
blocks: seq[bt.Block] blocks: seq[bt.Block]
manifest: Manifest
tree: CodexTree
manifestBlock: bt.Block
switch: Switch switch: Switch
peerStore: PeerCtxStore peerStore: PeerCtxStore
blockDiscovery: MockDiscovery blockDiscovery: MockDiscovery
@ -35,6 +44,10 @@ asyncchecksuite "Test Discovery Engine":
blocks.add(bt.Block.new(chunk).tryGet()) blocks.add(bt.Block.new(chunk).tryGet())
(manifest, tree) = makeManifestAndTree(blocks).tryGet()
manifestBlock = manifest.asBlock()
blocks.add(manifestBlock)
switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr})
network = BlockExcNetwork.new(switch) network = BlockExcNetwork.new(switch)
peerStore = PeerCtxStore.new() peerStore = PeerCtxStore.new()
@ -51,11 +64,11 @@ asyncchecksuite "Test Discovery Engine":
blockDiscovery, blockDiscovery,
pendingBlocks, pendingBlocks,
discoveryLoopSleep = 100.millis) discoveryLoopSleep = 100.millis)
wants = blocks.mapIt( pendingBlocks.getWantHandle(it.cid) ) wants = blocks.mapIt(pendingBlocks.getWantHandle(it.cid) )
blockDiscovery.findBlockProvidersHandler = blockDiscovery.findBlockProvidersHandler =
proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async, gcsafe.} = proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async, gcsafe.} =
pendingBlocks.resolve(blocks.filterIt( it.cid == cid).mapIt(BlockDelivery(blk: it, address: it.address))) pendingBlocks.resolve(blocks.filterIt(it.cid == cid).mapIt(BlockDelivery(blk: it, address: it.address)))
await discoveryEngine.start() await discoveryEngine.start()
await allFuturesThrowing(allFinished(wants)).wait(1.seconds) await allFuturesThrowing(allFinished(wants)).wait(1.seconds)
@ -63,7 +76,7 @@ asyncchecksuite "Test Discovery Engine":
test "Should Advertise Haves": test "Should Advertise Haves":
var var
localStore = CacheStore.new(blocks.mapIt( it )) localStore = CacheStore.new(blocks.mapIt(it))
discoveryEngine = DiscoveryEngine.new( discoveryEngine = DiscoveryEngine.new(
localStore, localStore,
peerStore, peerStore,
@ -72,8 +85,8 @@ asyncchecksuite "Test Discovery Engine":
pendingBlocks, pendingBlocks,
discoveryLoopSleep = 100.millis) discoveryLoopSleep = 100.millis)
haves = collect(initTable): haves = collect(initTable):
for b in blocks: for cid in @[manifestBlock.cid, manifest.treeCid]:
{ b.cid: newFuture[void]() } { cid: newFuture[void]() }
blockDiscovery.publishBlockProvideHandler = blockDiscovery.publishBlockProvideHandler =
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} = proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
@ -108,28 +121,6 @@ asyncchecksuite "Test Discovery Engine":
await want.wait(1.seconds) await want.wait(1.seconds)
await discoveryEngine.stop() await discoveryEngine.stop()
test "Should queue advertise request":
var
localStore = CacheStore.new(@[blocks[0]])
discoveryEngine = DiscoveryEngine.new(
localStore,
peerStore,
network,
blockDiscovery,
pendingBlocks,
discoveryLoopSleep = 100.millis)
have = newFuture[void]()
blockDiscovery.publishBlockProvideHandler =
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
check cid == blocks[0].cid
if not have.finished:
have.complete()
await discoveryEngine.start()
await have.wait(1.seconds)
await discoveryEngine.stop()
test "Should not request more than minPeersPerBlock": test "Should not request more than minPeersPerBlock":
var var
localStore = CacheStore.new() localStore = CacheStore.new()