logos-storage-nim/tests/storage/blockexchange/engine/testengine.nim

import std/[sequtils, options]

import pkg/chronos
import pkg/libp2p/routing_record
import pkg/codexdht/discv5/protocol as discv5

import pkg/storage/rng
import pkg/storage/blockexchange
import pkg/storage/stores
import pkg/storage/chunker
import pkg/storage/discovery
import pkg/storage/blocktype
import pkg/storage/merkletree
import pkg/storage/blockexchange/utils
import pkg/storage/blockexchange/engine/activedownload {.all.}
import pkg/storage/blockexchange/engine/downloadmanager {.all.}
import pkg/storage/blockexchange/protocol/constants

import ../../../asynctest
import ../../helpers
import ../../examples

asyncchecksuite "NetworkStore engine handlers":
  var
    peerId: PeerId
    chunker: Chunker
    blockDiscovery: Discovery
    peerStore: PeerContextStore
    downloadManager: DownloadManager
    network: BlockExcNetwork
    engine: BlockExcEngine
    discovery: DiscoveryEngine
    advertiser: Advertiser
    peerCtx: PeerContext
    localStore: BlockStore
    blocks: seq[Block]

  setup:
    chunker = RandomChunker.new(Rng.instance(), size = 1024'nb, chunkSize = 256'nb)

    while true:
      let chunk = await chunker.getBytes()
      if chunk.len <= 0:
        break

      blocks.add(Block.new(chunk).tryGet())

    peerId = PeerId.example
    blockDiscovery = Discovery.new()
    peerStore = PeerContextStore.new()
    downloadManager = DownloadManager.new()

    localStore = CacheStore.new()
    network = BlockExcNetwork()

    discovery = DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery)

    advertiser = Advertiser.new(localStore, blockDiscovery)

    engine = BlockExcEngine.new(
      localStore, network, discovery, advertiser, peerStore, downloadManager
    )

    peerCtx = PeerContext(id: peerId)
    engine.peers.add(peerCtx)

  test "Should handle want list":
    let
      tree = StorageMerkleTree.init(blocks.mapIt(it.cid)).tryGet
      rootCid = tree.rootCid.tryGet()

    for i, blk in blocks:
      (await localStore.putBlock(blk)).tryGet()
      (await localStore.putCidAndProof(rootCid, i, blk.cid, tree.getProof(i).tryGet())).tryGet()

    let
      done = newFuture[void]()
      wantList = makeWantList(rootCid, blocks.len)

    proc sendPresence(
        peerId: PeerId, presence: seq[BlockPresence]
    ) {.async: (raises: [CancelledError]).} =
      check presence.mapIt(it.address) == wantList.entries.mapIt(it.address)
      for p in presence:
        check p.kind in {BlockPresenceType.HaveRange, BlockPresenceType.Complete}
      done.complete()

    engine.network =
      BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence))

    await engine.wantListHandler(peerId, wantList)
    await done

  test "Should handle want list - `dont-have`":
    let
      done = newFuture[void]()
      treeCid = Cid.example
      wantList = makeWantList(treeCid, blocks.len, sendDontHave = true)

    proc sendPresence(
        peerId: PeerId, presence: seq[BlockPresence]
    ) {.async: (raises: [CancelledError]).} =
      check presence.mapIt(it.address) == wantList.entries.mapIt(it.address)
      for p in presence:
        check:
          p.kind == BlockPresenceType.DontHave

      done.complete()

    engine.network =
      BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence))

    await engine.wantListHandler(peerId, wantList)
    await done

  test "Should handle want list - `dont-have` some blocks":
    let
      tree = StorageMerkleTree.init(blocks.mapIt(it.cid)).tryGet
      rootCid = tree.rootCid.tryGet()

    for i in 0 ..< 2:
      (await engine.localStore.putBlock(blocks[i])).tryGet()

      (
        await engine.localStore.putCidAndProof(
          rootCid, i, blocks[i].cid, tree.getProof(i).tryGet()
        )
      ).tryGet()

    let
      done = newFuture[void]()
      wantList = makeWantList(rootCid, blocks.len, sendDontHave = true)

    proc sendPresence(
        peerId: PeerId, presence: seq[BlockPresence]
    ) {.async: (raises: [CancelledError]).} =
      for p in presence:
        if p.address.index >= 2:
          check p.kind == BlockPresenceType.DontHave
        else:
          check p.kind in {BlockPresenceType.HaveRange, BlockPresenceType.Complete}

      done.complete()

    engine.network =
      BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence))

    await engine.wantListHandler(peerId, wantList)

    await done

  test "Should handle block presence":
    proc sendWantList(
        id: PeerId,
        addresses: seq[BlockAddress],
        priority: int32 = 0,
        cancel: bool = false,
        wantType: WantType = WantType.WantHave,
        full: bool = false,
        sendDontHave: bool = false,
        rangeCount: uint64 = 0,
        downloadId: uint64 = 0,
    ) {.async: (raises: [CancelledError]).} =
      discard

    engine.network =
      BlockExcNetwork(request: BlockExcRequest(sendWantList: sendWantList))

    let
      md = testManifestDesc(blocks[0].cid, DefaultBlockSize.uint32, 1)
      address = BlockAddress(treeCid: md.manifest.treeCid, index: 0)
      desc = DownloadDesc(md: md, startIndex: address.index.uint64, count: 1)
      download = engine.downloadManager.startDownload(desc)

    discard download.getWantHandle(address)

    await engine.blockPresenceHandler(
      peerId,
      @[
        BlockPresence(
          address: address, kind: BlockPresenceType.Complete, downloadId: download.id
        )
      ],
    )

    let
      swarm = download.getSwarm()
      peerOpt = swarm.getPeer(peerId)
    check peerOpt.isSome

  test "Should handle range want list":
    let
      done = newFuture[void]()
      treeCid = Cid.example
      tree = StorageMerkleTree.init(blocks.mapIt(it.cid)).tryGet
      rootCid = tree.rootCid.tryGet()

    for i, blk in blocks:
      (await localStore.putBlock(blk)).tryGet()
      let proof = tree.getProof(i).tryGet()
      (await localStore.putCidAndProof(rootCid, i, blk.cid, proof)).tryGet()

    let wantList = WantList(
      entries: @[
        WantListEntry(
          address: BlockAddress(treeCid: rootCid, index: 0),
          priority: 0,
          cancel: false,
          wantType: WantType.WantHave,
          sendDontHave: false,
          rangeCount: blocks.len.uint64,
        )
      ],
      full: false,
    )

    proc sendPresence(
        peerId: PeerId, presence: seq[BlockPresence]
    ) {.async: (raises: [CancelledError]).} =
      check presence.len == 1
      check presence[0].kind == BlockPresenceType.HaveRange
      check presence[0].ranges.len > 0
      done.complete()

    engine.network =
      BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence))

    await engine.wantListHandler(peerId, wantList)
    await done

  test "Should not send presence for blocks not in range":
    let
      done = newFuture[void]()
      treeCid = Cid.example
      tree = StorageMerkleTree.init(blocks.mapIt(it.cid)).tryGet
      rootCid = tree.rootCid.tryGet()

    for i in 0 ..< 2:
      (await localStore.putBlock(blocks[i])).tryGet()
      let proof = tree.getProof(i).tryGet()
      (await localStore.putCidAndProof(rootCid, i, blocks[i].cid, proof)).tryGet()

    let wantList = WantList(
      entries: @[
        WantListEntry(
          address: BlockAddress(treeCid: rootCid, index: 0),
          priority: 0,
          cancel: false,
          wantType: WantType.WantHave,
          sendDontHave: false,
          rangeCount: blocks.len.uint64,
        )
      ],
      full: false,
    )

    proc sendPresence(
        peerId: PeerId, presence: seq[BlockPresence]
    ) {.async: (raises: [CancelledError]).} =
      check presence.len == 1
      check presence[0].kind == BlockPresenceType.HaveRange
      for (start, count) in presence[0].ranges:
        check start < 2
      done.complete()

    engine.network =
      BlockExcNetwork(request: BlockExcRequest(sendPresence: sendPresence))

    await engine.wantListHandler(peerId, wantList)
    await done

  test "WantBlocks: rejects range with count = 0":
    let req =
      WantBlocksRequest(requestId: 1, treeCid: Cid.example, ranges: @[(0'u64, 0'u64)])

    let blocks = await network.handlers.onWantBlocksRequest(peerId, req)
    check blocks.len == 0

  test "WantBlocks: rejects range with count > MaxBlocksPerBatch":
    let req = WantBlocksRequest(
      requestId: 1,
      treeCid: Cid.example,
      ranges: @[(0'u64, MaxBlocksPerBatch.uint64 + 1)],
    )

    let blocks = await network.handlers.onWantBlocksRequest(peerId, req)
    check blocks.len == 0

  test "WantBlocks: rejects range whose start+count overflows":
    let req = WantBlocksRequest(
      requestId: 1, treeCid: Cid.example, ranges: @[(uint64.high, 1'u64)]
    )

    let blocks = await network.handlers.onWantBlocksRequest(peerId, req)
    check blocks.len == 0

  test "WantBlocks: rejects range whose max index exceeds Natural":
    let req = WantBlocksRequest(
      requestId: 1, treeCid: Cid.example, ranges: @[(high(Natural).uint64 + 1, 1'u64)]
    )

    let blocks = await network.handlers.onWantBlocksRequest(peerId, req)
    check blocks.len == 0

  test "WantBlocks: rejects when total count across ranges exceeds cap":
    var ranges: seq[tuple[start: uint64, count: uint64]] = @[]
    let halfMaxBlocksPerBatchPlusOne = (MaxBlocksPerBatch div 2).uint64 + 1
    ranges.add((0'u64, halfMaxBlocksPerBatchPlusOne))
    ranges.add((10_000'u64, halfMaxBlocksPerBatchPlusOne))

    let req = WantBlocksRequest(requestId: 1, treeCid: Cid.example, ranges: ranges)

    let blocks = await network.handlers.onWantBlocksRequest(peerId, req)
    check blocks.len == 0

  test "WantBlocks: accepts a valid small request":
    let
      tree = StorageMerkleTree.init(blocks.mapIt(it.cid)).tryGet
      rootCid = tree.rootCid.tryGet()

    for i, blk in blocks:
      (await localStore.putBlock(blk)).tryGet()
      (await localStore.putCidAndProof(rootCid, i, blk.cid, tree.getProof(i).tryGet())).tryGet()

    let req = WantBlocksRequest(
      requestId: 1, treeCid: rootCid, ranges: @[(0'u64, blocks.len.uint64)]
    )

    let delivered = await network.handlers.onWantBlocksRequest(peerId, req)
    check delivered.len == blocks.len

suite "IsIndexInRanges":
  test "Empty ranges returns false":
    let ranges: seq[(uint64, uint64)] = @[]
    check not isIndexInRanges(0, ranges)
    check not isIndexInRanges(100, ranges)

  test "Single range - index inside":
    let ranges = @[(10'u64, 5'u64)]
    check isIndexInRanges(10, ranges, sortedRanges = true)
    check isIndexInRanges(12, ranges, sortedRanges = true)
    check isIndexInRanges(14, ranges, sortedRanges = true)

  test "Single range - index outside":
    let ranges = @[(10'u64, 5'u64)]
    check not isIndexInRanges(9, ranges, sortedRanges = true)
    check not isIndexInRanges(15, ranges, sortedRanges = true)
    check not isIndexInRanges(100, ranges, sortedRanges = true)

  test "Multiple sorted ranges - index in each":
    let ranges = @[(0'u64, 3'u64), (10'u64, 5'u64), (100'u64, 10'u64)]
    check isIndexInRanges(0, ranges, sortedRanges = true)
    check isIndexInRanges(2, ranges, sortedRanges = true)
    check isIndexInRanges(10, ranges, sortedRanges = true)
    check isIndexInRanges(14, ranges, sortedRanges = true)
    check isIndexInRanges(100, ranges, sortedRanges = true)
    check isIndexInRanges(109, ranges, sortedRanges = true)

  test "Multiple ranges - index in gaps":
    let ranges = @[(0'u64, 3'u64), (10'u64, 5'u64), (100'u64, 10'u64)]
    check not isIndexInRanges(3, ranges, sortedRanges = true)
    check not isIndexInRanges(9, ranges, sortedRanges = true)
    check not isIndexInRanges(15, ranges, sortedRanges = true)
    check not isIndexInRanges(99, ranges, sortedRanges = true)
    check not isIndexInRanges(110, ranges, sortedRanges = true)

  test "Unsorted ranges with sortedRanges=false":
    let ranges = @[(100'u64, 10'u64), (0'u64, 3'u64), (10'u64, 5'u64)]
    check isIndexInRanges(0, ranges, sortedRanges = false)
    check isIndexInRanges(2, ranges, sortedRanges = false)
    check isIndexInRanges(10, ranges, sortedRanges = false)
    check isIndexInRanges(105, ranges, sortedRanges = false)
    check not isIndexInRanges(50, ranges, sortedRanges = false)

  test "Adjacent ranges":
    let ranges = @[(0'u64, 5'u64), (5'u64, 5'u64), (10'u64, 5'u64)]
    for i in 0'u64 ..< 15:
      check isIndexInRanges(i, ranges, sortedRanges = true)
    check not isIndexInRanges(15, ranges, sortedRanges = true)

  test "Large range values":
    let ranges = @[(1_000_000_000'u64, 1_000_000'u64)]
    check isIndexInRanges(1_000_000_000, ranges, sortedRanges = true)
    check isIndexInRanges(1_000_500_000, ranges, sortedRanges = true)
    check not isIndexInRanges(999_999_999, ranges, sortedRanges = true)
    check not isIndexInRanges(1_001_000_000, ranges, sortedRanges = true)