adds testing setup for torrentdownloader

tests/codex/bittorrent/helpers.nim

@@ -0,0 +1,102 @@
+import pkg/chronos
+import pkg/libp2p/[cid, multicodec, multihash]
+import pkg/questionable/results
+
+import pkg/codex/stores/cachestore
+import pkg/codex/utils/iter
+
+import pkg/codex/manifest
+import pkg/codex/bittorrent/manifest
+
+proc torrentInfoForCodexManifest*(
+    localStore: BlockStore,
+    codexManifest: Manifest,
+    pieceLength = DefaultPieceLength.int,
+    name = string.none,
+): Future[?!BitTorrentInfo] {.async.} =
+  let treeCid = codexManifest.treeCid
+  let datasetSize = codexManifest.datasetSize
+  let blockSize = codexManifest.blockSize
+  let numOfBlocks = divUp(datasetSize, blockSize)
+  let blockIter = Iter.new(0 ..< numOfBlocks)
+  var blocks = newSeq[Block](numOfBlocks)
+  while not blockIter.finished:
+    let index = blockIter.next()
+    without blk =? (await localStore.getBlock(treeCid, index)), err:
+      return failure(err)
+    blocks[index] = blk
+  let
+    numOfBlocksPerPiece = pieceLength div BitTorrentBlockSize.int
+    numOfPieces = divUp(datasetSize, pieceLength.NBytes)
+
+  var
+    pieces: seq[MultiHash]
+    pieceHashCtx: sha1
+    pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece)
+
+  pieceHashCtx.init()
+
+  for blk in blocks:
+    if blk.data.len == 0:
+      break
+    if pieceIter.finished:
+      without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure,
+        err:
+        return failure(err)
+      pieces.add(mh)
+      pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece)
+      pieceHashCtx.init()
+    pieceHashCtx.update(blk.data)
+    discard pieceIter.next()
+
+  without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure, err:
+    return failure(err)
+  pieces.add(mh)
+
+  let info = BitTorrentInfo(
+    length: datasetSize.uint64,
+    pieceLength: pieceLength.uint32,
+    pieces: pieces,
+    name: name,
+  )
+
+  success info
+
+proc storeCodexManifest*(
+    codexManifest: Manifest, localStore: BlockStore
+): Future[?!Block] {.async.} =
+  without encodedManifest =? codexManifest.encode(), err:
+    trace "Unable to encode manifest", err = err.msg
+    return failure(err)
+
+  without blk =? Block.new(data = encodedManifest, codec = ManifestCodec), err:
+    trace "Unable to create block from manifest", err = err.msg
+    return failure(err)
+
+  if err =? (await localStore.putBlock(blk)).errorOption:
+    trace "Unable to store manifest block", cid = blk.cid, err = err.msg
+    return failure(err)
+
+  success blk
+
+proc storeTorrentManifest*(
+    torrentManifest: BitTorrentManifest, localStore: BlockStore
+): Future[?!Block] {.async.} =
+  let infoBencoded = torrentManifest.info.bencode()
+  let infoHash = MultiHash.digest($Sha1HashCodec, infoBencoded).tryGet
+  let encodedManifest = torrentManifest.encode()
+
+  without infoHashCid =? Cid.init(CIDv1, InfoHashV1Codec, infoHash).mapFailure, err:
+    trace "Unable to create CID for BitTorrent info hash", err = err.msg
+    return failure(err)
+
+  without blk =? Block.new(data = encodedManifest, cid = infoHashCid, verify = false),
+    err:
+    trace "Unable to create block from manifest", err = err.msg
+    return failure(err)
+
+  if err =? (await localStore.putBlock(blk)).errorOption:
+    trace "Unable to store BitTorrent manifest block", cid = blk.cid, err = err.msg
+    return failure(err)
+
+  success blk

tests/codex/bittorrent/testtorrentdownloader.nim

@@ -0,0 +1,100 @@
+import pkg/libp2p/[cid, multicodec, multihash]
+import pkg/questionable/results
+
+import pkg/codex/rng
+import pkg/codex/blockexchange
+import pkg/codex/stores
+import pkg/codex/discovery
+import pkg/codex/blocktype
+
+import pkg/codex/manifest
+import pkg/codex/bittorrent/manifest
+import pkg/codex/bittorrent/torrentdownloader
+
+import ../../asynctest
+import ./helpers
+import ../helpers
+import ../examples
+
+template setupDependencies() {.dirty.} =
+  var
+    rng: Rng
+    seckey: PrivateKey
+    peerId: PeerId
+    wallet: WalletRef
+    blockDiscovery: Discovery
+    peerStore: PeerCtxStore
+    pendingBlocks: PendingBlocksManager
+
+    network: BlockExcNetwork
+    localStore: CacheStore
+    discovery: DiscoveryEngine
+    advertiser: Advertiser
+    engine: BlockExcEngine
+    networkStore: NetworkStore
+
+  setup:
+    rng = Rng.instance()
+    seckey = PrivateKey.random(rng[]).tryGet()
+    peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet()
+    wallet = WalletRef.example
+    blockDiscovery = Discovery.new()
+    peerStore = PeerCtxStore.new()
+    pendingBlocks = PendingBlocksManager.new()
+
+    network = BlockExcNetwork()
+    localStore = CacheStore.new(chunkSize = BitTorrentBlockSize.NBytes)
+    discovery =
+      DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks)
+    advertiser = Advertiser.new(localStore, blockDiscovery)
+    engine = BlockExcEngine.new(
+      localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks
+    )
+    networkStore = NetworkStore.new(engine, localStore)
+
+asyncchecksuite "Torrent Downloader":
+  setupDependencies()
+
+  var
+    codexManifest: Manifest
+    torrentInfo: BitTorrentInfo
+    torrentManifest: BitTorrentManifest
+
+    blocks: seq[Block]
+    codexManifestBlock: Block
+    torrentManifestBlock: Block
+
+    torrentDownloader: TorrentDownloader
+
+  proc createTestData(datasetSize: int) {.async.} =
+    echo "datasetSize: ", datasetSize
+    blocks = await makeRandomBlocks(
+      datasetSize = datasetSize, blockSize = BitTorrentBlockSize.NBytes
+    )
+    for blk in blocks:
+      echo "block: ", blk.data.len
+    codexManifest = await storeDataGetManifest(localStore, blocks)
+    codexManifestBlock = (await storeCodexManifest(codexManifest, localStore)).tryGet()
+    torrentInfo = (
+      await torrentInfoForCodexManifest(
+        localStore, codexManifest, pieceLength = 64.KiBs.int, name = "data.bin".some
+      )
+    ).tryGet()
+    torrentManifest = newBitTorrentManifest(
+      info = torrentInfo, codexManifestCid = codexManifestBlock.cid
+    )
+    torrentManifestBlock =
+      (await storeTorrentManifest(torrentManifest, localStore)).tryGet()
+
+  setup:
+    await createTestData(datasetSize = 40.KiBs.int)
+
+    torrentDownloader =
+      newTorrentDownloader(torrentManifest, codexManifest, networkStore).tryGet()
+
+  test "correctly sets up the download queue":
+    echo "codeManifest: ", $codexManifest
+    echo "torrentInfo: ", $torrentInfo
+    echo "torrentManifest: ", $torrentManifest
+    echo "codexManifestBlockCid: ", $(codexManifestBlock.cid)
+    echo "torrentManifestBlockCid: ", $(torrentManifestBlock.cid)