diff --git a/tests/codex/bittorrent/helpers.nim b/tests/codex/bittorrent/helpers.nim new file mode 100644 index 00000000..10bcaf26 --- /dev/null +++ b/tests/codex/bittorrent/helpers.nim @@ -0,0 +1,102 @@ +import pkg/chronos +import pkg/libp2p/[cid, multicodec, multihash] +import pkg/questionable/results + +import pkg/codex/stores/cachestore +import pkg/codex/utils/iter + +import pkg/codex/manifest +import pkg/codex/bittorrent/manifest + +proc torrentInfoForCodexManifest*( + localStore: BlockStore, + codexManifest: Manifest, + pieceLength = DefaultPieceLength.int, + name = string.none, +): Future[?!BitTorrentInfo] {.async.} = + let treeCid = codexManifest.treeCid + let datasetSize = codexManifest.datasetSize + let blockSize = codexManifest.blockSize + let numOfBlocks = divUp(datasetSize, blockSize) + let blockIter = Iter.new(0 ..< numOfBlocks) + var blocks = newSeq[Block](numOfBlocks) + while not blockIter.finished: + let index = blockIter.next() + without blk =? (await localStore.getBlock(treeCid, index)), err: + return failure(err) + blocks[index] = blk + let + numOfBlocksPerPiece = pieceLength div BitTorrentBlockSize.int + numOfPieces = divUp(datasetSize, pieceLength.NBytes) + + var + pieces: seq[MultiHash] + pieceHashCtx: sha1 + pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece) + + pieceHashCtx.init() + + for blk in blocks: + if blk.data.len == 0: + break + if pieceIter.finished: + without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure, + err: + return failure(err) + pieces.add(mh) + pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece) + pieceHashCtx.init() + pieceHashCtx.update(blk.data) + discard pieceIter.next() + + without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure, err: + return failure(err) + pieces.add(mh) + + let info = BitTorrentInfo( + length: datasetSize.uint64, + pieceLength: pieceLength.uint32, + pieces: pieces, + name: name, + ) + + success info + +proc storeCodexManifest*( + codexManifest: Manifest, localStore: BlockStore +): Future[?!Block] {.async.} = + without encodedManifest =? codexManifest.encode(), err: + trace "Unable to encode manifest", err = err.msg + return failure(err) + + without blk =? Block.new(data = encodedManifest, codec = ManifestCodec), err: + trace "Unable to create block from manifest", err = err.msg + return failure(err) + + if err =? (await localStore.putBlock(blk)).errorOption: + trace "Unable to store manifest block", cid = blk.cid, err = err.msg + return failure(err) + + success blk + +proc storeTorrentManifest*( + torrentManifest: BitTorrentManifest, localStore: BlockStore +): Future[?!Block] {.async.} = + let infoBencoded = torrentManifest.info.bencode() + let infoHash = MultiHash.digest($Sha1HashCodec, infoBencoded).tryGet + let encodedManifest = torrentManifest.encode() + + without infoHashCid =? Cid.init(CIDv1, InfoHashV1Codec, infoHash).mapFailure, err: + trace "Unable to create CID for BitTorrent info hash", err = err.msg + return failure(err) + + without blk =? Block.new(data = encodedManifest, cid = infoHashCid, verify = false), + err: + trace "Unable to create block from manifest", err = err.msg + return failure(err) + + if err =? (await localStore.putBlock(blk)).errorOption: + trace "Unable to store BitTorrent manifest block", cid = blk.cid, err = err.msg + return failure(err) + + success blk diff --git a/tests/codex/bittorrent/testtorrentdownloader.nim b/tests/codex/bittorrent/testtorrentdownloader.nim new file mode 100644 index 00000000..379a3798 --- /dev/null +++ b/tests/codex/bittorrent/testtorrentdownloader.nim @@ -0,0 +1,100 @@ +import pkg/libp2p/[cid, multicodec, multihash] +import pkg/questionable/results + +import pkg/codex/rng +import pkg/codex/blockexchange +import pkg/codex/stores +import pkg/codex/discovery +import pkg/codex/blocktype + +import pkg/codex/manifest +import pkg/codex/bittorrent/manifest +import pkg/codex/bittorrent/torrentdownloader + +import ../../asynctest +import ./helpers +import ../helpers +import ../examples + +template setupDependencies() {.dirty.} = + var + rng: Rng + seckey: PrivateKey + peerId: PeerId + wallet: WalletRef + blockDiscovery: Discovery + peerStore: PeerCtxStore + pendingBlocks: PendingBlocksManager + + network: BlockExcNetwork + localStore: CacheStore + discovery: DiscoveryEngine + advertiser: Advertiser + engine: BlockExcEngine + networkStore: NetworkStore + + setup: + rng = Rng.instance() + seckey = PrivateKey.random(rng[]).tryGet() + peerId = PeerId.init(seckey.getPublicKey().tryGet()).tryGet() + wallet = WalletRef.example + blockDiscovery = Discovery.new() + peerStore = PeerCtxStore.new() + pendingBlocks = PendingBlocksManager.new() + + network = BlockExcNetwork() + localStore = CacheStore.new(chunkSize = BitTorrentBlockSize.NBytes) + discovery = + DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks) + advertiser = Advertiser.new(localStore, blockDiscovery) + engine = BlockExcEngine.new( + localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + ) + networkStore = NetworkStore.new(engine, localStore) + +asyncchecksuite "Torrent Downloader": + setupDependencies() + + var + codexManifest: Manifest + torrentInfo: BitTorrentInfo + torrentManifest: BitTorrentManifest + + blocks: seq[Block] + codexManifestBlock: Block + torrentManifestBlock: Block + + torrentDownloader: TorrentDownloader + + proc createTestData(datasetSize: int) {.async.} = + echo "datasetSize: ", datasetSize + blocks = await makeRandomBlocks( + datasetSize = datasetSize, blockSize = BitTorrentBlockSize.NBytes + ) + for blk in blocks: + echo "block: ", blk.data.len + codexManifest = await storeDataGetManifest(localStore, blocks) + codexManifestBlock = (await storeCodexManifest(codexManifest, localStore)).tryGet() + torrentInfo = ( + await torrentInfoForCodexManifest( + localStore, codexManifest, pieceLength = 64.KiBs.int, name = "data.bin".some + ) + ).tryGet() + torrentManifest = newBitTorrentManifest( + info = torrentInfo, codexManifestCid = codexManifestBlock.cid + ) + torrentManifestBlock = + (await storeTorrentManifest(torrentManifest, localStore)).tryGet() + + setup: + await createTestData(datasetSize = 40.KiBs.int) + + torrentDownloader = + newTorrentDownloader(torrentManifest, codexManifest, networkStore).tryGet() + + test "correctly sets up the download queue": + echo "codeManifest: ", $codexManifest + echo "torrentInfo: ", $torrentInfo + echo "torrentManifest: ", $torrentManifest + echo "codexManifestBlockCid: ", $(codexManifestBlock.cid) + echo "torrentManifestBlockCid: ", $(torrentManifestBlock.cid)