From 008b8950ef63da4e8aa2377cf406b15471ffa26a Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 12 Mar 2025 03:01:26 +0100 Subject: [PATCH] adds integration tests for BitTorrent --- tests/examples.nim | 6 +- tests/integration/codexclient.nim | 39 ++++++++- tests/integration/testbittorrent.nim | 126 +++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 tests/integration/testbittorrent.nim diff --git a/tests/examples.nim b/tests/examples.nim index 9ef4e292..fd33f4a2 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -88,10 +88,12 @@ proc example(_: type G2Point): G2Point = proc example*(_: type Groth16Proof): Groth16Proof = Groth16Proof(a: G1Point.example, b: G2Point.example, c: G1Point.example) -proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} = +proc example*( + _: type RandomChunker, blocks: int, blockSize = DefaultBlockSize.int +): Future[seq[byte]] {.async.} = let rng = Rng.instance() let chunker = RandomChunker.new( - rng, size = DefaultBlockSize * blocks.NBytes, chunkSize = DefaultBlockSize + rng, size = blockSize.NBytes * blocks.NBytes, chunkSize = blockSize ) var data: seq[byte] while (let moar = await chunker.getBytes(); moar != []): diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 17ed6dd4..23a6d83d 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -1,6 +1,5 @@ import std/strutils - -from pkg/libp2p import Cid, `$`, init +from pkg/libp2p import Cid, MultiHash, `$`, init, hex import pkg/stint import pkg/questionable/results import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable] @@ -120,9 +119,32 @@ proc upload*( proc upload*( client: CodexClient, bytes: seq[byte] -): Future[?!Cid] {.async: (raw: true).} = +): Future[?!Cid] {.async: (raw: true), raises: [CancelledError, HttpError].} = return client.upload(string.fromBytes(bytes)) +proc uploadTorrent*( + client: CodexClient, + contents: string, + filename = string.none, + contentType = "application/octet-stream", +): Future[?!MultiHash] {.async: (raises: [CancelledError, HttpError]).} = + var headers = newSeq[HttpHeaderTuple]() + if name =? filename: + headers = + @[ + ("Content-Disposition", "filename=\"" & name & "\""), + ("Content-Type", contentType), + ] + let response = + await client.post(client.baseurl & "/torrent", body = contents, headers = headers) + assert response.status == 200 + MultiHash.init((await response.body).hexToSeqByte).mapFailure + +proc uploadTorrent*( + client: CodexClient, bytes: seq[byte], filename = string.none +): Future[?!MultiHash] {.async: (raw: true), raises: [CancelledError, HttpError].} = + client.uploadTorrent(string.fromBytes(bytes), filename) + proc downloadRaw*( client: CodexClient, cid: string, local = false ): Future[HttpClientResponseRef] {. @@ -169,6 +191,17 @@ proc downloadManifestOnly*( success await response.body +proc downloadTorrentManifestOnly*( + client: CodexClient, infoHash: MultiHash +): Future[?!RestTorrentContent] = + let response = + await client.get(client.baseurl & "/torrent/" & infoHash.hex & "/network/manifest") + + if response.status != 200: + return failure($response.status) + + RestTorrentContent.fromJson(await response.body) + proc deleteRaw*( client: CodexClient, cid: string ): Future[HttpClientResponseRef] {. diff --git a/tests/integration/testbittorrent.nim b/tests/integration/testbittorrent.nim new file mode 100644 index 00000000..d47e6b36 --- /dev/null +++ b/tests/integration/testbittorrent.nim @@ -0,0 +1,126 @@ +import std/net +import std/sequtils +import pkg/nimcrypto +from pkg/libp2p import `==`, `$`, MultiHash, init +import pkg/codex/units +import pkg/codex/utils/iter +import pkg/codex/manifest +import pkg/codex/rest/json +import pkg/codex/bittorrent/manifest +import ./twonodes +import ../examples +import ../codex/examples +import json + +proc createInfoDictionaryForContent( + content: seq[byte], pieceLength = DefaultPieceLength.int, name = string.none +): ?!BitTorrentInfo = + let + numOfBlocksPerPiece = pieceLength div BitTorrentBlockSize.int + numOfPieces = divUp(content.len.NBytes, pieceLength.NBytes) + + var + pieces: seq[MultiHash] + pieceHashCtx: sha1 + pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece) + + echo "numOfBlocksPerPiece: ", numOfBlocksPerPiece + echo "numOfPieces: ", numOfPieces + pieceHashCtx.init() + + let chunks = content.distribute(num = numOfPieces, spread = false) + + echo "chunks: ", chunks.len + + for chunk in chunks: + echo "chunk: ", chunk.len + if chunk.len == 0: + break + if pieceIter.finished: + without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure, + err: + return failure(err) + pieces.add(mh) + pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece) + pieceHashCtx.init() + pieceHashCtx.update(chunk) + discard pieceIter.next() + + without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure, err: + return failure(err) + pieces.add(mh) + + let info = BitTorrentInfo( + length: content.len.uint64, + pieceLength: pieceLength.uint32, + pieces: pieces, + name: name, + ) + + success info + +twonodessuite "BitTorrent API": + test "uploading and downloading the content", twoNodesConfig: + let exampleContent = exampleString(100) + let infoHash = client1.uploadTorrent(exampleContent).tryGet + let downloadedContent = client1.downloadTorrent(infoHash).tryGet + check downloadedContent == exampleContent + + test "uploading and downloading the content (exactly one piece long)", twoNodesConfig: + let numOfBlocksPerPiece = int(DefaultPieceLength div BitTorrentBlockSize) + let bytes = await RandomChunker.example( + blocks = numOfBlocksPerPiece, blockSize = BitTorrentBlockSize.int + ) + + let infoHash = client1.uploadTorrent(bytes).tryGet + let downloadedContent = client1.downloadTorrent(infoHash).tryGet + check downloadedContent.toBytes == bytes + + test "uploading and downloading the content (exactly two pieces long)", twoNodesConfig: + let numOfBlocksPerPiece = int(DefaultPieceLength div BitTorrentBlockSize) + let bytes = await RandomChunker.example( + blocks = numOfBlocksPerPiece * 2, blockSize = BitTorrentBlockSize.int + ) + + let infoHash = client1.uploadTorrent(bytes).tryGet + let downloadedContent = client1.downloadTorrent(infoHash).tryGet + check downloadedContent.toBytes == bytes + + # use with debugging to see the content + # use: + # CodexConfigs.init(nodes = 2).debug().withLogTopics("restapi", "node").some + # in tests/integration/twonodes.nim + # await sleepAsync(2.seconds) + + test "retrieving torrent manifest for given info hash", twoNodesConfig: + let exampleFileName = "example.txt" + let exampleContent = exampleString(100) + let infoHash = client1.uploadTorrent( + contents = exampleContent, + filename = some exampleFileName, + contentType = "text/plain", + ).tryGet + + let expectedInfo = createInfoDictionaryForContent( + content = exampleContent.toBytes, name = some exampleFileName + ).tryGet + + let restTorrentContent = client1.downloadTorrentManifestOnly(infoHash).tryGet + let torrentManifest = restTorrentContent.torrentManifest + let info = torrentManifest.info + + check info == expectedInfo + + let response = + client1.downloadManifestOnly(cid = torrentManifest.codexManifestCid).tryGet + + echo "response: ", response + let restContent = RestContent.fromJson(response).tryGet + + check restContent.cid == torrentManifest.codexManifestCid + + let codexManifest = restContent.manifest + check codexManifest.datasetSize.uint64 == info.length + check codexManifest.blockSize == BitTorrentBlockSize + check codexManifest.filename == info.name + check codexManifest.mimetype == "text/plain".some