adds integration tests for BitTorrent

tests/examples.nim

@@ -88,10 +88,12 @@ proc example(_: type G2Point): G2Point =
 proc example*(_: type Groth16Proof): Groth16Proof =
   Groth16Proof(a: G1Point.example, b: G2Point.example, c: G1Point.example)
 
-proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} =
+proc example*(
+    _: type RandomChunker, blocks: int, blockSize = DefaultBlockSize.int
+): Future[seq[byte]] {.async.} =
   let rng = Rng.instance()
   let chunker = RandomChunker.new(
-    rng, size = DefaultBlockSize * blocks.NBytes, chunkSize = DefaultBlockSize
+    rng, size = blockSize.NBytes * blocks.NBytes, chunkSize = blockSize
   )
   var data: seq[byte]
   while (let moar = await chunker.getBytes(); moar != []):

tests/integration/codexclient.nim

@@ -1,6 +1,5 @@
 import std/strutils
-
-from pkg/libp2p import Cid, `$`, init
+from pkg/libp2p import Cid, MultiHash, `$`, init, hex
 import pkg/stint
 import pkg/questionable/results
 import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable]
@@ -120,9 +119,32 @@ proc upload*(
 
 proc upload*(
     client: CodexClient, bytes: seq[byte]
-): Future[?!Cid] {.async: (raw: true).} =
+): Future[?!Cid] {.async: (raw: true), raises: [CancelledError, HttpError].} =
   return client.upload(string.fromBytes(bytes))
 
+proc uploadTorrent*(
+    client: CodexClient,
+    contents: string,
+    filename = string.none,
+    contentType = "application/octet-stream",
+): Future[?!MultiHash] {.async: (raises: [CancelledError, HttpError]).} =
+  var headers = newSeq[HttpHeaderTuple]()
+  if name =? filename:
+    headers =
+      @[
+        ("Content-Disposition", "filename=\"" & name & "\""),
+        ("Content-Type", contentType),
+      ]
+  let response =
+    await client.post(client.baseurl & "/torrent", body = contents, headers = headers)
+  assert response.status == 200
+  MultiHash.init((await response.body).hexToSeqByte).mapFailure
+
+proc uploadTorrent*(
+    client: CodexClient, bytes: seq[byte], filename = string.none
+): Future[?!MultiHash] {.async: (raw: true), raises: [CancelledError, HttpError].} =
+  client.uploadTorrent(string.fromBytes(bytes), filename)
+
 proc downloadRaw*(
     client: CodexClient, cid: string, local = false
 ): Future[HttpClientResponseRef] {.
@@ -169,6 +191,17 @@ proc downloadManifestOnly*(
 
   success await response.body
 
+proc downloadTorrentManifestOnly*(
+    client: CodexClient, infoHash: MultiHash
+): Future[?!RestTorrentContent] =
+  let response =
+    await client.get(client.baseurl & "/torrent/" & infoHash.hex & "/network/manifest")
+
+  if response.status != 200:
+    return failure($response.status)
+
+  RestTorrentContent.fromJson(await response.body)
+
 proc deleteRaw*(
     client: CodexClient, cid: string
 ): Future[HttpClientResponseRef] {.

tests/integration/testbittorrent.nim

@@ -0,0 +1,126 @@
+import std/net
+import std/sequtils
+import pkg/nimcrypto
+from pkg/libp2p import `==`, `$`, MultiHash, init
+import pkg/codex/units
+import pkg/codex/utils/iter
+import pkg/codex/manifest
+import pkg/codex/rest/json
+import pkg/codex/bittorrent/manifest
+import ./twonodes
+import ../examples
+import ../codex/examples
+import json
+
+proc createInfoDictionaryForContent(
+    content: seq[byte], pieceLength = DefaultPieceLength.int, name = string.none
+): ?!BitTorrentInfo =
+  let
+    numOfBlocksPerPiece = pieceLength div BitTorrentBlockSize.int
+    numOfPieces = divUp(content.len.NBytes, pieceLength.NBytes)
+
+  var
+    pieces: seq[MultiHash]
+    pieceHashCtx: sha1
+    pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece)
+
+  echo "numOfBlocksPerPiece: ", numOfBlocksPerPiece
+  echo "numOfPieces: ", numOfPieces
+  pieceHashCtx.init()
+
+  let chunks = content.distribute(num = numOfPieces, spread = false)
+
+  echo "chunks: ", chunks.len
+
+  for chunk in chunks:
+    echo "chunk: ", chunk.len
+    if chunk.len == 0:
+      break
+    if pieceIter.finished:
+      without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure,
+        err:
+        return failure(err)
+      pieces.add(mh)
+      pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece)
+      pieceHashCtx.init()
+    pieceHashCtx.update(chunk)
+    discard pieceIter.next()
+
+  without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure, err:
+    return failure(err)
+  pieces.add(mh)
+
+  let info = BitTorrentInfo(
+    length: content.len.uint64,
+    pieceLength: pieceLength.uint32,
+    pieces: pieces,
+    name: name,
+  )
+
+  success info
+
+twonodessuite "BitTorrent API":
+  test "uploading and downloading the content", twoNodesConfig:
+    let exampleContent = exampleString(100)
+    let infoHash = client1.uploadTorrent(exampleContent).tryGet
+    let downloadedContent = client1.downloadTorrent(infoHash).tryGet
+    check downloadedContent == exampleContent
+
+  test "uploading and downloading the content (exactly one piece long)", twoNodesConfig:
+    let numOfBlocksPerPiece = int(DefaultPieceLength div BitTorrentBlockSize)
+    let bytes = await RandomChunker.example(
+      blocks = numOfBlocksPerPiece, blockSize = BitTorrentBlockSize.int
+    )
+
+    let infoHash = client1.uploadTorrent(bytes).tryGet
+    let downloadedContent = client1.downloadTorrent(infoHash).tryGet
+    check downloadedContent.toBytes == bytes
+
+  test "uploading and downloading the content (exactly two pieces long)", twoNodesConfig:
+    let numOfBlocksPerPiece = int(DefaultPieceLength div BitTorrentBlockSize)
+    let bytes = await RandomChunker.example(
+      blocks = numOfBlocksPerPiece * 2, blockSize = BitTorrentBlockSize.int
+    )
+
+    let infoHash = client1.uploadTorrent(bytes).tryGet
+    let downloadedContent = client1.downloadTorrent(infoHash).tryGet
+    check downloadedContent.toBytes == bytes
+
+    # use with debugging to see the content
+    # use:
+    # CodexConfigs.init(nodes = 2).debug().withLogTopics("restapi", "node").some
+    # in tests/integration/twonodes.nim
+    # await sleepAsync(2.seconds)
+
+  test "retrieving torrent manifest for given info hash", twoNodesConfig:
+    let exampleFileName = "example.txt"
+    let exampleContent = exampleString(100)
+    let infoHash = client1.uploadTorrent(
+      contents = exampleContent,
+      filename = some exampleFileName,
+      contentType = "text/plain",
+    ).tryGet
+
+    let expectedInfo = createInfoDictionaryForContent(
+      content = exampleContent.toBytes, name = some exampleFileName
+    ).tryGet
+
+    let restTorrentContent = client1.downloadTorrentManifestOnly(infoHash).tryGet
+    let torrentManifest = restTorrentContent.torrentManifest
+    let info = torrentManifest.info
+
+    check info == expectedInfo
+
+    let response =
+      client1.downloadManifestOnly(cid = torrentManifest.codexManifestCid).tryGet
+
+    echo "response: ", response
+    let restContent = RestContent.fromJson(response).tryGet
+
+    check restContent.cid == torrentManifest.codexManifestCid
+
+    let codexManifest = restContent.manifest
+    check codexManifest.datasetSize.uint64 == info.length
+    check codexManifest.blockSize == BitTorrentBlockSize
+    check codexManifest.filename == info.name
+    check codexManifest.mimetype == "text/plain".some