nim-codex/tests/codex/slots/helpers.nim

import std/sugar

import pkg/chronos
import pkg/libp2p/cid

import pkg/codex/codextypes
import pkg/codex/stores
import pkg/codex/merkletree
import pkg/codex/manifest
import pkg/codex/blocktype as bt
import pkg/codex/chunker
import pkg/codex/indexingstrategy
import pkg/codex/slots
import pkg/codex/rng

import ../helpers

proc storeManifest*(
    store: BlockStore, manifest: Manifest
): Future[?!bt.Block] {.async.} =
  without encodedVerifiable =? manifest.encode(), err:
    trace "Unable to encode manifest"
    return failure(err)

  without blk =? bt.Block.new(data = encodedVerifiable, codec = ManifestCodec), error:
    trace "Unable to create block from manifest"
    return failure(error)

  if err =? (await store.putBlock(blk)).errorOption:
    trace "Unable to store manifest block", cid = blk.cid, err = err.msg
    return failure(err)

  success blk

proc makeManifest*(
    cids: seq[Cid],
    datasetSize: NBytes,
    blockSize: NBytes,
    store: BlockStore,
    hcodec = Sha256HashCodec,
    dataCodec = BlockCodec,
): Future[?!Manifest] {.async.} =
  without tree =? CodexTree.init(cids), err:
    return failure(err)

  without treeCid =? tree.rootCid(CIDv1, dataCodec), err:
    return failure(err)

  for index, cid in cids:
    without proof =? tree.getProof(index), err:
      return failure(err)

    if err =? (await store.putCidAndProof(treeCid, index, cid, proof)).errorOption:
      # TODO add log here
      return failure(err)

  let manifest = Manifest.new(
    treeCid = treeCid,
    blockSize = blockSize,
    datasetSize = datasetSize,
    version = CIDv1,
    hcodec = hcodec,
    codec = dataCodec,
  )

  without manifestBlk =? await store.storeManifest(manifest), err:
    trace "Unable to store manifest"
    return failure(err)

  success manifest

proc createBlocks*(
    chunker: Chunker, store: BlockStore
): Future[seq[bt.Block]] {.async.} =
  collect(newSeq):
    while (let chunk = await chunker.getBytes(); chunk.len > 0):
      let blk = bt.Block.new(chunk).tryGet()
      discard await store.putBlock(blk)
      blk

proc createProtectedManifest*(
    datasetBlocks: seq[bt.Block],
    store: BlockStore,
    numDatasetBlocks: int,
    ecK: int,
    ecM: int,
    blockSize: NBytes,
    originalDatasetSize: int,
    totalDatasetSize: int,
): Future[tuple[manifest: Manifest, protected: Manifest]] {.async.} =
  let
    cids = datasetBlocks.mapIt(it.cid)
    datasetTree = CodexTree.init(cids[0 ..< numDatasetBlocks]).tryGet()
    datasetTreeCid = datasetTree.rootCid().tryGet()

    protectedTree = CodexTree.init(cids).tryGet()
    protectedTreeCid = protectedTree.rootCid().tryGet()

  for index, cid in cids[0 ..< numDatasetBlocks]:
    let proof = datasetTree.getProof(index).tryGet()
    (await store.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet

  for index, cid in cids:
    let proof = protectedTree.getProof(index).tryGet()
    (await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet

  let
    manifest = Manifest.new(
      treeCid = datasetTreeCid,
      blockSize = blockSize,
      datasetSize = originalDatasetSize.NBytes,
    )

    protectedManifest = Manifest.new(
      manifest = manifest,
      treeCid = protectedTreeCid,
      datasetSize = totalDatasetSize.NBytes,
      ecK = ecK,
      ecM = ecM,
      strategy = SteppedStrategy,
    )

    manifestBlock =
      bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet()

    protectedManifestBlock =
      bt.Block.new(protectedManifest.encode().tryGet(), codec = ManifestCodec).tryGet()

  (await store.putBlock(manifestBlock)).tryGet()
  (await store.putBlock(protectedManifestBlock)).tryGet()

  (manifest, protectedManifest)

proc createVerifiableManifest*(
    store: BlockStore,
    numDatasetBlocks: int,
    ecK: int,
    ecM: int,
    blockSize: NBytes,
    cellSize: NBytes,
): Future[tuple[manifest: Manifest, protected: Manifest, verifiable: Manifest]] {.
    async
.} =
  let
    numSlots = ecK + ecM
    numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM)
      # total number of blocks in the dataset after
      # EC (should will match number of slots)
    originalDatasetSize = numDatasetBlocks * blockSize.int
    totalDatasetSize = numTotalBlocks * blockSize.int

    chunker =
      RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
    datasetBlocks = await chunker.createBlocks(store)

    (manifest, protectedManifest) = await createProtectedManifest(
      datasetBlocks, store, numDatasetBlocks, ecK, ecM, blockSize, originalDatasetSize,
      totalDatasetSize,
    )

    builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet
    verifiableManifest = (await builder.buildManifest()).tryGet

  # build the slots and manifest
  (manifest, protectedManifest, verifiableManifest)