nim-dagger/tests/codex/slots/testslotbuilder.nim

import std/sequtils
import std/math
import std/importutils
import std/sugar

import pkg/chronos
import pkg/questionable/results
import pkg/codex/blocktype as bt
import pkg/codex/rng
import pkg/codex/stores
import pkg/codex/chunker
import pkg/codex/merkletree
import pkg/codex/manifest {.all.}
import pkg/codex/utils
import pkg/codex/utils/digest
import pkg/datastore
import pkg/poseidon2
import pkg/poseidon2/io
import constantine/math/io/io_fields

import ../../asynctest
import ../helpers
import ../examples
import ../merkletree/helpers

import pkg/codex/indexingstrategy {.all.}
import pkg/codex/slots {.all.}

privateAccess(SlotsBuilder) # enable access to private fields
privateAccess(Manifest) # enable access to private fields

suite "Slot builder":
  let
    blockSize = NBytes 1024
    cellSize = NBytes 64
    ecK = 3
    ecM = 2

    numSlots = ecK + ecM
    numDatasetBlocks = 100
    numBlockCells = (blockSize div cellSize).int

    numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM)                # total number of blocks in the dataset after
                                                                                  # EC (should will match number of slots)
    originalDatasetSize = numDatasetBlocks * blockSize.int                        # size of the dataset before EC
    totalDatasetSize = numTotalBlocks * blockSize.int                             # size of the dataset after EC
    numTotalSlotBlocks = nextPowerOfTwo(numTotalBlocks div numSlots)

    blockPadBytes =
      newSeq[byte](numBlockCells.nextPowerOfTwoPad * cellSize.int)                # power of two padding for blocks

    slotsPadLeafs =
      newSeqWith((numTotalBlocks div numSlots).nextPowerOfTwoPad, Poseidon2Zero)  # power of two padding for block roots

    rootsPadLeafs =
      newSeqWith(numSlots.nextPowerOfTwoPad, Poseidon2Zero)

  var
    datasetBlocks: seq[bt.Block]
    localStore: BlockStore
    manifest: Manifest
    protectedManifest: Manifest
    expectedEmptyCid: Cid
    slotBuilder: SlotsBuilder
    chunker: Chunker

  proc createBlocks(): Future[void] {.async.} =
    while true:
      let chunk = await chunker.getBytes()
      if chunk.len <= 0:
        break
      let blk = bt.Block.new(chunk).tryGet()
      datasetBlocks.add(blk)
      discard await localStore.putBlock(blk)

  proc createProtectedManifest(): Future[void] {.async.} =
    let
      cids = datasetBlocks.mapIt(it.cid)
      datasetTree = CodexTree.init(cids[0..<numDatasetBlocks]).tryGet()
      datasetTreeCid = datasetTree.rootCid().tryGet()

      protectedTree = CodexTree.init(cids).tryGet()
      protectedTreeCid = protectedTree.rootCid().tryGet()

    for index, cid in cids[0..<numDatasetBlocks]:
      let proof = datasetTree.getProof(index).tryget()
      (await localStore.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet

    for index, cid in cids:
      let proof = protectedTree.getProof(index).tryget()
      (await localStore.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet

    manifest = Manifest.new(
      treeCid = datasetTreeCid,
      blockSize = blockSize.NBytes,
      datasetSize = originalDatasetSize.NBytes)

    protectedManifest = Manifest.new(
      manifest = manifest,
      treeCid = protectedTreeCid,
      datasetSize = totalDatasetSize.NBytes,
      ecK = ecK,
      ecM = ecM)

    let
      manifestBlock = bt.Block.new(
        manifest.encode().tryGet(),
        codec = ManifestCodec).tryGet()

      protectedManifestBlock = bt.Block.new(
        protectedManifest.encode().tryGet(),
        codec = ManifestCodec).tryGet()

    (await localStore.putBlock(manifestBlock)).tryGet()
    (await localStore.putBlock(protectedManifestBlock)).tryGet()

    expectedEmptyCid = emptyCid(
      protectedManifest.version,
      protectedManifest.hcodec,
      protectedManifest.codec).tryGet()

  setup:
    let
      repoDs = SQLiteDatastore.new(Memory).tryGet()
      metaDs = SQLiteDatastore.new(Memory).tryGet()
    localStore = RepoStore.new(repoDs, metaDs)

    chunker = RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize)
    await createBlocks()
    await createProtectedManifest()

  teardown:
    await localStore.close()

    # TODO: THIS IS A BUG IN asynctest, because it doesn't release the
    #       objects after the test is done, so we need to do it manually
    #
    # Need to reset all objects because otherwise they get
    # captured by the test runner closures, not good!
    reset(datasetBlocks)
    reset(localStore)
    reset(manifest)
    reset(protectedManifest)
    reset(expectedEmptyCid)
    reset(slotBuilder)
    reset(chunker)

  test "Can only create slotBuilder with protected manifest":
    let
      unprotectedManifest = Manifest.new(
        treeCid = Cid.example,
        blockSize = blockSize.NBytes,
        datasetSize = originalDatasetSize.NBytes)

    check:
      SlotsBuilder.new(localStore, unprotectedManifest, cellSize = cellSize)
        .error.msg == "Can only create SlotsBuilder using protected manifests."

  test "Number of blocks must be devisable by number of slots":
    let
      mismatchManifest = Manifest.new(
        manifest = Manifest.new(
          treeCid = Cid.example,
          blockSize = blockSize.NBytes,
          datasetSize = originalDatasetSize.NBytes),
        treeCid = Cid.example,
        datasetSize = totalDatasetSize.NBytes,
        ecK = ecK - 1,
        ecM = ecM)

    check:
      SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
        .error.msg == "Number of blocks must be divisable by number of slots."

  test "Block size must be divisable by cell size":
    let
      mismatchManifest = Manifest.new(
        manifest = Manifest.new(
          treeCid = Cid.example,
          blockSize = (blockSize + 1).NBytes,
          datasetSize = (originalDatasetSize - 1).NBytes),
        treeCid = Cid.example,
        datasetSize = (totalDatasetSize - 1).NBytes,
        ecK = ecK,
        ecM = ecM)

    check:
      SlotsBuilder.new(localStore, mismatchManifest, cellSize = cellSize)
        .error.msg == "Block size must be divisable by cell size."

  test "Should build correct slot builder":
    slotBuilder = SlotsBuilder.new(
      localStore,
      protectedManifest,
      cellSize = cellSize).tryGet()

    check:
      slotBuilder.numBlockPadBytes == blockPadBytes.len
      slotBuilder.numSlotsPadLeafs == slotsPadLeafs.len
      slotBuilder.numRootsPadLeafs == rootsPadLeafs.len

  test "Should build slot hashes for all slots":
    let
      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

    for i in 0 ..< numSlots:
      let
        expectedBlock = steppedStrategy
          .getIndicies(i)
          .mapIt( datasetBlocks[it] )

        expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
          for blk in expectedBlock:
            SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)

        cellHashes = (await slotBuilder.getBlockHashes(i)).tryGet()

      check:
        expectedHashes == cellHashes

  test "Should build slot trees for all slots":
    let
      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

    for i in 0 ..< numSlots:
      let
        expectedBlock = steppedStrategy
          .getIndicies(i)
          .mapIt( datasetBlocks[it] )

        expectedHashes: seq[Poseidon2Hash] = collect(newSeq):
          for blk in expectedBlock:
            SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)
        expectedRoot = Merkle.digest(expectedHashes & slotsPadLeafs)

        slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()

      check:
        expectedRoot == slotTree.root().tryGet()

  test "Should persist trees for all slots":
    let
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

    for i in 0 ..< numSlots:
      let
        slotTree = (await slotBuilder.buildSlotTree(i)).tryGet()
        slotRoot = (await slotBuilder.buildSlot(i)).tryGet()
        slotCid = slotRoot.toSlotCid().tryGet()

      for cellIndex in 0..<numTotalSlotBlocks:
        let
          (cellCid, proof) = (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet()
          verifiableProof = proof.toVerifiableProof().tryGet()
          posProof = slotTree.getProof(cellIndex).tryGet

        check:
          verifiableProof.index == posProof.index
          verifiableProof.nleaves == posProof.nleaves
          verifiableProof.path == posProof.path

  test "Should build correct verification root":
    let
      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

    (await slotBuilder.buildSlots()).tryGet
    let
      slotsHashes = collect(newSeq):
        for i in 0 ..< numSlots:
          let
            expectedBlocks = steppedStrategy
              .getIndicies(i)
              .mapIt( datasetBlocks[it] )

            slotHashes: seq[Poseidon2Hash] = collect(newSeq):
              for blk in expectedBlocks:
                SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)

          Merkle.digest(slotHashes & slotsPadLeafs)

      expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
      rootHash = slotBuilder.buildVerifyTree(slotBuilder.slotRoots).tryGet().root.tryGet()

    check:
      expectedRoot == rootHash

  test "Should build correct verification root manifest":
    let
      steppedStrategy = SteppedIndexingStrategy.new(0, numTotalBlocks - 1, numSlots)
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

      slotsHashes = collect(newSeq):
        for i in 0 ..< numSlots:
          let
            expectedBlocks = steppedStrategy
              .getIndicies(i)
              .mapIt( datasetBlocks[it] )

            slotHashes: seq[Poseidon2Hash] = collect(newSeq):
              for blk in expectedBlocks:
                SpongeMerkle.digest(blk.data & blockPadBytes, cellSize.int)

          Merkle.digest(slotHashes & slotsPadLeafs)

      expectedRoot = Merkle.digest(slotsHashes & rootsPadLeafs)
      manifest = (await slotBuilder.buildManifest()).tryGet()
      mhash = manifest.verifyRoot.mhash.tryGet()
      mhashBytes = mhash.digestBytes
      rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get

    check:
      expectedRoot == rootHash

  test "Should not build from verifiable manifest with 0 slots":
    var
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()
      verifyManifest = (await slotBuilder.buildManifest()).tryGet()

    verifyManifest.slotRoots = @[]
    check SlotsBuilder.new(
        localStore,
        verifyManifest,
        cellSize = cellSize).isErr

  test "Should not build from verifiable manifest with incorrect number of slots":
    var
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

      verifyManifest = (await slotBuilder.buildManifest()).tryGet()

    verifyManifest.slotRoots.del(
      verifyManifest.slotRoots.len - 1
    )

    check SlotsBuilder.new(
        localStore,
        verifyManifest,
        cellSize = cellSize).isErr

  test "Should not build from verifiable manifest with invalid verify root":
    let
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

      verifyManifest = (await slotBuilder.buildManifest()).tryGet()
      offset = verifyManifest.verifyRoot.data.buffer.len div 2

    rng.shuffle(
      Rng.instance,
      verifyManifest.verifyRoot.data.buffer)

    check SlotsBuilder.new(
        localStore,
        verifyManifest,
        cellSize = cellSize).isErr

  test "Should build from verifiable manifest":
    let
      slotBuilder = SlotsBuilder.new(
        localStore,
        protectedManifest,
        cellSize = cellSize).tryGet()

      verifyManifest = (await slotBuilder.buildManifest()).tryGet()

      verificationBuilder = SlotsBuilder.new(
        localStore,
        verifyManifest,
        cellSize = cellSize).tryGet()

    check:
      slotBuilder.slotRoots == verificationBuilder.slotRoots
      slotBuilder.verifyRoot == verificationBuilder.verifyRoot