nim-blockstore/tests/test_dataset.nim

import std/[unittest, os, options]
import chronos
import results
import ../blockstore/errors
import ../blockstore/cid
import ../blockstore/blocks
import ../blockstore/dataset
import ../blockstore/blockmap

const
  TestDir = getTempDir() / "nim_blockstore_dataset_test"
  DbPath = TestDir / "db"
  BlocksDir = TestDir / "blocks"

proc cleanup() =
  if dirExists(TestDir):
    removeDir(TestDir)

proc createTestDataset(store: DatasetStore, blockCount: int, chunkSize: int = 4096): Future[BResult[Dataset]] {.async.} =
  let builderResult = store.startDataset(chunkSize.uint32, some("test"))
  if builderResult.isErr:
    return err(builderResult.error)

  var builder = builderResult.value

  for i in 0 ..< blockCount:
    var data = newSeq[byte](chunkSize)
    for j in 0 ..< chunkSize:
      data[j] = byte((i * chunkSize + j) mod 256)

    let blkResult = newBlock(data)
    if blkResult.isErr:
      return err(blkResult.error)

    let addResult = await builder.addBlock(blkResult.value)
    if addResult.isErr:
      return err(addResult.error)

  return await builder.finalize()

proc runDeleteExistingDataset() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir)
  doAssert storeResult.isOk, "Failed to create store: " & $storeResult.error
  let store = storeResult.value
  defer: store.close()

  let datasetResult = await createTestDataset(store, 5)
  doAssert datasetResult.isOk, "Failed to create dataset: " & $datasetResult.error
  let dataset = datasetResult.value

  let manifestCid = dataset.manifestCid

  let getResult1 = await store.getDataset(dataset.treeCid)
  doAssert getResult1.isOk
  doAssert getResult1.value.isSome, "Dataset should exist before deletion"

  let deleteResult = await store.deleteDataset(manifestCid)
  doAssert deleteResult.isOk, "Delete should succeed: " & $deleteResult.error

  let getResult2 = await store.getDataset(dataset.treeCid)
  doAssert getResult2.isOk
  doAssert getResult2.value.isNone, "Dataset should not exist after deletion"

  cleanup()

proc runDeleteNonExistentDataset() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir)
  doAssert storeResult.isOk
  let store = storeResult.value
  defer: store.close()

  let fakeCidResult = cidFromString("bagazuayseaka5yn4pfmebc7bqkkoij6wb5x3o4jlvzq7flqhd63qalnrskwvy")
  doAssert fakeCidResult.isOk
  let fakeCid = fakeCidResult.value

  let deleteResult = await store.deleteDataset(fakeCid)
  doAssert deleteResult.isErr, "Delete should fail for non-existent dataset"
  doAssert deleteResult.error.kind == DatasetNotFound

  cleanup()

proc runStorageReleasedAfterDeletion() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir)
  doAssert storeResult.isOk
  let store = storeResult.value
  defer: store.close()

  let usedBefore = store.used()

  let datasetResult = await createTestDataset(store, 10, 4096)
  doAssert datasetResult.isOk
  let dataset = datasetResult.value

  let usedAfterCreate = store.used()
  doAssert usedAfterCreate > usedBefore, "Storage should increase after adding dataset"

  let deleteResult = await store.deleteDataset(dataset.manifestCid)
  doAssert deleteResult.isOk

  # Wait for pending deletions to be processed by the worker - hoping that
  # 500milli will do the job
  for _ in 0 ..< 10:
    await sleepAsync(50.milliseconds)
    if store.used() < usedAfterCreate:
      break

  let usedAfterDelete = store.used()
  doAssert usedAfterDelete < usedAfterCreate, "Storage should decrease after deletion"

  cleanup()

proc runMultipleDatasetsDeletion() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir)
  doAssert storeResult.isOk
  let store = storeResult.value
  defer: store.close()

  let dataset1Result = await createTestDataset(store, 3, 4096)
  doAssert dataset1Result.isOk
  let dataset1 = dataset1Result.value

  let dataset2Result = await createTestDataset(store, 4, 4096)
  doAssert dataset2Result.isOk
  let dataset2 = dataset2Result.value

  let get1Before = await store.getDataset(dataset1.treeCid)
  let get2Before = await store.getDataset(dataset2.treeCid)
  doAssert get1Before.isOk and get1Before.value.isSome
  doAssert get2Before.isOk and get2Before.value.isSome

  let delete1Result = await store.deleteDataset(dataset1.manifestCid)
  doAssert delete1Result.isOk

  let get1After = await store.getDataset(dataset1.treeCid)
  let get2After = await store.getDataset(dataset2.treeCid)
  doAssert get1After.isOk and get1After.value.isNone, "Dataset 1 should be deleted"
  doAssert get2After.isOk and get2After.value.isSome, "Dataset 2 should still exist"

  let delete2Result = await store.deleteDataset(dataset2.manifestCid)
  doAssert delete2Result.isOk

  let get2Final = await store.getDataset(dataset2.treeCid)
  doAssert get2Final.isOk and get2Final.value.isNone, "Dataset 2 should be deleted"

  cleanup()

proc runDeleteDatasetWithManyBlocks() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir)
  doAssert storeResult.isOk
  let store = storeResult.value
  defer: store.close()

  let datasetResult = await createTestDataset(store, 100, 4096)
  doAssert datasetResult.isOk
  let dataset = datasetResult.value

  doAssert dataset.blockCount == 100

  let deleteResult = await store.deleteDataset(dataset.manifestCid)
  doAssert deleteResult.isOk, "Delete should succeed for dataset with many blocks"

  let getResult = await store.getDataset(dataset.treeCid)
  doAssert getResult.isOk and getResult.value.isNone

  cleanup()

proc runMappedBlockmapBasic() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir, blockmapBackend = bmFile)
  doAssert storeResult.isOk, "Failed to create store with mapped blockmap: " & $storeResult.error
  let store = storeResult.value
  defer: store.close()

  let datasetResult = await createTestDataset(store, 10)
  doAssert datasetResult.isOk, "Failed to create dataset with mapped blockmap: " & $datasetResult.error
  let dataset = datasetResult.value

  doAssert dataset.blockCount == 10
  doAssert dataset.completed() == 10

  for i in 0 ..< 10:
    let blockResult = await dataset.getBlock(i)
    doAssert blockResult.isOk
    doAssert blockResult.value.isSome

  cleanup()

proc runMappedBlockmapRanges() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir, blockmapBackend = bmFile)
  doAssert storeResult.isOk
  let store = storeResult.value
  defer: store.close()

  let datasetResult = await createTestDataset(store, 20)
  doAssert datasetResult.isOk
  let dataset = datasetResult.value

  let ranges = dataset.getBlockmapRanges()
  doAssert ranges.len >= 1, "Expected at least one range"

  var totalBlocks: uint64 = 0
  for r in ranges:
    totalBlocks += r.count
  doAssert totalBlocks == 20, "Expected 20 blocks in ranges"

  cleanup()

proc runMappedBlockmapPersistence() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  var treeCid: Cid
  block:
    let storeResult = newDatasetStore(DbPath, BlocksDir, blockmapBackend = bmFile)
    doAssert storeResult.isOk
    let store = storeResult.value

    let datasetResult = await createTestDataset(store, 15)
    doAssert datasetResult.isOk
    treeCid = datasetResult.value.treeCid

    store.close()

  block:
    let storeResult = newDatasetStore(DbPath, BlocksDir, blockmapBackend = bmFile)
    doAssert storeResult.isOk
    let store = storeResult.value
    defer: store.close()

    let getResult = await store.getDataset(treeCid)
    doAssert getResult.isOk
    doAssert getResult.value.isSome, "Dataset should persist after reopen"

    let dataset = getResult.value.get()
    doAssert dataset.blockCount == 15
    doAssert dataset.completed() == 15

  cleanup()

proc runMappedBlockmapDeletion() {.async.} =
  cleanup()
  createDir(TestDir)
  createDir(BlocksDir)

  let storeResult = newDatasetStore(DbPath, BlocksDir, blockmapBackend = bmFile)
  doAssert storeResult.isOk
  let store = storeResult.value
  defer: store.close()

  let datasetResult = await createTestDataset(store, 5)
  doAssert datasetResult.isOk
  let dataset = datasetResult.value

  let manifestCid = dataset.manifestCid

  let deleteResult = await store.deleteDataset(manifestCid)
  doAssert deleteResult.isOk

  let getResult = await store.getDataset(dataset.treeCid)
  doAssert getResult.isOk
  doAssert getResult.value.isNone, "Dataset should not exist after deletion"

  cleanup()

suite "Dataset deletion tests":
  setup:
    cleanup()

  teardown:
    cleanup()

  test "delete existing dataset":
    waitFor runDeleteExistingDataset()

  test "delete non-existent dataset returns error":
    waitFor runDeleteNonExistentDataset()

  test "storage released after deletion":
    waitFor runStorageReleasedAfterDeletion()

  test "delete one dataset doesn't affect others":
    waitFor runMultipleDatasetsDeletion()

  test "delete dataset with many blocks":
    waitFor runDeleteDatasetWithManyBlocks()

suite "Mapped blockmap backend tests":
  setup:
    cleanup()

  teardown:
    cleanup()

  test "basic dataset operations with mapped blockmap":
    waitFor runMappedBlockmapBasic()

  test "blockmap ranges work with mapped backend":
    waitFor runMappedBlockmapRanges()

  test "mapped blockmap persists across reopens":
    waitFor runMappedBlockmapPersistence()

  test "mapped blockmap files deleted with dataset":
    waitFor runMappedBlockmapDeletion()