Can select slot blocks

codex/manifest/manifest.nim

@@ -46,11 +46,11 @@ type
     else:
       discard
 
-    # WIP:
-    isSlot* {.serialize.}: bool
-    datasetSlotIndex*: int
-    originalProtectedTreeCid*: Cid
-    originalProtectedDatasetSize*: NBytes
+    # # WIP:
+    # isSlot* {.serialize.}: bool
+    # datasetSlotIndex*: int
+    # originalProtectedTreeCid*: Cid
+    # originalProtectedDatasetSize*: NBytes
 
 ############################################################
 # Accessors

codex/slotbuilder/slotbuilder.nim

@@ -31,93 +31,120 @@ proc new*(
     numberOfSlotBlocks: numberOfSlotBlocks
   ))
 
-proc getTreeLeaf(self: SlotBuilder, datasetTreeCid: Cid, datasetBlockIndex: int): Future[?!MultiHash] {.async.} =
+proc getTreeLeafCid(self: SlotBuilder, datasetTreeCid: Cid, datasetBlockIndex: int): Future[?!Cid] {.async.} =
   without slotBlockCid =? await self.blockStore.getCid(datasetTreeCid, datasetBlockIndex), err:
     error "Failed to get block for tree at index", index=datasetBlockIndex, tree=datasetTreeCid
     return failure(err)
 
-  without slotBlockLeaf =? slotBlockCid.mhash:
-    error "Failed to get multihash from slot block CID", slotBlockCid
-    return failure("Failed to get multihash from slot block CID")
+  # without slotBlockLeaf =? slotBlockCid.mhash:
+  #   error "Failed to get multihash from slot block CID", slotBlockCid
+  #   return failure("Failed to get multihash from slot block CID")
 
-  return success(slotBlockLeaf)
+  return success(slotBlockCid)
 
-proc createAndSaveSlotTree(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} =
+proc selectSlotBlocks*(self: SlotBuilder, datasetSlotIndex: int): Future[?!seq[Cid]] {.async.} =
+  var cids = newSeq[Cid]()
+  let
+    datasetTreeCid = self.manifest.treeCid
+    blockCount = self.manifest.blocksCount
+    numberOfSlots = self.manifest.ecK
+    strategy = SteppedIndexingStrategy.new(0, blockCount - 1, numberOfSlots)
+
+  for index in strategy.getIndicies(datasetSlotIndex):
+    without slotBlockCid =? await self.getTreeLeafCid(datasetTreeCid, index), err:
+      return failure(err)
+    cids.add(slotBlockCid)
+
+  return success(cids)
+
+proc createAndSaveSlotTree*(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} =
   without var builder =? MerkleTreeBuilder.init(), err:
     return failure(err)
 
-  let
-    datasetTreeCid = self.manifest.treeCid
-    datasetBlockIndexStart = datasetSlotIndex * self.numberOfSlotBlocks
-    datasetBlockIndexEnd = datasetBlockIndexStart + self.numberOfSlotBlocks
+  raiseAssert("not implemented")
 
-  for index in datasetBlockIndexStart ..< datasetBlockIndexEnd:
-    without slotBlockLeaf =? await self.getTreeLeaf(datasetTreeCid, index), err:
-      return failure(err)
-    if builder.addLeaf(slotBlockLeaf).isErr:
-      error "Failed to add slotBlockCid to slot tree builder"
-      return failure("Failed to add slotBlockCid to slot tree builder")
+  # select slot blocks
 
-  without slotTree =? builder.build(), err:
-    error "Failed to build slot tree"
-    return failure(err)
+  # pad till cells are power of two
 
-  if (await self.blockStore.putAllProofs(slotTree)).isErr:
-    error "Failed to store slot tree"
-    return failure("Failed to store slot tree")
+  # build tree
 
-  return success(slotTree)
+  # save tree
 
-proc createSlotManifest*(self: SlotBuilder, datasetSlotIndex: int): Future[?!Manifest] {.async.} =
-  without slotTree =? await self.createAndSaveSlotTree(datasetSlotIndex), err:
-    error "Failed to create slot tree"
-    return failure(err)
+  # return tree
 
-  without slotTreeRootCid =? slotTree.rootCid, err:
-    error "Failed to get root CID from slot tree"
-    return failure(err)
+  # let
+  #   datasetTreeCid = self.manifest.treeCid
+  #   datasetBlockIndexStart = datasetSlotIndex * self.numberOfSlotBlocks
+  #   datasetBlockIndexEnd = datasetBlockIndexStart + self.numberOfSlotBlocks
 
-  var slotManifest = Manifest.new(
-    treeCid = slotTreeRootCid,
-    datasetSize = self.numberOfSlotBlocks.NBytes * self.manifest.blockSize,
-    blockSize = self.manifest.blockSize,
-    version = self.manifest.version,
-    hcodec = self.manifest.hcodec,
-    codec = self.manifest.codec,
-    ecK = self.manifest.ecK, # should change this = EC params of first ECing. there's be another!
-    ecM = self.manifest.ecK,
-    originalTreeCid = self.manifest.originalTreeCid,
-    originalDatasetSize = self.manifest.originalDatasetSize
-  )
+  # for index in datasetBlockIndexStart ..< datasetBlockIndexEnd:
+  #   without slotBlockLeaf =? await self.getTreeLeafCid(datasetTreeCid, index), err:
+  #     return failure(err)
+  #   if builder.addLeaf(slotBlockLeaf).isErr:
+  #     error "Failed to add slotBlockCid to slot tree builder"
+  #     return failure("Failed to add slotBlockCid to slot tree builder")
 
-   #treeCid: Cid
-   # datasetSize: NBytes
-   # blockSize: NBytes
-   # version: CidVersion
-   # hcodec: MultiCodec
-   # codex: MultiCodec
-   # ecK: int
-   # ecM: int
-   # originalTreeCid: Cid
-   # originalDatasetSize: NBytes
+  # without slotTree =? builder.build(), err:
+  #   error "Failed to build slot tree"
+  #   return failure(err)
 
-#  treeCid: Cid
-# datasetSize: NBytes
-# blockSize: NBytes
-#          version: CidVersion
-# hcodec: MultiCodec
-# codec: MultiCodec
-# ecK: int
-#          ecM: int
-# originalTreeCid: Cid
-# originalDatasetSize: NBytes): Manifest
-#   first type mismatch at position: 7
+  # if (await self.blockStore.putAllProofs(slotTree)).isErr:
+  #   error "Failed to store slot tree"
+  #   return failure("Failed to store slot tree")
+
+  # return success(slotTree)
+
+# proc createSlotManifest*(self: SlotBuilder, datasetSlotIndex: int): Future[?!Manifest] {.async.} =
+#   without slotTree =? await self.createAndSaveSlotTree(datasetSlotIndex), err:
+#     error "Failed to create slot tree"
+#     return failure(err)
+
+#   without slotTreeRootCid =? slotTree.rootCid, err:
+#     error "Failed to get root CID from slot tree"
+#     return failure(err)
+
+#   var slotManifest = Manifest.new(
+#     treeCid = slotTreeRootCid,
+#     datasetSize = self.numberOfSlotBlocks.NBytes * self.manifest.blockSize,
+#     blockSize = self.manifest.blockSize,
+#     version = self.manifest.version,
+#     hcodec = self.manifest.hcodec,
+#     codec = self.manifest.codec,
+#     ecK = self.manifest.ecK, # should change this = EC params of first ECing. there's be another!
+#     ecM = self.manifest.ecK,
+#     originalTreeCid = self.manifest.originalTreeCid,
+#     originalDatasetSize = self.manifest.originalDatasetSize
+#   )
+
+#    #treeCid: Cid
+#    # datasetSize: NBytes
+#    # blockSize: NBytes
+#    # version: CidVersion
+#    # hcodec: MultiCodec
+#    # codex: MultiCodec
+#    # ecK: int
+#    # ecM: int
+#    # originalTreeCid: Cid
+#    # originalDatasetSize: NBytes
+
+# #  treeCid: Cid
+# # datasetSize: NBytes
+# # blockSize: NBytes
+# #          version: CidVersion
+# # hcodec: MultiCodec
+# # codec: MultiCodec
+# # ecK: int
+# #          ecM: int
+# # originalTreeCid: Cid
+# # originalDatasetSize: NBytes): Manifest
+# #   first type mismatch at position: 7
 
 
 
-  slotManifest.isSlot = true
-  slotManifest.datasetSlotIndex = datasetSlotIndex
-  slotManifest.originalProtectedTreeCid = self.manifest.treeCid
-  slotManifest.originalProtectedDatasetSize = self.manifest.datasetSize
+#   slotManifest.isSlot = true
+#   slotManifest.datasetSlotIndex = datasetSlotIndex
+#   slotManifest.originalProtectedTreeCid = self.manifest.treeCid
+#   slotManifest.originalProtectedDatasetSize = self.manifest.datasetSize
 
-  return success(slotManifest)
+#   return success(slotManifest)

tests/codex/slotbuilder/testslotbuilder.nim

@@ -11,6 +11,7 @@ import pkg/codex/merkletree
 import ../helpers
 import ../examples
 
+import codex/manifest/indexingstrategy
 import codex/slotbuilder/slotbuilder
 
 asyncchecksuite "Slot builder":
@@ -18,7 +19,8 @@ asyncchecksuite "Slot builder":
     blockSize = 64 * 1024
     numberOfSlotBlocks = 6
     numberOfSlots = 5
-    datasetSize = numberOfSlotBlocks * numberOfSlots * blockSize
+    numberOfDatasetBlocks = numberOfSlotBlocks * numberOfSlots
+    datasetSize = numberOfDatasetBlocks * blockSize
     chunker = RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize)
 
   var
@@ -89,8 +91,20 @@ asyncchecksuite "Slot builder":
     check:
       SlotBuilder.new(localStore, mismatchManifest).isErr
 
+  for i in 0 ..< numberOfSlots:
+    test "Can select slot block CIDs (index: " & $i & ")":
+      let
+        steppedStrategy = SteppedIndexingStrategy.new(0, numberOfDatasetBlocks - 1, numberOfSlots)
+        expectedDatasetBlockIndicies = steppedStrategy.getIndicies(i)
+        expectedBlockCids = expectedDatasetBlockIndicies.mapIt(datasetBlocks[it].cid)
+
+        slotBlockCids = (await slotBuilder.selectSlotBlocks(i)).tryGet()
+
+      check:
+        expectedBlockCids == slotBlockCids
+
   # for i in 0 ..< numberOfSlots:
-  #   test "Can create slot manifest given index (" & $i & ")":
+  #   test "Can create slot tree given index (" & $i & ")":
   #     let
   #       selectStart = i * numberOfSlotBlocks
   #       selectEnd = selectStart + numberOfSlotBlocks