diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index a9b97c83..ed6415f3 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -46,11 +46,11 @@ type else: discard - # WIP: - isSlot* {.serialize.}: bool - datasetSlotIndex*: int - originalProtectedTreeCid*: Cid - originalProtectedDatasetSize*: NBytes + # # WIP: + # isSlot* {.serialize.}: bool + # datasetSlotIndex*: int + # originalProtectedTreeCid*: Cid + # originalProtectedDatasetSize*: NBytes ############################################################ # Accessors diff --git a/codex/slotbuilder/slotbuilder.nim b/codex/slotbuilder/slotbuilder.nim index c2e02b04..8cc6f102 100644 --- a/codex/slotbuilder/slotbuilder.nim +++ b/codex/slotbuilder/slotbuilder.nim @@ -31,93 +31,120 @@ proc new*( numberOfSlotBlocks: numberOfSlotBlocks )) -proc getTreeLeaf(self: SlotBuilder, datasetTreeCid: Cid, datasetBlockIndex: int): Future[?!MultiHash] {.async.} = +proc getTreeLeafCid(self: SlotBuilder, datasetTreeCid: Cid, datasetBlockIndex: int): Future[?!Cid] {.async.} = without slotBlockCid =? await self.blockStore.getCid(datasetTreeCid, datasetBlockIndex), err: error "Failed to get block for tree at index", index=datasetBlockIndex, tree=datasetTreeCid return failure(err) - without slotBlockLeaf =? slotBlockCid.mhash: - error "Failed to get multihash from slot block CID", slotBlockCid - return failure("Failed to get multihash from slot block CID") + # without slotBlockLeaf =? slotBlockCid.mhash: + # error "Failed to get multihash from slot block CID", slotBlockCid + # return failure("Failed to get multihash from slot block CID") - return success(slotBlockLeaf) + return success(slotBlockCid) -proc createAndSaveSlotTree(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} = +proc selectSlotBlocks*(self: SlotBuilder, datasetSlotIndex: int): Future[?!seq[Cid]] {.async.} = + var cids = newSeq[Cid]() + let + datasetTreeCid = self.manifest.treeCid + blockCount = self.manifest.blocksCount + numberOfSlots = self.manifest.ecK + strategy = SteppedIndexingStrategy.new(0, blockCount - 1, numberOfSlots) + + for index in strategy.getIndicies(datasetSlotIndex): + without slotBlockCid =? await self.getTreeLeafCid(datasetTreeCid, index), err: + return failure(err) + cids.add(slotBlockCid) + + return success(cids) + +proc createAndSaveSlotTree*(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} = without var builder =? MerkleTreeBuilder.init(), err: return failure(err) - let - datasetTreeCid = self.manifest.treeCid - datasetBlockIndexStart = datasetSlotIndex * self.numberOfSlotBlocks - datasetBlockIndexEnd = datasetBlockIndexStart + self.numberOfSlotBlocks + raiseAssert("not implemented") - for index in datasetBlockIndexStart ..< datasetBlockIndexEnd: - without slotBlockLeaf =? await self.getTreeLeaf(datasetTreeCid, index), err: - return failure(err) - if builder.addLeaf(slotBlockLeaf).isErr: - error "Failed to add slotBlockCid to slot tree builder" - return failure("Failed to add slotBlockCid to slot tree builder") + # select slot blocks - without slotTree =? builder.build(), err: - error "Failed to build slot tree" - return failure(err) + # pad till cells are power of two - if (await self.blockStore.putAllProofs(slotTree)).isErr: - error "Failed to store slot tree" - return failure("Failed to store slot tree") + # build tree - return success(slotTree) + # save tree -proc createSlotManifest*(self: SlotBuilder, datasetSlotIndex: int): Future[?!Manifest] {.async.} = - without slotTree =? await self.createAndSaveSlotTree(datasetSlotIndex), err: - error "Failed to create slot tree" - return failure(err) + # return tree - without slotTreeRootCid =? slotTree.rootCid, err: - error "Failed to get root CID from slot tree" - return failure(err) + # let + # datasetTreeCid = self.manifest.treeCid + # datasetBlockIndexStart = datasetSlotIndex * self.numberOfSlotBlocks + # datasetBlockIndexEnd = datasetBlockIndexStart + self.numberOfSlotBlocks - var slotManifest = Manifest.new( - treeCid = slotTreeRootCid, - datasetSize = self.numberOfSlotBlocks.NBytes * self.manifest.blockSize, - blockSize = self.manifest.blockSize, - version = self.manifest.version, - hcodec = self.manifest.hcodec, - codec = self.manifest.codec, - ecK = self.manifest.ecK, # should change this = EC params of first ECing. there's be another! - ecM = self.manifest.ecK, - originalTreeCid = self.manifest.originalTreeCid, - originalDatasetSize = self.manifest.originalDatasetSize - ) + # for index in datasetBlockIndexStart ..< datasetBlockIndexEnd: + # without slotBlockLeaf =? await self.getTreeLeafCid(datasetTreeCid, index), err: + # return failure(err) + # if builder.addLeaf(slotBlockLeaf).isErr: + # error "Failed to add slotBlockCid to slot tree builder" + # return failure("Failed to add slotBlockCid to slot tree builder") - #treeCid: Cid - # datasetSize: NBytes - # blockSize: NBytes - # version: CidVersion - # hcodec: MultiCodec - # codex: MultiCodec - # ecK: int - # ecM: int - # originalTreeCid: Cid - # originalDatasetSize: NBytes + # without slotTree =? builder.build(), err: + # error "Failed to build slot tree" + # return failure(err) -# treeCid: Cid -# datasetSize: NBytes -# blockSize: NBytes -# version: CidVersion -# hcodec: MultiCodec -# codec: MultiCodec -# ecK: int -# ecM: int -# originalTreeCid: Cid -# originalDatasetSize: NBytes): Manifest -# first type mismatch at position: 7 + # if (await self.blockStore.putAllProofs(slotTree)).isErr: + # error "Failed to store slot tree" + # return failure("Failed to store slot tree") + + # return success(slotTree) + +# proc createSlotManifest*(self: SlotBuilder, datasetSlotIndex: int): Future[?!Manifest] {.async.} = +# without slotTree =? await self.createAndSaveSlotTree(datasetSlotIndex), err: +# error "Failed to create slot tree" +# return failure(err) + +# without slotTreeRootCid =? slotTree.rootCid, err: +# error "Failed to get root CID from slot tree" +# return failure(err) + +# var slotManifest = Manifest.new( +# treeCid = slotTreeRootCid, +# datasetSize = self.numberOfSlotBlocks.NBytes * self.manifest.blockSize, +# blockSize = self.manifest.blockSize, +# version = self.manifest.version, +# hcodec = self.manifest.hcodec, +# codec = self.manifest.codec, +# ecK = self.manifest.ecK, # should change this = EC params of first ECing. there's be another! +# ecM = self.manifest.ecK, +# originalTreeCid = self.manifest.originalTreeCid, +# originalDatasetSize = self.manifest.originalDatasetSize +# ) + +# #treeCid: Cid +# # datasetSize: NBytes +# # blockSize: NBytes +# # version: CidVersion +# # hcodec: MultiCodec +# # codex: MultiCodec +# # ecK: int +# # ecM: int +# # originalTreeCid: Cid +# # originalDatasetSize: NBytes + +# # treeCid: Cid +# # datasetSize: NBytes +# # blockSize: NBytes +# # version: CidVersion +# # hcodec: MultiCodec +# # codec: MultiCodec +# # ecK: int +# # ecM: int +# # originalTreeCid: Cid +# # originalDatasetSize: NBytes): Manifest +# # first type mismatch at position: 7 - slotManifest.isSlot = true - slotManifest.datasetSlotIndex = datasetSlotIndex - slotManifest.originalProtectedTreeCid = self.manifest.treeCid - slotManifest.originalProtectedDatasetSize = self.manifest.datasetSize +# slotManifest.isSlot = true +# slotManifest.datasetSlotIndex = datasetSlotIndex +# slotManifest.originalProtectedTreeCid = self.manifest.treeCid +# slotManifest.originalProtectedDatasetSize = self.manifest.datasetSize - return success(slotManifest) +# return success(slotManifest) diff --git a/tests/codex/slotbuilder/testslotbuilder.nim b/tests/codex/slotbuilder/testslotbuilder.nim index 960bdd14..5f749a6b 100644 --- a/tests/codex/slotbuilder/testslotbuilder.nim +++ b/tests/codex/slotbuilder/testslotbuilder.nim @@ -11,6 +11,7 @@ import pkg/codex/merkletree import ../helpers import ../examples +import codex/manifest/indexingstrategy import codex/slotbuilder/slotbuilder asyncchecksuite "Slot builder": @@ -18,7 +19,8 @@ asyncchecksuite "Slot builder": blockSize = 64 * 1024 numberOfSlotBlocks = 6 numberOfSlots = 5 - datasetSize = numberOfSlotBlocks * numberOfSlots * blockSize + numberOfDatasetBlocks = numberOfSlotBlocks * numberOfSlots + datasetSize = numberOfDatasetBlocks * blockSize chunker = RandomChunker.new(Rng.instance(), size = datasetSize, chunkSize = blockSize) var @@ -89,8 +91,20 @@ asyncchecksuite "Slot builder": check: SlotBuilder.new(localStore, mismatchManifest).isErr + for i in 0 ..< numberOfSlots: + test "Can select slot block CIDs (index: " & $i & ")": + let + steppedStrategy = SteppedIndexingStrategy.new(0, numberOfDatasetBlocks - 1, numberOfSlots) + expectedDatasetBlockIndicies = steppedStrategy.getIndicies(i) + expectedBlockCids = expectedDatasetBlockIndicies.mapIt(datasetBlocks[it].cid) + + slotBlockCids = (await slotBuilder.selectSlotBlocks(i)).tryGet() + + check: + expectedBlockCids == slotBlockCids + # for i in 0 ..< numberOfSlots: - # test "Can create slot manifest given index (" & $i & ")": + # test "Can create slot tree given index (" & $i & ")": # let # selectStart = i * numberOfSlotBlocks # selectEnd = selectStart + numberOfSlotBlocks