From eead39ba1c14ae2d99f9f8b754b90898e00005c2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 5 Dec 2023 16:48:15 +0100 Subject: [PATCH] finishes implementing slot builder --- codex/manifest/manifest.nim | 6 -- codex/slotbuilder/slotbuilder.nim | 96 ++------------------- tests/codex/slotbuilder/testslotbuilder.nim | 52 ++++++----- 3 files changed, 31 insertions(+), 123 deletions(-) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index ed6415f3..fcc2a704 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -46,12 +46,6 @@ type else: discard - # # WIP: - # isSlot* {.serialize.}: bool - # datasetSlotIndex*: int - # originalProtectedTreeCid*: Cid - # originalProtectedDatasetSize*: NBytes - ############################################################ # Accessors ############################################################ diff --git a/codex/slotbuilder/slotbuilder.nim b/codex/slotbuilder/slotbuilder.nim index 0737ae8f..cc912b1b 100644 --- a/codex/slotbuilder/slotbuilder.nim +++ b/codex/slotbuilder/slotbuilder.nim @@ -143,95 +143,11 @@ proc buildSlotTree*(self: SlotBuilder, slotBlocks: seq[Cid], numberOfPaddingCell return success(slotTree) -proc createAndSaveSlotTree*(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} = +proc createSlotTree*(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} = + without slotBlocks =? await self.selectSlotBlocks(datasetSlotIndex), err: + error "Failed to select slot blocks" + return failure(err) + let numberOfPaddingCells = self.calculateNumberOfPaddingCells(slotBlocks.len) - raiseAssert("not implemented") - - # select slot blocks - - # pad till cells are power of two - # -> get number of padding cells - # -> convert to number of padding blocks - - # build tree - - # save tree - - # return tree - - # let - # datasetTreeCid = self.manifest.treeCid - # datasetBlockIndexStart = datasetSlotIndex * self.numberOfSlotBlocks - # datasetBlockIndexEnd = datasetBlockIndexStart + self.numberOfSlotBlocks - - # for index in datasetBlockIndexStart ..< datasetBlockIndexEnd: - # without slotBlockLeaf =? await self.getTreeLeafCid(datasetTreeCid, index), err: - # return failure(err) - # if builder.addLeaf(slotBlockLeaf).isErr: - # error "Failed to add slotBlockCid to slot tree builder" - # return failure("Failed to add slotBlockCid to slot tree builder") - - # without slotTree =? builder.build(), err: - # error "Failed to build slot tree" - # return failure(err) - - # if (await self.blockStore.putAllProofs(slotTree)).isErr: - # error "Failed to store slot tree" - # return failure("Failed to store slot tree") - - # return success(slotTree) - -# proc createSlotManifest*(self: SlotBuilder, datasetSlotIndex: int): Future[?!Manifest] {.async.} = -# without slotTree =? await self.createAndSaveSlotTree(datasetSlotIndex), err: -# error "Failed to create slot tree" -# return failure(err) - -# without slotTreeRootCid =? slotTree.rootCid, err: -# error "Failed to get root CID from slot tree" -# return failure(err) - -# var slotManifest = Manifest.new( -# treeCid = slotTreeRootCid, -# datasetSize = self.numberOfSlotBlocks.NBytes * self.manifest.blockSize, -# blockSize = self.manifest.blockSize, -# version = self.manifest.version, -# hcodec = self.manifest.hcodec, -# codec = self.manifest.codec, -# ecK = self.manifest.ecK, # should change this = EC params of first ECing. there's be another! -# ecM = self.manifest.ecK, -# originalTreeCid = self.manifest.originalTreeCid, -# originalDatasetSize = self.manifest.originalDatasetSize -# ) - -# #treeCid: Cid -# # datasetSize: NBytes -# # blockSize: NBytes -# # version: CidVersion -# # hcodec: MultiCodec -# # codex: MultiCodec -# # ecK: int -# # ecM: int -# # originalTreeCid: Cid -# # originalDatasetSize: NBytes - -# # treeCid: Cid -# # datasetSize: NBytes -# # blockSize: NBytes -# # version: CidVersion -# # hcodec: MultiCodec -# # codec: MultiCodec -# # ecK: int -# # ecM: int -# # originalTreeCid: Cid -# # originalDatasetSize: NBytes): Manifest -# # first type mismatch at position: 7 - - - -# slotManifest.isSlot = true -# slotManifest.datasetSlotIndex = datasetSlotIndex -# slotManifest.originalProtectedTreeCid = self.manifest.treeCid -# slotManifest.originalProtectedDatasetSize = self.manifest.datasetSize - -# return success(slotManifest) + return await self.buildSlotTree(slotBlocks, numberOfPaddingCells) diff --git a/tests/codex/slotbuilder/testslotbuilder.nim b/tests/codex/slotbuilder/testslotbuilder.nim index f6c75421..45018a84 100644 --- a/tests/codex/slotbuilder/testslotbuilder.nim +++ b/tests/codex/slotbuilder/testslotbuilder.nim @@ -8,6 +8,7 @@ import pkg/codex/rng import pkg/codex/stores import pkg/codex/chunker import pkg/codex/merkletree +import pkg/codex/utils import ../helpers import ../examples @@ -29,6 +30,7 @@ asyncchecksuite "Slot builder": datasetBlocks: seq[bt.Block] localStore = CacheStore.new() protectedManifest: Manifest + expectedEmptyCid: Cid slotBuilder: SlotBuilder proc createBlocks(): Future[void] {.async.} = @@ -63,6 +65,7 @@ asyncchecksuite "Slot builder": let manifestBlock = bt.Block.new(protectedManifest.encode().tryGet(), codec = DagPBCodec).tryGet() discard await localStore.putBlock(manifestBlock) + expectedEmptyCid = emptyCid(protectedManifest.version, protectedManifest.hcodec, protectedManifest.codec).tryget() setup: await createBlocks() @@ -134,11 +137,31 @@ asyncchecksuite "Slot builder": check: expectedBlockCids == slotBlockCids + test "Can create slot tree (index: " & $i & ")": + let + expectedSlotBlockCids = (await slotBuilder.selectSlotBlocks(i)).tryGet() + expectedNumPadBlocks = divUp(slotBuilder.calculateNumberOfPaddingCells(expectedSlotBlockCids.len), numberOfCellsPerBlock) + + slotTree = (await slotBuilder.createSlotTree(i)).tryGet() + + check: + # Tree size + slotTree.leavesCount == expectedSlotBlockCids.len + expectedNumPadBlocks + + for i in 0 ..< numberOfSlotBlocks: + check: + # Each slot block + slotTree.getLeafCid(i).tryget() == expectedSlotBlockCids[i] + + for i in 0 ..< expectedNumPadBlocks: + check: + # Each pad block + slotTree.getLeafCid(numberOfSlotBlocks + i).tryget() == expectedEmptyCid + test "Can create slot tree": let slotBlockCids = datasetBlocks[0 ..< numberOfSlotBlocks].mapIt(it.cid) numPadCells = numberOfCellsPerBlock div 2 # We expect 1 pad block. - expectedEmptyCid = emptyCid(protectedManifest.version, protectedManifest.hcodec, protectedManifest.codec) slotTree = (await slotBuilder.buildSlotTree(slotBlockCids, numPadCells)).tryGet() @@ -153,32 +176,7 @@ asyncchecksuite "Slot builder": check: # 1 pad block - slotTree.getLeafCid(numberOfSlotBlocks) == expectedEmptyCid + slotTree.getLeafCid(numberOfSlotBlocks).tryget() == expectedEmptyCid - - - # test "Can create slot tree (index: " & $i & ")": - # let - # slotBlockCids = - # m = (await slotBuilder.buildSlotTree(slotBlockCids, numPadCells)).tryGet() - - # check: - # m.treeCid # check - # m.datasetSize == (numberOfSlotBlocks * blockSize).NBytes - # m.blockSize == blockSize - # m.version == manifest.version - # m.hcodec == manifest.hcodec - # m.codec == manifest.codec - # #m.ecK == ?? - # #m.ecM == ?? - # m.originalTreeCid == manifest.originalTreeCid - # m.originalDatasetSize = manifest.originalDatasetSize - - # m.isSlot == true - # m.datasetSlotIndex == i - # m.originalProtectedTreeCide == manifest.treeCid - # m.originalProtectedDatasetSize == manifest.datasetSize - -