From b17202c8a7ff72338d1af5e1853cd1c2539b932f Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Nov 2023 14:18:09 +0100 Subject: [PATCH] implements getting slot blocks by index --- codex/proof/slotblocks.nim | 20 +++- tests/codex/proof/testslotblocks.nim | 137 +++++++++++++++++++++------ 2 files changed, 125 insertions(+), 32 deletions(-) diff --git a/codex/proof/slotblocks.nim b/codex/proof/slotblocks.nim index e17c3b17..7d8ab1d9 100644 --- a/codex/proof/slotblocks.nim +++ b/codex/proof/slotblocks.nim @@ -11,7 +11,7 @@ import ../contracts/requests import ../stores/blockstore import ../manifest -proc getTreeCidForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Cid] {.async.} = +proc getManifestForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Manifest] {.async.} = without manifestBlockCid =? Cid.init(slot.request.content.cid).mapFailure, err: error "Unable to init CID from slot.content.cid" return failure err @@ -24,8 +24,20 @@ proc getTreeCidForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Cid] {.asy error "Unable to decode manifest" return failure("Unable to decode manifest") - return success(manifest.treeCid) + return success(manifest) -proc getSlotBlock*(slot: Slot, blockstore: BlockStore, treeCid: Cid, slotBlockIndex: int): Future[?!Block] {.async.} = - raiseAssert("a") +proc getIndexForSlotBlock*(slot: Slot, blockSize: NBytes, slotBlockIndex: int): uint64 = + let + slotSize = slot.request.ask.slotSize.truncate(uint64) + blocksInSlot = slotSize div blockSize.uint64 + slotIndex = slot.slotIndex.truncate(uint64) + return (slotIndex * blocksInSlot) + slotBlockIndex.uint64 + +proc getSlotBlock*(slot: Slot, blockstore: BlockStore, slotBlockIndex: int): Future[?!Block] {.async.} = + without manifest =? (await getManifestForSlot(slot, blockstore)), err: + error "Failed to get manifest for slot" + return failure(err) + + let datasetIndex = getIndexForSlotBlock(slot, manifest.blockSize, slotBlockIndex) + return await blockstore.getBlock(manifest.treeCid, datasetIndex) diff --git a/tests/codex/proof/testslotblocks.nim b/tests/codex/proof/testslotblocks.nim index 4a170304..e3a0acc3 100644 --- a/tests/codex/proof/testslotblocks.nim +++ b/tests/codex/proof/testslotblocks.nim @@ -22,6 +22,7 @@ import pkg/codex/clock import pkg/codex/utils/asynciter import pkg/codex/contracts/requests import pkg/codex/contracts +import pkg/codex/merkletree import pkg/codex/proof/slotblocks @@ -33,8 +34,7 @@ let numberOfSlotBlocks = 16 slotIndex = 3 - -asyncchecksuite "Test slotblocks": +asyncchecksuite "Test slotblocks - manifest": let localStore = CacheStore.new() manifest = Manifest.new( @@ -56,51 +56,132 @@ asyncchecksuite "Test slotblocks": slotIndex: u256(slotIndex) ) - # let chunker = RandomChunker.new(rng.Rng.instance(), - # size = bytesPerBlock * numberOfSlotBlocks, - # chunkSize = bytesPerBlock) - - # var slotBlocks: seq[bt.Block] - - # proc createSlotBlocks(): Future[void] {.async.} = - # while true: - # let chunk = await chunker.getBytes() - # if chunk.len <= 0: - # break - # slotBlocks.add(bt.Block.new(chunk).tryGet()) - setup: discard await localStore.putBlock(manifestBlock) - # await createSlotBlocks() - test "Can get tree root for slot": - let cid = (await getTreeCidForSlot(slot, localStore)).tryGet() + test "Can get manifest for slot": + let m = (await getManifestForSlot(slot, localStore)).tryGet() check: - cid == manifest.treeCid + m.treeCid == manifest.treeCid - test "Can fail to get tree root for invalid cid": + test "Can fail to get manifest for invalid cid": slot.request.content.cid = "invalid" - let cid = (await getTreeCidForSlot(slot, localStore)) + let m = (await getManifestForSlot(slot, localStore)) check: - cid.isErr + m.isErr - test "Can fail to get tree root when manifest block not found": + test "Can fail to get manifest when manifest block not found": let emptyStore = CacheStore.new() - cid = (await getTreeCidForSlot(slot, emptyStore)) + m = (await getManifestForSlot(slot, emptyStore)) check: - cid.isErr + m.isErr - test "Can fail to get tree root when manifest fails to decode": + test "Can fail to get manifest when manifest fails to decode": manifestBlock.data = @[] - let cid = (await getTreeCidForSlot(slot, localStore)) + let m = (await getManifestForSlot(slot, localStore)) check: - cid.isErr + m.isErr +asyncchecksuite "Test slotblocks - slot blocks by index": + let + totalNumberOfSlots = 4 + localStore = CacheStore.new() + chunker = RandomChunker.new(rng.Rng.instance(), + size = bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots, + chunkSize = bytesPerBlock) + var + manifest: Manifest + manifestBlock: bt.Block + slot: Slot + datasetBlocks: seq[bt.Block] + + proc createDatasetBlocks(): Future[void] {.async.} = + while true: + let chunk = await chunker.getBytes() + if chunk.len <= 0: + break + let b = bt.Block.new(chunk).tryGet() + datasetBlocks.add(b) + discard await localStore.putBlock(b) + + proc createManifest(): Future[void] {.async.} = + let + cids = datasetBlocks.mapIt(it.cid) + tree = MerkleTree.init(cids).tryGet() + treeCid = tree.rootCid().tryGet() + + for index, cid in cids: + let proof = tree.getProof(index).tryget() + discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof) + + manifest = Manifest.new( + treeCid = treeCid, + blockSize = bytesPerBlock.NBytes, + datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes) + manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet() + + + proc createSlot(): void = + slot = Slot( + request: StorageRequest( + ask: StorageAsk( + slotSize: u256(bytesPerBlock * numberOfSlotBlocks) + ), + content: StorageContent( + cid: $manifestBlock.cid + ), + ), + slotIndex: u256(slotIndex) + ) + + setup: + await createDatasetBlocks() + await createManifest() + createSlot() + discard await localStore.putBlock(manifestBlock) + + test "Can get index for slot block": + proc getIndex(i: int): uint64 = + getIndexForSlotBlock(slot, bytesPerBlock.NBytes, i) + + proc getExpected(i: int): uint64 = + (slotIndex * numberOfSlotBlocks + i).uint64 + + check: + getIndex(0) == getExpected(0) + getIndex(0) == 48 + getIndex(1) == getExpected(1) + getIndex(1) == 49 + getIndex(10) == getExpected(10) + getIndex(10) == 58 + + test "Can get slot block by index": + proc getBlocks(i: int): Future[(bt.Block, bt.Block)] {.async.} = + let + slotBlock = (await getSlotBlock(slot, localStore, 3)).tryget() + expectedIndex = getIndexForSlotBlock(slot, bytesPerBlock.NBytes, 3) + expectedBlock = datasetBlocks[expectedIndex] + return (slotBlock, expectedBlock) + + let (slotBlock0, expectedBlock0) = await getBlocks(0) + let (slotBlock3, expectedBlock3) = await getBlocks(3) + let (slotBlockLast5, expectedBlockLast5) = await getBlocks(numberOfSlotBlocks - 3) + let (slotBlockLast, expectedBlockLast) = await getBlocks(numberOfSlotBlocks - 1) + + check: + slotBlock0.cid == expectedBlock0.cid + slotBlock0.data == expectedBlock0.data + slotBlock3.cid == expectedBlock3.cid + slotBlock3.data == expectedBlock3.data + slotBlockLast5.cid == expectedBlockLast5.cid + slotBlockLast5.data == expectedBlockLast5.data + slotBlockLast.cid == expectedBlockLast.cid + slotBlockLast.data == expectedBlockLast.data