implements getting slot blocks by index
This commit is contained in:
benbierens
Dmitriy Ryajov
parent
f767d2b4d0
commit
b17202c8a7
|
|
@ -11,7 +11,7 @@ import ../contracts/requests
|
|||
import ../stores/blockstore
|
||||
import ../manifest
|
||||
|
||||
proc getTreeCidForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Cid] {.async.} =
|
||||
proc getManifestForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Manifest] {.async.} =
|
||||
without manifestBlockCid =? Cid.init(slot.request.content.cid).mapFailure, err:
|
||||
error "Unable to init CID from slot.content.cid"
|
||||
return failure err
|
||||
|
|
@ -24,8 +24,20 @@ proc getTreeCidForSlot*(slot: Slot, blockstore: BlockStore): Future[?!Cid] {.asy
|
|||
error "Unable to decode manifest"
|
||||
return failure("Unable to decode manifest")
|
||||
|
||||
return success(manifest.treeCid)
|
||||
return success(manifest)
|
||||
|
||||
proc getSlotBlock*(slot: Slot, blockstore: BlockStore, treeCid: Cid, slotBlockIndex: int): Future[?!Block] {.async.} =
|
||||
raiseAssert("a")
|
||||
proc getIndexForSlotBlock*(slot: Slot, blockSize: NBytes, slotBlockIndex: int): uint64 =
|
||||
let
|
||||
slotSize = slot.request.ask.slotSize.truncate(uint64)
|
||||
blocksInSlot = slotSize div blockSize.uint64
|
||||
slotIndex = slot.slotIndex.truncate(uint64)
|
||||
|
||||
return (slotIndex * blocksInSlot) + slotBlockIndex.uint64
|
||||
|
||||
proc getSlotBlock*(slot: Slot, blockstore: BlockStore, slotBlockIndex: int): Future[?!Block] {.async.} =
|
||||
without manifest =? (await getManifestForSlot(slot, blockstore)), err:
|
||||
error "Failed to get manifest for slot"
|
||||
return failure(err)
|
||||
|
||||
let datasetIndex = getIndexForSlotBlock(slot, manifest.blockSize, slotBlockIndex)
|
||||
return await blockstore.getBlock(manifest.treeCid, datasetIndex)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import pkg/codex/clock
|
|||
import pkg/codex/utils/asynciter
|
||||
import pkg/codex/contracts/requests
|
||||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
|
||||
import pkg/codex/proof/slotblocks
|
||||
|
||||
|
|
@ -33,8 +34,7 @@ let
|
|||
numberOfSlotBlocks = 16
|
||||
slotIndex = 3
|
||||
|
||||
|
||||
asyncchecksuite "Test slotblocks":
|
||||
asyncchecksuite "Test slotblocks - manifest":
|
||||
let
|
||||
localStore = CacheStore.new()
|
||||
manifest = Manifest.new(
|
||||
|
|
@ -56,51 +56,132 @@ asyncchecksuite "Test slotblocks":
|
|||
slotIndex: u256(slotIndex)
|
||||
)
|
||||
|
||||
# let chunker = RandomChunker.new(rng.Rng.instance(),
|
||||
# size = bytesPerBlock * numberOfSlotBlocks,
|
||||
# chunkSize = bytesPerBlock)
|
||||
|
||||
# var slotBlocks: seq[bt.Block]
|
||||
|
||||
# proc createSlotBlocks(): Future[void] {.async.} =
|
||||
# while true:
|
||||
# let chunk = await chunker.getBytes()
|
||||
# if chunk.len <= 0:
|
||||
# break
|
||||
# slotBlocks.add(bt.Block.new(chunk).tryGet())
|
||||
|
||||
setup:
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
# await createSlotBlocks()
|
||||
|
||||
test "Can get tree root for slot":
|
||||
let cid = (await getTreeCidForSlot(slot, localStore)).tryGet()
|
||||
test "Can get manifest for slot":
|
||||
let m = (await getManifestForSlot(slot, localStore)).tryGet()
|
||||
|
||||
check:
|
||||
cid == manifest.treeCid
|
||||
m.treeCid == manifest.treeCid
|
||||
|
||||
test "Can fail to get tree root for invalid cid":
|
||||
test "Can fail to get manifest for invalid cid":
|
||||
slot.request.content.cid = "invalid"
|
||||
let cid = (await getTreeCidForSlot(slot, localStore))
|
||||
let m = (await getManifestForSlot(slot, localStore))
|
||||
|
||||
check:
|
||||
cid.isErr
|
||||
m.isErr
|
||||
|
||||
test "Can fail to get tree root when manifest block not found":
|
||||
test "Can fail to get manifest when manifest block not found":
|
||||
let
|
||||
emptyStore = CacheStore.new()
|
||||
cid = (await getTreeCidForSlot(slot, emptyStore))
|
||||
m = (await getManifestForSlot(slot, emptyStore))
|
||||
|
||||
check:
|
||||
cid.isErr
|
||||
m.isErr
|
||||
|
||||
test "Can fail to get tree root when manifest fails to decode":
|
||||
test "Can fail to get manifest when manifest fails to decode":
|
||||
manifestBlock.data = @[]
|
||||
|
||||
let cid = (await getTreeCidForSlot(slot, localStore))
|
||||
let m = (await getManifestForSlot(slot, localStore))
|
||||
|
||||
check:
|
||||
cid.isErr
|
||||
m.isErr
|
||||
|
||||
|
||||
asyncchecksuite "Test slotblocks - slot blocks by index":
|
||||
let
|
||||
totalNumberOfSlots = 4
|
||||
localStore = CacheStore.new()
|
||||
chunker = RandomChunker.new(rng.Rng.instance(),
|
||||
size = bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots,
|
||||
chunkSize = bytesPerBlock)
|
||||
|
||||
var
|
||||
manifest: Manifest
|
||||
manifestBlock: bt.Block
|
||||
slot: Slot
|
||||
datasetBlocks: seq[bt.Block]
|
||||
|
||||
proc createDatasetBlocks(): Future[void] {.async.} =
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
let b = bt.Block.new(chunk).tryGet()
|
||||
datasetBlocks.add(b)
|
||||
discard await localStore.putBlock(b)
|
||||
|
||||
proc createManifest(): Future[void] {.async.} =
|
||||
let
|
||||
cids = datasetBlocks.mapIt(it.cid)
|
||||
tree = MerkleTree.init(cids).tryGet()
|
||||
treeCid = tree.rootCid().tryGet()
|
||||
|
||||
for index, cid in cids:
|
||||
let proof = tree.getProof(index).tryget()
|
||||
discard await localStore.putBlockCidAndProof(treeCid, index, cid, proof)
|
||||
|
||||
manifest = Manifest.new(
|
||||
treeCid = treeCid,
|
||||
blockSize = bytesPerBlock.NBytes,
|
||||
datasetSize = (bytesPerBlock * numberOfSlotBlocks * totalNumberOfSlots).NBytes)
|
||||
manifestBlock = bt.Block.new(manifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
|
||||
|
||||
proc createSlot(): void =
|
||||
slot = Slot(
|
||||
request: StorageRequest(
|
||||
ask: StorageAsk(
|
||||
slotSize: u256(bytesPerBlock * numberOfSlotBlocks)
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: $manifestBlock.cid
|
||||
),
|
||||
),
|
||||
slotIndex: u256(slotIndex)
|
||||
)
|
||||
|
||||
setup:
|
||||
await createDatasetBlocks()
|
||||
await createManifest()
|
||||
createSlot()
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
|
||||
test "Can get index for slot block":
|
||||
proc getIndex(i: int): uint64 =
|
||||
getIndexForSlotBlock(slot, bytesPerBlock.NBytes, i)
|
||||
|
||||
proc getExpected(i: int): uint64 =
|
||||
(slotIndex * numberOfSlotBlocks + i).uint64
|
||||
|
||||
check:
|
||||
getIndex(0) == getExpected(0)
|
||||
getIndex(0) == 48
|
||||
getIndex(1) == getExpected(1)
|
||||
getIndex(1) == 49
|
||||
getIndex(10) == getExpected(10)
|
||||
getIndex(10) == 58
|
||||
|
||||
test "Can get slot block by index":
|
||||
proc getBlocks(i: int): Future[(bt.Block, bt.Block)] {.async.} =
|
||||
let
|
||||
slotBlock = (await getSlotBlock(slot, localStore, 3)).tryget()
|
||||
expectedIndex = getIndexForSlotBlock(slot, bytesPerBlock.NBytes, 3)
|
||||
expectedBlock = datasetBlocks[expectedIndex]
|
||||
return (slotBlock, expectedBlock)
|
||||
|
||||
let (slotBlock0, expectedBlock0) = await getBlocks(0)
|
||||
let (slotBlock3, expectedBlock3) = await getBlocks(3)
|
||||
let (slotBlockLast5, expectedBlockLast5) = await getBlocks(numberOfSlotBlocks - 3)
|
||||
let (slotBlockLast, expectedBlockLast) = await getBlocks(numberOfSlotBlocks - 1)
|
||||
|
||||
check:
|
||||
slotBlock0.cid == expectedBlock0.cid
|
||||
slotBlock0.data == expectedBlock0.data
|
||||
slotBlock3.cid == expectedBlock3.cid
|
||||
slotBlock3.data == expectedBlock3.data
|
||||
slotBlockLast5.cid == expectedBlockLast5.cid
|
||||
slotBlockLast5.data == expectedBlockLast5.data
|
||||
slotBlockLast.cid == expectedBlockLast.cid
|
||||
slotBlockLast.data == expectedBlockLast.data
|
||||
|
|
|
|||
Loading…
Reference in New Issue