mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-22 15:23:07 +00:00
finishes implementing slot builder
This commit is contained in:
benbierens
Dmitriy Ryajov
parent
e7a39dcb6e
commit
eead39ba1c
@ -46,12 +46,6 @@ type
|
||||
else:
|
||||
discard
|
||||
|
||||
# # WIP:
|
||||
# isSlot* {.serialize.}: bool
|
||||
# datasetSlotIndex*: int
|
||||
# originalProtectedTreeCid*: Cid
|
||||
# originalProtectedDatasetSize*: NBytes
|
||||
|
||||
############################################################
|
||||
# Accessors
|
||||
############################################################
|
||||
|
||||
@ -143,95 +143,11 @@ proc buildSlotTree*(self: SlotBuilder, slotBlocks: seq[Cid], numberOfPaddingCell
|
||||
|
||||
return success(slotTree)
|
||||
|
||||
proc createAndSaveSlotTree*(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} =
|
||||
proc createSlotTree*(self: SlotBuilder, datasetSlotIndex: int): Future[?!MerkleTree] {.async.} =
|
||||
without slotBlocks =? await self.selectSlotBlocks(datasetSlotIndex), err:
|
||||
error "Failed to select slot blocks"
|
||||
return failure(err)
|
||||
|
||||
let numberOfPaddingCells = self.calculateNumberOfPaddingCells(slotBlocks.len)
|
||||
|
||||
raiseAssert("not implemented")
|
||||
|
||||
# select slot blocks
|
||||
|
||||
# pad till cells are power of two
|
||||
# -> get number of padding cells
|
||||
# -> convert to number of padding blocks
|
||||
|
||||
# build tree
|
||||
|
||||
# save tree
|
||||
|
||||
# return tree
|
||||
|
||||
# let
|
||||
# datasetTreeCid = self.manifest.treeCid
|
||||
# datasetBlockIndexStart = datasetSlotIndex * self.numberOfSlotBlocks
|
||||
# datasetBlockIndexEnd = datasetBlockIndexStart + self.numberOfSlotBlocks
|
||||
|
||||
# for index in datasetBlockIndexStart ..< datasetBlockIndexEnd:
|
||||
# without slotBlockLeaf =? await self.getTreeLeafCid(datasetTreeCid, index), err:
|
||||
# return failure(err)
|
||||
# if builder.addLeaf(slotBlockLeaf).isErr:
|
||||
# error "Failed to add slotBlockCid to slot tree builder"
|
||||
# return failure("Failed to add slotBlockCid to slot tree builder")
|
||||
|
||||
# without slotTree =? builder.build(), err:
|
||||
# error "Failed to build slot tree"
|
||||
# return failure(err)
|
||||
|
||||
# if (await self.blockStore.putAllProofs(slotTree)).isErr:
|
||||
# error "Failed to store slot tree"
|
||||
# return failure("Failed to store slot tree")
|
||||
|
||||
# return success(slotTree)
|
||||
|
||||
# proc createSlotManifest*(self: SlotBuilder, datasetSlotIndex: int): Future[?!Manifest] {.async.} =
|
||||
# without slotTree =? await self.createAndSaveSlotTree(datasetSlotIndex), err:
|
||||
# error "Failed to create slot tree"
|
||||
# return failure(err)
|
||||
|
||||
# without slotTreeRootCid =? slotTree.rootCid, err:
|
||||
# error "Failed to get root CID from slot tree"
|
||||
# return failure(err)
|
||||
|
||||
# var slotManifest = Manifest.new(
|
||||
# treeCid = slotTreeRootCid,
|
||||
# datasetSize = self.numberOfSlotBlocks.NBytes * self.manifest.blockSize,
|
||||
# blockSize = self.manifest.blockSize,
|
||||
# version = self.manifest.version,
|
||||
# hcodec = self.manifest.hcodec,
|
||||
# codec = self.manifest.codec,
|
||||
# ecK = self.manifest.ecK, # should change this = EC params of first ECing. there's be another!
|
||||
# ecM = self.manifest.ecK,
|
||||
# originalTreeCid = self.manifest.originalTreeCid,
|
||||
# originalDatasetSize = self.manifest.originalDatasetSize
|
||||
# )
|
||||
|
||||
# #treeCid: Cid
|
||||
# # datasetSize: NBytes
|
||||
# # blockSize: NBytes
|
||||
# # version: CidVersion
|
||||
# # hcodec: MultiCodec
|
||||
# # codex: MultiCodec
|
||||
# # ecK: int
|
||||
# # ecM: int
|
||||
# # originalTreeCid: Cid
|
||||
# # originalDatasetSize: NBytes
|
||||
|
||||
# # treeCid: Cid
|
||||
# # datasetSize: NBytes
|
||||
# # blockSize: NBytes
|
||||
# # version: CidVersion
|
||||
# # hcodec: MultiCodec
|
||||
# # codec: MultiCodec
|
||||
# # ecK: int
|
||||
# # ecM: int
|
||||
# # originalTreeCid: Cid
|
||||
# # originalDatasetSize: NBytes): Manifest
|
||||
# # first type mismatch at position: 7
|
||||
|
||||
|
||||
|
||||
# slotManifest.isSlot = true
|
||||
# slotManifest.datasetSlotIndex = datasetSlotIndex
|
||||
# slotManifest.originalProtectedTreeCid = self.manifest.treeCid
|
||||
# slotManifest.originalProtectedDatasetSize = self.manifest.datasetSize
|
||||
|
||||
# return success(slotManifest)
|
||||
return await self.buildSlotTree(slotBlocks, numberOfPaddingCells)
|
||||
|
||||
@ -8,6 +8,7 @@ import pkg/codex/rng
|
||||
import pkg/codex/stores
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils
|
||||
|
||||
import ../helpers
|
||||
import ../examples
|
||||
@ -29,6 +30,7 @@ asyncchecksuite "Slot builder":
|
||||
datasetBlocks: seq[bt.Block]
|
||||
localStore = CacheStore.new()
|
||||
protectedManifest: Manifest
|
||||
expectedEmptyCid: Cid
|
||||
slotBuilder: SlotBuilder
|
||||
|
||||
proc createBlocks(): Future[void] {.async.} =
|
||||
@ -63,6 +65,7 @@ asyncchecksuite "Slot builder":
|
||||
|
||||
let manifestBlock = bt.Block.new(protectedManifest.encode().tryGet(), codec = DagPBCodec).tryGet()
|
||||
discard await localStore.putBlock(manifestBlock)
|
||||
expectedEmptyCid = emptyCid(protectedManifest.version, protectedManifest.hcodec, protectedManifest.codec).tryget()
|
||||
|
||||
setup:
|
||||
await createBlocks()
|
||||
@ -134,11 +137,31 @@ asyncchecksuite "Slot builder":
|
||||
check:
|
||||
expectedBlockCids == slotBlockCids
|
||||
|
||||
test "Can create slot tree (index: " & $i & ")":
|
||||
let
|
||||
expectedSlotBlockCids = (await slotBuilder.selectSlotBlocks(i)).tryGet()
|
||||
expectedNumPadBlocks = divUp(slotBuilder.calculateNumberOfPaddingCells(expectedSlotBlockCids.len), numberOfCellsPerBlock)
|
||||
|
||||
slotTree = (await slotBuilder.createSlotTree(i)).tryGet()
|
||||
|
||||
check:
|
||||
# Tree size
|
||||
slotTree.leavesCount == expectedSlotBlockCids.len + expectedNumPadBlocks
|
||||
|
||||
for i in 0 ..< numberOfSlotBlocks:
|
||||
check:
|
||||
# Each slot block
|
||||
slotTree.getLeafCid(i).tryget() == expectedSlotBlockCids[i]
|
||||
|
||||
for i in 0 ..< expectedNumPadBlocks:
|
||||
check:
|
||||
# Each pad block
|
||||
slotTree.getLeafCid(numberOfSlotBlocks + i).tryget() == expectedEmptyCid
|
||||
|
||||
test "Can create slot tree":
|
||||
let
|
||||
slotBlockCids = datasetBlocks[0 ..< numberOfSlotBlocks].mapIt(it.cid)
|
||||
numPadCells = numberOfCellsPerBlock div 2 # We expect 1 pad block.
|
||||
expectedEmptyCid = emptyCid(protectedManifest.version, protectedManifest.hcodec, protectedManifest.codec)
|
||||
|
||||
slotTree = (await slotBuilder.buildSlotTree(slotBlockCids, numPadCells)).tryGet()
|
||||
|
||||
@ -153,32 +176,7 @@ asyncchecksuite "Slot builder":
|
||||
|
||||
check:
|
||||
# 1 pad block
|
||||
slotTree.getLeafCid(numberOfSlotBlocks) == expectedEmptyCid
|
||||
slotTree.getLeafCid(numberOfSlotBlocks).tryget() == expectedEmptyCid
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# test "Can create slot tree (index: " & $i & ")":
|
||||
# let
|
||||
# slotBlockCids =
|
||||
# m = (await slotBuilder.buildSlotTree(slotBlockCids, numPadCells)).tryGet()
|
||||
|
||||
# check:
|
||||
# m.treeCid # check
|
||||
# m.datasetSize == (numberOfSlotBlocks * blockSize).NBytes
|
||||
# m.blockSize == blockSize
|
||||
# m.version == manifest.version
|
||||
# m.hcodec == manifest.hcodec
|
||||
# m.codec == manifest.codec
|
||||
# #m.ecK == ??
|
||||
# #m.ecM == ??
|
||||
# m.originalTreeCid == manifest.originalTreeCid
|
||||
# m.originalDatasetSize = manifest.originalDatasetSize
|
||||
|
||||
# m.isSlot == true
|
||||
# m.datasetSlotIndex == i
|
||||
# m.originalProtectedTreeCide == manifest.treeCid
|
||||
# m.originalProtectedDatasetSize == manifest.datasetSize
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user