diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 1bab089a..6996353c 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -73,11 +73,12 @@ type EncodingParams = object ecK: int ecM: int - rounded: int + interleave: int + rounded: int # padded number of blocks steps: int - blocksCount: int + blocksCount: int # total number of blocks including padding en EC -func indexToPos(steps, idx, step: int): int {.inline.} = +func oldIndexToPos(params: EncodingParams, idx: int): int {.inline.} = ## Convert an index to a position in the encoded ## dataset ## `idx` - the index to convert @@ -85,7 +86,39 @@ func indexToPos(steps, idx, step: int): int {.inline.} = ## `pos` - the position in the encoded dataset ## - (idx - step) div steps + #(idx div params.interleave) mod (params.ecK + params.ecM) + (idx div params.interleave) mod (params.ecK) + +func newIndexToPos(encoded: Manifest, idx: int): int {.inline.} = + (idx div encoded.interleave) mod (encoded.ecK + encoded.ecM) + +func newIndex(params: EncodingParams, step, column, pos: int): int = + #params.rounded + step + params.steps * pos + #step + params.steps * pos + step * params.interleave * (params.ecK + params.ecM) + pos * params.interleave + column + +func newIndex(params: Manifest, step, column, pos: int): int = + #params.rounded + step + params.steps * pos + #step + encoded.steps * pos + step * params.interleave * (params.ecK + params.ecM) + pos * params.interleave + column + +func oldIndex(params: EncodingParams, step, column, pos: int): int = + #params.rounded + step + params.steps * pos + #step + encoded.steps * pos + step * params.interleave * (params.ecK) + pos * params.interleave + column + +func oldIndex(params: Manifest, step, column, pos: int): int = + #params.rounded + step + params.steps * pos + #step + encoded.steps * pos + step * params.interleave * (params.ecK) + pos * params.interleave + column + +iterator oldIndices(params: EncodingParams, step, column: int): int = + for i in 0 ..< params.ecK: + yield oldIndex(params, step, column, i) + +iterator newIndices(params: Manifest, step, column: int): int = + for i in 0 ..< params.ecK + params.ecM: + yield newIndex(params, step, column, i) proc getPendingBlocks( self: Erasure, @@ -118,6 +151,7 @@ proc prepareEncodingData( manifest: Manifest, params: EncodingParams, step: int, + column: int, data: ref seq[seq[byte]], cids: ref seq[Cid], emptyBlock: seq[byte]): Future[?!int] {.async.} = @@ -125,7 +159,7 @@ proc prepareEncodingData( ## let - indicies = toSeq(countup(step, params.rounded - 1, params.steps)) + indicies = toSeq(oldIndices(params, step, column)) pendingBlocksIter = self.getPendingBlocks(manifest, indicies.filterIt(it < manifest.blocksCount)) var resolved = 0 @@ -134,20 +168,22 @@ proc prepareEncodingData( without blk =? blkOrErr, err: warn "Failed retreiving a block", treeCid = manifest.treeCid, idx, msg = err.msg continue - - let pos = indexToPos(params.steps, idx, step) + + let pos = oldIndexToPos(params, idx) + let newidx = newIndex(params, step, column, pos) shallowCopy(data[pos], if blk.isEmpty: emptyBlock else: blk.data) - cids[idx] = blk.cid + cids[newidx] = blk.cid resolved.inc() for idx in indicies.filterIt(it >= manifest.blocksCount): - let pos = indexToPos(params.steps, idx, step) - trace "Padding with empty block", idx + let pos = oldIndexToPos(params, idx) + let newidx = newIndex(params, step, column, pos) + trace "Padding with empty block", idx, newidx shallowCopy(data[pos], emptyBlock) without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec), err: return failure(err) - cids[idx] = emptyBlockCid + cids[newidx] = emptyBlockCid success(resolved) @@ -155,6 +191,7 @@ proc prepareDecodingData( self: Erasure, encoded: Manifest, step: int, + column: int, data: ref seq[seq[byte]], parityData: ref seq[seq[byte]], cids: ref seq[Cid], @@ -168,8 +205,8 @@ proc prepareDecodingData( ## `emptyBlock` - the empty block to be used for padding ## - let - indicies = toSeq(countup(step, encoded.blocksCount - 1, encoded.steps)) + let + indicies = toSeq(newIndices(encoded, step, column)) pendingBlocksIter = self.getPendingBlocks(encoded, indicies) var @@ -188,7 +225,7 @@ proc prepareDecodingData( continue let - pos = indexToPos(encoded.steps, idx, step) + pos = newIndexToPos(encoded, idx) logScope: cid = blk.cid @@ -198,7 +235,7 @@ proc prepareDecodingData( empty = blk.isEmpty cids[idx] = blk.cid - if idx >= encoded.rounded: + if pos >= encoded.ecK: trace "Retrieved parity block" shallowCopy(parityData[pos - encoded.ecK], if blk.isEmpty: emptyBlock else: blk.data) parityPieces.inc @@ -211,18 +248,25 @@ proc prepareDecodingData( return success (dataPieces, parityPieces) -proc init(_: type EncodingParams, manifest: Manifest, ecK: int, ecM: int): ?!EncodingParams = +proc init(_: type EncodingParams, manifest: Manifest, ecK, ecM, interl: int): ?!EncodingParams = + ## Calculate erasure coding parameters. + ## interl: if 0, use the interleaving resulting in a single "step". Otherwise use the given value, and calculate the number of steps needed. if ecK > manifest.blocksCount: return failure("Unable to encode manifest, not enough blocks, ecK = " & $ecK & ", blocksCount = " & $manifest.blocksCount) + let interleave = + if interl == 0: divUp(manifest.blocksCount, ecK) + else: interl + let - rounded = roundUp(manifest.blocksCount, ecK) - steps = divUp(manifest.blocksCount, ecK) - blocksCount = rounded + (steps * ecM) + rounded = roundUp(manifest.blocksCount, interleave * ecK) + steps = divUp(manifest.blocksCount, interleave * ecK) + blocksCount = rounded + (steps * interleave * ecM) EncodingParams( ecK: ecK, ecM: ecM, + interleave: interleave, rounded: rounded, steps: steps, blocksCount: blocksCount @@ -244,6 +288,7 @@ proc encodeData( blocks_count = params.blocksCount ecK = params.ecK ecM = params.ecM + interleave = params.interleave var cids = seq[Cid].new() @@ -254,42 +299,42 @@ proc encodeData( try: for step in 0..= encoded.ecK: - trace "Retrieved all the required data blocks" - continue + if dataPieces >= encoded.ecK: + trace "Retrieved all the required data blocks" + continue - trace "Erasure decoding data" - if ( - let err = decoder.decode(data[], parityData[], recovered); - err.isErr): - trace "Unable to decode data!", err = $err.error - return failure($err.error) + trace "Erasure decoding data" + if ( + let err = decoder.decode(data[], parityData[], recovered); + err.isErr): + trace "Unable to decode data!", err = $err.error + return failure($err.error) - for i in 0..