diff --git a/tests/codex/testerasure2d.nim b/tests/codex/testerasure2d.nim new file mode 100644 index 00000000..46febeb4 --- /dev/null +++ b/tests/codex/testerasure2d.nim @@ -0,0 +1,94 @@ +import std/sequtils +from std/math import sqrt + +import pkg/asynctest +import pkg/chronos +import pkg/libp2p +import pkg/questionable/results + +import pkg/codex/erasure +import pkg/codex/manifest +import pkg/codex/stores +import pkg/codex/blocktype as bt +import pkg/codex/rng + +import ./helpers + +suite "2D Erasure encode/decode": + + const BlockSize = 1024 + const dataSetSize = BlockSize * 123 + + var rng: Rng + var chunker: Chunker + var manifest: Manifest + var store: BlockStore + var erasure: Erasure + + setup: + rng = Rng.instance() + chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize) + manifest = !Manifest.new(blockSize = BlockSize) + store = CacheStore.new(cacheSize = (dataSetSize * 2), chunkSize = BlockSize) + erasure = Erasure.new(store, leoEncoderProvider2D, leoDecoderProvider2D) + + while ( + let chunk = await chunker.getBytes(); + chunk.len > 0): + + let blk = bt.Block.new(chunk).tryGet() + manifest.add(blk.cid) + (await store.putBlock(blk)).tryGet() + + proc encode(buffers, parity: int): Future[Manifest] {.async.} = + let + encoded = (await erasure.encode( + manifest, + buffers, + parity)).tryGet() + + check: + encoded.len mod (buffers + parity) == 0 + encoded.rounded == manifest.len - 1 + buffers - (manifest.len - 1) mod buffers + encoded.steps == encoded.rounded div buffers + + echo(manifest.blocks) + echo(encoded.blocks) + + return encoded + + test "Should tolerate losing M1 data blocks in a single random column": + const + k1 = 3 + k2 = k1 + m1 = 1 + m2 = m1 + buffers = k1 * k2 + parity = (k1+m1) * (k2+m2) - buffers + + echo "encode" + + let encoded = await encode(buffers, parity) + + var + column = rng.rand(encoded.steps - 1) # random column + dropped: seq[Cid] + + echo ("steps", encoded.steps) + for _ in 0 ..< m1: + echo ("column", column) + dropped.add(encoded[column]) + (await store.delBlock(encoded[column])).tryGet() + column.inc(encoded.steps) + + var + decoded = (await erasure.decode(encoded, parity + buffers - dropped.len)).tryGet() + + check: + decoded.cid.tryGet() == manifest.cid.tryGet() + decoded.cid.tryGet() == encoded.originalCid + decoded.len == encoded.originalLen + + for d in dropped: + let present = await store.hasBlock(d) + check present.tryGet()