codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix erasure tests

This commit is contained in:
Dmitriy Ryajov 2023-10-20 12:30:47 -06:00
parent 778093d12e
commit 12b5922daa
No known key found for this signature in database
GPG Key ID: DA8C680CE7C657A4
1 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tests/codex/testerasure.nim
View File

@ -10,6 +10,7 @@ import pkg/codex/manifest
import pkg/codex/stores import pkg/codex/stores
import pkg/codex/blocktype as bt import pkg/codex/blocktype as bt
import pkg/codex/rng import pkg/codex/rng
import pkg/codex/utils
import ./helpers import ./helpers
@ -51,7 +52,7 @@ asyncchecksuite "Erasure encode/decode":
check: check:
encoded.len mod (buffers + parity) == 0 encoded.len mod (buffers + parity) == 0
encoded.rounded == (manifest.len + (buffers - (manifest.len mod buffers))) encoded.rounded == roundUp(manifest.len, buffers)
encoded.steps == encoded.rounded div buffers encoded.steps == encoded.rounded div buffers
return encoded return encoded
@ -64,13 +65,13 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
var var
column = rng.rand((encoded.len - 1) div encoded.steps) # random column column = rng.rand((encoded.len div encoded.steps) - 1) # random column
dropped: seq[Cid] dropped: seq[(int, Cid)]
for _ in 0..<encoded.ecM: for _ in 0..<encoded.ecM:
dropped.add(encoded[column]) dropped.add((column, encoded[column]))
(await store.delBlock(encoded[column])).tryGet() (await store.delBlock(encoded[column])).tryGet()
column.inc(encoded.steps - 1) column = (column + encoded.steps) mod encoded.len # wrap around
var var
decoded = (await erasure.decode(encoded)).tryGet() decoded = (await erasure.decode(encoded)).tryGet()
@ -81,8 +82,9 @@ asyncchecksuite "Erasure encode/decode":
decoded.len == encoded.originalLen decoded.len == encoded.originalLen
for d in dropped: for d in dropped:
let present = await store.hasBlock(d) if d[0] < manifest.len: # we don't support returning parity blocks yet
check present.tryGet() let present = await store.hasBlock(d[1])
check present.tryGet()
test "Should not tolerate losing more than M data blocks in a single random column": test "Should not tolerate losing more than M data blocks in a single random column":
const const
@ -92,13 +94,13 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
var var
column = rng.rand((encoded.len - 1) div encoded.steps) # random column column = rng.rand((encoded.len div encoded.steps) - 1) # random column
dropped: seq[Cid] dropped: seq[Cid]
for _ in 0..<encoded.ecM + 1: for _ in 0..<encoded.ecM + 1:
dropped.add(encoded[column]) dropped.add(encoded[column])
(await store.delBlock(encoded[column])).tryGet() (await store.delBlock(encoded[column])).tryGet()
column.inc(encoded.steps) column = (column + encoded.steps) mod encoded.len # wrap around
var var
decoded: Manifest decoded: Manifest
@ -149,7 +151,7 @@ asyncchecksuite "Erasure encode/decode":
blocks: seq[int] blocks: seq[int]
offset = 0 offset = 0
while offset < encoded.steps - 1: while offset < encoded.steps:
let let
blockIdx = toSeq(countup(offset, encoded.len - 1, encoded.steps)) blockIdx = toSeq(countup(offset, encoded.len - 1, encoded.steps))
@ -179,6 +181,7 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
# loose M original (systematic) symbols/blocks
for b in encoded.blocks[0..<encoded.steps * encoded.ecM]: for b in encoded.blocks[0..<encoded.steps * encoded.ecM]:
(await store.delBlock(b)).tryGet() (await store.delBlock(b)).tryGet()
@ -195,6 +198,7 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
# loose M parity (all!) symbols/blocks from the dataset
for b in encoded.blocks[^(encoded.steps * encoded.ecM)..^1]: for b in encoded.blocks[^(encoded.steps * encoded.ecM)..^1]:
(await store.delBlock(b)).tryGet() (await store.delBlock(b)).tryGet()