status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

backport changes from fix-erasure-tests (#618)

This commit is contained in:
Dmitriy Ryajov 2023-11-14 10:53:06 -06:00 committed by GitHub
parent a1725594a8
commit 06bb21bfc7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
tests/codex/testerasure.nim
View File

@ -1,4 +1,5 @@
import std/sequtils import std/sequtils
import std/sugar
import pkg/asynctest import pkg/asynctest
import pkg/chronos import pkg/chronos
@ -10,6 +11,7 @@ import pkg/codex/manifest
import pkg/codex/stores import pkg/codex/stores
import pkg/codex/blocktype as bt import pkg/codex/blocktype as bt
import pkg/codex/rng import pkg/codex/rng
import pkg/codex/utils
import ./helpers import ./helpers
@ -42,7 +44,7 @@ asyncchecksuite "Erasure encode/decode":
check: check:
encoded.blocksCount mod (buffers + parity) == 0 encoded.blocksCount mod (buffers + parity) == 0
encoded.rounded == (manifest.blocksCount + (buffers - (manifest.blocksCount mod buffers))) encoded.rounded == roundUp(manifest.blocksCount, buffers)
encoded.steps == encoded.rounded div buffers encoded.steps == encoded.rounded div buffers
return encoded return encoded
@ -55,14 +57,14 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
var var
column = rng.rand((encoded.blocksCount - 1) div encoded.steps) # random column column = rng.rand((encoded.blocksCount div encoded.steps) - 1) # random column
dropped: seq[int] dropped: seq[int]
for _ in 0..<encoded.ecM: for _ in 0..<encoded.ecM:
dropped.add(column) dropped.add(column)
(await store.delBlock(encoded.treeCid, column)).tryGet() (await store.delBlock(encoded.treeCid, column)).tryGet()
(await store.delBlock(manifest.treeCid, column)).tryGet() (await store.delBlock(manifest.treeCid, column)).tryGet()
column.inc(encoded.steps - 1) column = (column + encoded.steps) mod encoded.blocksCount # wrap around
var var
decoded = (await erasure.decode(encoded)).tryGet() decoded = (await erasure.decode(encoded)).tryGet()
@ -73,8 +75,9 @@ asyncchecksuite "Erasure encode/decode":
decoded.blocksCount == encoded.originalBlocksCount decoded.blocksCount == encoded.originalBlocksCount
for d in dropped: for d in dropped:
let present = await store.hasBlock(manifest.treeCid, d) if d < manifest.blocksCount: # we don't support returning parity blocks yet
check present.tryGet() let present = await store.hasBlock(manifest.treeCid, d)
check present.tryGet()
test "Should not tolerate losing more than M data blocks in a single random column": test "Should not tolerate losing more than M data blocks in a single random column":
const const
@ -84,14 +87,14 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
var var
column = rng.rand((encoded.blocksCount - 1) div encoded.steps) # random column column = rng.rand((encoded.blocksCount div encoded.steps) - 1) # random column
dropped: seq[int] dropped: seq[int]
for _ in 0..<encoded.ecM + 1: for _ in 0..<encoded.ecM + 1:
dropped.add(column) dropped.add(column)
(await store.delBlock(encoded.treeCid, column)).tryGet() (await store.delBlock(encoded.treeCid, column)).tryGet()
(await store.delBlock(manifest.treeCid, column)).tryGet() (await store.delBlock(manifest.treeCid, column)).tryGet()
column.inc(encoded.steps) column = (column + encoded.steps) mod encoded.blocksCount # wrap around
var var
decoded: Manifest decoded: Manifest
@ -144,7 +147,7 @@ asyncchecksuite "Erasure encode/decode":
blocks: seq[int] blocks: seq[int]
offset = 0 offset = 0
while offset < encoded.steps - 1: while offset < encoded.steps:
let let
blockIdx = toSeq(countup(offset, encoded.blocksCount - 1, encoded.steps)) blockIdx = toSeq(countup(offset, encoded.blocksCount - 1, encoded.steps))
@ -177,7 +180,8 @@ asyncchecksuite "Erasure encode/decode":
let encoded = await encode(buffers, parity) let encoded = await encode(buffers, parity)
for b in 0..<encoded.steps * encoded.ecM: # loose M original (systematic) symbols/blocks
for b in 0..<(encoded.steps * encoded.ecM):
(await store.delBlock(encoded.treeCid, b)).tryGet() (await store.delBlock(encoded.treeCid, b)).tryGet()
(await store.delBlock(manifest.treeCid, b)).tryGet() (await store.delBlock(manifest.treeCid, b)).tryGet()
@ -192,9 +196,14 @@ asyncchecksuite "Erasure encode/decode":
buffers = 20 buffers = 20
parity = 10 parity = 10
let encoded = await encode(buffers, parity) let
encoded = await encode(buffers, parity)
blocks = collect:
for i in 0..encoded.blocksCount:
i
for b in (encoded.blocksCount - encoded.steps * encoded.ecM)..<encoded.blocksCount: # loose M parity (all!) symbols/blocks from the dataset
for b in blocks[^(encoded.steps * encoded.ecM)..^1]:
(await store.delBlock(encoded.treeCid, b)).tryGet() (await store.delBlock(encoded.treeCid, b)).tryGet()
(await store.delBlock(manifest.treeCid, b)).tryGet() (await store.delBlock(manifest.treeCid, b)).tryGet()