mirror of
https://github.com/logos-storage/nim-leopard.git
synced 2026-01-02 13:43:08 +00:00
chore: orc support
Refactor public API to use `var openArray[seq[byte]]` instead of raw pointers for ORC compatibility. The previous API required callers to extract raw pointers from seqs before calling encode/decode. With refc and shallowCopy, reference counting kept buffers alive. ORC doesn't track raw pointers and can reallocate seq buffers at yield points, leaving pointers dangling. The new openArray API borrows data safely. Also updates CI to Nim 2.2.4 and bumps version to 0.2.0. Part of https://github.com/logos-storage/nim-leopard/issues/24 Signed-off-by: Chrysostomos Nanakos <chris@include.gr>
This commit is contained in:
Chrysostomos Nanakos
parent
0478b12df9
commit
f1878d6aaa
7
.github/workflows/test.yml
vendored
7
.github/workflows/test.yml
vendored
@ -23,7 +23,7 @@ jobs:
|
||||
runner: windows-latest
|
||||
}
|
||||
# Earliest supported and latest nim
|
||||
nim: [1.6.18, "stable"]
|
||||
nim: [binary:2.2.4]
|
||||
name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim }}
|
||||
runs-on: ${{ matrix.platform.runner }}
|
||||
defaults:
|
||||
@ -33,10 +33,9 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: jiro4989/setup-nim-action@v2
|
||||
- uses: iffy/install-nim@v5
|
||||
with:
|
||||
nim-version: ${{matrix.nim}}
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
version: ${{ matrix.nim }}
|
||||
- name: Install
|
||||
run: nimble install -y
|
||||
- name: Build and run tests
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
mode = ScriptMode.Verbose
|
||||
|
||||
packageName = "leopard"
|
||||
version = "0.1.1"
|
||||
version = "0.2.0"
|
||||
author = "Status Research & Development GmbH"
|
||||
description = "A wrapper for Leopard-RS"
|
||||
license = "Apache License 2.0 or MIT"
|
||||
|
||||
@ -16,27 +16,27 @@ import ./utils
|
||||
|
||||
export wrapper, results
|
||||
|
||||
const
|
||||
BuffMultiples* = 64
|
||||
const BuffMultiples* = 64
|
||||
|
||||
type
|
||||
LeoBufferPtr* = ptr UncheckedArray[byte]
|
||||
|
||||
LeoCoderKind* {.pure.} = enum
|
||||
Encoder,
|
||||
Encoder
|
||||
Decoder
|
||||
|
||||
Leo* = object of RootObj
|
||||
bufSize*: int # size of the buffer in multiples of 64
|
||||
buffers*: int # total number of data buffers (K)
|
||||
parity*: int # total number of parity buffers (M)
|
||||
dataBufferPtr: seq[LeoBufferPtr] # buffer where data is copied before encoding
|
||||
workBufferCount: int # number of parity work buffers
|
||||
workBufferPtr: seq[LeoBufferPtr] # buffer where parity data is written during encoding or before decoding
|
||||
bufSize*: int # size of the buffer in multiples of 64
|
||||
buffers*: int # total number of data buffers (K)
|
||||
parity*: int # total number of parity buffers (M)
|
||||
dataBufferPtr: seq[LeoBufferPtr] # buffer where data is copied before encoding
|
||||
workBufferCount: int # number of parity work buffers
|
||||
workBufferPtr: seq[LeoBufferPtr]
|
||||
# buffer where parity data is written during encoding or before decoding
|
||||
case kind: LeoCoderKind
|
||||
of LeoCoderKind.Decoder:
|
||||
decodeBufferCount: int # number of decoding work buffers
|
||||
decodeBufferPtr: seq[LeoBufferPtr] # work buffer used for decoding
|
||||
decodeBufferCount: int # number of decoding work buffers
|
||||
decodeBufferPtr: seq[LeoBufferPtr] # work buffer used for decoding
|
||||
of LeoCoderKind.Encoder:
|
||||
discard
|
||||
|
||||
@ -44,9 +44,8 @@ type
|
||||
LeoDecoder* = object of Leo
|
||||
|
||||
func encode*(
|
||||
self: var LeoEncoder,
|
||||
data,parity: ptr UncheckedArray[ptr UncheckedArray[byte]],
|
||||
dataLen,parityLen: int ): Result[void, cstring] =
|
||||
self: var LeoEncoder, data, parity: var openArray[seq[byte]]
|
||||
): Result[void, cstring] =
|
||||
## Encode a list of buffers in `data` into a number of `bufSize` sized
|
||||
## `parity` buffers
|
||||
##
|
||||
@ -54,43 +53,40 @@ func encode*(
|
||||
## `parity` - list of parity `buffers` of size `bufSize`
|
||||
##
|
||||
|
||||
if dataLen != self.buffers:
|
||||
if data.len != self.buffers:
|
||||
return err("Number of data buffers should match!")
|
||||
|
||||
if parityLen != self.parity:
|
||||
if parity.len != self.parity:
|
||||
return err("Number of parity buffers should match!")
|
||||
|
||||
# zero encode work buffer to avoid corrupting with previous run
|
||||
for i in 0..<self.workBufferCount:
|
||||
for i in 0 ..< self.workBufferCount:
|
||||
zeroMem(self.workBufferPtr[i], self.bufSize)
|
||||
|
||||
# copy data into aligned buffer
|
||||
for i in 0..<dataLen:
|
||||
for i in 0 ..< data.len:
|
||||
copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize)
|
||||
|
||||
let
|
||||
res = leoEncode(
|
||||
self.bufSize.culonglong,
|
||||
self.buffers.cuint,
|
||||
self.parity.cuint,
|
||||
self.workBufferCount.cuint,
|
||||
cast[LeoDataPtr](addr self.dataBufferPtr[0]),
|
||||
cast[ptr pointer](addr self.workBufferPtr[0]))
|
||||
let res = leoEncode(
|
||||
self.bufSize.culonglong,
|
||||
self.buffers.cuint,
|
||||
self.parity.cuint,
|
||||
self.workBufferCount.cuint,
|
||||
cast[LeoDataPtr](addr self.dataBufferPtr[0]),
|
||||
cast[ptr pointer](addr self.workBufferPtr[0]),
|
||||
)
|
||||
|
||||
if ord(res) != ord(LeopardSuccess):
|
||||
return err(leoResultString(res.LeopardResult))
|
||||
|
||||
for i in 0..<parityLen:
|
||||
copyMem(parity[i], self.workBufferPtr[i], self.bufSize)
|
||||
for i in 0 ..< parity.len:
|
||||
copyMem(addr parity[i][0], self.workBufferPtr[i], self.bufSize)
|
||||
|
||||
return ok()
|
||||
|
||||
func decode*(
|
||||
self: var LeoDecoder,
|
||||
data,
|
||||
parity,
|
||||
recovered: ptr UncheckedArray[ptr UncheckedArray[byte]],
|
||||
dataLen,parityLen,recoveredLen: int): Result[void, cstring] =
|
||||
self: var LeoDecoder, data, parity, recovered: var openArray[seq[byte]]
|
||||
): Result[void, cstring] =
|
||||
## Decode a list of buffers in `data` and `parity` into a list
|
||||
## of `recovered` buffers of `bufSize`. The list of `recovered`
|
||||
## buffers should be match the `Leo.buffers`
|
||||
@ -100,55 +96,55 @@ func decode*(
|
||||
## `recovered` - list of recovered `buffers` of size `bufSize`
|
||||
##
|
||||
|
||||
if dataLen != self.buffers:
|
||||
if data.len != self.buffers:
|
||||
return err("Number of data buffers should match!")
|
||||
|
||||
if parityLen != self.parity:
|
||||
if parity.len != self.parity:
|
||||
return err("Number of parity buffers should match!")
|
||||
|
||||
if recoveredLen != self.buffers:
|
||||
if recovered.len != self.buffers:
|
||||
return err("Number of recovered buffers should match buffers!")
|
||||
|
||||
# clean out work and data buffers
|
||||
for i in 0..<self.workBufferCount:
|
||||
for i in 0 ..< self.workBufferCount:
|
||||
zeroMem(self.workBufferPtr[i], self.bufSize)
|
||||
|
||||
for i in 0..<self.decodeBufferCount:
|
||||
for i in 0 ..< self.decodeBufferCount:
|
||||
zeroMem(self.decodeBufferPtr[i], self.bufSize)
|
||||
|
||||
for i in 0..<dataLen:
|
||||
for i in 0 ..< data.len:
|
||||
zeroMem(self.dataBufferPtr[i], self.bufSize)
|
||||
|
||||
# this is needed because erasures are nil pointers
|
||||
var
|
||||
dataPtr = newSeq[LeoBufferPtr](dataLen)
|
||||
dataPtr = newSeq[LeoBufferPtr](data.len)
|
||||
parityPtr = newSeq[LeoBufferPtr](self.workBufferCount)
|
||||
|
||||
# copy data into aligned buffer
|
||||
for i in 0..<dataLen:
|
||||
if not data[i].isNil:
|
||||
copyMem(self.dataBufferPtr[i],addr data[i][0], self.bufSize)
|
||||
for i in 0 ..< data.len:
|
||||
if data[i].len > 0:
|
||||
copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize)
|
||||
dataPtr[i] = self.dataBufferPtr[i]
|
||||
else:
|
||||
dataPtr[i] = nil
|
||||
|
||||
# copy parity into aligned buffer
|
||||
for i in 0..<self.workBufferCount:
|
||||
if i < parityLen and not parity[i].isNil:
|
||||
for i in 0 ..< self.workBufferCount:
|
||||
if i < parity.len and parity[i].len > 0:
|
||||
copyMem(self.workBufferPtr[i], addr parity[i][0], self.bufSize)
|
||||
parityPtr[i] = self.workBufferPtr[i]
|
||||
else:
|
||||
parityPtr[i] = nil
|
||||
|
||||
let
|
||||
res = leoDecode(
|
||||
self.bufSize.culonglong,
|
||||
self.buffers.cuint,
|
||||
self.parity.cuint,
|
||||
self.decodeBufferCount.cuint,
|
||||
cast[LeoDataPtr](addr dataPtr[0]),
|
||||
cast[LeoDataPtr](addr parityPtr[0]),
|
||||
cast[ptr pointer](addr self.decodeBufferPtr[0]))
|
||||
let res = leoDecode(
|
||||
self.bufSize.culonglong,
|
||||
self.buffers.cuint,
|
||||
self.parity.cuint,
|
||||
self.decodeBufferCount.cuint,
|
||||
cast[LeoDataPtr](addr dataPtr[0]),
|
||||
cast[LeoDataPtr](addr parityPtr[0]),
|
||||
cast[ptr pointer](addr self.decodeBufferPtr[0]),
|
||||
)
|
||||
|
||||
if ord(res) != ord(LeopardSuccess):
|
||||
return err(leoResultString(res.LeopardResult))
|
||||
@ -193,11 +189,8 @@ func free*(self: var Leo) =
|
||||
# self.free()
|
||||
|
||||
proc init[TT: Leo](
|
||||
T: type TT,
|
||||
bufSize,
|
||||
buffers,
|
||||
parity: int,
|
||||
kind: LeoCoderKind): Result[T, cstring] =
|
||||
T: type TT, bufSize, buffers, parity: int, kind: LeoCoderKind
|
||||
): Result[T, cstring] =
|
||||
if bufSize mod BuffMultiples != 0:
|
||||
return err("bufSize should be multiples of 64 bytes!")
|
||||
|
||||
@ -219,46 +212,33 @@ proc init[TT: Leo](
|
||||
if (let res = leoInit(); res.ord != LeopardSuccess.ord):
|
||||
return err(leoResultString(res.LeopardResult))
|
||||
|
||||
var
|
||||
self = T(
|
||||
kind: kind,
|
||||
bufSize: bufSize,
|
||||
buffers: buffers,
|
||||
parity: parity)
|
||||
var self = T(kind: kind, bufSize: bufSize, buffers: buffers, parity: parity)
|
||||
|
||||
self.workBufferCount = leoEncodeWorkCount(
|
||||
buffers.cuint,
|
||||
parity.cuint).int
|
||||
self.workBufferCount = leoEncodeWorkCount(buffers.cuint, parity.cuint).int
|
||||
|
||||
# initialize encode work buffers
|
||||
for _ in 0..<self.workBufferCount:
|
||||
for _ in 0 ..< self.workBufferCount:
|
||||
self.workBufferPtr.add(cast[LeoBufferPtr](self.bufSize.leoAlloc()))
|
||||
|
||||
# initialize data buffers
|
||||
for _ in 0..<self.buffers:
|
||||
for _ in 0 ..< self.buffers:
|
||||
self.dataBufferPtr.add(cast[LeoBufferPtr](self.bufSize.leoAlloc()))
|
||||
|
||||
if self.kind == LeoCoderKind.Decoder:
|
||||
self.decodeBufferCount = leoDecodeWorkCount(
|
||||
buffers.cuint,
|
||||
parity.cuint).int
|
||||
self.decodeBufferCount = leoDecodeWorkCount(buffers.cuint, parity.cuint).int
|
||||
|
||||
# initialize decode work buffers
|
||||
for _ in 0..<self.decodeBufferCount:
|
||||
for _ in 0 ..< self.decodeBufferCount:
|
||||
self.decodeBufferPtr.add(cast[LeoBufferPtr](self.bufSize.leoAlloc()))
|
||||
|
||||
ok(self)
|
||||
|
||||
proc init*(
|
||||
T: type LeoEncoder,
|
||||
bufSize,
|
||||
buffers,
|
||||
parity: int): Result[LeoEncoder, cstring] =
|
||||
T: type LeoEncoder, bufSize, buffers, parity: int
|
||||
): Result[LeoEncoder, cstring] =
|
||||
LeoEncoder.init(bufSize, buffers, parity, LeoCoderKind.Encoder)
|
||||
|
||||
proc init*(
|
||||
T: type LeoDecoder,
|
||||
bufSize,
|
||||
buffers,
|
||||
parity: int): Result[LeoDecoder, cstring] =
|
||||
T: type LeoDecoder, bufSize, buffers, parity: int
|
||||
): Result[LeoDecoder, cstring] =
|
||||
LeoDecoder.init(bufSize, buffers, parity, LeoCoderKind.Decoder)
|
||||
|
||||
@ -24,22 +24,22 @@ proc randomCRCPacket*(data: var openArray[byte]) =
|
||||
|
||||
copyMem(addr data[4], unsafeAddr crc, sizeof(crc))
|
||||
|
||||
proc checkCRCPacket*(data: ptr UncheckedArray[byte], len: int): bool =
|
||||
if len < 16:
|
||||
for i in 1..<len:
|
||||
if data[i] != data[0]:
|
||||
proc checkCRCPacket*(data: openArray[byte]): bool =
|
||||
if data.len < 16:
|
||||
for d in data[1 .. data.high]:
|
||||
if d != data[0]:
|
||||
raise (ref Defect)(msg: "Packet don't match")
|
||||
else:
|
||||
var
|
||||
crc = len.uint32
|
||||
crc = data.len.uint32
|
||||
packCrc: uint32
|
||||
packSize: uint32
|
||||
|
||||
copyMem(addr packSize, unsafeAddr data[0], sizeof(packSize))
|
||||
if packSize != len.uint:
|
||||
if packSize != data.len.uint:
|
||||
raise (ref Defect)(msg: "Packet size don't match!")
|
||||
|
||||
for i in 4..<len:
|
||||
for i in 4..< data.len:
|
||||
let v = data[i]
|
||||
crc = (crc shl 3) and (crc shr (32 - 3))
|
||||
crc += v
|
||||
@ -49,43 +49,19 @@ proc checkCRCPacket*(data: ptr UncheckedArray[byte], len: int): bool =
|
||||
if packCrc == crc:
|
||||
return true
|
||||
|
||||
proc dropRandomIdx*(bufs: ptr UncheckedArray[ptr UncheckedArray[byte]], bufsLen,dropCount: int) =
|
||||
proc dropRandomIdx*(bufs: var openArray[seq[byte]], dropCount: int) =
|
||||
var
|
||||
count = 0
|
||||
dups: seq[int]
|
||||
size = bufsLen
|
||||
size = bufs.len
|
||||
|
||||
while count < dropCount:
|
||||
let i = rand(0..<size)
|
||||
if dups.find(i) == -1:
|
||||
dups.add(i)
|
||||
bufs[i]=nil
|
||||
bufs[i].setLen(0)
|
||||
count.inc
|
||||
|
||||
proc createDoubleArray*(
|
||||
outerLen, innerLen: int
|
||||
): ptr UncheckedArray[ptr UncheckedArray[byte]] =
|
||||
# Allocate outer array
|
||||
result = cast[ptr UncheckedArray[ptr UncheckedArray[byte]]](alloc0(
|
||||
sizeof(ptr UncheckedArray[byte]) * outerLen
|
||||
))
|
||||
|
||||
# Allocate each inner array
|
||||
for i in 0 ..< outerLen:
|
||||
result[i] = cast[ptr UncheckedArray[byte]](alloc0(sizeof(byte) * innerLen))
|
||||
|
||||
proc freeDoubleArray*(
|
||||
arr: ptr UncheckedArray[ptr UncheckedArray[byte]], outerLen: int
|
||||
) =
|
||||
# Free each inner array
|
||||
for i in 0 ..< outerLen:
|
||||
if not arr[i].isNil:
|
||||
dealloc(arr[i])
|
||||
|
||||
# Free outer array
|
||||
if not arr.isNil:
|
||||
dealloc(arr)
|
||||
|
||||
proc testPackets*(
|
||||
buffers,
|
||||
parity,
|
||||
@ -96,37 +72,34 @@ proc testPackets*(
|
||||
decoder: var LeoDecoder): Result[void, cstring] =
|
||||
|
||||
var
|
||||
dataBuf = createDoubleArray(buffers, bufSize)
|
||||
parityBuf = createDoubleArray(parity, bufSize)
|
||||
recoveredBuf = createDoubleArray(buffers, bufSize)
|
||||
|
||||
defer:
|
||||
freeDoubleArray(dataBuf, buffers)
|
||||
freeDoubleArray(parityBuf, parity)
|
||||
freeDoubleArray(recoveredBuf, buffers)
|
||||
|
||||
|
||||
dataBuf = newSeqOfCap[seq[byte]](buffers)
|
||||
parityBuf = newSeqOfCap[seq[byte]](parity)
|
||||
recoveredBuf = newSeqOfCap[seq[byte]](buffers)
|
||||
|
||||
for i in 0..<buffers:
|
||||
var
|
||||
dataSeq = newSeq[byte](bufSize)
|
||||
var dataSeq = newSeq[byte](bufSize)
|
||||
|
||||
randomCRCPacket(dataSeq)
|
||||
copyMem(dataBuf[i],addr dataSeq[0],bufSize)
|
||||
dataBuf.add(dataSeq)
|
||||
|
||||
encoder.encode(dataBuf, parityBuf,buffers,parity).tryGet()
|
||||
recoveredBuf.add(newSeq[byte](bufSize))
|
||||
|
||||
for _ in 0 ..< parity:
|
||||
parityBuf.add(newSeq[byte](bufSize))
|
||||
|
||||
encoder.encode(dataBuf, parityBuf).tryGet()
|
||||
|
||||
if dataLosses > 0:
|
||||
dropRandomIdx(dataBuf,buffers, dataLosses)
|
||||
dropRandomIdx(dataBuf, dataLosses)
|
||||
|
||||
if parityLosses > 0:
|
||||
dropRandomIdx(parityBuf,parity,parityLosses)
|
||||
dropRandomIdx(parityBuf, parityLosses)
|
||||
|
||||
decoder.decode(dataBuf, parityBuf, recoveredBuf,buffers,parity,buffers).tryGet()
|
||||
decoder.decode(dataBuf, parityBuf, recoveredBuf).tryGet()
|
||||
|
||||
for i in 0..<buffers:
|
||||
if dataBuf[i].isNil:
|
||||
if not checkCRCPacket(recoveredBuf[i],bufSize):
|
||||
for i, d in dataBuf:
|
||||
if d.len <= 0:
|
||||
if not checkCRCPacket(recoveredBuf[i]):
|
||||
return err(("Check failed for packet " & $i).cstring)
|
||||
|
||||
ok()
|
||||
|
||||
@ -18,86 +18,65 @@ suite "Leopard Parametrization":
|
||||
test "Should not allow invalid data/parity buffer counts":
|
||||
check:
|
||||
LeoEncoder.init(64, 1, 2).error ==
|
||||
"number of parity buffers cannot exceed number of data buffers!"
|
||||
"number of parity buffers cannot exceed number of data buffers!"
|
||||
|
||||
test "Should not allow data + parity to exceed 65536":
|
||||
check:
|
||||
LeoEncoder.init(64, 65536 + 1, 0).error ==
|
||||
"number of parity and data buffers cannot exceed 65536!"
|
||||
"number of parity and data buffers cannot exceed 65536!"
|
||||
|
||||
LeoEncoder.init(64, 32768 + 1, 32768).error ==
|
||||
"number of parity and data buffers cannot exceed 65536!"
|
||||
"number of parity and data buffers cannot exceed 65536!"
|
||||
|
||||
test "Should not allow encoding with invalid data buffer counts":
|
||||
var
|
||||
dataLen =3
|
||||
parityLen = 2
|
||||
leo = LeoEncoder.init(64, 4, 2).tryGet()
|
||||
data = createDoubleArray(dataLen, 64)
|
||||
parity = createDoubleArray(parityLen, 64)
|
||||
defer:
|
||||
freeDoubleArray(data, dataLen)
|
||||
freeDoubleArray(parity, parityLen)
|
||||
data = newSeq[seq[byte]](3)
|
||||
parity = newSeq[seq[byte]](2)
|
||||
|
||||
check:
|
||||
leo.encode(data, parity,dataLen,parityLen).error == "Number of data buffers should match!"
|
||||
leo.encode(data, parity).error == "Number of data buffers should match!"
|
||||
|
||||
test "Should not allow encoding with invalid parity buffer counts":
|
||||
var
|
||||
dataLen =4
|
||||
parityLen = 3
|
||||
leo = LeoEncoder.init(64, 4, 2).tryGet()
|
||||
data = createDoubleArray(dataLen, 64)
|
||||
parity = createDoubleArray(parityLen, 64)
|
||||
|
||||
defer:
|
||||
freeDoubleArray(data, dataLen)
|
||||
freeDoubleArray(parity, parityLen)
|
||||
data = newSeq[seq[byte]](4)
|
||||
parity = newSeq[seq[byte]](3)
|
||||
|
||||
check:
|
||||
leo.encode(data, parity,dataLen,parityLen).error == "Number of parity buffers should match!"
|
||||
leo.encode(data, parity).error == "Number of parity buffers should match!"
|
||||
|
||||
test "Should not allow decoding with invalid data buffer counts":
|
||||
var
|
||||
dataLen =3
|
||||
parityLen = 2
|
||||
leo = LeoDecoder.init(64, 4, 2).tryGet()
|
||||
data = createDoubleArray(dataLen, 64)
|
||||
parity = createDoubleArray(parityLen, 64)
|
||||
recovered = createDoubleArray(dataLen, 64)
|
||||
|
||||
defer:
|
||||
freeDoubleArray(data, dataLen)
|
||||
freeDoubleArray(parity, parityLen)
|
||||
freeDoubleArray(recovered, dataLen)
|
||||
data = newSeq[seq[byte]](3)
|
||||
parity = newSeq[seq[byte]](2)
|
||||
recovered = newSeq[seq[byte]](3)
|
||||
|
||||
check:
|
||||
leo.decode(data, parity, recovered,dataLen,parityLen,dataLen).error == "Number of data buffers should match!"
|
||||
leo.decode(data, parity, recovered).error == "Number of data buffers should match!"
|
||||
|
||||
test "Should not allow decoding with invalid data buffer counts":
|
||||
var
|
||||
dataLen =4
|
||||
parityLen = 1
|
||||
recoveredLen = 3
|
||||
leo = LeoDecoder.init(64, 4, 2).tryGet()
|
||||
data = createDoubleArray(dataLen, 64)
|
||||
parity = createDoubleArray(parityLen, 64)
|
||||
recovered = createDoubleArray(recoveredLen, 64)
|
||||
data = newSeq[seq[byte]](4)
|
||||
parity = newSeq[seq[byte]](1)
|
||||
recovered = newSeq[seq[byte]](3)
|
||||
|
||||
check:
|
||||
leo.decode(data, parity, recovered,dataLen,parityLen,recoveredLen).error == "Number of parity buffers should match!"
|
||||
leo.decode(data, parity, recovered).error ==
|
||||
"Number of parity buffers should match!"
|
||||
|
||||
test "Should not allow decoding with invalid data buffer counts":
|
||||
var
|
||||
dataLen =4
|
||||
parityLen = 2
|
||||
recoveredLen = 3
|
||||
leo = LeoDecoder.init(64, 4, 2).tryGet()
|
||||
data = createDoubleArray(dataLen, 64)
|
||||
parity = createDoubleArray(parityLen, 64)
|
||||
recovered = createDoubleArray(recoveredLen, 64)
|
||||
data = newSeq[seq[byte]](4)
|
||||
parity = newSeq[seq[byte]](2)
|
||||
recovered = newSeq[seq[byte]](3)
|
||||
|
||||
check:
|
||||
leo.decode(data, parity, recovered,dataLen,parityLen,recoveredLen).error == "Number of recovered buffers should match buffers!"
|
||||
leo.decode(data, parity, recovered).error ==
|
||||
"Number of recovered buffers should match buffers!"
|
||||
|
||||
suite "Leopard simple Encode/Decode":
|
||||
const
|
||||
@ -109,76 +88,67 @@ suite "Leopard simple Encode/Decode":
|
||||
var
|
||||
encoder: LeoEncoder
|
||||
decoder: LeoDecoder
|
||||
data: ptr UncheckedArray[ptr UncheckedArray[byte]]
|
||||
parity: ptr UncheckedArray[ptr UncheckedArray[byte]]
|
||||
recovered: ptr UncheckedArray[ptr UncheckedArray[byte]]
|
||||
data: seq[seq[byte]]
|
||||
parity: seq[seq[byte]]
|
||||
recovered: seq[seq[byte]]
|
||||
|
||||
setup:
|
||||
encoder = LeoEncoder.init(BufferSize, DataCount, ParityCount).tryGet()
|
||||
decoder = LeoDecoder.init(BufferSize, DataCount, ParityCount).tryGet()
|
||||
data = createDoubleArray(DataCount, BufferSize)
|
||||
parity = createDoubleArray(ParityCount, BufferSize)
|
||||
recovered = createDoubleArray(DataCount, BufferSize)
|
||||
data = newSeq[seq[byte]](DataCount)
|
||||
parity = newSeq[seq[byte]](ParityCount)
|
||||
recovered = newSeq[seq[byte]](DataCount)
|
||||
|
||||
teardown:
|
||||
freeDoubleArray(data, DataCount)
|
||||
freeDoubleArray(parity, ParityCount)
|
||||
freeDoubleArray(recovered, DataCount)
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
|
||||
test "Test 2 data loses out of 4 possible":
|
||||
for i in 0..<DataCount:
|
||||
var
|
||||
str = TestString & " " & $i
|
||||
|
||||
copyMem(data[i], addr str[0], str.len)
|
||||
|
||||
|
||||
encoder.encode(data, parity,DataCount,ParityCount).tryGet()
|
||||
|
||||
var
|
||||
data1 =cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
|
||||
data2 = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
|
||||
|
||||
defer:
|
||||
deallocShared(data1)
|
||||
deallocShared(data2)
|
||||
|
||||
copyMem(data1,data[0], BufferSize)
|
||||
copyMem(data2,data[1], BufferSize)
|
||||
|
||||
data[0]=nil
|
||||
data[1]=nil
|
||||
|
||||
decoder.decode(data, parity, recovered,DataCount,ParityCount,DataCount).tryGet()
|
||||
|
||||
check equalMem(recovered[0], data1, BufferSize)
|
||||
check equalMem(recovered[1], data2, BufferSize)
|
||||
|
||||
test "Test 1 data and 1 parity loss out of 4 possible":
|
||||
for i in 0..<DataCount:
|
||||
var
|
||||
str = TestString & " " & $i
|
||||
for i in 0 ..< DataCount:
|
||||
data[i] = newSeq[byte](BufferSize)
|
||||
recovered[i] = newSeq[byte](BufferSize)
|
||||
var str = TestString & " " & $i
|
||||
|
||||
copyMem(addr data[i][0], addr str[0], str.len)
|
||||
|
||||
encoder.encode(data, parity,DataCount,ParityCount).tryGet()
|
||||
for i in 0 ..< ParityCount:
|
||||
parity[i] = newSeq[byte](BufferSize)
|
||||
|
||||
encoder.encode(data, parity).tryGet()
|
||||
|
||||
var data1 = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
|
||||
var
|
||||
data1 = data[0]
|
||||
data2 = data[1]
|
||||
|
||||
defer: deallocShared(data1)
|
||||
data[0].setLen(0)
|
||||
data[1].setLen(0)
|
||||
|
||||
copyMem(data1,data[0], BufferSize)
|
||||
decoder.decode(data, parity, recovered).tryGet()
|
||||
|
||||
data[0]=nil
|
||||
parity[0]=nil
|
||||
check recovered[0] == data1
|
||||
check recovered[1] == data2
|
||||
|
||||
decoder.decode(data, parity, recovered,DataCount,ParityCount,DataCount).tryGet()
|
||||
test "Test 1 data and 1 parity loss out of 4 possible":
|
||||
for i in 0 ..< DataCount:
|
||||
data[i] = newSeq[byte](BufferSize)
|
||||
recovered[i] = newSeq[byte](BufferSize)
|
||||
|
||||
check equalMem(recovered[0], data1, BufferSize)
|
||||
var str = TestString & " " & $i
|
||||
|
||||
copyMem(addr data[i][0], addr str[0], str.len)
|
||||
|
||||
for i in 0 ..< ParityCount:
|
||||
parity[i] = newSeq[byte](BufferSize)
|
||||
|
||||
encoder.encode(data, parity).tryGet()
|
||||
|
||||
var data1 = data[0]
|
||||
|
||||
data[0].setLen(0)
|
||||
parity[0].setLen(0)
|
||||
|
||||
decoder.decode(data, parity, recovered).tryGet()
|
||||
check recovered[0] == data1
|
||||
|
||||
suite "Leopard Encode/Decode":
|
||||
test "bufSize = 4096, K = 800, M = 200 - drop data = 200 data":
|
||||
@ -228,7 +198,8 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
@ -246,7 +217,8 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
@ -264,7 +236,8 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
@ -282,7 +255,8 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
@ -300,7 +274,8 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
@ -318,7 +293,8 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
@ -336,23 +312,25 @@ suite "Leopard Encode/Decode":
|
||||
try:
|
||||
encoder = LeoEncoder.init(bufSize, buffers, parity).tryGet()
|
||||
decoder = LeoDecoder.init(bufSize, buffers, parity).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder).tryGet()
|
||||
testPackets(buffers, parity, bufSize, dataLoses, parityLoses, encoder, decoder)
|
||||
.tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
|
||||
suite "Leopard use same encoder/decoder multiple times":
|
||||
var
|
||||
encoder: LeoEncoder
|
||||
decoder: LeoDecoder
|
||||
var
|
||||
encoder: LeoEncoder
|
||||
decoder: LeoDecoder
|
||||
|
||||
try:
|
||||
encoder = LeoEncoder.init(4096, 800, 800).tryGet()
|
||||
decoder = LeoDecoder.init(4096, 800, 800).tryGet()
|
||||
for i in 0..10:
|
||||
let lost = 40 * i
|
||||
test "Encode/Decode using same encoder/decoder - lost data = " & $lost & " lost parity = " & $lost:
|
||||
testPackets(800, 800, 4096, 40 * i, 40 * i, encoder, decoder).tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
try:
|
||||
encoder = LeoEncoder.init(4096, 800, 800).tryGet()
|
||||
decoder = LeoDecoder.init(4096, 800, 800).tryGet()
|
||||
for i in 0 .. 10:
|
||||
let lost = 40 * i
|
||||
test "Encode/Decode using same encoder/decoder - lost data = " & $lost &
|
||||
" lost parity = " & $lost:
|
||||
testPackets(800, 800, 4096, 40 * i, 40 * i, encoder, decoder).tryGet()
|
||||
finally:
|
||||
encoder.free()
|
||||
decoder.free()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user