logos-storage/nim-leopard
1
0
Fork 0
mirror of https://github.com/logos-storage/nim-leopard.git synced 2026-01-02 05:33:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enhance encode/decode procs for multithreading support (#20)

Browse Source 
* update encode proc signature

* add new decode proc

* remove old encode/decode procs

* fix tests
This commit is contained in:
munna0908 2025-02-12 00:44:18 +05:30 committed by GitHub
parent aa5f8d7748
commit 03f71498fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 150 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
leopard/leopard.nim
View File

@ -48,8 +48,8 @@ type
func encode*(
self: var LeoEncoder,
data,
parity: var openArray[seq[byte]]): Result[void, cstring] =
data,parity: ptr UncheckedArray[ptr UncheckedArray[byte]],
dataLen,parityLen: int ): Result[void, cstring] =
## Encode a list of buffers in `data` into a number of `bufSize` sized
## `parity` buffers
##
@ -57,10 +57,10 @@ func encode*(
## `parity` - list of parity `buffers` of size `bufSize`
##
if data.len != self.buffers:
if dataLen != self.buffers:
return err("Number of data buffers should match!")
if parity.len != self.parity:
if parityLen != self.parity:
return err("Number of parity buffers should match!")
# zero encode work buffer to avoid corrupting with previous run
@ -68,7 +68,7 @@ func encode*(
zeroMem(self.workBufferPtr[i], self.bufSize)
# copy data into aligned buffer
for i in 0..<data.len:
for i in 0..<dataLen:
copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize)
let
@ -83,8 +83,8 @@ func encode*(
if ord(res) != ord(LeopardSuccess):
return err(leoResultString(res.LeopardResult))
for i in 0..<parity.len:
copyMem(addr parity[i][0], self.workBufferPtr[i], self.bufSize)
for i in 0..<parityLen:
copyMem(parity[i], self.workBufferPtr[i], self.bufSize)
return ok()
@ -92,7 +92,8 @@ func decode*(
self: var LeoDecoder,
data,
parity,
recovered: var openArray[seq[byte]]): Result[void, cstring] =
recovered: ptr UncheckedArray[ptr UncheckedArray[byte]],
dataLen,parityLen,recoveredLen: int): Result[void, cstring] =
## Decode a list of buffers in `data` and `parity` into a list
## of `recovered` buffers of `bufSize`. The list of `recovered`
## buffers should be match the `Leo.buffers`
@ -102,13 +103,13 @@ func decode*(
## `recovered` - list of recovered `buffers` of size `bufSize`
##
if data.len != self.buffers:
if dataLen != self.buffers:
return err("Number of data buffers should match!")
if parity.len != self.parity:
if parityLen != self.parity:
return err("Number of parity buffers should match!")
if recovered.len != self.buffers:
if recoveredLen != self.buffers:
return err("Number of recovered buffers should match buffers!")
# clean out work and data buffers
@ -118,25 +119,25 @@ func decode*(
for i in 0..<self.decodeBufferCount:
zeroMem(self.decodeBufferPtr[i], self.bufSize)
for i in 0..<data.len:
for i in 0..<dataLen:
zeroMem(self.dataBufferPtr[i], self.bufSize)
# this is needed because erasures are nil pointers
var
dataPtr = newSeq[LeoBufferPtr](data.len)
dataPtr = newSeq[LeoBufferPtr](dataLen)
parityPtr = newSeq[LeoBufferPtr](self.workBufferCount)
# copy data into aligned buffer
for i in 0..<data.len:
if data[i].len > 0:
copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize)
for i in 0..<dataLen:
if not data[i].isNil:
copyMem(self.dataBufferPtr[i],addr data[i][0], self.bufSize)
dataPtr[i] = self.dataBufferPtr[i]
else:
dataPtr[i] = nil
# copy parity into aligned buffer
for i in 0..<self.workBufferCount:
if i < parity.len and parity[i].len > 0:
if i < parityLen and not parity[i].isNil:
copyMem(self.workBufferPtr[i], addr parity[i][0], self.bufSize)
parityPtr[i] = self.workBufferPtr[i]
else:

80
tests/helpers.nim
View File

@ -24,22 +24,22 @@ proc randomCRCPacket*(data: var openArray[byte]) =
copyMem(addr data[4], unsafeAddr crc, sizeof(crc))
proc checkCRCPacket*(data: openArray[byte]): bool =
if data.len < 16:
for d in data[1..data.high]:
if d != data[0]:
proc checkCRCPacket*(data: ptr UncheckedArray[byte], len: int): bool =
if len < 16:
for i in 1..<len:
if data[i] != data[0]:
raise (ref Defect)(msg: "Packet don't match")
else:
var
crc = data.len.uint32
crc = len.uint32
packCrc: uint32
packSize: uint32
copyMem(addr packSize, unsafeAddr data[0], sizeof(packSize))
if packSize != data.len.uint:
if packSize != len.uint:
raise (ref Defect)(msg: "Packet size don't match!")
for i in 4..<data.len:
for i in 4..<len:
let v = data[i]
crc = (crc shl 3) and (crc shr (32 - 3))
crc += v
@ -49,19 +49,43 @@ proc checkCRCPacket*(data: openArray[byte]): bool =
if packCrc == crc:
return true
proc dropRandomIdx*(bufs: var openArray[seq[byte]], dropCount: int) =
proc dropRandomIdx*(bufs: ptr UncheckedArray[ptr UncheckedArray[byte]], bufsLen,dropCount: int) =
var
count = 0
dups: seq[int]
size = bufs.len
size = bufsLen
while count < dropCount:
let i = rand(0..<size)
if dups.find(i) == -1:
dups.add(i)
bufs[i].setLen(0)
bufs[i]=nil
count.inc
proc createDoubleArray*(
outerLen, innerLen: int
): ptr UncheckedArray[ptr UncheckedArray[byte]] =
# Allocate outer array
result = cast[ptr UncheckedArray[ptr UncheckedArray[byte]]](alloc0(
sizeof(ptr UncheckedArray[byte]) * outerLen
))
# Allocate each inner array
for i in 0 ..< outerLen:
result[i] = cast[ptr UncheckedArray[byte]](alloc0(sizeof(byte) * innerLen))
proc freeDoubleArray*(
arr: ptr UncheckedArray[ptr UncheckedArray[byte]], outerLen: int
) =
# Free each inner array
for i in 0 ..< outerLen:
if not arr[i].isNil:
dealloc(arr[i])
# Free outer array
if not arr.isNil:
dealloc(arr)
proc testPackets*(
buffers,
parity,
@ -72,35 +96,37 @@ proc testPackets*(
decoder: var LeoDecoder): Result[void, cstring] =
var
dataBuf = newSeqOfCap[seq[byte]](buffers)
parityBuf = newSeqOfCap[seq[byte]](parity)
recoveredBuf = newSeqOfCap[seq[byte]](buffers)
dataBuf = createDoubleArray(buffers, bufSize)
parityBuf = createDoubleArray(parity, bufSize)
recoveredBuf = createDoubleArray(buffers, bufSize)
defer:
freeDoubleArray(dataBuf, buffers)
freeDoubleArray(parityBuf, parity)
freeDoubleArray(recoveredBuf, buffers)
for _ in 0..<buffers:
for i in 0..<buffers:
var
dataSeq = newSeq[byte](bufSize)
randomCRCPacket(dataSeq)
dataBuf.add(dataSeq)
copyMem(dataBuf[i],addr dataSeq[0],bufSize)
recoveredBuf.add(newSeq[byte](bufSize))
for _ in 0..<parity:
parityBuf.add(newSeq[byte](bufSize))
encoder.encode(dataBuf, parityBuf).tryGet()
encoder.encode(dataBuf, parityBuf,buffers,parity).tryGet()
if dataLosses > 0:
dropRandomIdx(dataBuf, dataLosses)
dropRandomIdx(dataBuf,buffers, dataLosses)
if parityLosses > 0:
dropRandomIdx(parityBuf, parityLosses)
dropRandomIdx(parityBuf,parity,parityLosses)
decoder.decode(dataBuf, parityBuf, recoveredBuf).tryGet()
decoder.decode(dataBuf, parityBuf, recoveredBuf,buffers,parity,buffers).tryGet()
for i, d in dataBuf:
if d.len <= 0:
if not checkCRCPacket(recoveredBuf[i]):
for i in 0..<buffers:
if dataBuf[i].isNil:
if not checkCRCPacket(recoveredBuf[i],bufSize):
return err(("Check failed for packet " & $i).cstring)
ok()

129
tests/testleopard.nim
View File

@ -30,51 +30,74 @@ suite "Leopard Parametrization":
test "Should not allow encoding with invalid data buffer counts":
var
dataLen =3
parityLen = 2
leo = LeoEncoder.init(64, 4, 2).tryGet()
data = newSeq[seq[byte]](3)
parity = newSeq[seq[byte]](2)
data = createDoubleArray(dataLen, 64)
parity = createDoubleArray(parityLen, 64)
defer:
freeDoubleArray(data, dataLen)
freeDoubleArray(parity, parityLen)
check:
leo.encode(data, parity).error == "Number of data buffers should match!"
leo.encode(data, parity,dataLen,parityLen).error == "Number of data buffers should match!"
test "Should not allow encoding with invalid parity buffer counts":
var
dataLen =4
parityLen = 3
leo = LeoEncoder.init(64, 4, 2).tryGet()
data = newSeq[seq[byte]](4)
parity = newSeq[seq[byte]](3)
data = createDoubleArray(dataLen, 64)
parity = createDoubleArray(parityLen, 64)
defer:
freeDoubleArray(data, dataLen)
freeDoubleArray(parity, parityLen)
check:
leo.encode(data, parity).error == "Number of parity buffers should match!"
leo.encode(data, parity,dataLen,parityLen).error == "Number of parity buffers should match!"
test "Should not allow decoding with invalid data buffer counts":
var
dataLen =3
parityLen = 2
leo = LeoDecoder.init(64, 4, 2).tryGet()
data = newSeq[seq[byte]](3)
parity = newSeq[seq[byte]](2)
recovered = newSeq[seq[byte]](3)
data = createDoubleArray(dataLen, 64)
parity = createDoubleArray(parityLen, 64)
recovered = createDoubleArray(dataLen, 64)
defer:
freeDoubleArray(data, dataLen)
freeDoubleArray(parity, parityLen)
freeDoubleArray(recovered, dataLen)
check:
leo.decode(data, parity, recovered).error == "Number of data buffers should match!"
leo.decode(data, parity, recovered,dataLen,parityLen,dataLen).error == "Number of data buffers should match!"
test "Should not allow decoding with invalid data buffer counts":
var
dataLen =4
parityLen = 1
recoveredLen = 3
leo = LeoDecoder.init(64, 4, 2).tryGet()
data = newSeq[seq[byte]](4)
parity = newSeq[seq[byte]](1)
recovered = newSeq[seq[byte]](3)
data = createDoubleArray(dataLen, 64)
parity = createDoubleArray(parityLen, 64)
recovered = createDoubleArray(recoveredLen, 64)
check:
leo.decode(data, parity, recovered).error == "Number of parity buffers should match!"
leo.decode(data, parity, recovered,dataLen,parityLen,recoveredLen).error == "Number of parity buffers should match!"
test "Should not allow decoding with invalid data buffer counts":
var
dataLen =4
parityLen = 2
recoveredLen = 3
leo = LeoDecoder.init(64, 4, 2).tryGet()
data = newSeq[seq[byte]](4)
parity = newSeq[seq[byte]](2)
recovered = newSeq[seq[byte]](3)
data = createDoubleArray(dataLen, 64)
parity = createDoubleArray(parityLen, 64)
recovered = createDoubleArray(recoveredLen, 64)
check:
leo.decode(data, parity, recovered).error == "Number of recovered buffers should match buffers!"
leo.decode(data, parity, recovered,dataLen,parityLen,recoveredLen).error == "Number of recovered buffers should match buffers!"
suite "Leopard simple Encode/Decode":
const
@ -86,70 +109,76 @@ suite "Leopard simple Encode/Decode":
var
encoder: LeoEncoder
decoder: LeoDecoder
data: seq[seq[byte]]
parity: seq[seq[byte]]
recovered: seq[seq[byte]]
data: ptr UncheckedArray[ptr UncheckedArray[byte]]
parity: ptr UncheckedArray[ptr UncheckedArray[byte]]
recovered: ptr UncheckedArray[ptr UncheckedArray[byte]]
setup:
encoder = LeoEncoder.init(BufferSize, DataCount, ParityCount).tryGet()
decoder = LeoDecoder.init(BufferSize, DataCount, ParityCount).tryGet()
data = newSeq[seq[byte]](DataCount)
parity = newSeq[seq[byte]](ParityCount)
recovered = newSeq[seq[byte]](DataCount)
data = createDoubleArray(DataCount, BufferSize)
parity = createDoubleArray(ParityCount, BufferSize)
recovered = createDoubleArray(DataCount, BufferSize)
teardown:
freeDoubleArray(data, DataCount)
freeDoubleArray(parity, ParityCount)
freeDoubleArray(recovered, DataCount)
encoder.free()
decoder.free()
test "Test 2 data loses out of 4 possible":
for i in 0..<DataCount:
data[i] = newSeq[byte](BufferSize)
recovered[i] = newSeq[byte](BufferSize)
var
str = TestString & " " & $i
copyMem(addr data[i][0], addr str[0], str.len)
copyMem(data[i], addr str[0], str.len)
for i in 0..<ParityCount:
parity[i] = newSeq[byte](BufferSize)
encoder.encode(data, parity).tryGet()
encoder.encode(data, parity,DataCount,ParityCount).tryGet()
var
data1 = data[0]
data2 = data[1]
data1 =cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
data2 = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
data[0].setLen(0)
data[1].setLen(0)
defer:
deallocShared(data1)
deallocShared(data2)
copyMem(data1,data[0], BufferSize)
copyMem(data2,data[1], BufferSize)
data[0]=nil
data[1]=nil
decoder.decode(data, parity, recovered).tryGet()
decoder.decode(data, parity, recovered,DataCount,ParityCount,DataCount).tryGet()
check recovered[0] == data1
check recovered[1] == data2
check equalMem(recovered[0], data1, BufferSize)
check equalMem(recovered[1], data2, BufferSize)
test "Test 1 data and 1 parity loss out of 4 possible":
for i in 0..<DataCount:
data[i] = newSeq[byte](BufferSize)
recovered[i] = newSeq[byte](BufferSize)
var
str = TestString & " " & $i
copyMem(addr data[i][0], addr str[0], str.len)
for i in 0..<ParityCount:
parity[i] = newSeq[byte](BufferSize)
encoder.encode(data, parity,DataCount,ParityCount).tryGet()
encoder.encode(data, parity).tryGet()
var data1 = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
var
data1 = data[0]
defer: deallocShared(data1)
data[0].setLen(0)
parity[0].setLen(0)
copyMem(data1,data[0], BufferSize)
data[0]=nil
parity[0]=nil
decoder.decode(data, parity, recovered,DataCount,ParityCount,DataCount).tryGet()
check equalMem(recovered[0], data1, BufferSize)
decoder.decode(data, parity, recovered).tryGet()
check recovered[0] == data1
suite "Leopard Encode/Decode":
test "bufSize = 4096, K = 800, M = 200 - drop data = 200 data":