Revert "feat: enhance encode/decode procs for multithreading support (#20)"

This reverts commit 03f71498fc413e650f8f1a6010be741972c2d3d6.

leopard/leopard.nim

@@ -48,8 +48,8 @@ type
 
 func encode*(
   self: var LeoEncoder,
-  data,parity: ptr UncheckedArray[ptr UncheckedArray[byte]],
+  data,
-  dataLen,parityLen: int ): Result[void, cstring] =
+  parity: var openArray[seq[byte]]): Result[void, cstring] =
   ## Encode a list of buffers in `data` into a number of `bufSize` sized
   ## `parity` buffers
   ##
@@ -57,10 +57,10 @@ func encode*(
   ## `parity` - list of parity `buffers` of size `bufSize`
   ##
 
-  if dataLen != self.buffers:
+  if data.len != self.buffers:
     return err("Number of data buffers should match!")
 
-  if parityLen != self.parity:
+  if parity.len != self.parity:
     return err("Number of parity buffers should match!")
 
   # zero encode work buffer to avoid corrupting with previous run
@@ -68,7 +68,7 @@ func encode*(
     zeroMem(self.workBufferPtr[i], self.bufSize)
 
   # copy data into aligned buffer
-  for i in 0..<dataLen:
+  for i in 0..<data.len:
     copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize)
 
   let
@@ -83,8 +83,8 @@ func encode*(
   if ord(res) != ord(LeopardSuccess):
     return err(leoResultString(res.LeopardResult))
 
-  for i in 0..<parityLen:
+  for i in 0..<parity.len:
-    copyMem(parity[i], self.workBufferPtr[i], self.bufSize)
+    copyMem(addr parity[i][0], self.workBufferPtr[i], self.bufSize)
 
   return ok()
 
@@ -92,8 +92,7 @@ func decode*(
   self: var LeoDecoder,
   data,
   parity,
-  recovered: ptr UncheckedArray[ptr UncheckedArray[byte]],
+  recovered: var openArray[seq[byte]]): Result[void, cstring] =
-  dataLen,parityLen,recoveredLen: int): Result[void, cstring] =
   ## Decode a list of buffers in `data` and `parity` into a list
   ## of `recovered` buffers of `bufSize`. The list of `recovered`
   ## buffers should be match the `Leo.buffers`
@@ -103,13 +102,13 @@ func decode*(
   ## `recovered`  - list of recovered `buffers` of size `bufSize`
   ##
 
-  if dataLen != self.buffers:
+  if data.len != self.buffers:
     return err("Number of data buffers should match!")
 
-  if parityLen != self.parity:
+  if parity.len != self.parity:
     return err("Number of parity buffers should match!")
 
-  if recoveredLen != self.buffers:
+  if recovered.len != self.buffers:
     return err("Number of recovered buffers should match buffers!")
 
   # clean out work and data buffers
@@ -119,25 +118,25 @@ func decode*(
   for i in 0..<self.decodeBufferCount:
     zeroMem(self.decodeBufferPtr[i], self.bufSize)
 
-  for i in 0..<dataLen:
+  for i in 0..<data.len:
     zeroMem(self.dataBufferPtr[i], self.bufSize)
 
   # this is needed because erasures are nil pointers
   var
-    dataPtr = newSeq[LeoBufferPtr](dataLen)
+    dataPtr = newSeq[LeoBufferPtr](data.len)
     parityPtr = newSeq[LeoBufferPtr](self.workBufferCount)
 
   # copy data into aligned buffer
-  for i in 0..<dataLen:
+  for i in 0..<data.len:
-    if not data[i].isNil:
+    if data[i].len > 0:
-      copyMem(self.dataBufferPtr[i],addr data[i][0], self.bufSize)
+      copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize)
       dataPtr[i] = self.dataBufferPtr[i]
     else:
       dataPtr[i] = nil
 
   # copy parity into aligned buffer
   for i in 0..<self.workBufferCount:
-    if i < parityLen and not parity[i].isNil:
+    if i < parity.len and parity[i].len > 0:
       copyMem(self.workBufferPtr[i], addr parity[i][0], self.bufSize)
       parityPtr[i] = self.workBufferPtr[i]
     else:

tests/helpers.nim

@@ -24,22 +24,22 @@ proc randomCRCPacket*(data: var openArray[byte]) =
 
     copyMem(addr data[4], unsafeAddr crc, sizeof(crc))
 
-proc checkCRCPacket*(data: ptr UncheckedArray[byte], len: int): bool =
+proc checkCRCPacket*(data: openArray[byte]): bool =
-  if len < 16:
+  if data.len < 16:
-    for i in 1..<len:
+    for d in data[1..data.high]:
-      if data[i] != data[0]:
+      if d != data[0]:
         raise (ref Defect)(msg: "Packet don't match")
   else:
     var
-      crc = len.uint32
+      crc = data.len.uint32
       packCrc: uint32
       packSize: uint32
 
     copyMem(addr packSize, unsafeAddr data[0], sizeof(packSize))
-    if packSize != len.uint:
+    if packSize != data.len.uint:
       raise (ref Defect)(msg: "Packet size don't match!")
 
-    for i in 4..<len:
+    for i in 4..<data.len:
       let v = data[i]
       crc = (crc shl 3) and (crc shr (32 - 3))
       crc += v
@@ -49,43 +49,19 @@ proc checkCRCPacket*(data: ptr UncheckedArray[byte], len: int): bool =
     if packCrc == crc:
       return true
 
-proc dropRandomIdx*(bufs: ptr UncheckedArray[ptr UncheckedArray[byte]], bufsLen,dropCount: int) =
+proc dropRandomIdx*(bufs: var openArray[seq[byte]], dropCount: int) =
   var
     count = 0
     dups: seq[int]
-    size = bufsLen
+    size = bufs.len
 
   while count < dropCount:
     let i = rand(0..<size)
     if dups.find(i) == -1:
       dups.add(i)
-      bufs[i]=nil
+      bufs[i].setLen(0)
       count.inc
 
-proc createDoubleArray*(
-    outerLen, innerLen: int
-): ptr UncheckedArray[ptr UncheckedArray[byte]] =
-  # Allocate outer array
-  result = cast[ptr UncheckedArray[ptr UncheckedArray[byte]]](alloc0(
-    sizeof(ptr UncheckedArray[byte]) * outerLen
-  ))
-
-  # Allocate each inner array
-  for i in 0 ..< outerLen:
-    result[i] = cast[ptr UncheckedArray[byte]](alloc0(sizeof(byte) * innerLen))
-
-proc freeDoubleArray*(
-    arr: ptr UncheckedArray[ptr UncheckedArray[byte]], outerLen: int
-) =
-  # Free each inner array
-  for i in 0 ..< outerLen:
-    if not arr[i].isNil:
-      dealloc(arr[i])
-
-  # Free outer array
-  if not arr.isNil:
-    dealloc(arr)
-
 proc testPackets*(
   buffers,
   parity,
@@ -96,37 +72,35 @@ proc testPackets*(
   decoder: var LeoDecoder): Result[void, cstring] =
 
   var
-    dataBuf = createDoubleArray(buffers, bufSize)
+    dataBuf = newSeqOfCap[seq[byte]](buffers)
-    parityBuf = createDoubleArray(parity, bufSize)
+    parityBuf = newSeqOfCap[seq[byte]](parity)
-    recoveredBuf = createDoubleArray(buffers, bufSize)
+    recoveredBuf = newSeqOfCap[seq[byte]](buffers)
-  
-  defer: 
-    freeDoubleArray(dataBuf, buffers)
-    freeDoubleArray(parityBuf, parity)
-    freeDoubleArray(recoveredBuf, buffers)
 
-
+  for _ in 0..<buffers:
-
-  for i in 0..<buffers:
     var
       dataSeq = newSeq[byte](bufSize)
 
     randomCRCPacket(dataSeq)
-    copyMem(dataBuf[i],addr dataSeq[0],bufSize)
+    dataBuf.add(dataSeq)
 
-  encoder.encode(dataBuf, parityBuf,buffers,parity).tryGet()
+    recoveredBuf.add(newSeq[byte](bufSize))
+
+  for _ in 0..<parity:
+    parityBuf.add(newSeq[byte](bufSize))
+
+  encoder.encode(dataBuf, parityBuf).tryGet()
 
   if dataLosses > 0:
-    dropRandomIdx(dataBuf,buffers, dataLosses)
+    dropRandomIdx(dataBuf, dataLosses)
 
   if parityLosses > 0:
-    dropRandomIdx(parityBuf,parity,parityLosses)
+    dropRandomIdx(parityBuf, parityLosses)
 
-  decoder.decode(dataBuf, parityBuf, recoveredBuf,buffers,parity,buffers).tryGet()
+  decoder.decode(dataBuf, parityBuf, recoveredBuf).tryGet()
 
-  for i in 0..<buffers:
+  for i, d in dataBuf:
-    if dataBuf[i].isNil:
+    if d.len <= 0:
-      if not checkCRCPacket(recoveredBuf[i],bufSize):
+      if not checkCRCPacket(recoveredBuf[i]):
         return err(("Check failed for packet " & $i).cstring)
 
   ok()

tests/testleopard.nim

@@ -30,74 +30,51 @@ suite "Leopard Parametrization":
 
   test "Should not allow encoding with invalid data buffer counts":
     var
-      dataLen =3
-      parityLen = 2
       leo = LeoEncoder.init(64, 4, 2).tryGet()
-      data = createDoubleArray(dataLen, 64)
+      data = newSeq[seq[byte]](3)
-      parity = createDoubleArray(parityLen, 64)
+      parity = newSeq[seq[byte]](2)
-    defer:
+
-      freeDoubleArray(data, dataLen)
-      freeDoubleArray(parity, parityLen)
     check:
-      leo.encode(data, parity,dataLen,parityLen).error == "Number of data buffers should match!"
+      leo.encode(data, parity).error == "Number of data buffers should match!"
 
   test "Should not allow encoding with invalid parity buffer counts":
     var
-      dataLen =4
-      parityLen = 3
       leo = LeoEncoder.init(64, 4, 2).tryGet()
-      data = createDoubleArray(dataLen, 64)
+      data = newSeq[seq[byte]](4)
-      parity = createDoubleArray(parityLen, 64)
+      parity = newSeq[seq[byte]](3)
-    
-    defer:
-      freeDoubleArray(data, dataLen)
-      freeDoubleArray(parity, parityLen)
 
     check:
-      leo.encode(data, parity,dataLen,parityLen).error == "Number of parity buffers should match!"
+      leo.encode(data, parity).error == "Number of parity buffers should match!"
 
   test "Should not allow decoding with invalid data buffer counts":
     var
-      dataLen =3
-      parityLen = 2
       leo = LeoDecoder.init(64, 4, 2).tryGet()
-      data = createDoubleArray(dataLen, 64)
+      data = newSeq[seq[byte]](3)
-      parity = createDoubleArray(parityLen, 64)
+      parity = newSeq[seq[byte]](2)
-      recovered = createDoubleArray(dataLen, 64)
+      recovered = newSeq[seq[byte]](3)
-    
-    defer:
-      freeDoubleArray(data, dataLen)
-      freeDoubleArray(parity, parityLen)
-      freeDoubleArray(recovered, dataLen)
 
     check:
-      leo.decode(data, parity, recovered,dataLen,parityLen,dataLen).error == "Number of data buffers should match!"
+      leo.decode(data, parity, recovered).error == "Number of data buffers should match!"
 
   test "Should not allow decoding with invalid data buffer counts":
     var
-      dataLen =4
-      parityLen = 1
-      recoveredLen = 3
       leo = LeoDecoder.init(64, 4, 2).tryGet()
-      data = createDoubleArray(dataLen, 64)
+      data = newSeq[seq[byte]](4)
-      parity = createDoubleArray(parityLen, 64)
+      parity = newSeq[seq[byte]](1)
-      recovered = createDoubleArray(recoveredLen, 64)
+      recovered = newSeq[seq[byte]](3)
 
     check:
-      leo.decode(data, parity, recovered,dataLen,parityLen,recoveredLen).error == "Number of parity buffers should match!"
+      leo.decode(data, parity, recovered).error == "Number of parity buffers should match!"
 
   test "Should not allow decoding with invalid data buffer counts":
     var
-      dataLen =4
-      parityLen = 2
-      recoveredLen = 3
       leo = LeoDecoder.init(64, 4, 2).tryGet()
-      data = createDoubleArray(dataLen, 64)
+      data = newSeq[seq[byte]](4)
-      parity = createDoubleArray(parityLen, 64)
+      parity = newSeq[seq[byte]](2)
-      recovered = createDoubleArray(recoveredLen, 64)
+      recovered = newSeq[seq[byte]](3)
 
     check:
-      leo.decode(data, parity, recovered,dataLen,parityLen,recoveredLen).error == "Number of recovered buffers should match buffers!"
+      leo.decode(data, parity, recovered).error == "Number of recovered buffers should match buffers!"
 
 suite "Leopard simple Encode/Decode":
   const
@@ -109,76 +86,70 @@ suite "Leopard simple Encode/Decode":
   var
     encoder: LeoEncoder
     decoder: LeoDecoder
-    data: ptr UncheckedArray[ptr UncheckedArray[byte]]
+    data: seq[seq[byte]]
-    parity: ptr UncheckedArray[ptr UncheckedArray[byte]]
+    parity: seq[seq[byte]]
-    recovered: ptr UncheckedArray[ptr UncheckedArray[byte]]
+    recovered: seq[seq[byte]]
 
   setup:
     encoder = LeoEncoder.init(BufferSize, DataCount, ParityCount).tryGet()
     decoder = LeoDecoder.init(BufferSize, DataCount, ParityCount).tryGet()
-    data = createDoubleArray(DataCount, BufferSize)
+    data = newSeq[seq[byte]](DataCount)
-    parity = createDoubleArray(ParityCount, BufferSize)
+    parity = newSeq[seq[byte]](ParityCount)
-    recovered = createDoubleArray(DataCount, BufferSize)
+    recovered = newSeq[seq[byte]](DataCount)
 
   teardown:
-    freeDoubleArray(data, DataCount)
-    freeDoubleArray(parity, ParityCount)
-    freeDoubleArray(recovered, DataCount)
     encoder.free()
     decoder.free()
 
   test "Test 2 data loses out of 4 possible":
     for i in 0..<DataCount:
-      var
+      data[i] = newSeq[byte](BufferSize)
-        str = TestString & " " & $i
+      recovered[i] = newSeq[byte](BufferSize)
-
-      copyMem(data[i], addr str[0], str.len)
-
-
-    encoder.encode(data, parity,DataCount,ParityCount).tryGet()
-
-    var
-      data1 =cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
-      data2 = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
-
-    defer: 
-      deallocShared(data1)
-      deallocShared(data2)
-    
-    copyMem(data1,data[0], BufferSize)
-    copyMem(data2,data[1], BufferSize)
-    
-    data[0]=nil
-    data[1]=nil
-
-    decoder.decode(data, parity, recovered,DataCount,ParityCount,DataCount).tryGet()
-
-    check equalMem(recovered[0], data1, BufferSize)
-    check equalMem(recovered[1], data2, BufferSize)
-
-  test "Test 1 data and 1 parity loss out of 4 possible":
-    for i in 0..<DataCount:
       var
         str = TestString & " " & $i
 
       copyMem(addr data[i][0], addr str[0], str.len)
 
-    encoder.encode(data, parity,DataCount,ParityCount).tryGet()
+    for i in 0..<ParityCount:
+      parity[i] = newSeq[byte](BufferSize)
 
-    
+    encoder.encode(data, parity).tryGet()
-    var data1 = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * BufferSize))
 
-    defer: deallocShared(data1)
+    var
+      data1 = data[0]
+      data2 = data[1]
 
-    copyMem(data1,data[0], BufferSize)
+    data[0].setLen(0)
+    data[1].setLen(0)
 
-    data[0]=nil
+    decoder.decode(data, parity, recovered).tryGet()
-    parity[0]=nil
 
-    decoder.decode(data, parity, recovered,DataCount,ParityCount,DataCount).tryGet()
+    check recovered[0] == data1
+    check recovered[1] == data2
 
-    check equalMem(recovered[0], data1, BufferSize)
+  test "Test 1 data and 1 parity loss out of 4 possible":
+    for i in 0..<DataCount:
+      data[i] = newSeq[byte](BufferSize)
+      recovered[i] = newSeq[byte](BufferSize)
 
+      var
+        str = TestString & " " & $i
+
+      copyMem(addr data[i][0], addr str[0], str.len)
+
+    for i in 0..<ParityCount:
+      parity[i] = newSeq[byte](BufferSize)
+
+    encoder.encode(data, parity).tryGet()
+
+    var
+      data1 = data[0]
+
+    data[0].setLen(0)
+    parity[0].setLen(0)
+
+    decoder.decode(data, parity, recovered).tryGet()
+    check recovered[0] == data1
 
 suite "Leopard Encode/Decode":
   test "bufSize = 4096, K = 800, M = 200 - drop data = 200 data":