mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-11 09:53:12 +00:00
adding 2D encoding provider
Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
This commit is contained in:
Csaba Kiraly
parent
03113f60d1
commit
6adb498cbc
@ -9,11 +9,27 @@
|
||||
|
||||
import ./erasure/erasure
|
||||
import ./erasure/backends/leopard
|
||||
import ./erasure/backends/leopard2d
|
||||
|
||||
export erasure
|
||||
|
||||
func leoEncoderProvider*(size, buffers, parity: int): EncoderBackend {.raises: [Defect].} =
|
||||
## size: blockSize in bytes
|
||||
## buffers: RS K
|
||||
## parity: RS M=N-K
|
||||
LeoEncoderBackend.new(size, buffers, parity)
|
||||
|
||||
func leoDecoderProvider*(size, buffers, parity: int): DecoderBackend {.raises: [Defect].} =
|
||||
LeoDecoderBackend.new(size, buffers, parity)
|
||||
|
||||
func leoEncoderProvider2D*(blocksize, buffers, parity : int): EncoderBackend {.raises: [Defect].} =
|
||||
LeoEncoderBackend2D.new(blocksize, buffers, parity)
|
||||
|
||||
func leoEncoderProvider2D*(blocksize, k1, m1, k2, m2 : int): EncoderBackend {.raises: [Defect].} =
|
||||
LeoEncoderBackend2D.new(blocksize, k1, m1, k2, m2)
|
||||
|
||||
func leoDecoderProvider2D*(blocksize, buffers, parity : int): DecoderBackend {.raises: [Defect].} =
|
||||
LeoDecoderBackend2D.new(blocksize, buffers, parity)
|
||||
|
||||
func leoDecoderProvider2D*(blocksize, k1, m1, k2, m2 : int): DecoderBackend {.raises: [Defect].} =
|
||||
LeoDecoderBackend2D.new(blocksize, k1, m1, k2, m2)
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
## those terms.
|
||||
|
||||
import std/options
|
||||
from std/math import sqrt
|
||||
import std/sequtils
|
||||
|
||||
import pkg/leopard
|
||||
import pkg/stew/results
|
||||
@ -15,72 +17,204 @@ import pkg/stew/results
|
||||
import ../backend
|
||||
|
||||
type
|
||||
LeoEncoderBackend* = ref object of EncoderBackend
|
||||
encoder*: Option[LeoEncoder]
|
||||
LeoEncoderBackend2D* = ref object of EncoderBackend
|
||||
encoder1*: Option[LeoEncoder]
|
||||
encoder2*: Option[LeoEncoder]
|
||||
k1*, m1*, k2*, m2*: int
|
||||
|
||||
LeoDecoderBackend* = ref object of DecoderBackend
|
||||
decoder*: Option[LeoDecoder]
|
||||
LeoDecoderBackend2D* = ref object of DecoderBackend
|
||||
decoder1*: Option[LeoDecoder]
|
||||
decoder2*: Option[LeoDecoder]
|
||||
k1*, m1*, k2*, m2*: int
|
||||
|
||||
method encode*(
|
||||
self: LeoEncoderBackend,
|
||||
self: LeoEncoderBackend2D,
|
||||
data,
|
||||
parity: var openArray[seq[byte]]): Result[void, cstring] =
|
||||
## Encode using 2D RS encoding.
|
||||
|
||||
if parity.len == 0:
|
||||
return ok()
|
||||
|
||||
var encoder = if self.encoder.isNone:
|
||||
self.encoder = (? LeoEncoder.init(
|
||||
var encoder1 = if self.encoder1.isNone:
|
||||
self.encoder1 = (? LeoEncoder.init(
|
||||
self.blockSize,
|
||||
self.buffers,
|
||||
self.parity)).some
|
||||
self.encoder.get()
|
||||
self.k1,
|
||||
self.m1)).some
|
||||
self.encoder1.get()
|
||||
else:
|
||||
self.encoder.get()
|
||||
self.encoder1.get()
|
||||
|
||||
encoder.encode(data, parity)
|
||||
for i in 0 ..< self.k2:
|
||||
#encoder1.encode(data[i*self.k1 ..< (i+1)+self.k1], parity[i*self.m1 ..< (i+1)*self.m1])
|
||||
var
|
||||
data1 = newSeq[seq[byte]](self.k1)
|
||||
parity1 = newSeq[seq[byte]](self.m1)
|
||||
for j in 0 ..< self.k1:
|
||||
shallowCopy(data1[j], data[i*self.k1 + j])
|
||||
for j in 0 ..< self.m1:
|
||||
shallowCopy(parity1[j], parity[i*self.m1 + j])
|
||||
let res = encoder1.encode(data1, parity1)
|
||||
if res.isErr:
|
||||
return res
|
||||
|
||||
var encoder2 = if self.encoder2.isNone:
|
||||
self.encoder2 = (? LeoEncoder.init(
|
||||
self.blockSize,
|
||||
self.k1,
|
||||
self.m1)).some
|
||||
self.encoder2.get()
|
||||
else:
|
||||
self.encoder2.get()
|
||||
|
||||
for i in 0 ..< self.k1 + self.m1:
|
||||
var
|
||||
data2 = newSeq[seq[byte]](self.k2)
|
||||
parity2 = newSeq[seq[byte]](self.m2)
|
||||
for j in 0 ..< self.k2:
|
||||
if i < self.k1:
|
||||
shallowCopy(data2[j], data[i + self.k1 * j])
|
||||
else:
|
||||
shallowCopy(data2[j], parity[i + self.m1 * j])
|
||||
for j in 0 ..< self.m2:
|
||||
shallowCopy(parity2[j], parity[self.k2 * self.m1 + i + self.m2 * j])
|
||||
let res = encoder2.encode(data2, parity2)
|
||||
if res.isErr:
|
||||
return res
|
||||
|
||||
ok()
|
||||
|
||||
method decode*(
|
||||
self: LeoDecoderBackend,
|
||||
self: LeoDecoderBackend2D,
|
||||
data,
|
||||
parity,
|
||||
recovered: var openArray[seq[byte]]): Result[void, cstring] =
|
||||
## Decode 2D RS encoded data. Only do one simple recovery attempt,
|
||||
## since underlying encoder misses recovery of parity segments.
|
||||
|
||||
var decoder = if self.decoder.isNone:
|
||||
self.decoder = (? LeoDecoder.init(
|
||||
var decoder1 = if self.decoder1.isNone:
|
||||
self.decoder1 = (? LeoDecoder.init(
|
||||
self.blockSize,
|
||||
self.buffers,
|
||||
self.parity)).some
|
||||
self.decoder.get()
|
||||
self.k1,
|
||||
self.m1)).some
|
||||
self.decoder1.get()
|
||||
else:
|
||||
self.decoder.get()
|
||||
self.decoder1.get()
|
||||
|
||||
decoder.decode(data, parity, recovered)
|
||||
var
|
||||
missing = 0
|
||||
repaired = 0
|
||||
|
||||
method release*(self: LeoEncoderBackend) =
|
||||
if self.encoder.isSome:
|
||||
self.encoder.get().free()
|
||||
for i in 0 ..< self.k2:
|
||||
var
|
||||
data1 = newSeq[seq[byte]](self.k1)
|
||||
parity1 = newSeq[seq[byte]](self.m1)
|
||||
recovered1 = newSeq[seq[byte]](self.k1)
|
||||
for j in 0 ..< self.k1:
|
||||
shallowCopy(data1[j], data[i*self.k1 + j])
|
||||
shallowCopy(recovered1[j], recovered[i*self.k1 + j])
|
||||
if data1[j].len == 0:
|
||||
missing += 1
|
||||
for j in 0 ..< self.m1:
|
||||
shallowCopy(parity1[j], parity[i*self.m1 + j])
|
||||
if parity1[j].len == 0:
|
||||
missing += 1
|
||||
|
||||
method release*(self: LeoDecoderBackend) =
|
||||
if self.decoder.isSome:
|
||||
self.decoder.get().free()
|
||||
let res = decoder1.decode(data1, parity1, recovered1)
|
||||
if res.isOk():
|
||||
#missing parity is not recovered by the Leopard decode API
|
||||
for j in 0 ..< self.k1:
|
||||
if data1[j].len == 0:
|
||||
repaired += 1
|
||||
|
||||
if missing == repaired:
|
||||
ok()
|
||||
else:
|
||||
err("can't repair in a single round")
|
||||
|
||||
method release*(self: LeoEncoderBackend2D) =
|
||||
if self.encoder1.isSome:
|
||||
self.encoder1.get().free()
|
||||
if self.encoder2.isSome:
|
||||
self.encoder2.get().free()
|
||||
|
||||
method release*(self: LeoDecoderBackend2D) =
|
||||
if self.decoder1.isSome:
|
||||
self.decoder1.get().free()
|
||||
if self.decoder2.isSome:
|
||||
self.decoder2.get().free()
|
||||
|
||||
func new*(
|
||||
T: type LeoEncoderBackend,
|
||||
T: type LeoEncoderBackend2D,
|
||||
blockSize,
|
||||
buffers,
|
||||
parity: int): T =
|
||||
k1,
|
||||
m1,
|
||||
k2,
|
||||
m2: int): T =
|
||||
## Initialize 2D encoder.
|
||||
T(
|
||||
blockSize: blockSize,
|
||||
buffers: buffers,
|
||||
parity: parity)
|
||||
buffers: k1*k2, # store K and M for compatibility
|
||||
parity: (k1+m1) * (k2*m2) - k1*k2,
|
||||
k1: k1,
|
||||
m1: m1,
|
||||
k2: k2,
|
||||
m2: m2
|
||||
)
|
||||
|
||||
func new*(
|
||||
T: type LeoDecoderBackend,
|
||||
T: type LeoEncoderBackend2D,
|
||||
blockSize,
|
||||
buffers,
|
||||
parity: int): T =
|
||||
k,
|
||||
m: int): T =
|
||||
## Initialize 2D encoder using "product" k and m, assuming these are squares.
|
||||
## TODO: check that params are actually squares.
|
||||
let
|
||||
k1, k2 = sqrt(k.float).int
|
||||
m1, m2 = sqrt((k + m).float).int - k1
|
||||
T(
|
||||
blockSize: blockSize,
|
||||
buffers: buffers,
|
||||
parity: parity)
|
||||
buffers: k,
|
||||
parity: m,
|
||||
k1: k1,
|
||||
m1: m1,
|
||||
k2: k2,
|
||||
m2: m2
|
||||
)
|
||||
|
||||
func new*(
|
||||
T: type LeoDecoderBackend2D,
|
||||
blockSize,
|
||||
k1,
|
||||
m1,
|
||||
k2,
|
||||
m2: int): T =
|
||||
T(
|
||||
blockSize: blockSize,
|
||||
buffers: k1*k2, # store K and M for compatibility
|
||||
parity: (k1+m1) * (k2*m2) - k1*k2,
|
||||
k1: k1,
|
||||
m1: m1,
|
||||
k2: k2,
|
||||
m2: m2
|
||||
)
|
||||
|
||||
## TODO: initalize using sqrt, failing if not squares
|
||||
func new*(
|
||||
T: type LeoDecoderBackend2D,
|
||||
blockSize,
|
||||
k,
|
||||
m: int): T =
|
||||
let
|
||||
k1, k2 = sqrt(k.float).int
|
||||
m1, m2 = sqrt((k + m).float).int - k1
|
||||
## TODO check
|
||||
T(
|
||||
blockSize: blockSize,
|
||||
buffers: k,
|
||||
parity: m,
|
||||
k1: k1,
|
||||
m1: m1,
|
||||
k2: k2,
|
||||
m2: m2
|
||||
)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user