mirror of
https://github.com/logos-storage/nim-leopard.git
synced 2026-01-05 23:23:07 +00:00
initial implementation and tests
This commit is contained in:
parent
4d89e44e0d
commit
0d2fae23bf
5
.editorconfig
Normal file
5
.editorconfig
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
[*]
|
||||||
|
indent_style = space
|
||||||
|
insert_final_newline = true
|
||||||
|
indent_size = 2
|
||||||
|
trim_trailing_whitespace = true
|
||||||
1
.gitattributes
vendored
Normal file
1
.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
* text=auto eol=lf
|
||||||
154
.github/workflows/test.yml
vendored
Normal file
154
.github/workflows/test.yml
vendored
Normal file
@ -0,0 +1,154 @@
|
|||||||
|
name: Tests
|
||||||
|
|
||||||
|
on: [pull_request, push]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
tests:
|
||||||
|
env:
|
||||||
|
NPROC: 2
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
cache_nonce: [ 1 ]
|
||||||
|
nim_version: [ 1.2.18, 1.4.8, 1.6.4 ]
|
||||||
|
platform:
|
||||||
|
- {
|
||||||
|
icon: 🐧,
|
||||||
|
label: Linux,
|
||||||
|
os: ubuntu,
|
||||||
|
shell: bash --noprofile --norc -eo pipefail
|
||||||
|
}
|
||||||
|
- {
|
||||||
|
icon: 🍎,
|
||||||
|
label: macOS,
|
||||||
|
os: macos,
|
||||||
|
shell: bash --noprofile --norc -eo pipefail
|
||||||
|
}
|
||||||
|
- {
|
||||||
|
icon: 🏁,
|
||||||
|
label: Windows,
|
||||||
|
os: windows,
|
||||||
|
shell: msys2
|
||||||
|
}
|
||||||
|
name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim_version }}
|
||||||
|
runs-on: ${{ matrix.platform.os }}-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: ${{ matrix.platform.shell }} {0}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
# - name: Install tools and libraries via APT (Linux)
|
||||||
|
# if: matrix.platform.os == 'ubuntu'
|
||||||
|
# run: |
|
||||||
|
# sudo apt update
|
||||||
|
# sudo apt install -y \
|
||||||
|
# ...
|
||||||
|
|
||||||
|
- name: Install tools and libraries via Homebrew (macOS)
|
||||||
|
if: matrix.platform.os == 'macos'
|
||||||
|
run: |
|
||||||
|
brew update
|
||||||
|
brew install \
|
||||||
|
findutils \
|
||||||
|
libomp
|
||||||
|
|
||||||
|
- name: Install tools and libraries via MSYS2 (Windows)
|
||||||
|
if: matrix.platform.os == 'windows'
|
||||||
|
uses: msys2/setup-msys2@v2
|
||||||
|
with:
|
||||||
|
msystem: UCRT64
|
||||||
|
install: >
|
||||||
|
base-devel
|
||||||
|
git
|
||||||
|
mingw-w64-ucrt-x86_64-cmake
|
||||||
|
mingw-w64-ucrt-x86_64-toolchain
|
||||||
|
|
||||||
|
- name: Checkout sources from GitHub
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
submodules: true
|
||||||
|
|
||||||
|
- name: Calculate cache member paths
|
||||||
|
id: calc-paths
|
||||||
|
run: |
|
||||||
|
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||||
|
echo "::set-output name=bash_env::$(cygpath -m "${HOME}")/.bash_env"
|
||||||
|
echo "::set-output name=choosenim::$(cygpath -m "${USERPROFILE}")/.choosenim"
|
||||||
|
echo "::set-output name=nimble::$(cygpath -m "${HOME}")/.nimble"
|
||||||
|
else
|
||||||
|
echo "::set-output name=bash_env::${HOME}/.bash_env"
|
||||||
|
echo "::set-output name=choosenim::${HOME}/.choosenim"
|
||||||
|
echo "::set-output name=nimble::${HOME}/.nimble"
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Restore choosenim and Nim tooling from cache
|
||||||
|
id: choosenim-nim-tooling-cache
|
||||||
|
uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
${{ steps.calc-paths.outputs.bash_env }}
|
||||||
|
${{ steps.calc-paths.outputs.choosenim }}
|
||||||
|
${{ steps.calc-paths.outputs.nimble }}/bin
|
||||||
|
key: ${{ matrix.platform.os }}-nim_version:${{ matrix.nim_version }}-cache_nonce:${{ matrix.cache_nonce }}
|
||||||
|
|
||||||
|
- name: Install choosenim and Nim tooling
|
||||||
|
if: steps.choosenim-nim-tooling-cache.outputs.cache-hit != 'true'
|
||||||
|
run: |
|
||||||
|
mkdir -p "${HOME}/Downloads"
|
||||||
|
cd "${HOME}/Downloads"
|
||||||
|
curl https://nim-lang.org/choosenim/init.sh -sSf -O
|
||||||
|
chmod +x init.sh
|
||||||
|
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||||
|
mkdir -p "$(cygpath "${USERPROFILE}")/.nimble/bin"
|
||||||
|
fi
|
||||||
|
CHOOSENIM_CHOOSE_VERSION=${{ matrix.nim_version }} ./init.sh -y
|
||||||
|
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||||
|
mv "$(cygpath "${USERPROFILE}")/.nimble" "${HOME}/"
|
||||||
|
# intention is to rely only on libs provided by the OS and MSYS2 env
|
||||||
|
rm -rf "${HOME}/.nimble/bin/"*.dll
|
||||||
|
rm -rf "${HOME}/.nimble/bin/"*.pem
|
||||||
|
fi
|
||||||
|
echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env"
|
||||||
|
echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env"
|
||||||
|
|
||||||
|
- name: Install project dependencies
|
||||||
|
run: |
|
||||||
|
source "${HOME}/.bash_env"
|
||||||
|
cd "${NIMBLE_DIR}/bin"
|
||||||
|
# delete broken symlinks, which can arise because e.g. the cache
|
||||||
|
# restored a symlink that points to an executable within
|
||||||
|
# ../pkgs/foo-1.2.3/ but the project's .nimble file has been updated
|
||||||
|
# to install foo-#head. In the case of a broken symlink, nimble's
|
||||||
|
# auto-overwrite fails (only sometimes? only on macOS?)
|
||||||
|
if [[ ${{ matrix.platform.os }} = macos ]]; then
|
||||||
|
gfind . -xtype l -delete
|
||||||
|
else
|
||||||
|
find . -xtype l -delete
|
||||||
|
fi
|
||||||
|
cd -
|
||||||
|
nimble --accept install
|
||||||
|
|
||||||
|
- name: Build and run tests
|
||||||
|
run: |
|
||||||
|
source "${HOME}/.bash_env"
|
||||||
|
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||||
|
touch tests/test_leopard.exe
|
||||||
|
else
|
||||||
|
touch tests/test_leopard
|
||||||
|
fi
|
||||||
|
if [[ ${{ matrix.platform.os }} = macos ]]; then
|
||||||
|
export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}"
|
||||||
|
export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib"
|
||||||
|
nimble test -d:verbose -d:release -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib"
|
||||||
|
else
|
||||||
|
nimble test -d:verbose -d:release
|
||||||
|
fi
|
||||||
|
if [[ ${{ matrix.platform.os }} = macos ]]; then
|
||||||
|
echo
|
||||||
|
echo otool -L tests/test_leopard
|
||||||
|
otool -L tests/test_leopard
|
||||||
|
else
|
||||||
|
echo
|
||||||
|
echo ldd tests/test_leopard
|
||||||
|
ldd tests/test_leopard
|
||||||
|
fi
|
||||||
13
.gitignore
vendored
Normal file
13
.gitignore
vendored
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
*
|
||||||
|
!*/
|
||||||
|
!*.*
|
||||||
|
*.a
|
||||||
|
*.dll
|
||||||
|
*.dylib
|
||||||
|
*.exe
|
||||||
|
*.so
|
||||||
|
.DS_Store
|
||||||
|
.idea
|
||||||
|
.vscode
|
||||||
|
leopard.nims
|
||||||
|
TODO
|
||||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -1,5 +1,5 @@
|
|||||||
[submodule "vendor/leopard"]
|
[submodule "vendor/leopard"]
|
||||||
path = vendor/leopard
|
path = vendor/leopard
|
||||||
url = https://github.com/catid/leopard.git
|
url = https://github.com/status-im/leopard.git
|
||||||
ignore = untracked
|
ignore = untracked
|
||||||
branch = master
|
branch = master
|
||||||
|
|||||||
@ -3,6 +3,7 @@
|
|||||||
[](https://opensource.org/licenses/Apache-2.0)
|
[](https://opensource.org/licenses/Apache-2.0)
|
||||||
[](https://opensource.org/licenses/MIT)
|
[](https://opensource.org/licenses/MIT)
|
||||||
[](https://github.com/status-im/nim-leopard#stability)
|
[](https://github.com/status-im/nim-leopard#stability)
|
||||||
|
[](https://github.com/status-im/nim-leopard/actions?query=workflow%3ATests+branch%3Ainitial_impl)
|
||||||
|
|
||||||
Nim wrapper for [Leopard-RS](https://github.com/catid/leopard): a fast library for [Reed-Solomon](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) erasure correction coding.
|
Nim wrapper for [Leopard-RS](https://github.com/catid/leopard): a fast library for [Reed-Solomon](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) erasure correction coding.
|
||||||
|
|
||||||
|
|||||||
307
leopard.nim
307
leopard.nim
@ -0,0 +1,307 @@
|
|||||||
|
import pkg/stew/ptrops
|
||||||
|
import pkg/stew/results
|
||||||
|
import pkg/upraises
|
||||||
|
|
||||||
|
import ./leopard/wrapper
|
||||||
|
|
||||||
|
export results
|
||||||
|
|
||||||
|
push: {.upraises: [].}
|
||||||
|
|
||||||
|
const
|
||||||
|
LeopardBadCodeMsg = "Bad RS code"
|
||||||
|
LeopardInconsistentSizeMsg =
|
||||||
|
"Buffer sizes must all be the same multiple of 64 bytes"
|
||||||
|
LeopardNeedLessDataMsg = "Too much recovery data received"
|
||||||
|
LeopardNotEnoughDataMsg = "Buffer counts are too low"
|
||||||
|
|
||||||
|
MinBufferSize* = 64.uint
|
||||||
|
|
||||||
|
type
|
||||||
|
Data* = seq[seq[byte]]
|
||||||
|
|
||||||
|
LeopardDefect* = object of Defect
|
||||||
|
|
||||||
|
# It should not be necessary to redefine LeopardResult, but if that's not
|
||||||
|
# done here then defining LeopardError as `object of CatchableError` will
|
||||||
|
# cause a mystery crash at compile-time (symbol not found). Can workaround by
|
||||||
|
# defining as just `object`, but then when trying to work with LeopardResult
|
||||||
|
# errors in e.g. tests/test_leopard.nim the same mystery crash happens at
|
||||||
|
# compile-time. The problem may be related to use of importcpp in
|
||||||
|
# leopard/wrapper.nim, so it could be a compiler bug. By redefining
|
||||||
|
# LeopardResult in this module (and casting wrapper.LeopardResult values) the
|
||||||
|
# the problem is avoided.
|
||||||
|
LeopardResult* = enum
|
||||||
|
LeopardNotEnoughData = -11.cint # Buffer counts are too low
|
||||||
|
LeopardNeedLessData = -10.cint # Too much recovery data received
|
||||||
|
LeopardInconsistentSize = -9.cint # Buffer sizes must all be the same multiple of 64 bytes
|
||||||
|
LeopardBadCode = -8.cint # Bad RS code
|
||||||
|
LeopardCallInitialize = wrapper.LeopardCallInitialize
|
||||||
|
LeopardPlatform = wrapper.LeopardPlatform
|
||||||
|
LeopardInvalidInput = wrapper.LeopardInvalidInput
|
||||||
|
LeopardInvalidCounts = wrapper.LeopardInvalidCounts
|
||||||
|
LeopardInvalidSize = wrapper.LeopardInvalidSize
|
||||||
|
LeopardTooMuchData = wrapper.LeopardTooMuchData
|
||||||
|
LeopardNeedMoreData = wrapper.LeopardNeedMoreData
|
||||||
|
LeopardSuccess = wrapper.LeopardSuccess
|
||||||
|
|
||||||
|
LeopardError* = object of CatchableError
|
||||||
|
code*: LeopardResult
|
||||||
|
|
||||||
|
ParityData* = Data
|
||||||
|
|
||||||
|
ReedSolomonCode* = tuple[codeword, data, parity: uint] # symbol counts
|
||||||
|
|
||||||
|
# https://github.com/catid/leopard/issues/12
|
||||||
|
# https://www.cs.cmu.edu/~guyb/realworld/reedsolomon/reed_solomon_codes.html
|
||||||
|
#
|
||||||
|
# RS(255,239)
|
||||||
|
# ---------------------------------
|
||||||
|
# codeword symbols = 255
|
||||||
|
# data symbols = 239
|
||||||
|
# parity symbols = 255 - 239 = 16
|
||||||
|
|
||||||
|
proc RS*(codeword, data: Positive): ReedSolomonCode =
|
||||||
|
var
|
||||||
|
parity = codeword - data
|
||||||
|
|
||||||
|
if parity <= 0: parity = 0
|
||||||
|
(codeword: codeword.uint, data: data.uint, parity: parity.uint)
|
||||||
|
|
||||||
|
when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
|
||||||
|
const
|
||||||
|
header = "<stdlib.h>"
|
||||||
|
|
||||||
|
proc c_malloc(size: csize_t): pointer {.importc: "malloc", header: header.}
|
||||||
|
proc c_free(p: pointer) {.importc: "free", header: header.}
|
||||||
|
|
||||||
|
proc SIMDSafeAllocate(size: int): pointer {.inline.} =
|
||||||
|
var
|
||||||
|
data =
|
||||||
|
when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
|
||||||
|
c_malloc(LEO_ALIGN_BYTES + size.uint)
|
||||||
|
else:
|
||||||
|
allocShared(LEO_ALIGN_BYTES + size.uint)
|
||||||
|
|
||||||
|
doffset = cast[uint](data) mod LEO_ALIGN_BYTES
|
||||||
|
|
||||||
|
data = offset(data, (LEO_ALIGN_BYTES + doffset).int)
|
||||||
|
|
||||||
|
var
|
||||||
|
offsetPtr = cast[pointer](cast[uint](data) - 1)
|
||||||
|
|
||||||
|
moveMem(offsetPtr, addr doffset, sizeof(doffset))
|
||||||
|
data
|
||||||
|
|
||||||
|
proc SIMDSafeFree(data: pointer) {.inline.} =
|
||||||
|
var
|
||||||
|
data = data
|
||||||
|
|
||||||
|
if not data.isNil:
|
||||||
|
let
|
||||||
|
offset = cast[uint](data) - 1
|
||||||
|
|
||||||
|
if offset >= LEO_ALIGN_BYTES: return
|
||||||
|
|
||||||
|
data = cast[pointer](cast[uint](data) - (LEO_ALIGN_BYTES - offset))
|
||||||
|
|
||||||
|
when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
|
||||||
|
c_free data
|
||||||
|
else:
|
||||||
|
deallocShared data
|
||||||
|
|
||||||
|
proc leoInit*() =
|
||||||
|
if wrapper.leoInit() != 0:
|
||||||
|
raise (ref LeopardDefect)(msg: "Leopard-RS failed to initialize")
|
||||||
|
|
||||||
|
proc encode*(code: ReedSolomonCode, data: Data):
|
||||||
|
Result[ParityData, LeopardError] =
|
||||||
|
if code.parity < 1 or code.parity > code.data:
|
||||||
|
return err LeopardError(code: LeopardBadCode, msg: LeopardBadCodeMsg)
|
||||||
|
|
||||||
|
var
|
||||||
|
data = data
|
||||||
|
|
||||||
|
let
|
||||||
|
symbolBytes = data[0].len
|
||||||
|
|
||||||
|
if data.len < code.data.int:
|
||||||
|
return err LeopardError(code: LeopardNotEnoughData,
|
||||||
|
msg: LeopardNotEnoughDataMsg)
|
||||||
|
|
||||||
|
elif data.len > code.data.int:
|
||||||
|
return err LeopardError(code: LeopardTooMuchData,
|
||||||
|
msg: $leoResultString(wrapper.LeopardTooMuchData))
|
||||||
|
|
||||||
|
if symbolBytes < MinBufferSize.int or symbolBytes mod MinBufferSize.int != 0:
|
||||||
|
return err LeopardError(code: LeopardInvalidSize,
|
||||||
|
msg: $leoResultString(wrapper.LeopardInvalidSize))
|
||||||
|
|
||||||
|
var
|
||||||
|
enData = newSeq[pointer](code.data)
|
||||||
|
|
||||||
|
for i in 0..<code.data:
|
||||||
|
if data[i].len != symbolBytes:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||||
|
return err LeopardError(code: LeopardInconsistentSize,
|
||||||
|
msg: LeopardInconsistentSizeMsg)
|
||||||
|
|
||||||
|
enData[i] = SIMDSafeAllocate symbolBytes
|
||||||
|
moveMem(enData[i], addr data[i][0], symbolBytes)
|
||||||
|
|
||||||
|
let
|
||||||
|
workCount = leoEncodeWorkCount(code.data.cuint, code.parity.cuint)
|
||||||
|
|
||||||
|
if workCount == 0:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||||
|
return err LeopardError(code: LeopardInvalidInput,
|
||||||
|
msg: $leoResultString(wrapper.LeopardInvalidInput))
|
||||||
|
|
||||||
|
var
|
||||||
|
workData = newSeq[pointer](workCount)
|
||||||
|
|
||||||
|
for i in 0..<workCount:
|
||||||
|
workData[i] = SIMDSafeAllocate symbolBytes
|
||||||
|
|
||||||
|
let
|
||||||
|
encodeRes = leoEncode(
|
||||||
|
symbolBytes.uint64,
|
||||||
|
code.data.cuint,
|
||||||
|
code.parity.cuint,
|
||||||
|
workCount,
|
||||||
|
addr enData[0],
|
||||||
|
addr workData[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
if encodeRes != wrapper.LeopardSuccess:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||||
|
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||||
|
return err LeopardError(code: cast[LeopardResult](encodeRes),
|
||||||
|
msg: $leoResultString(encodeRes))
|
||||||
|
|
||||||
|
var
|
||||||
|
parityData: ParityData
|
||||||
|
|
||||||
|
newSeq(parityData, code.parity)
|
||||||
|
for i in 0..<code.parity:
|
||||||
|
newSeq(parityData[i], symbolBytes)
|
||||||
|
moveMem(addr parityData[i][0], workData[i], symbolBytes)
|
||||||
|
|
||||||
|
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||||
|
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||||
|
|
||||||
|
ok parityData
|
||||||
|
|
||||||
|
proc decode*(code: ReedSolomonCode, data: Data, parityData: ParityData,
|
||||||
|
symbolBytes: uint): Result[Data, LeopardError] =
|
||||||
|
if code.parity < 1 or code.parity > code.data:
|
||||||
|
return err LeopardError(code: LeopardBadCode, msg: LeopardBadCodeMsg)
|
||||||
|
|
||||||
|
var
|
||||||
|
data = data
|
||||||
|
parityData = parityData
|
||||||
|
holes: seq[int]
|
||||||
|
|
||||||
|
if data.len < code.data.int:
|
||||||
|
return err LeopardError(code: LeopardNotEnoughData,
|
||||||
|
msg: LeopardNotEnoughDataMsg)
|
||||||
|
|
||||||
|
elif data.len > code.data.int:
|
||||||
|
return err LeopardError(code: LeopardTooMuchData,
|
||||||
|
msg: $leoResultString(wrapper.LeopardTooMuchData))
|
||||||
|
|
||||||
|
if parityData.len < code.parity.int:
|
||||||
|
return err LeopardError(code: LeopardNeedMoreData,
|
||||||
|
msg: $leoResultString(wrapper.LeopardNeedMoreData))
|
||||||
|
|
||||||
|
elif parityData.len > code.parity.int:
|
||||||
|
return err LeopardError(code: LeopardNeedLessData,
|
||||||
|
msg: LeopardNeedLessDataMsg)
|
||||||
|
|
||||||
|
if symbolBytes < MinBufferSize or symbolBytes mod MinBufferSize != 0:
|
||||||
|
return err LeopardError(code: LeopardInvalidSize,
|
||||||
|
msg: $leoResultString(wrapper.LeopardInvalidSize))
|
||||||
|
|
||||||
|
var
|
||||||
|
deData = newSeq[pointer](code.data)
|
||||||
|
|
||||||
|
for i in 0..<code.data:
|
||||||
|
if data[i].len != 0:
|
||||||
|
if data[i].len != symbolBytes.int:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||||
|
return err LeopardError(code: LeopardInconsistentSize,
|
||||||
|
msg: LeopardInconsistentSizeMsg)
|
||||||
|
|
||||||
|
deData[i] = SIMDSafeAllocate symbolBytes.int
|
||||||
|
moveMem(deData[i], addr data[i][0], symbolBytes)
|
||||||
|
|
||||||
|
else:
|
||||||
|
holes.add i.int
|
||||||
|
|
||||||
|
if holes.len == 0:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||||
|
return ok data
|
||||||
|
|
||||||
|
var
|
||||||
|
paData = newSeq[pointer](code.parity)
|
||||||
|
|
||||||
|
for i in 0..<code.parity:
|
||||||
|
if parityData[i].len != 0:
|
||||||
|
if parityData[i].len != symbolBytes.int:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||||
|
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||||
|
return err LeopardError(code: LeopardInconsistentSize,
|
||||||
|
msg: LeopardInconsistentSizeMsg)
|
||||||
|
|
||||||
|
paData[i] = SIMDSafeAllocate symbolBytes.int
|
||||||
|
moveMem(paData[i], addr parityData[i][0], symbolBytes)
|
||||||
|
|
||||||
|
let
|
||||||
|
workCount = leoDecodeWorkCount(code.data.cuint, code.parity.cuint)
|
||||||
|
|
||||||
|
if workCount == 0:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||||
|
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||||
|
return err LeopardError(code: LeopardInvalidInput,
|
||||||
|
msg: $leoResultString(wrapper.LeopardInvalidInput))
|
||||||
|
|
||||||
|
var
|
||||||
|
workData = newSeq[pointer](workCount)
|
||||||
|
|
||||||
|
for i in 0..<workCount:
|
||||||
|
workData[i] = SIMDSafeAllocate symbolBytes.int
|
||||||
|
|
||||||
|
let
|
||||||
|
decodeRes = leoDecode(
|
||||||
|
symbolBytes.uint64,
|
||||||
|
code.data.cuint,
|
||||||
|
code.parity.cuint,
|
||||||
|
workCount,
|
||||||
|
addr deData[0],
|
||||||
|
addr paData[0],
|
||||||
|
addr workData[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
if decodeRes != wrapper.LeopardSuccess:
|
||||||
|
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||||
|
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||||
|
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||||
|
return err LeopardError(code: cast[LeopardResult](decodeRes),
|
||||||
|
msg: $leoResultString(decodeRes))
|
||||||
|
|
||||||
|
var
|
||||||
|
recoveredData: Data
|
||||||
|
|
||||||
|
newSeq(recoveredData, workCount)
|
||||||
|
for i in 0..<workCount:
|
||||||
|
newSeq(recoveredData[i], symbolBytes)
|
||||||
|
moveMem(addr recoveredData[i][0], workData[i], symbolBytes)
|
||||||
|
|
||||||
|
for i in holes:
|
||||||
|
data[i] = recoveredData[i]
|
||||||
|
|
||||||
|
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||||
|
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||||
|
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||||
|
|
||||||
|
ok data
|
||||||
@ -5,7 +5,9 @@ version = "0.0.1"
|
|||||||
author = "Status Research & Development GmbH"
|
author = "Status Research & Development GmbH"
|
||||||
description = "A wrapper for Leopard-RS"
|
description = "A wrapper for Leopard-RS"
|
||||||
license = "Apache License 2.0 or MIT"
|
license = "Apache License 2.0 or MIT"
|
||||||
|
installDirs = @["vendor"]
|
||||||
|
|
||||||
requires "nim >= 1.2.0",
|
requires "nim >= 1.2.0",
|
||||||
"stew#head",
|
"stew",
|
||||||
"unittest2"
|
"unittest2",
|
||||||
|
"upraises >= 0.1.0 & < 0.2.0"
|
||||||
|
|||||||
@ -0,0 +1,293 @@
|
|||||||
|
## Copyright (c) 2017 Christopher A. Taylor. All rights reserved.
|
||||||
|
##
|
||||||
|
## Redistribution and use in source and binary forms, with or without
|
||||||
|
## modification, are permitted provided that the following conditions are met:
|
||||||
|
##
|
||||||
|
## * Redistributions of source code must retain the above copyright notice,
|
||||||
|
## this list of conditions and the following disclaimer.
|
||||||
|
## * Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
## this list of conditions and the following disclaimer in the documentation
|
||||||
|
## and/or other materials provided with the distribution.
|
||||||
|
## * Neither the name of Leopard-RS nor the names of its contributors may be
|
||||||
|
## used to endorse or promote products derived from this software without
|
||||||
|
## specific prior written permission.
|
||||||
|
##
|
||||||
|
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||||
|
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||||
|
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||||
|
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||||
|
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||||
|
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||||
|
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||||
|
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||||
|
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
|
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||||
|
## POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
|
||||||
|
## Leopard-RS
|
||||||
|
## MDS Reed-Solomon Erasure Correction Codes for Large Data in C
|
||||||
|
##
|
||||||
|
## Algorithms are described in LeopardCommon.h
|
||||||
|
##
|
||||||
|
##
|
||||||
|
## Inspired by discussion with:
|
||||||
|
##
|
||||||
|
## Sian-Jhen Lin <sjhenglin@gmail.com> : Author of {1} {3}, basis for Leopard
|
||||||
|
## Bulat Ziganshin <bulat.ziganshin@gmail.com> : Author of FastECC
|
||||||
|
## Yutaka Sawada <tenfon@outlook.jp> : Author of MultiPar
|
||||||
|
##
|
||||||
|
##
|
||||||
|
## References:
|
||||||
|
##
|
||||||
|
## {1} S.-J. Lin, T. Y. Al-Naffouri, Y. S. Han, and W.-H. Chung,
|
||||||
|
## "Novel Polynomial Basis with Fast Fourier Transform
|
||||||
|
## and Its Application to Reed-Solomon Erasure Codes"
|
||||||
|
## IEEE Trans. on Information Theory, pp. 6284-6299, November, 2016.
|
||||||
|
##
|
||||||
|
## {2} D. G. Cantor, "On arithmetical algorithms over finite fields",
|
||||||
|
## Journal of Combinatorial Theory, Series A, vol. 50, no. 2, pp. 285-300, 1989.
|
||||||
|
##
|
||||||
|
## {3} Sian-Jheng Lin, Wei-Ho Chung, "An Efficient (n, k) Information
|
||||||
|
## Dispersal Algorithm for High Code Rate System over Fermat Fields,"
|
||||||
|
## IEEE Commun. Lett., vol.16, no.12, pp. 2036-2039, Dec. 2012.
|
||||||
|
##
|
||||||
|
## {4} Plank, J. S., Greenan, K. M., Miller, E. L., "Screaming fast Galois Field
|
||||||
|
## arithmetic using Intel SIMD instructions." In: FAST-2013: 11th Usenix
|
||||||
|
## Conference on File and Storage Technologies, San Jose, 2013
|
||||||
|
|
||||||
|
|
||||||
|
import upraises
|
||||||
|
push: {.upraises: [].}
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Build configuration
|
||||||
|
|
||||||
|
import std/compilesettings
|
||||||
|
import std/os
|
||||||
|
import std/strutils
|
||||||
|
|
||||||
|
const
|
||||||
|
LeopardCmakeFlags {.strdefine.} =
|
||||||
|
when defined(macosx):
|
||||||
|
"-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off"
|
||||||
|
elif defined(windows):
|
||||||
|
"-G\"MSYS Makefiles\" -DCMAKE_BUILD_TYPE=Release"
|
||||||
|
else:
|
||||||
|
"-DCMAKE_BUILD_TYPE=Release"
|
||||||
|
|
||||||
|
LeopardDir {.strdefine.} =
|
||||||
|
joinPath(currentSourcePath.parentDir.parentDir, "vendor", "leopard")
|
||||||
|
|
||||||
|
buildDir = joinPath(querySetting(nimcacheDir), "vendor_leopard")
|
||||||
|
|
||||||
|
LeopardHeader {.strdefine.} = "leopard.h"
|
||||||
|
|
||||||
|
LeopardLib {.strdefine.} = joinPath(buildDir, "liblibleopard.a")
|
||||||
|
|
||||||
|
LeopardCompilerFlags {.strdefine.} =
|
||||||
|
when defined(macosx):
|
||||||
|
"-I" & LeopardDir
|
||||||
|
else:
|
||||||
|
"-I" & LeopardDir & " -fopenmp"
|
||||||
|
|
||||||
|
LeopardLinkerFlags {.strdefine.} =
|
||||||
|
when defined(macosx):
|
||||||
|
LeopardLib
|
||||||
|
else:
|
||||||
|
LeopardLib & " -fopenmp"
|
||||||
|
|
||||||
|
LeopardExtraCompilerFlags {.strdefine.} = ""
|
||||||
|
|
||||||
|
LeopardExtraLinkerFlags {.strdefine.} = ""
|
||||||
|
|
||||||
|
static:
|
||||||
|
if defined(windows):
|
||||||
|
func pathUnix2Win(path: string): string =
|
||||||
|
gorge("cygpath -w " & path.strip).strip
|
||||||
|
|
||||||
|
func pathWin2Unix(path: string): string =
|
||||||
|
gorge("cygpath " & path.strip).strip
|
||||||
|
|
||||||
|
proc bash(cmd: varargs[string]): string =
|
||||||
|
gorge(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'")
|
||||||
|
|
||||||
|
proc bashEx(cmd: varargs[string]): tuple[output: string, exitCode: int] =
|
||||||
|
gorgeEx(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'")
|
||||||
|
|
||||||
|
let
|
||||||
|
buildDirUnix = buildDir.pathWin2Unix
|
||||||
|
leopardDirUnix = LeopardDir.pathWin2Unix
|
||||||
|
if defined(LeopardRebuild): discard bash("rm -rf", buildDirUnix)
|
||||||
|
if (bashEx("ls", LeopardLib.pathWin2Unix)).exitCode != 0:
|
||||||
|
discard bash("mkdir -p", buildDirUnix)
|
||||||
|
let cmd =
|
||||||
|
@["cd", buildDirUnix, "&& cmake", leopardDirUnix, LeopardCmakeFlags,
|
||||||
|
"&& make"]
|
||||||
|
echo "\nBuilding Leopard-RS: " & cmd.join(" ")
|
||||||
|
let (output, exitCode) = bashEx cmd
|
||||||
|
echo output
|
||||||
|
if exitCode != 0:
|
||||||
|
discard bash("rm -rf", buildDirUnix)
|
||||||
|
raise (ref Defect)(msg: "Failed to build Leopard-RS")
|
||||||
|
else:
|
||||||
|
if defined(LeopardRebuild): discard gorge "rm -rf " & buildDir
|
||||||
|
if gorgeEx("ls " & LeopardLib).exitCode != 0:
|
||||||
|
discard gorge "mkdir -p " & buildDir
|
||||||
|
let cmd =
|
||||||
|
"cd " & buildDir & " && cmake " & LeopardDir & " " & LeopardCmakeFlags &
|
||||||
|
" && make"
|
||||||
|
echo "\nBuilding Leopard-RS: " & cmd
|
||||||
|
let (output, exitCode) = gorgeEx cmd
|
||||||
|
echo output
|
||||||
|
if exitCode != 0:
|
||||||
|
discard gorge "rm -rf " & buildDir
|
||||||
|
raise (ref Defect)(msg: "Failed to build Leopard-RS")
|
||||||
|
|
||||||
|
{.passC: LeopardCompilerFlags & " " & LeopardExtraCompilerFlags.}
|
||||||
|
{.passL: LeopardLinkerFlags & " " & LeopardExtraLinkerFlags.}
|
||||||
|
|
||||||
|
{.pragma: leo, cdecl, header: LeopardHeader.}
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Library version
|
||||||
|
|
||||||
|
var LEO_VERSION* {.header: LeopardHeader, importc.}: int
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Platform/Architecture
|
||||||
|
|
||||||
|
# maybe should detect AVX2 and set to 32 if detected, 16 otherwise:
|
||||||
|
# https://github.com/catid/leopard/blob/master/LeopardCommon.h#L247-L253
|
||||||
|
# https://github.com/mratsim/Arraymancer/blob/master/src/arraymancer/laser/cpuinfo_x86.nim#L220
|
||||||
|
const LEO_ALIGN_BYTES* = 16
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Initialization API
|
||||||
|
|
||||||
|
## leoInit()
|
||||||
|
##
|
||||||
|
## Perform static initialization for the library, verifying that the platform
|
||||||
|
## is supported.
|
||||||
|
##
|
||||||
|
## Returns 0 on success and other values on failure.
|
||||||
|
|
||||||
|
proc leoInit*(): cint {.leo, importcpp: "leo_init".}
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Shared Constants / Datatypes
|
||||||
|
|
||||||
|
## Results
|
||||||
|
type
|
||||||
|
LeopardResult* = enum
|
||||||
|
LeopardCallInitialize = -7.cint ## Call leoInit() first
|
||||||
|
LeopardPlatform = -6.cint ## Platform is unsupported
|
||||||
|
LeopardInvalidInput = -5.cint ## A function parameter was invalid
|
||||||
|
LeopardInvalidCounts = -4.cint ## Invalid counts provided
|
||||||
|
LeopardInvalidSize = -3.cint ## Buffer size must be multiple of 64 bytes
|
||||||
|
LeopardTooMuchData = -2.cint ## Buffer counts are too high
|
||||||
|
LeopardNeedMoreData = -1.cint ## Not enough recovery data received
|
||||||
|
LeopardSuccess = 0.cint ## Operation succeeded
|
||||||
|
|
||||||
|
## Convert Leopard result to string
|
||||||
|
func leoResultString*(res: LeopardResult): cstring
|
||||||
|
{.leo, importc: "leo_result_string".}
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Encoder API
|
||||||
|
|
||||||
|
## leoEncodeWorkCount()
|
||||||
|
##
|
||||||
|
## Calculate the number of work data buffers to provide to leoEncode().
|
||||||
|
##
|
||||||
|
## The sum of originalCount + recoveryCount must not exceed 65536.
|
||||||
|
##
|
||||||
|
## Returns the workCount value to pass into leoEncode().
|
||||||
|
## Returns 0 on invalid input.
|
||||||
|
|
||||||
|
func leoEncodeWorkCount*(originalCount, recoveryCount: cuint): cuint
|
||||||
|
{.leo, importc: "leo_encode_work_count".}
|
||||||
|
|
||||||
|
## leoEncode()
|
||||||
|
##
|
||||||
|
## Generate recovery data.
|
||||||
|
##
|
||||||
|
## bufferBytes: Number of bytes in each data buffer.
|
||||||
|
## originalCount: Number of original data buffers provided.
|
||||||
|
## recoveryCount: Number of desired recovery data buffers.
|
||||||
|
## workCount: Number of work data buffers, from leoEncodeWorkCount().
|
||||||
|
## originalData: Array of pointers to original data buffers.
|
||||||
|
## workData: Array of pointers to work data buffers.
|
||||||
|
##
|
||||||
|
## The sum of originalCount + recoveryCount must not exceed 65536.
|
||||||
|
## The recoveryCount <= originalCount.
|
||||||
|
##
|
||||||
|
## The value of bufferBytes must be a multiple of 64.
|
||||||
|
## Each buffer should have the same number of bytes.
|
||||||
|
## Even the last piece must be rounded up to the block size.
|
||||||
|
##
|
||||||
|
## Returns LeopardSuccess on success.
|
||||||
|
## The first set of recoveryCount buffers in workData will be the result.
|
||||||
|
## Returns other values on errors.
|
||||||
|
|
||||||
|
proc leoEncode*(
|
||||||
|
bufferBytes: uint64, ## Number of bytes in each data buffer
|
||||||
|
originalCount: cuint, ## Number of originalData[] buffer pointers
|
||||||
|
recoveryCount: cuint, ## Number of recovery data buffer pointers
|
||||||
|
## (readable post-call from start of workData[])
|
||||||
|
workCount: cuint, ## Number of workData[] buffer pointers
|
||||||
|
originalData: pointer, ## Array of pointers to original data buffers
|
||||||
|
workData: pointer, ## Array of pointers to work data buffers
|
||||||
|
): LeopardResult {.leo, importc: "leo_encode".}
|
||||||
|
|
||||||
|
|
||||||
|
## -----------------------------------------------------------------------------
|
||||||
|
## Decoder API
|
||||||
|
|
||||||
|
## leoDecodeWorkCount()
|
||||||
|
##
|
||||||
|
## Calculate the number of work data buffers to provide to leoDecode().
|
||||||
|
##
|
||||||
|
## The sum of originalCount + recoveryCount must not exceed 65536.
|
||||||
|
##
|
||||||
|
## Returns the workCount value to pass into leoDecode().
|
||||||
|
## Returns 0 on invalid input.
|
||||||
|
|
||||||
|
func leoDecodeWorkCount*(originalCount, recoveryCount: cuint): cuint
|
||||||
|
{.leo, importc: "leo_decode_work_count".}
|
||||||
|
|
||||||
|
## leoDecode()
|
||||||
|
##
|
||||||
|
## Decode original data from recovery data.
|
||||||
|
##
|
||||||
|
## bufferBytes: Number of bytes in each data buffer.
|
||||||
|
## originalCount: Number of original data buffers provided.
|
||||||
|
## recoveryCount: Number of recovery data buffers provided.
|
||||||
|
## workCount: Number of work data buffers, from leoDecodeWorkCount().
|
||||||
|
## originalData: Array of pointers to original data buffers.
|
||||||
|
## recoveryData: Array of pointers to recovery data buffers.
|
||||||
|
## workData: Array of pointers to work data buffers.
|
||||||
|
##
|
||||||
|
## Lost original/recovery data should be set to NULL.
|
||||||
|
##
|
||||||
|
## The sum of recoveryCount + the number of non-NULL original data must be at
|
||||||
|
## least originalCount in order to perform recovery.
|
||||||
|
##
|
||||||
|
## Returns LeopardSuccess on success.
|
||||||
|
## Returns other values on errors.
|
||||||
|
|
||||||
|
proc leoDecode*(
|
||||||
|
bufferBytes: uint64, ## Number of bytes in each data buffer
|
||||||
|
originalCount: cuint, ## Number of originalData[] buffer pointers
|
||||||
|
recoveryCount: cuint, ## Number of recoveryData[] buffer pointers
|
||||||
|
workCount: cuint, ## Number of workData[] buffer pointers
|
||||||
|
originalData: pointer, ## Array of pointers to original data buffers
|
||||||
|
recoveryData: pointer, ## Array of pointers to recovery data buffers
|
||||||
|
workData: pointer, ## Array of pointers to work data buffers
|
||||||
|
): LeopardResult {.leo, importc: "leo_decode".}
|
||||||
@ -0,0 +1,558 @@
|
|||||||
|
import std/random
|
||||||
|
|
||||||
|
import pkg/leopard
|
||||||
|
import pkg/unittest2
|
||||||
|
|
||||||
|
randomize()
|
||||||
|
|
||||||
|
proc genData(outerLen, innerLen: uint): Data =
|
||||||
|
var
|
||||||
|
data = newSeqOfCap[seq[byte]](outerLen)
|
||||||
|
|
||||||
|
for i in 0..<outerLen.int:
|
||||||
|
data.add newSeqUninitialized[byte](innerLen)
|
||||||
|
for j in 0..<innerLen:
|
||||||
|
data[i][j] = rand(255).byte
|
||||||
|
|
||||||
|
data
|
||||||
|
|
||||||
|
var
|
||||||
|
initialized = false
|
||||||
|
|
||||||
|
suite "Initialization":
|
||||||
|
test "encode and decode should fail if Leopard-RS is not initialized":
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
|
||||||
|
var
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
|
||||||
|
let
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
# Related to a subtle race re: decode being called with data that has no
|
||||||
|
# holes while Leopard-RS is not initialized, i.e. it would succeed by
|
||||||
|
# simply returning the data without a call to leoDecode.
|
||||||
|
data[0] = @[]
|
||||||
|
|
||||||
|
let
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check:
|
||||||
|
encodeRes.isErr
|
||||||
|
encodeRes.error.code == LeopardCallInitialize
|
||||||
|
decodeRes.isErr
|
||||||
|
decodeRes.error.code == LeopardCallInitialize
|
||||||
|
|
||||||
|
test "initialization should succeed else raise a Defect":
|
||||||
|
leoInit()
|
||||||
|
initialized = true
|
||||||
|
|
||||||
|
check: initialized
|
||||||
|
|
||||||
|
suite "Encoder":
|
||||||
|
test "should fail if RS code is nonsensical or is so per Leopard-RS":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
|
||||||
|
var
|
||||||
|
rsCode = RS(5,5)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isErr
|
||||||
|
if encodeRes.isErr:
|
||||||
|
check: encodeRes.error.code == LeopardBadCode
|
||||||
|
|
||||||
|
rsCode = RS(5,10)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isErr
|
||||||
|
if encodeRes.isErr:
|
||||||
|
check: encodeRes.error.code == LeopardBadCode
|
||||||
|
|
||||||
|
rsCode = RS(110,10)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isErr
|
||||||
|
if encodeRes.isErr:
|
||||||
|
check: encodeRes.error.code == LeopardBadCode
|
||||||
|
|
||||||
|
test "should fail if outer length of data does not match the RS code":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
notEnoughData = genData(rsCode.data - 1, symbolBytes)
|
||||||
|
tooMuchData = genData(rsCode.data + 1, symbolBytes)
|
||||||
|
notEnoughEncodeRes = rsCode.encode notEnoughData
|
||||||
|
tooMuchEncodeRes = rsCode.encode tooMuchData
|
||||||
|
|
||||||
|
check:
|
||||||
|
notEnoughEncodeRes.isErr
|
||||||
|
tooMuchEncodeRes.isErr
|
||||||
|
if notEnoughEncodeRes.isErr:
|
||||||
|
check: notEnoughEncodeRes.error.code == LeopardNotEnoughData
|
||||||
|
if tooMuchEncodeRes.isErr:
|
||||||
|
check: tooMuchEncodeRes.error.code == LeopardTooMuchData
|
||||||
|
|
||||||
|
test "should fail if length of data[0] is less than minimum buffer size":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize - 5
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isErr
|
||||||
|
if encodeRes.isErr:
|
||||||
|
check: encodeRes.error.code == LeopardInvalidSize
|
||||||
|
|
||||||
|
test "should fail if length of data[0] is not a multiple of minimum buffer size":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize * 2 + 1
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isErr
|
||||||
|
if encodeRes.isErr:
|
||||||
|
check: encodeRes.error.code == LeopardInvalidSize
|
||||||
|
|
||||||
|
test "should fail if length of data[0+N] does not equal length of data[0]":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
|
||||||
|
var
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
|
||||||
|
data[3] = @[1.byte, 2.byte, 3.byte]
|
||||||
|
|
||||||
|
let
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isErr
|
||||||
|
if encodeRes.isErr:
|
||||||
|
check: encodeRes.error.code == LeopardInconsistentSize
|
||||||
|
|
||||||
|
# With the current setup in leopard.nim it seems it's not possible to call
|
||||||
|
# encode with an RS code that would result in leoEncodeWorkCount being called
|
||||||
|
# with invalid parameters, i.e. that would result in it returning 0, because
|
||||||
|
# a Result error will always be returned before leoEncodeWorkCount is called.
|
||||||
|
|
||||||
|
# test "should fail if RS code parameters yield invalid parameters for leoEncodWorkCount":
|
||||||
|
# check: initialized
|
||||||
|
# if not initialized: return
|
||||||
|
#
|
||||||
|
# let
|
||||||
|
# rsCode = RS(?,?)
|
||||||
|
# symbolBytes = MinBufferSize
|
||||||
|
# data = genData(rsCode.data, symbolBytes)
|
||||||
|
# encodeRes = rsCode.encode data
|
||||||
|
#
|
||||||
|
# check: encodeRes.isErr
|
||||||
|
# if encodeRes.isErr:
|
||||||
|
# check: encodeRes.error.code == LeopardInvalidInput
|
||||||
|
|
||||||
|
test "should succeed if RS code and data yield valid parameters for leoEncode":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: encodeRes.isOk
|
||||||
|
|
||||||
|
suite "Decoder":
|
||||||
|
test "should fail if RS code is nonsensical or is so per Leopard-RS":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
|
||||||
|
var
|
||||||
|
rsCode = RS(5,5)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
parityData: ParityData
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardBadCode
|
||||||
|
|
||||||
|
rsCode = RS(5,10)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardBadCode
|
||||||
|
|
||||||
|
rsCode = RS(110,10)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardBadCode
|
||||||
|
|
||||||
|
test "should fail if outer length of data does not match the RS code":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
notEnoughData = genData(rsCode.data - 1, symbolBytes)
|
||||||
|
tooMuchData = genData(rsCode.data + 1, symbolBytes)
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
notEnoughDecodeRes = rsCode.decode(notEnoughData, parityData, symbolBytes)
|
||||||
|
tooMuchDecodeRes = rsCode.decode(tooMuchData, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check:
|
||||||
|
notEnoughDecodeRes.isErr
|
||||||
|
tooMuchDecodeRes.isErr
|
||||||
|
if notEnoughDecodeRes.isErr:
|
||||||
|
check: notEnoughDecodeRes.error.code == LeopardNotEnoughData
|
||||||
|
if tooMuchDecodeRes.isErr:
|
||||||
|
check: tooMuchDecodeRes.error.code == LeopardTooMuchData
|
||||||
|
|
||||||
|
test "should fail if outer length of parityData does not match the RS code":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
notEnoughParityData = genData(rsCode.parity - 1, symbolBytes)
|
||||||
|
tooMuchParityData = genData(rsCode.parity + 1, symbolBytes)
|
||||||
|
notEnoughDecodeRes = rsCode.decode(data, notEnoughParityData, symbolBytes)
|
||||||
|
tooMuchDecodeRes = rsCode.decode(data, tooMuchParityData, symbolBytes)
|
||||||
|
|
||||||
|
check:
|
||||||
|
notEnoughDecodeRes.isErr
|
||||||
|
tooMuchDecodeRes.isErr
|
||||||
|
if notEnoughDecodeRes.isErr:
|
||||||
|
check: notEnoughDecodeRes.error.code == LeopardNeedMoreData
|
||||||
|
if tooMuchDecodeRes.isErr:
|
||||||
|
check: tooMuchDecodeRes.error.code == LeopardNeedLessData
|
||||||
|
|
||||||
|
test "should fail if symbolBytes is less than minimum buffer size":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize - 5
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardInvalidSize
|
||||||
|
|
||||||
|
test "should fail if symbolBytes is not a multiple of minimum buffer size":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize * 2 + 1
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardInvalidSize
|
||||||
|
|
||||||
|
test "should fail if length of data[0+N] is not zero and does not equal symbolBytes":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
|
||||||
|
var
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
|
||||||
|
data[3] = @[1.byte, 2.byte, 3.byte]
|
||||||
|
|
||||||
|
let
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardInconsistentSize
|
||||||
|
|
||||||
|
test "should fail if there are data losses and length of parityData[0+N] is not zero and does not equal symbolBytes":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
|
||||||
|
var
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
|
||||||
|
data[3] = @[]
|
||||||
|
parityData[1] = @[1.byte, 2.byte, 3.byte]
|
||||||
|
|
||||||
|
let
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardInconsistentSize
|
||||||
|
|
||||||
|
# With the current setup in leopard.nim it seems it's not possible to call
|
||||||
|
# decode with an RS code that would result in leoDecodeWorkCount being called
|
||||||
|
# with invalid parameters, i.e. that would result in it returning 0, because
|
||||||
|
# a Result error will always be returned before leoDecodeWorkCount is called.
|
||||||
|
|
||||||
|
# test "should fail if there are data losses and RS code parameters yield invalid parameters for leoDecodWorkCount":
|
||||||
|
# check: initialized
|
||||||
|
# if not initialized: return
|
||||||
|
#
|
||||||
|
# let
|
||||||
|
# rsCode = RS(?,?)
|
||||||
|
# symbolBytes = MinBufferSize
|
||||||
|
# parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
#
|
||||||
|
# var
|
||||||
|
# data = genData(rsCode.data, symbolBytes)
|
||||||
|
#
|
||||||
|
# data[0] = @[]
|
||||||
|
#
|
||||||
|
# let
|
||||||
|
# decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
#
|
||||||
|
# check: decodeRes.isErr
|
||||||
|
# if decodeRes.isErr:
|
||||||
|
# check: decodeRes.error.code == LeopardInvalidInput
|
||||||
|
|
||||||
|
test "should succeed if there are no data losses even when all parity data is lost":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
let
|
||||||
|
rsCode = RS(8,5)
|
||||||
|
symbolBytes = MinBufferSize
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
|
||||||
|
var
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isOk
|
||||||
|
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
parityData[1] = @[]
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isOk
|
||||||
|
|
||||||
|
parityData = genData(rsCode.parity, symbolBytes)
|
||||||
|
for i in 0..<parityData.len: parityData[i] = @[]
|
||||||
|
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isOk
|
||||||
|
|
||||||
|
suite "Encode + Decode":
|
||||||
|
test "should fail to recover data when losses exceed tolerance":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
var i = 0
|
||||||
|
while i < 1000:
|
||||||
|
let
|
||||||
|
# together dataSymbols = 256+, paritySymbols = 17+, symbolBytes = 64+
|
||||||
|
# seem to consistently trigger parallel processing with OpenMP
|
||||||
|
dataSymbols = rand(256..320)
|
||||||
|
paritySymbols = rand(17..dataSymbols)
|
||||||
|
codewordSymbols = dataSymbols + paritySymbols
|
||||||
|
symbolBytesMultip = rand(1..8)
|
||||||
|
symbolBytes = MinBufferSize * symbolBytesMultip.uint
|
||||||
|
rsCode = RS(codewordSymbols, dataSymbols)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
losses = paritySymbols + 1
|
||||||
|
parityDataHoleCount =
|
||||||
|
if (losses - 1) == 0: 0 else: rand(1..(losses - 1))
|
||||||
|
dataHoleCount = losses - parityDataHoleCount
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: dataHoleCount + parityDataHoleCount == losses
|
||||||
|
|
||||||
|
check: encodeRes.isOk
|
||||||
|
if encodeRes.isOk:
|
||||||
|
let
|
||||||
|
parityData = encodeRes.get
|
||||||
|
|
||||||
|
var
|
||||||
|
dataWithHoles = data
|
||||||
|
parityDataWithHoles = parityData
|
||||||
|
|
||||||
|
var
|
||||||
|
dataHoles: seq[int]
|
||||||
|
|
||||||
|
for i in 1..dataHoleCount:
|
||||||
|
while true:
|
||||||
|
let
|
||||||
|
j = rand(dataSymbols - 1)
|
||||||
|
|
||||||
|
if dataHoles.find(j) == -1:
|
||||||
|
dataHoles.add j
|
||||||
|
break
|
||||||
|
|
||||||
|
check: dataHoles.len == dataHoleCount
|
||||||
|
|
||||||
|
for i in dataHoles:
|
||||||
|
dataWithHoles[i] = @[]
|
||||||
|
|
||||||
|
var
|
||||||
|
parityDataHoles: seq[int]
|
||||||
|
|
||||||
|
for i in 1..parityDataHoleCount:
|
||||||
|
while true:
|
||||||
|
let
|
||||||
|
j = rand(paritySymbols - 1)
|
||||||
|
|
||||||
|
if parityDataHoles.find(j) == -1:
|
||||||
|
parityDataHoles.add j
|
||||||
|
break
|
||||||
|
|
||||||
|
check: parityDataHoles.len == parityDataHoleCount
|
||||||
|
|
||||||
|
for i in parityDataHoles:
|
||||||
|
parityDataWithHoles[i] = @[]
|
||||||
|
|
||||||
|
let
|
||||||
|
decodeRes = rsCode.decode(dataWithHoles, parityDataWithHoles,
|
||||||
|
symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isErr
|
||||||
|
if decodeRes.isErr:
|
||||||
|
check: decodeRes.error.code == LeopardNeedMoreData
|
||||||
|
|
||||||
|
else:
|
||||||
|
echo "encode error message: " & encodeRes.error.msg
|
||||||
|
|
||||||
|
inc i
|
||||||
|
|
||||||
|
test "should recover data otherwise":
|
||||||
|
check: initialized
|
||||||
|
if not initialized: return
|
||||||
|
|
||||||
|
var i = 0
|
||||||
|
while i < 1000:
|
||||||
|
let
|
||||||
|
# together dataSymbols = 256+, paritySymbols = 17+, symbolBytes = 64+
|
||||||
|
# seem to consistently trigger parallel processing with OpenMP
|
||||||
|
dataSymbols = rand(256..320)
|
||||||
|
paritySymbols = rand(17..dataSymbols)
|
||||||
|
codewordSymbols = dataSymbols + paritySymbols
|
||||||
|
symbolBytesMultip = rand(1..8)
|
||||||
|
symbolBytes = MinBufferSize * symbolBytesMultip.uint
|
||||||
|
rsCode = RS(codewordSymbols, dataSymbols)
|
||||||
|
data = genData(rsCode.data, symbolBytes)
|
||||||
|
losses = rand(1..paritySymbols)
|
||||||
|
parityDataHoleCount =
|
||||||
|
if (losses - 1) == 0: 0 else: rand(1..(losses - 1))
|
||||||
|
dataHoleCount = losses - parityDataHoleCount
|
||||||
|
encodeRes = rsCode.encode data
|
||||||
|
|
||||||
|
check: dataHoleCount + parityDataHoleCount == losses
|
||||||
|
|
||||||
|
check: encodeRes.isOk
|
||||||
|
if encodeRes.isOk:
|
||||||
|
let
|
||||||
|
parityData = encodeRes.get
|
||||||
|
|
||||||
|
var
|
||||||
|
dataWithHoles = data
|
||||||
|
parityDataWithHoles = parityData
|
||||||
|
|
||||||
|
var
|
||||||
|
dataHoles: seq[int]
|
||||||
|
|
||||||
|
for i in 1..dataHoleCount:
|
||||||
|
while true:
|
||||||
|
let
|
||||||
|
j = rand(dataSymbols - 1)
|
||||||
|
|
||||||
|
if dataHoles.find(j) == -1:
|
||||||
|
dataHoles.add j
|
||||||
|
break
|
||||||
|
|
||||||
|
check: dataHoles.len == dataHoleCount
|
||||||
|
|
||||||
|
for i in dataHoles:
|
||||||
|
dataWithHoles[i] = @[]
|
||||||
|
|
||||||
|
var
|
||||||
|
parityDataHoles: seq[int]
|
||||||
|
|
||||||
|
for i in 1..parityDataHoleCount:
|
||||||
|
while true:
|
||||||
|
let
|
||||||
|
j = rand(paritySymbols - 1)
|
||||||
|
|
||||||
|
if parityDataHoles.find(j) == -1:
|
||||||
|
parityDataHoles.add j
|
||||||
|
break
|
||||||
|
|
||||||
|
check: parityDataHoles.len == parityDataHoleCount
|
||||||
|
|
||||||
|
for i in parityDataHoles:
|
||||||
|
parityDataWithHoles[i] = @[]
|
||||||
|
|
||||||
|
let
|
||||||
|
decodeRes = rsCode.decode(dataWithHoles, parityDataWithHoles,
|
||||||
|
symbolBytes)
|
||||||
|
|
||||||
|
check: decodeRes.isOk
|
||||||
|
if decodeRes.isOk:
|
||||||
|
let
|
||||||
|
decodedData = decodeRes.get
|
||||||
|
|
||||||
|
check:
|
||||||
|
decodedData != dataWithHoles
|
||||||
|
decodedData == data
|
||||||
|
|
||||||
|
else:
|
||||||
|
echo "decode error message: " & decodeRes.error.msg
|
||||||
|
|
||||||
|
else:
|
||||||
|
echo "encode error message: " & encodeRes.error.msg
|
||||||
|
|
||||||
|
inc i
|
||||||
|
|
||||||
|
echo ""
|
||||||
Loading…
x
Reference in New Issue
Block a user