mirror of
https://github.com/logos-storage/nim-leopard.git
synced 2026-01-02 13:43:08 +00:00
initial implementation and tests
This commit is contained in:
Michael Bradley, Jr
parent
4d89e44e0d
commit
0d2fae23bf
5
.editorconfig
Normal file
5
.editorconfig
Normal file
@ -0,0 +1,5 @@
|
||||
[*]
|
||||
indent_style = space
|
||||
insert_final_newline = true
|
||||
indent_size = 2
|
||||
trim_trailing_whitespace = true
|
||||
1
.gitattributes
vendored
Normal file
1
.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
* text=auto eol=lf
|
||||
154
.github/workflows/test.yml
vendored
Normal file
154
.github/workflows/test.yml
vendored
Normal file
@ -0,0 +1,154 @@
|
||||
name: Tests
|
||||
|
||||
on: [pull_request, push]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
env:
|
||||
NPROC: 2
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
cache_nonce: [ 1 ]
|
||||
nim_version: [ 1.2.18, 1.4.8, 1.6.4 ]
|
||||
platform:
|
||||
- {
|
||||
icon: 🐧,
|
||||
label: Linux,
|
||||
os: ubuntu,
|
||||
shell: bash --noprofile --norc -eo pipefail
|
||||
}
|
||||
- {
|
||||
icon: 🍎,
|
||||
label: macOS,
|
||||
os: macos,
|
||||
shell: bash --noprofile --norc -eo pipefail
|
||||
}
|
||||
- {
|
||||
icon: 🏁,
|
||||
label: Windows,
|
||||
os: windows,
|
||||
shell: msys2
|
||||
}
|
||||
name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim_version }}
|
||||
runs-on: ${{ matrix.platform.os }}-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: ${{ matrix.platform.shell }} {0}
|
||||
|
||||
steps:
|
||||
# - name: Install tools and libraries via APT (Linux)
|
||||
# if: matrix.platform.os == 'ubuntu'
|
||||
# run: |
|
||||
# sudo apt update
|
||||
# sudo apt install -y \
|
||||
# ...
|
||||
|
||||
- name: Install tools and libraries via Homebrew (macOS)
|
||||
if: matrix.platform.os == 'macos'
|
||||
run: |
|
||||
brew update
|
||||
brew install \
|
||||
findutils \
|
||||
libomp
|
||||
|
||||
- name: Install tools and libraries via MSYS2 (Windows)
|
||||
if: matrix.platform.os == 'windows'
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: UCRT64
|
||||
install: >
|
||||
base-devel
|
||||
git
|
||||
mingw-w64-ucrt-x86_64-cmake
|
||||
mingw-w64-ucrt-x86_64-toolchain
|
||||
|
||||
- name: Checkout sources from GitHub
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Calculate cache member paths
|
||||
id: calc-paths
|
||||
run: |
|
||||
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||
echo "::set-output name=bash_env::$(cygpath -m "${HOME}")/.bash_env"
|
||||
echo "::set-output name=choosenim::$(cygpath -m "${USERPROFILE}")/.choosenim"
|
||||
echo "::set-output name=nimble::$(cygpath -m "${HOME}")/.nimble"
|
||||
else
|
||||
echo "::set-output name=bash_env::${HOME}/.bash_env"
|
||||
echo "::set-output name=choosenim::${HOME}/.choosenim"
|
||||
echo "::set-output name=nimble::${HOME}/.nimble"
|
||||
fi
|
||||
|
||||
- name: Restore choosenim and Nim tooling from cache
|
||||
id: choosenim-nim-tooling-cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
${{ steps.calc-paths.outputs.bash_env }}
|
||||
${{ steps.calc-paths.outputs.choosenim }}
|
||||
${{ steps.calc-paths.outputs.nimble }}/bin
|
||||
key: ${{ matrix.platform.os }}-nim_version:${{ matrix.nim_version }}-cache_nonce:${{ matrix.cache_nonce }}
|
||||
|
||||
- name: Install choosenim and Nim tooling
|
||||
if: steps.choosenim-nim-tooling-cache.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
mkdir -p "${HOME}/Downloads"
|
||||
cd "${HOME}/Downloads"
|
||||
curl https://nim-lang.org/choosenim/init.sh -sSf -O
|
||||
chmod +x init.sh
|
||||
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||
mkdir -p "$(cygpath "${USERPROFILE}")/.nimble/bin"
|
||||
fi
|
||||
CHOOSENIM_CHOOSE_VERSION=${{ matrix.nim_version }} ./init.sh -y
|
||||
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||
mv "$(cygpath "${USERPROFILE}")/.nimble" "${HOME}/"
|
||||
# intention is to rely only on libs provided by the OS and MSYS2 env
|
||||
rm -rf "${HOME}/.nimble/bin/"*.dll
|
||||
rm -rf "${HOME}/.nimble/bin/"*.pem
|
||||
fi
|
||||
echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env"
|
||||
echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env"
|
||||
|
||||
- name: Install project dependencies
|
||||
run: |
|
||||
source "${HOME}/.bash_env"
|
||||
cd "${NIMBLE_DIR}/bin"
|
||||
# delete broken symlinks, which can arise because e.g. the cache
|
||||
# restored a symlink that points to an executable within
|
||||
# ../pkgs/foo-1.2.3/ but the project's .nimble file has been updated
|
||||
# to install foo-#head. In the case of a broken symlink, nimble's
|
||||
# auto-overwrite fails (only sometimes? only on macOS?)
|
||||
if [[ ${{ matrix.platform.os }} = macos ]]; then
|
||||
gfind . -xtype l -delete
|
||||
else
|
||||
find . -xtype l -delete
|
||||
fi
|
||||
cd -
|
||||
nimble --accept install
|
||||
|
||||
- name: Build and run tests
|
||||
run: |
|
||||
source "${HOME}/.bash_env"
|
||||
if [[ ${{ matrix.platform.os }} = windows ]]; then
|
||||
touch tests/test_leopard.exe
|
||||
else
|
||||
touch tests/test_leopard
|
||||
fi
|
||||
if [[ ${{ matrix.platform.os }} = macos ]]; then
|
||||
export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}"
|
||||
export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib"
|
||||
nimble test -d:verbose -d:release -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib"
|
||||
else
|
||||
nimble test -d:verbose -d:release
|
||||
fi
|
||||
if [[ ${{ matrix.platform.os }} = macos ]]; then
|
||||
echo
|
||||
echo otool -L tests/test_leopard
|
||||
otool -L tests/test_leopard
|
||||
else
|
||||
echo
|
||||
echo ldd tests/test_leopard
|
||||
ldd tests/test_leopard
|
||||
fi
|
||||
13
.gitignore
vendored
Normal file
13
.gitignore
vendored
Normal file
@ -0,0 +1,13 @@
|
||||
*
|
||||
!*/
|
||||
!*.*
|
||||
*.a
|
||||
*.dll
|
||||
*.dylib
|
||||
*.exe
|
||||
*.so
|
||||
.DS_Store
|
||||
.idea
|
||||
.vscode
|
||||
leopard.nims
|
||||
TODO
|
||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -1,5 +1,5 @@
|
||||
[submodule "vendor/leopard"]
|
||||
path = vendor/leopard
|
||||
url = https://github.com/catid/leopard.git
|
||||
url = https://github.com/status-im/leopard.git
|
||||
ignore = untracked
|
||||
branch = master
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
[](https://opensource.org/licenses/Apache-2.0)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/status-im/nim-leopard#stability)
|
||||
[](https://github.com/status-im/nim-leopard/actions?query=workflow%3ATests+branch%3Ainitial_impl)
|
||||
|
||||
Nim wrapper for [Leopard-RS](https://github.com/catid/leopard): a fast library for [Reed-Solomon](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) erasure correction coding.
|
||||
|
||||
|
||||
307
leopard.nim
307
leopard.nim
@ -0,0 +1,307 @@
|
||||
import pkg/stew/ptrops
|
||||
import pkg/stew/results
|
||||
import pkg/upraises
|
||||
|
||||
import ./leopard/wrapper
|
||||
|
||||
export results
|
||||
|
||||
push: {.upraises: [].}
|
||||
|
||||
const
|
||||
LeopardBadCodeMsg = "Bad RS code"
|
||||
LeopardInconsistentSizeMsg =
|
||||
"Buffer sizes must all be the same multiple of 64 bytes"
|
||||
LeopardNeedLessDataMsg = "Too much recovery data received"
|
||||
LeopardNotEnoughDataMsg = "Buffer counts are too low"
|
||||
|
||||
MinBufferSize* = 64.uint
|
||||
|
||||
type
|
||||
Data* = seq[seq[byte]]
|
||||
|
||||
LeopardDefect* = object of Defect
|
||||
|
||||
# It should not be necessary to redefine LeopardResult, but if that's not
|
||||
# done here then defining LeopardError as `object of CatchableError` will
|
||||
# cause a mystery crash at compile-time (symbol not found). Can workaround by
|
||||
# defining as just `object`, but then when trying to work with LeopardResult
|
||||
# errors in e.g. tests/test_leopard.nim the same mystery crash happens at
|
||||
# compile-time. The problem may be related to use of importcpp in
|
||||
# leopard/wrapper.nim, so it could be a compiler bug. By redefining
|
||||
# LeopardResult in this module (and casting wrapper.LeopardResult values) the
|
||||
# the problem is avoided.
|
||||
LeopardResult* = enum
|
||||
LeopardNotEnoughData = -11.cint # Buffer counts are too low
|
||||
LeopardNeedLessData = -10.cint # Too much recovery data received
|
||||
LeopardInconsistentSize = -9.cint # Buffer sizes must all be the same multiple of 64 bytes
|
||||
LeopardBadCode = -8.cint # Bad RS code
|
||||
LeopardCallInitialize = wrapper.LeopardCallInitialize
|
||||
LeopardPlatform = wrapper.LeopardPlatform
|
||||
LeopardInvalidInput = wrapper.LeopardInvalidInput
|
||||
LeopardInvalidCounts = wrapper.LeopardInvalidCounts
|
||||
LeopardInvalidSize = wrapper.LeopardInvalidSize
|
||||
LeopardTooMuchData = wrapper.LeopardTooMuchData
|
||||
LeopardNeedMoreData = wrapper.LeopardNeedMoreData
|
||||
LeopardSuccess = wrapper.LeopardSuccess
|
||||
|
||||
LeopardError* = object of CatchableError
|
||||
code*: LeopardResult
|
||||
|
||||
ParityData* = Data
|
||||
|
||||
ReedSolomonCode* = tuple[codeword, data, parity: uint] # symbol counts
|
||||
|
||||
# https://github.com/catid/leopard/issues/12
|
||||
# https://www.cs.cmu.edu/~guyb/realworld/reedsolomon/reed_solomon_codes.html
|
||||
#
|
||||
# RS(255,239)
|
||||
# ---------------------------------
|
||||
# codeword symbols = 255
|
||||
# data symbols = 239
|
||||
# parity symbols = 255 - 239 = 16
|
||||
|
||||
proc RS*(codeword, data: Positive): ReedSolomonCode =
|
||||
var
|
||||
parity = codeword - data
|
||||
|
||||
if parity <= 0: parity = 0
|
||||
(codeword: codeword.uint, data: data.uint, parity: parity.uint)
|
||||
|
||||
when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
|
||||
const
|
||||
header = "<stdlib.h>"
|
||||
|
||||
proc c_malloc(size: csize_t): pointer {.importc: "malloc", header: header.}
|
||||
proc c_free(p: pointer) {.importc: "free", header: header.}
|
||||
|
||||
proc SIMDSafeAllocate(size: int): pointer {.inline.} =
|
||||
var
|
||||
data =
|
||||
when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
|
||||
c_malloc(LEO_ALIGN_BYTES + size.uint)
|
||||
else:
|
||||
allocShared(LEO_ALIGN_BYTES + size.uint)
|
||||
|
||||
doffset = cast[uint](data) mod LEO_ALIGN_BYTES
|
||||
|
||||
data = offset(data, (LEO_ALIGN_BYTES + doffset).int)
|
||||
|
||||
var
|
||||
offsetPtr = cast[pointer](cast[uint](data) - 1)
|
||||
|
||||
moveMem(offsetPtr, addr doffset, sizeof(doffset))
|
||||
data
|
||||
|
||||
proc SIMDSafeFree(data: pointer) {.inline.} =
|
||||
var
|
||||
data = data
|
||||
|
||||
if not data.isNil:
|
||||
let
|
||||
offset = cast[uint](data) - 1
|
||||
|
||||
if offset >= LEO_ALIGN_BYTES: return
|
||||
|
||||
data = cast[pointer](cast[uint](data) - (LEO_ALIGN_BYTES - offset))
|
||||
|
||||
when (NimMajor, NimMinor, NimPatch) < (1, 4, 0):
|
||||
c_free data
|
||||
else:
|
||||
deallocShared data
|
||||
|
||||
proc leoInit*() =
|
||||
if wrapper.leoInit() != 0:
|
||||
raise (ref LeopardDefect)(msg: "Leopard-RS failed to initialize")
|
||||
|
||||
proc encode*(code: ReedSolomonCode, data: Data):
|
||||
Result[ParityData, LeopardError] =
|
||||
if code.parity < 1 or code.parity > code.data:
|
||||
return err LeopardError(code: LeopardBadCode, msg: LeopardBadCodeMsg)
|
||||
|
||||
var
|
||||
data = data
|
||||
|
||||
let
|
||||
symbolBytes = data[0].len
|
||||
|
||||
if data.len < code.data.int:
|
||||
return err LeopardError(code: LeopardNotEnoughData,
|
||||
msg: LeopardNotEnoughDataMsg)
|
||||
|
||||
elif data.len > code.data.int:
|
||||
return err LeopardError(code: LeopardTooMuchData,
|
||||
msg: $leoResultString(wrapper.LeopardTooMuchData))
|
||||
|
||||
if symbolBytes < MinBufferSize.int or symbolBytes mod MinBufferSize.int != 0:
|
||||
return err LeopardError(code: LeopardInvalidSize,
|
||||
msg: $leoResultString(wrapper.LeopardInvalidSize))
|
||||
|
||||
var
|
||||
enData = newSeq[pointer](code.data)
|
||||
|
||||
for i in 0..<code.data:
|
||||
if data[i].len != symbolBytes:
|
||||
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||
return err LeopardError(code: LeopardInconsistentSize,
|
||||
msg: LeopardInconsistentSizeMsg)
|
||||
|
||||
enData[i] = SIMDSafeAllocate symbolBytes
|
||||
moveMem(enData[i], addr data[i][0], symbolBytes)
|
||||
|
||||
let
|
||||
workCount = leoEncodeWorkCount(code.data.cuint, code.parity.cuint)
|
||||
|
||||
if workCount == 0:
|
||||
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||
return err LeopardError(code: LeopardInvalidInput,
|
||||
msg: $leoResultString(wrapper.LeopardInvalidInput))
|
||||
|
||||
var
|
||||
workData = newSeq[pointer](workCount)
|
||||
|
||||
for i in 0..<workCount:
|
||||
workData[i] = SIMDSafeAllocate symbolBytes
|
||||
|
||||
let
|
||||
encodeRes = leoEncode(
|
||||
symbolBytes.uint64,
|
||||
code.data.cuint,
|
||||
code.parity.cuint,
|
||||
workCount,
|
||||
addr enData[0],
|
||||
addr workData[0]
|
||||
)
|
||||
|
||||
if encodeRes != wrapper.LeopardSuccess:
|
||||
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||
return err LeopardError(code: cast[LeopardResult](encodeRes),
|
||||
msg: $leoResultString(encodeRes))
|
||||
|
||||
var
|
||||
parityData: ParityData
|
||||
|
||||
newSeq(parityData, code.parity)
|
||||
for i in 0..<code.parity:
|
||||
newSeq(parityData[i], symbolBytes)
|
||||
moveMem(addr parityData[i][0], workData[i], symbolBytes)
|
||||
|
||||
for i in 0..<code.data: SIMDSafeFree enData[i]
|
||||
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||
|
||||
ok parityData
|
||||
|
||||
proc decode*(code: ReedSolomonCode, data: Data, parityData: ParityData,
|
||||
symbolBytes: uint): Result[Data, LeopardError] =
|
||||
if code.parity < 1 or code.parity > code.data:
|
||||
return err LeopardError(code: LeopardBadCode, msg: LeopardBadCodeMsg)
|
||||
|
||||
var
|
||||
data = data
|
||||
parityData = parityData
|
||||
holes: seq[int]
|
||||
|
||||
if data.len < code.data.int:
|
||||
return err LeopardError(code: LeopardNotEnoughData,
|
||||
msg: LeopardNotEnoughDataMsg)
|
||||
|
||||
elif data.len > code.data.int:
|
||||
return err LeopardError(code: LeopardTooMuchData,
|
||||
msg: $leoResultString(wrapper.LeopardTooMuchData))
|
||||
|
||||
if parityData.len < code.parity.int:
|
||||
return err LeopardError(code: LeopardNeedMoreData,
|
||||
msg: $leoResultString(wrapper.LeopardNeedMoreData))
|
||||
|
||||
elif parityData.len > code.parity.int:
|
||||
return err LeopardError(code: LeopardNeedLessData,
|
||||
msg: LeopardNeedLessDataMsg)
|
||||
|
||||
if symbolBytes < MinBufferSize or symbolBytes mod MinBufferSize != 0:
|
||||
return err LeopardError(code: LeopardInvalidSize,
|
||||
msg: $leoResultString(wrapper.LeopardInvalidSize))
|
||||
|
||||
var
|
||||
deData = newSeq[pointer](code.data)
|
||||
|
||||
for i in 0..<code.data:
|
||||
if data[i].len != 0:
|
||||
if data[i].len != symbolBytes.int:
|
||||
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||
return err LeopardError(code: LeopardInconsistentSize,
|
||||
msg: LeopardInconsistentSizeMsg)
|
||||
|
||||
deData[i] = SIMDSafeAllocate symbolBytes.int
|
||||
moveMem(deData[i], addr data[i][0], symbolBytes)
|
||||
|
||||
else:
|
||||
holes.add i.int
|
||||
|
||||
if holes.len == 0:
|
||||
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||
return ok data
|
||||
|
||||
var
|
||||
paData = newSeq[pointer](code.parity)
|
||||
|
||||
for i in 0..<code.parity:
|
||||
if parityData[i].len != 0:
|
||||
if parityData[i].len != symbolBytes.int:
|
||||
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||
return err LeopardError(code: LeopardInconsistentSize,
|
||||
msg: LeopardInconsistentSizeMsg)
|
||||
|
||||
paData[i] = SIMDSafeAllocate symbolBytes.int
|
||||
moveMem(paData[i], addr parityData[i][0], symbolBytes)
|
||||
|
||||
let
|
||||
workCount = leoDecodeWorkCount(code.data.cuint, code.parity.cuint)
|
||||
|
||||
if workCount == 0:
|
||||
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||
return err LeopardError(code: LeopardInvalidInput,
|
||||
msg: $leoResultString(wrapper.LeopardInvalidInput))
|
||||
|
||||
var
|
||||
workData = newSeq[pointer](workCount)
|
||||
|
||||
for i in 0..<workCount:
|
||||
workData[i] = SIMDSafeAllocate symbolBytes.int
|
||||
|
||||
let
|
||||
decodeRes = leoDecode(
|
||||
symbolBytes.uint64,
|
||||
code.data.cuint,
|
||||
code.parity.cuint,
|
||||
workCount,
|
||||
addr deData[0],
|
||||
addr paData[0],
|
||||
addr workData[0]
|
||||
)
|
||||
|
||||
if decodeRes != wrapper.LeopardSuccess:
|
||||
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||
return err LeopardError(code: cast[LeopardResult](decodeRes),
|
||||
msg: $leoResultString(decodeRes))
|
||||
|
||||
var
|
||||
recoveredData: Data
|
||||
|
||||
newSeq(recoveredData, workCount)
|
||||
for i in 0..<workCount:
|
||||
newSeq(recoveredData[i], symbolBytes)
|
||||
moveMem(addr recoveredData[i][0], workData[i], symbolBytes)
|
||||
|
||||
for i in holes:
|
||||
data[i] = recoveredData[i]
|
||||
|
||||
for i in 0..<code.data: SIMDSafeFree deData[i]
|
||||
for i in 0..<code.parity: SIMDSafeFree paData[i]
|
||||
for i in 0..<workCount: SIMDSafeFree workData[i]
|
||||
|
||||
ok data
|
||||
@ -5,7 +5,9 @@ version = "0.0.1"
|
||||
author = "Status Research & Development GmbH"
|
||||
description = "A wrapper for Leopard-RS"
|
||||
license = "Apache License 2.0 or MIT"
|
||||
installDirs = @["vendor"]
|
||||
|
||||
requires "nim >= 1.2.0",
|
||||
"stew#head",
|
||||
"unittest2"
|
||||
"stew",
|
||||
"unittest2",
|
||||
"upraises >= 0.1.0 & < 0.2.0"
|
||||
|
||||
@ -0,0 +1,293 @@
|
||||
## Copyright (c) 2017 Christopher A. Taylor. All rights reserved.
|
||||
##
|
||||
## Redistribution and use in source and binary forms, with or without
|
||||
## modification, are permitted provided that the following conditions are met:
|
||||
##
|
||||
## * Redistributions of source code must retain the above copyright notice,
|
||||
## this list of conditions and the following disclaimer.
|
||||
## * Redistributions in binary form must reproduce the above copyright notice,
|
||||
## this list of conditions and the following disclaimer in the documentation
|
||||
## and/or other materials provided with the distribution.
|
||||
## * Neither the name of Leopard-RS nor the names of its contributors may be
|
||||
## used to endorse or promote products derived from this software without
|
||||
## specific prior written permission.
|
||||
##
|
||||
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
## POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
## Leopard-RS
|
||||
## MDS Reed-Solomon Erasure Correction Codes for Large Data in C
|
||||
##
|
||||
## Algorithms are described in LeopardCommon.h
|
||||
##
|
||||
##
|
||||
## Inspired by discussion with:
|
||||
##
|
||||
## Sian-Jhen Lin <sjhenglin@gmail.com> : Author of {1} {3}, basis for Leopard
|
||||
## Bulat Ziganshin <bulat.ziganshin@gmail.com> : Author of FastECC
|
||||
## Yutaka Sawada <tenfon@outlook.jp> : Author of MultiPar
|
||||
##
|
||||
##
|
||||
## References:
|
||||
##
|
||||
## {1} S.-J. Lin, T. Y. Al-Naffouri, Y. S. Han, and W.-H. Chung,
|
||||
## "Novel Polynomial Basis with Fast Fourier Transform
|
||||
## and Its Application to Reed-Solomon Erasure Codes"
|
||||
## IEEE Trans. on Information Theory, pp. 6284-6299, November, 2016.
|
||||
##
|
||||
## {2} D. G. Cantor, "On arithmetical algorithms over finite fields",
|
||||
## Journal of Combinatorial Theory, Series A, vol. 50, no. 2, pp. 285-300, 1989.
|
||||
##
|
||||
## {3} Sian-Jheng Lin, Wei-Ho Chung, "An Efficient (n, k) Information
|
||||
## Dispersal Algorithm for High Code Rate System over Fermat Fields,"
|
||||
## IEEE Commun. Lett., vol.16, no.12, pp. 2036-2039, Dec. 2012.
|
||||
##
|
||||
## {4} Plank, J. S., Greenan, K. M., Miller, E. L., "Screaming fast Galois Field
|
||||
## arithmetic using Intel SIMD instructions." In: FAST-2013: 11th Usenix
|
||||
## Conference on File and Storage Technologies, San Jose, 2013
|
||||
|
||||
|
||||
import upraises
|
||||
push: {.upraises: [].}
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Build configuration
|
||||
|
||||
import std/compilesettings
|
||||
import std/os
|
||||
import std/strutils
|
||||
|
||||
const
|
||||
LeopardCmakeFlags {.strdefine.} =
|
||||
when defined(macosx):
|
||||
"-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off"
|
||||
elif defined(windows):
|
||||
"-G\"MSYS Makefiles\" -DCMAKE_BUILD_TYPE=Release"
|
||||
else:
|
||||
"-DCMAKE_BUILD_TYPE=Release"
|
||||
|
||||
LeopardDir {.strdefine.} =
|
||||
joinPath(currentSourcePath.parentDir.parentDir, "vendor", "leopard")
|
||||
|
||||
buildDir = joinPath(querySetting(nimcacheDir), "vendor_leopard")
|
||||
|
||||
LeopardHeader {.strdefine.} = "leopard.h"
|
||||
|
||||
LeopardLib {.strdefine.} = joinPath(buildDir, "liblibleopard.a")
|
||||
|
||||
LeopardCompilerFlags {.strdefine.} =
|
||||
when defined(macosx):
|
||||
"-I" & LeopardDir
|
||||
else:
|
||||
"-I" & LeopardDir & " -fopenmp"
|
||||
|
||||
LeopardLinkerFlags {.strdefine.} =
|
||||
when defined(macosx):
|
||||
LeopardLib
|
||||
else:
|
||||
LeopardLib & " -fopenmp"
|
||||
|
||||
LeopardExtraCompilerFlags {.strdefine.} = ""
|
||||
|
||||
LeopardExtraLinkerFlags {.strdefine.} = ""
|
||||
|
||||
static:
|
||||
if defined(windows):
|
||||
func pathUnix2Win(path: string): string =
|
||||
gorge("cygpath -w " & path.strip).strip
|
||||
|
||||
func pathWin2Unix(path: string): string =
|
||||
gorge("cygpath " & path.strip).strip
|
||||
|
||||
proc bash(cmd: varargs[string]): string =
|
||||
gorge(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'")
|
||||
|
||||
proc bashEx(cmd: varargs[string]): tuple[output: string, exitCode: int] =
|
||||
gorgeEx(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'")
|
||||
|
||||
let
|
||||
buildDirUnix = buildDir.pathWin2Unix
|
||||
leopardDirUnix = LeopardDir.pathWin2Unix
|
||||
if defined(LeopardRebuild): discard bash("rm -rf", buildDirUnix)
|
||||
if (bashEx("ls", LeopardLib.pathWin2Unix)).exitCode != 0:
|
||||
discard bash("mkdir -p", buildDirUnix)
|
||||
let cmd =
|
||||
@["cd", buildDirUnix, "&& cmake", leopardDirUnix, LeopardCmakeFlags,
|
||||
"&& make"]
|
||||
echo "\nBuilding Leopard-RS: " & cmd.join(" ")
|
||||
let (output, exitCode) = bashEx cmd
|
||||
echo output
|
||||
if exitCode != 0:
|
||||
discard bash("rm -rf", buildDirUnix)
|
||||
raise (ref Defect)(msg: "Failed to build Leopard-RS")
|
||||
else:
|
||||
if defined(LeopardRebuild): discard gorge "rm -rf " & buildDir
|
||||
if gorgeEx("ls " & LeopardLib).exitCode != 0:
|
||||
discard gorge "mkdir -p " & buildDir
|
||||
let cmd =
|
||||
"cd " & buildDir & " && cmake " & LeopardDir & " " & LeopardCmakeFlags &
|
||||
" && make"
|
||||
echo "\nBuilding Leopard-RS: " & cmd
|
||||
let (output, exitCode) = gorgeEx cmd
|
||||
echo output
|
||||
if exitCode != 0:
|
||||
discard gorge "rm -rf " & buildDir
|
||||
raise (ref Defect)(msg: "Failed to build Leopard-RS")
|
||||
|
||||
{.passC: LeopardCompilerFlags & " " & LeopardExtraCompilerFlags.}
|
||||
{.passL: LeopardLinkerFlags & " " & LeopardExtraLinkerFlags.}
|
||||
|
||||
{.pragma: leo, cdecl, header: LeopardHeader.}
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Library version
|
||||
|
||||
var LEO_VERSION* {.header: LeopardHeader, importc.}: int
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Platform/Architecture
|
||||
|
||||
# maybe should detect AVX2 and set to 32 if detected, 16 otherwise:
|
||||
# https://github.com/catid/leopard/blob/master/LeopardCommon.h#L247-L253
|
||||
# https://github.com/mratsim/Arraymancer/blob/master/src/arraymancer/laser/cpuinfo_x86.nim#L220
|
||||
const LEO_ALIGN_BYTES* = 16
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Initialization API
|
||||
|
||||
## leoInit()
|
||||
##
|
||||
## Perform static initialization for the library, verifying that the platform
|
||||
## is supported.
|
||||
##
|
||||
## Returns 0 on success and other values on failure.
|
||||
|
||||
proc leoInit*(): cint {.leo, importcpp: "leo_init".}
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Shared Constants / Datatypes
|
||||
|
||||
## Results
|
||||
type
|
||||
LeopardResult* = enum
|
||||
LeopardCallInitialize = -7.cint ## Call leoInit() first
|
||||
LeopardPlatform = -6.cint ## Platform is unsupported
|
||||
LeopardInvalidInput = -5.cint ## A function parameter was invalid
|
||||
LeopardInvalidCounts = -4.cint ## Invalid counts provided
|
||||
LeopardInvalidSize = -3.cint ## Buffer size must be multiple of 64 bytes
|
||||
LeopardTooMuchData = -2.cint ## Buffer counts are too high
|
||||
LeopardNeedMoreData = -1.cint ## Not enough recovery data received
|
||||
LeopardSuccess = 0.cint ## Operation succeeded
|
||||
|
||||
## Convert Leopard result to string
|
||||
func leoResultString*(res: LeopardResult): cstring
|
||||
{.leo, importc: "leo_result_string".}
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Encoder API
|
||||
|
||||
## leoEncodeWorkCount()
|
||||
##
|
||||
## Calculate the number of work data buffers to provide to leoEncode().
|
||||
##
|
||||
## The sum of originalCount + recoveryCount must not exceed 65536.
|
||||
##
|
||||
## Returns the workCount value to pass into leoEncode().
|
||||
## Returns 0 on invalid input.
|
||||
|
||||
func leoEncodeWorkCount*(originalCount, recoveryCount: cuint): cuint
|
||||
{.leo, importc: "leo_encode_work_count".}
|
||||
|
||||
## leoEncode()
|
||||
##
|
||||
## Generate recovery data.
|
||||
##
|
||||
## bufferBytes: Number of bytes in each data buffer.
|
||||
## originalCount: Number of original data buffers provided.
|
||||
## recoveryCount: Number of desired recovery data buffers.
|
||||
## workCount: Number of work data buffers, from leoEncodeWorkCount().
|
||||
## originalData: Array of pointers to original data buffers.
|
||||
## workData: Array of pointers to work data buffers.
|
||||
##
|
||||
## The sum of originalCount + recoveryCount must not exceed 65536.
|
||||
## The recoveryCount <= originalCount.
|
||||
##
|
||||
## The value of bufferBytes must be a multiple of 64.
|
||||
## Each buffer should have the same number of bytes.
|
||||
## Even the last piece must be rounded up to the block size.
|
||||
##
|
||||
## Returns LeopardSuccess on success.
|
||||
## The first set of recoveryCount buffers in workData will be the result.
|
||||
## Returns other values on errors.
|
||||
|
||||
proc leoEncode*(
|
||||
bufferBytes: uint64, ## Number of bytes in each data buffer
|
||||
originalCount: cuint, ## Number of originalData[] buffer pointers
|
||||
recoveryCount: cuint, ## Number of recovery data buffer pointers
|
||||
## (readable post-call from start of workData[])
|
||||
workCount: cuint, ## Number of workData[] buffer pointers
|
||||
originalData: pointer, ## Array of pointers to original data buffers
|
||||
workData: pointer, ## Array of pointers to work data buffers
|
||||
): LeopardResult {.leo, importc: "leo_encode".}
|
||||
|
||||
|
||||
## -----------------------------------------------------------------------------
|
||||
## Decoder API
|
||||
|
||||
## leoDecodeWorkCount()
|
||||
##
|
||||
## Calculate the number of work data buffers to provide to leoDecode().
|
||||
##
|
||||
## The sum of originalCount + recoveryCount must not exceed 65536.
|
||||
##
|
||||
## Returns the workCount value to pass into leoDecode().
|
||||
## Returns 0 on invalid input.
|
||||
|
||||
func leoDecodeWorkCount*(originalCount, recoveryCount: cuint): cuint
|
||||
{.leo, importc: "leo_decode_work_count".}
|
||||
|
||||
## leoDecode()
|
||||
##
|
||||
## Decode original data from recovery data.
|
||||
##
|
||||
## bufferBytes: Number of bytes in each data buffer.
|
||||
## originalCount: Number of original data buffers provided.
|
||||
## recoveryCount: Number of recovery data buffers provided.
|
||||
## workCount: Number of work data buffers, from leoDecodeWorkCount().
|
||||
## originalData: Array of pointers to original data buffers.
|
||||
## recoveryData: Array of pointers to recovery data buffers.
|
||||
## workData: Array of pointers to work data buffers.
|
||||
##
|
||||
## Lost original/recovery data should be set to NULL.
|
||||
##
|
||||
## The sum of recoveryCount + the number of non-NULL original data must be at
|
||||
## least originalCount in order to perform recovery.
|
||||
##
|
||||
## Returns LeopardSuccess on success.
|
||||
## Returns other values on errors.
|
||||
|
||||
proc leoDecode*(
|
||||
bufferBytes: uint64, ## Number of bytes in each data buffer
|
||||
originalCount: cuint, ## Number of originalData[] buffer pointers
|
||||
recoveryCount: cuint, ## Number of recoveryData[] buffer pointers
|
||||
workCount: cuint, ## Number of workData[] buffer pointers
|
||||
originalData: pointer, ## Array of pointers to original data buffers
|
||||
recoveryData: pointer, ## Array of pointers to recovery data buffers
|
||||
workData: pointer, ## Array of pointers to work data buffers
|
||||
): LeopardResult {.leo, importc: "leo_decode".}
|
||||
@ -0,0 +1,558 @@
|
||||
import std/random
|
||||
|
||||
import pkg/leopard
|
||||
import pkg/unittest2
|
||||
|
||||
randomize()
|
||||
|
||||
proc genData(outerLen, innerLen: uint): Data =
|
||||
var
|
||||
data = newSeqOfCap[seq[byte]](outerLen)
|
||||
|
||||
for i in 0..<outerLen.int:
|
||||
data.add newSeqUninitialized[byte](innerLen)
|
||||
for j in 0..<innerLen:
|
||||
data[i][j] = rand(255).byte
|
||||
|
||||
data
|
||||
|
||||
var
|
||||
initialized = false
|
||||
|
||||
suite "Initialization":
|
||||
test "encode and decode should fail if Leopard-RS is not initialized":
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
|
||||
var
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
|
||||
let
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
# Related to a subtle race re: decode being called with data that has no
|
||||
# holes while Leopard-RS is not initialized, i.e. it would succeed by
|
||||
# simply returning the data without a call to leoDecode.
|
||||
data[0] = @[]
|
||||
|
||||
let
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check:
|
||||
encodeRes.isErr
|
||||
encodeRes.error.code == LeopardCallInitialize
|
||||
decodeRes.isErr
|
||||
decodeRes.error.code == LeopardCallInitialize
|
||||
|
||||
test "initialization should succeed else raise a Defect":
|
||||
leoInit()
|
||||
initialized = true
|
||||
|
||||
check: initialized
|
||||
|
||||
suite "Encoder":
|
||||
test "should fail if RS code is nonsensical or is so per Leopard-RS":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
symbolBytes = MinBufferSize
|
||||
|
||||
var
|
||||
rsCode = RS(5,5)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isErr
|
||||
if encodeRes.isErr:
|
||||
check: encodeRes.error.code == LeopardBadCode
|
||||
|
||||
rsCode = RS(5,10)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isErr
|
||||
if encodeRes.isErr:
|
||||
check: encodeRes.error.code == LeopardBadCode
|
||||
|
||||
rsCode = RS(110,10)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isErr
|
||||
if encodeRes.isErr:
|
||||
check: encodeRes.error.code == LeopardBadCode
|
||||
|
||||
test "should fail if outer length of data does not match the RS code":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
notEnoughData = genData(rsCode.data - 1, symbolBytes)
|
||||
tooMuchData = genData(rsCode.data + 1, symbolBytes)
|
||||
notEnoughEncodeRes = rsCode.encode notEnoughData
|
||||
tooMuchEncodeRes = rsCode.encode tooMuchData
|
||||
|
||||
check:
|
||||
notEnoughEncodeRes.isErr
|
||||
tooMuchEncodeRes.isErr
|
||||
if notEnoughEncodeRes.isErr:
|
||||
check: notEnoughEncodeRes.error.code == LeopardNotEnoughData
|
||||
if tooMuchEncodeRes.isErr:
|
||||
check: tooMuchEncodeRes.error.code == LeopardTooMuchData
|
||||
|
||||
test "should fail if length of data[0] is less than minimum buffer size":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize - 5
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isErr
|
||||
if encodeRes.isErr:
|
||||
check: encodeRes.error.code == LeopardInvalidSize
|
||||
|
||||
test "should fail if length of data[0] is not a multiple of minimum buffer size":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize * 2 + 1
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isErr
|
||||
if encodeRes.isErr:
|
||||
check: encodeRes.error.code == LeopardInvalidSize
|
||||
|
||||
test "should fail if length of data[0+N] does not equal length of data[0]":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
|
||||
var
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
|
||||
data[3] = @[1.byte, 2.byte, 3.byte]
|
||||
|
||||
let
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isErr
|
||||
if encodeRes.isErr:
|
||||
check: encodeRes.error.code == LeopardInconsistentSize
|
||||
|
||||
# With the current setup in leopard.nim it seems it's not possible to call
|
||||
# encode with an RS code that would result in leoEncodeWorkCount being called
|
||||
# with invalid parameters, i.e. that would result in it returning 0, because
|
||||
# a Result error will always be returned before leoEncodeWorkCount is called.
|
||||
|
||||
# test "should fail if RS code parameters yield invalid parameters for leoEncodWorkCount":
|
||||
# check: initialized
|
||||
# if not initialized: return
|
||||
#
|
||||
# let
|
||||
# rsCode = RS(?,?)
|
||||
# symbolBytes = MinBufferSize
|
||||
# data = genData(rsCode.data, symbolBytes)
|
||||
# encodeRes = rsCode.encode data
|
||||
#
|
||||
# check: encodeRes.isErr
|
||||
# if encodeRes.isErr:
|
||||
# check: encodeRes.error.code == LeopardInvalidInput
|
||||
|
||||
test "should succeed if RS code and data yield valid parameters for leoEncode":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: encodeRes.isOk
|
||||
|
||||
suite "Decoder":
|
||||
test "should fail if RS code is nonsensical or is so per Leopard-RS":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
symbolBytes = MinBufferSize
|
||||
|
||||
var
|
||||
rsCode = RS(5,5)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
parityData: ParityData
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardBadCode
|
||||
|
||||
rsCode = RS(5,10)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardBadCode
|
||||
|
||||
rsCode = RS(110,10)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardBadCode
|
||||
|
||||
test "should fail if outer length of data does not match the RS code":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
notEnoughData = genData(rsCode.data - 1, symbolBytes)
|
||||
tooMuchData = genData(rsCode.data + 1, symbolBytes)
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
notEnoughDecodeRes = rsCode.decode(notEnoughData, parityData, symbolBytes)
|
||||
tooMuchDecodeRes = rsCode.decode(tooMuchData, parityData, symbolBytes)
|
||||
|
||||
check:
|
||||
notEnoughDecodeRes.isErr
|
||||
tooMuchDecodeRes.isErr
|
||||
if notEnoughDecodeRes.isErr:
|
||||
check: notEnoughDecodeRes.error.code == LeopardNotEnoughData
|
||||
if tooMuchDecodeRes.isErr:
|
||||
check: tooMuchDecodeRes.error.code == LeopardTooMuchData
|
||||
|
||||
test "should fail if outer length of parityData does not match the RS code":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
notEnoughParityData = genData(rsCode.parity - 1, symbolBytes)
|
||||
tooMuchParityData = genData(rsCode.parity + 1, symbolBytes)
|
||||
notEnoughDecodeRes = rsCode.decode(data, notEnoughParityData, symbolBytes)
|
||||
tooMuchDecodeRes = rsCode.decode(data, tooMuchParityData, symbolBytes)
|
||||
|
||||
check:
|
||||
notEnoughDecodeRes.isErr
|
||||
tooMuchDecodeRes.isErr
|
||||
if notEnoughDecodeRes.isErr:
|
||||
check: notEnoughDecodeRes.error.code == LeopardNeedMoreData
|
||||
if tooMuchDecodeRes.isErr:
|
||||
check: tooMuchDecodeRes.error.code == LeopardNeedLessData
|
||||
|
||||
test "should fail if symbolBytes is less than minimum buffer size":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize - 5
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardInvalidSize
|
||||
|
||||
test "should fail if symbolBytes is not a multiple of minimum buffer size":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize * 2 + 1
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardInvalidSize
|
||||
|
||||
test "should fail if length of data[0+N] is not zero and does not equal symbolBytes":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
|
||||
var
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
|
||||
data[3] = @[1.byte, 2.byte, 3.byte]
|
||||
|
||||
let
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardInconsistentSize
|
||||
|
||||
test "should fail if there are data losses and length of parityData[0+N] is not zero and does not equal symbolBytes":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
|
||||
var
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
|
||||
data[3] = @[]
|
||||
parityData[1] = @[1.byte, 2.byte, 3.byte]
|
||||
|
||||
let
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardInconsistentSize
|
||||
|
||||
# With the current setup in leopard.nim it seems it's not possible to call
|
||||
# decode with an RS code that would result in leoDecodeWorkCount being called
|
||||
# with invalid parameters, i.e. that would result in it returning 0, because
|
||||
# a Result error will always be returned before leoDecodeWorkCount is called.
|
||||
|
||||
# test "should fail if there are data losses and RS code parameters yield invalid parameters for leoDecodWorkCount":
|
||||
# check: initialized
|
||||
# if not initialized: return
|
||||
#
|
||||
# let
|
||||
# rsCode = RS(?,?)
|
||||
# symbolBytes = MinBufferSize
|
||||
# parityData = genData(rsCode.parity, symbolBytes)
|
||||
#
|
||||
# var
|
||||
# data = genData(rsCode.data, symbolBytes)
|
||||
#
|
||||
# data[0] = @[]
|
||||
#
|
||||
# let
|
||||
# decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
#
|
||||
# check: decodeRes.isErr
|
||||
# if decodeRes.isErr:
|
||||
# check: decodeRes.error.code == LeopardInvalidInput
|
||||
|
||||
test "should succeed if there are no data losses even when all parity data is lost":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
let
|
||||
rsCode = RS(8,5)
|
||||
symbolBytes = MinBufferSize
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
|
||||
var
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isOk
|
||||
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
parityData[1] = @[]
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isOk
|
||||
|
||||
parityData = genData(rsCode.parity, symbolBytes)
|
||||
for i in 0..<parityData.len: parityData[i] = @[]
|
||||
decodeRes = rsCode.decode(data, parityData, symbolBytes)
|
||||
|
||||
check: decodeRes.isOk
|
||||
|
||||
suite "Encode + Decode":
|
||||
test "should fail to recover data when losses exceed tolerance":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
var i = 0
|
||||
while i < 1000:
|
||||
let
|
||||
# together dataSymbols = 256+, paritySymbols = 17+, symbolBytes = 64+
|
||||
# seem to consistently trigger parallel processing with OpenMP
|
||||
dataSymbols = rand(256..320)
|
||||
paritySymbols = rand(17..dataSymbols)
|
||||
codewordSymbols = dataSymbols + paritySymbols
|
||||
symbolBytesMultip = rand(1..8)
|
||||
symbolBytes = MinBufferSize * symbolBytesMultip.uint
|
||||
rsCode = RS(codewordSymbols, dataSymbols)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
losses = paritySymbols + 1
|
||||
parityDataHoleCount =
|
||||
if (losses - 1) == 0: 0 else: rand(1..(losses - 1))
|
||||
dataHoleCount = losses - parityDataHoleCount
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: dataHoleCount + parityDataHoleCount == losses
|
||||
|
||||
check: encodeRes.isOk
|
||||
if encodeRes.isOk:
|
||||
let
|
||||
parityData = encodeRes.get
|
||||
|
||||
var
|
||||
dataWithHoles = data
|
||||
parityDataWithHoles = parityData
|
||||
|
||||
var
|
||||
dataHoles: seq[int]
|
||||
|
||||
for i in 1..dataHoleCount:
|
||||
while true:
|
||||
let
|
||||
j = rand(dataSymbols - 1)
|
||||
|
||||
if dataHoles.find(j) == -1:
|
||||
dataHoles.add j
|
||||
break
|
||||
|
||||
check: dataHoles.len == dataHoleCount
|
||||
|
||||
for i in dataHoles:
|
||||
dataWithHoles[i] = @[]
|
||||
|
||||
var
|
||||
parityDataHoles: seq[int]
|
||||
|
||||
for i in 1..parityDataHoleCount:
|
||||
while true:
|
||||
let
|
||||
j = rand(paritySymbols - 1)
|
||||
|
||||
if parityDataHoles.find(j) == -1:
|
||||
parityDataHoles.add j
|
||||
break
|
||||
|
||||
check: parityDataHoles.len == parityDataHoleCount
|
||||
|
||||
for i in parityDataHoles:
|
||||
parityDataWithHoles[i] = @[]
|
||||
|
||||
let
|
||||
decodeRes = rsCode.decode(dataWithHoles, parityDataWithHoles,
|
||||
symbolBytes)
|
||||
|
||||
check: decodeRes.isErr
|
||||
if decodeRes.isErr:
|
||||
check: decodeRes.error.code == LeopardNeedMoreData
|
||||
|
||||
else:
|
||||
echo "encode error message: " & encodeRes.error.msg
|
||||
|
||||
inc i
|
||||
|
||||
test "should recover data otherwise":
|
||||
check: initialized
|
||||
if not initialized: return
|
||||
|
||||
var i = 0
|
||||
while i < 1000:
|
||||
let
|
||||
# together dataSymbols = 256+, paritySymbols = 17+, symbolBytes = 64+
|
||||
# seem to consistently trigger parallel processing with OpenMP
|
||||
dataSymbols = rand(256..320)
|
||||
paritySymbols = rand(17..dataSymbols)
|
||||
codewordSymbols = dataSymbols + paritySymbols
|
||||
symbolBytesMultip = rand(1..8)
|
||||
symbolBytes = MinBufferSize * symbolBytesMultip.uint
|
||||
rsCode = RS(codewordSymbols, dataSymbols)
|
||||
data = genData(rsCode.data, symbolBytes)
|
||||
losses = rand(1..paritySymbols)
|
||||
parityDataHoleCount =
|
||||
if (losses - 1) == 0: 0 else: rand(1..(losses - 1))
|
||||
dataHoleCount = losses - parityDataHoleCount
|
||||
encodeRes = rsCode.encode data
|
||||
|
||||
check: dataHoleCount + parityDataHoleCount == losses
|
||||
|
||||
check: encodeRes.isOk
|
||||
if encodeRes.isOk:
|
||||
let
|
||||
parityData = encodeRes.get
|
||||
|
||||
var
|
||||
dataWithHoles = data
|
||||
parityDataWithHoles = parityData
|
||||
|
||||
var
|
||||
dataHoles: seq[int]
|
||||
|
||||
for i in 1..dataHoleCount:
|
||||
while true:
|
||||
let
|
||||
j = rand(dataSymbols - 1)
|
||||
|
||||
if dataHoles.find(j) == -1:
|
||||
dataHoles.add j
|
||||
break
|
||||
|
||||
check: dataHoles.len == dataHoleCount
|
||||
|
||||
for i in dataHoles:
|
||||
dataWithHoles[i] = @[]
|
||||
|
||||
var
|
||||
parityDataHoles: seq[int]
|
||||
|
||||
for i in 1..parityDataHoleCount:
|
||||
while true:
|
||||
let
|
||||
j = rand(paritySymbols - 1)
|
||||
|
||||
if parityDataHoles.find(j) == -1:
|
||||
parityDataHoles.add j
|
||||
break
|
||||
|
||||
check: parityDataHoles.len == parityDataHoleCount
|
||||
|
||||
for i in parityDataHoles:
|
||||
parityDataWithHoles[i] = @[]
|
||||
|
||||
let
|
||||
decodeRes = rsCode.decode(dataWithHoles, parityDataWithHoles,
|
||||
symbolBytes)
|
||||
|
||||
check: decodeRes.isOk
|
||||
if decodeRes.isOk:
|
||||
let
|
||||
decodedData = decodeRes.get
|
||||
|
||||
check:
|
||||
decodedData != dataWithHoles
|
||||
decodedData == data
|
||||
|
||||
else:
|
||||
echo "decode error message: " & decodeRes.error.msg
|
||||
|
||||
else:
|
||||
echo "encode error message: " & encodeRes.error.msg
|
||||
|
||||
inc i
|
||||
|
||||
echo ""
|
||||
Loading…
x
Reference in New Issue
Block a user