logos-storage/nim-leopard
1
0
Fork 0
mirror of https://github.com/logos-storage/nim-leopard.git synced 2026-01-03 06:03:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial implementation of low-level wrapper and tests

Browse Source
This commit is contained in:
Michael Bradley, Jr 2022-03-07 23:57:13 -06:00
parent 4d89e44e0d
commit b2ef831911
No known key found for this signature in database
GPG Key ID: 9FCA591DA4CE7D0D
14 changed files with 829 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.editorconfig Normal file
View File

@ -0,0 +1,5 @@
[*]
indent_style = space
insert_final_newline = true
indent_size = 2
trim_trailing_whitespace = true

1
.gitattributes vendored Normal file
View File

@ -0,0 +1 @@
* text=auto eol=lf

154
.github/workflows/test.yml vendored Normal file
View File

@ -0,0 +1,154 @@
name: Tests
on: [pull_request, push]
jobs:
tests:
env:
NPROC: 2
strategy:
fail-fast: false
matrix:
cache_nonce: [ 1 ]
nim_version: [ 1.2.18, 1.4.8, 1.6.4 ]
platform:
- {
icon: 🐧,
label: Linux,
os: ubuntu,
shell: bash --noprofile --norc -eo pipefail
}
- {
icon: 🍎,
label: macOS,
os: macos,
shell: bash --noprofile --norc -eo pipefail
}
- {
icon: 🏁,
label: Windows,
os: windows,
shell: msys2
}
name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim_version }}
runs-on: ${{ matrix.platform.os }}-latest
defaults:
run:
shell: ${{ matrix.platform.shell }} {0}
steps:
# - name: Install tools and libraries via APT (Linux)
# if: matrix.platform.os == 'ubuntu'
# run: |
# sudo apt update
# sudo apt install -y \
# ...
- name: Install tools and libraries via Homebrew (macOS)
if: matrix.platform.os == 'macos'
run: |
brew update
brew install \
findutils \
libomp
- name: Install tools and libraries via MSYS2 (Windows)
if: matrix.platform.os == 'windows'
uses: msys2/setup-msys2@v2
with:
msystem: UCRT64
install: >
base-devel
git
mingw-w64-ucrt-x86_64-cmake
mingw-w64-ucrt-x86_64-toolchain
- name: Checkout sources from GitHub
uses: actions/checkout@v2
with:
submodules: true
- name: Calculate cache member paths
id: calc-paths
run: |
if [[ ${{ matrix.platform.os }} = windows ]]; then
echo "::set-output name=bash_env::$(cygpath -m "${HOME}")/.bash_env"
echo "::set-output name=choosenim::$(cygpath -m "${USERPROFILE}")/.choosenim"
echo "::set-output name=nimble::$(cygpath -m "${HOME}")/.nimble"
else
echo "::set-output name=bash_env::${HOME}/.bash_env"
echo "::set-output name=choosenim::${HOME}/.choosenim"
echo "::set-output name=nimble::${HOME}/.nimble"
fi
- name: Restore choosenim and Nim tooling from cache
id: choosenim-nim-tooling-cache
uses: actions/cache@v2
with:
path: |
${{ steps.calc-paths.outputs.bash_env }}
${{ steps.calc-paths.outputs.choosenim }}
${{ steps.calc-paths.outputs.nimble }}/bin
key: ${{ matrix.platform.os }}-nim_version:${{ matrix.nim_version }}-cache_nonce:${{ matrix.cache_nonce }}
- name: Install choosenim and Nim tooling
if: steps.choosenim-nim-tooling-cache.outputs.cache-hit != 'true'
run: |
mkdir -p "${HOME}/Downloads"
cd "${HOME}/Downloads"
curl https://nim-lang.org/choosenim/init.sh -sSf -O
chmod +x init.sh
if [[ ${{ matrix.platform.os }} = windows ]]; then
mkdir -p "$(cygpath "${USERPROFILE}")/.nimble/bin"
fi
CHOOSENIM_CHOOSE_VERSION=${{ matrix.nim_version }} ./init.sh -y
if [[ ${{ matrix.platform.os }} = windows ]]; then
mv "$(cygpath "${USERPROFILE}")/.nimble" "${HOME}/"
# intention is to rely only on libs provided by the OS and MSYS2 env
rm -rf "${HOME}/.nimble/bin/"*.dll
rm -rf "${HOME}/.nimble/bin/"*.pem
fi
echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env"
echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env"
- name: Install project dependencies
run: |
source "${HOME}/.bash_env"
cd "${NIMBLE_DIR}/bin"
# delete broken symlinks, which can arise because e.g. the cache
# restored a symlink that points to an executable within
# ../pkgs/foo-1.2.3/ but the project's .nimble file has been updated
# to install foo-#head; in the case of a broken symlink, nimble's
# auto-overwrite fails
if [[ ${{ matrix.platform.os }} = macos ]]; then
gfind . -xtype l -delete
else
find . -xtype l -delete
fi
cd -
nimble --accept install
- name: Build and run tests
run: |
source "${HOME}/.bash_env"
if [[ ${{ matrix.platform.os }} = windows ]]; then
touch tests/test_leopard.exe
else
touch tests/test_leopard
fi
if [[ ${{ matrix.platform.os }} = macos ]]; then
export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}"
export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib"
nimble test -d:verbose -d:release -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib"
else
nimble test -d:verbose -d:release
fi
if [[ ${{ matrix.platform.os }} = macos ]]; then
echo
echo otool -L tests/test_leopard
otool -L tests/test_leopard
else
echo
echo ldd tests/test_leopard
ldd tests/test_leopard
fi

13
.gitignore vendored Normal file
View File

@ -0,0 +1,13 @@
*
!*/
!*.*
*.a
*.dll
*.dylib
*.exe
*.so
.DS_Store
.idea
.vscode
TODO
leopard.nims

2
.gitmodules vendored
View File

@ -1,5 +1,5 @@
[submodule "vendor/leopard"]
path = vendor/leopard
url = https://github.com/catid/leopard.git
url = https://github.com/status-im/leopard.git
ignore = untracked
branch = master

2
LICENSE-MIT
View File

@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2018 Status Research & Development GmbH
Copyright (c) 2022 Status Research & Development GmbH
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

77
README.md
View File

@ -2,21 +2,92 @@
[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](https://github.com/status-im/nim-leopard#stability)
[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability)
[![Tests (GitHub Actions)](https://github.com/status-im/nim-leopard/workflows/Tests/badge.svg?branch=main)](https://github.com/status-im/nim-leopard/actions?query=workflow%3ATests+branch%3Amain)
Nim wrapper for [Leopard-RS](https://github.com/catid/leopard): a fast library for [Reed-Solomon](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) erasure correction coding.
## Requirements
* Same as Leopard-RS' requirements, e.g. CMake 3.7 or newer.
* Nim 1.2 or newer.
## Installation
With [Nimble](https://github.com/nim-lang/nimble)
```text
$ nimble install leopard
```
In a project's `.nimble` file
```nim
requires "leopard >= 0.0.1 & < 0.0.2"
```
In a [nimbus-build-system](https://github.com/status-im/nimbus-build-system) project
```text
$ git submodule add https://github.com/status-im/nim-leopard.git vendor/nim-leopard
$ make update
```
### Submodule
#### Init
[status-im/leopard](https://github.com/status-im/leopard), a fork of [catid/leopard](https://github.com/catid/leopard) (Leopard-RS), is a submodule of nim-leopard.
When nim-leopard is installed with `nimble install leopard`, or as a dependency in a Nimble project, or vendored in a nimbus-build-system project, submodule init is handled automatically.
In a standalone `git clone` of nim-leopard, it's necessary to init the submodule before running `nimble develop` or `nimble install` in the root of the clone
```text
$ git submodule update --init
```
#### Build
The submodule is automatically built (in the `nimcache` dir) and statically linked during compilation of any Nim module that has `import leopard` or `import leopard/wrapper`.
If the `nimcache` dir is set to a custom value, it must be an absolute path.
For the build to work on Windows, `nimble` or `nim c` must be run from a Bash shell, e.g. Git Bash or an MSYS2 shell, and all needed tools (`cmake`, `make`, compiler, etc.) must be available in and suitable for that environment.
##### OpenMP
Leopard-RS' `CMakeLists.txt` checks for [OpenMP](https://en.wikipedia.org/wiki/OpenMP) support. If it is available then it is enabled in the build of `libleopard.a`.
Build toolchains commonly installed on Linux and Windows come with support for OpenMP.
The clang compiler that ships with Apple's Xcode does not support OpenMP, but the one installed with `brew install llvm` does support it, though it's also necessary to `brew install libomp`.
So, on macOS, when running `nimble test` of nim-leopard or compiling a project that imports nim-leopard:
* If libomp is not installed and Xcode clang is used, no extra flags need to be passed to the Nim compiler. OpenMP support will not be enabled in `libleopard.a`.
* If libomp is installed and Xcode clang is used, this flag should be passed to `nim c`
```text
-d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off"
```
* If the intent is to use brew-installed clang + libomp, the shell environment should be modified
```text
$ export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}"
$ export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib"
```
and these flags should be passed to `nim c`
```text
-d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib"
```
## Usage
TODO
### OpenMP
When OpenMP is enabled, whether or not parallel processing kicks in depends on the symbol and byte counts. `originalCount == 239` and `recoveryCount == 17` with `bufferBytes == 64` seems to be the lower bound for triggering parallel processing on a local machine with a 64-bit Intel processor.
## Versioning
nim-leopard generally follows the upstream master branch.
nim-leopard generally follows the upstream `master` branch of [status-im/leopard](https://github.com/status-im/leopard) such that changes there will result in a version bump for this project.
## Stability
The API provided by this package is currently marked as experimental. Until it is marked as stable, it may be subject to breaking changes across any version bump.
nim-leopard is currently marked as experimental and may be subject to breaking changes across any version bump until it is marked as stable.
## License

2
config.nims Normal file
View File

@ -0,0 +1,2 @@
--threads:on
--tlsEmulation:off

6
leopard.nim
View File

@ -0,0 +1,6 @@
import pkg/stew/ptrops
import ./leopard/aligned
import ./leopard/wrapper
export aligned, ptrops, wrapper

6
leopard.nimble
View File

@ -5,7 +5,9 @@ version = "0.0.1"
author = "Status Research & Development GmbH"
description = "A wrapper for Leopard-RS"
license = "Apache License 2.0 or MIT"
installDirs = @["vendor"]
requires "nim >= 1.2.0",
"stew#head",
"unittest2"
"stew",
"unittest2",
"upraises >= 0.1.0 & < 0.2.0"

49
leopard/aligned.nim Normal file
View File

@ -0,0 +1,49 @@
# allocAligned, freeAligned, and helpers adapted from mratsim/weave:
# https://github.com/mratsim/weave/blob/master/weave/memory/allocs.nim
func isPowerOfTwo(n: int): bool {.inline.} =
(n and (n - 1)) == 0
func roundNextMultipleOf(x, n: Natural): int {.inline.} =
(x + n - 1) and not (n - 1)
when defined(windows):
proc aligned_alloc_windows(size, alignment: csize_t): pointer
{.header: "<malloc.h>", importc: "_aligned_malloc", sideeffect.}
proc aligned_free_windows(p: pointer)
{.header: "<malloc.h>", importc: "_aligned_free", sideeffect.}
proc freeAligned*(p: pointer) =
if not p.isNil:
aligned_free_windows(p)
elif defined(osx):
proc posix_memalign(mem: var pointer, alignment, size: csize_t)
{.header: "<stdlib.h>", importc, sideeffect.}
proc aligned_alloc(alignment, size: csize_t): pointer {.inline.} =
posix_memalign(result, alignment, size)
else:
proc aligned_alloc(alignment, size: csize_t): pointer
{.header: "<stdlib.h>", importc, sideeffect.}
when not defined(windows):
proc c_free(p: pointer) {.header: "<stdlib.h>", importc: "free".}
proc freeAligned*(p: pointer) {.inline.} =
if not p.isNil:
c_free(p)
proc allocAligned*(size: int, alignment: static Natural): pointer {.inline.} =
static:
assert alignment.isPowerOfTwo()
let
requiredMem = size.roundNextMultipleOf(alignment)
when defined(windows):
aligned_alloc_windows(csize_t requiredMem, csize_t alignment)
else:
aligned_alloc(csize_t alignment, csize_t requiredMem)

10
leopard/results.nim Normal file
View File

@ -0,0 +1,10 @@
type
LeopardResult* = enum
LeopardCallInitialize = -7.cint ## Call leoInit() first
LeopardPlatform = -6.cint ## Platform is unsupported
LeopardInvalidInput = -5.cint ## A function parameter was invalid
LeopardInvalidCounts = -4.cint ## Invalid counts provided
LeopardInvalidSize = -3.cint ## Buffer size must be multiple of 64 bytes
LeopardTooMuchData = -2.cint ## Buffer counts are too high
LeopardNeedMoreData = -1.cint ## Not enough recovery data received
LeopardSuccess = 0.cint ## Operation succeeded

286
leopard/wrapper.nim
View File

@ -0,0 +1,286 @@
## Copyright (c) 2017 Christopher A. Taylor. All rights reserved.
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted provided that the following conditions are met:
##
## * Redistributions of source code must retain the above copyright notice,
## this list of conditions and the following disclaimer.
## * Redistributions in binary form must reproduce the above copyright notice,
## this list of conditions and the following disclaimer in the documentation
## and/or other materials provided with the distribution.
## * Neither the name of Leopard-RS nor the names of its contributors may be
## used to endorse or promote products derived from this software without
## specific prior written permission.
##
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
## POSSIBILITY OF SUCH DAMAGE.
## Leopard-RS
## MDS Reed-Solomon Erasure Correction Codes for Large Data in C
##
## Algorithms are described in LeopardCommon.h
##
##
## Inspired by discussion with:
##
## Sian-Jhen Lin <sjhenglin@gmail.com> : Author of {1} {3}, basis for Leopard
## Bulat Ziganshin <bulat.ziganshin@gmail.com> : Author of FastECC
## Yutaka Sawada <tenfon@outlook.jp> : Author of MultiPar
##
##
## References:
##
## {1} S.-J. Lin, T. Y. Al-Naffouri, Y. S. Han, and W.-H. Chung,
## "Novel Polynomial Basis with Fast Fourier Transform
## and Its Application to Reed-Solomon Erasure Codes"
## IEEE Trans. on Information Theory, pp. 6284-6299, November, 2016.
##
## {2} D. G. Cantor, "On arithmetical algorithms over finite fields",
## Journal of Combinatorial Theory, Series A, vol. 50, no. 2, pp. 285-300, 1989.
##
## {3} Sian-Jheng Lin, Wei-Ho Chung, "An Efficient (n, k) Information
## Dispersal Algorithm for High Code Rate System over Fermat Fields,"
## IEEE Commun. Lett., vol.16, no.12, pp. 2036-2039, Dec. 2012.
##
## {4} Plank, J. S., Greenan, K. M., Miller, E. L., "Screaming fast Galois Field
## arithmetic using Intel SIMD instructions." In: FAST-2013: 11th Usenix
## Conference on File and Storage Technologies, San Jose, 2013
import upraises
push: {.upraises: [].}
## -----------------------------------------------------------------------------
## Build configuration
import std/compilesettings
import std/os
import std/strutils
const
LeopardCmakeFlags {.strdefine.} =
when defined(macosx):
"-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off"
elif defined(windows):
"-G\"MSYS Makefiles\" -DCMAKE_BUILD_TYPE=Release"
else:
"-DCMAKE_BUILD_TYPE=Release"
LeopardDir {.strdefine.} =
joinPath(currentSourcePath.parentDir.parentDir, "vendor", "leopard")
buildDir = joinPath(querySetting(nimcacheDir), "vendor_leopard")
LeopardHeader {.strdefine.} = "leopard.h"
LeopardLib {.strdefine.} = joinPath(buildDir, "liblibleopard.a")
LeopardCompilerFlags {.strdefine.} =
when defined(macosx):
"-I" & LeopardDir
else:
"-I" & LeopardDir & " -fopenmp"
LeopardLinkerFlags {.strdefine.} =
when defined(macosx):
LeopardLib
else:
LeopardLib & " -fopenmp"
LeopardExtraCompilerFlags {.strdefine.} = ""
LeopardExtraLinkerFlags {.strdefine.} = ""
static:
if defined(windows):
func pathUnix2Win(path: string): string =
gorge("cygpath -w " & path.strip).strip
func pathWin2Unix(path: string): string =
gorge("cygpath " & path.strip).strip
proc bash(cmd: varargs[string]): string =
gorge(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'")
proc bashEx(cmd: varargs[string]): tuple[output: string, exitCode: int] =
gorgeEx(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'")
let
buildDirUnix = buildDir.pathWin2Unix
leopardDirUnix = LeopardDir.pathWin2Unix
if defined(LeopardRebuild): discard bash("rm -rf", buildDirUnix)
if (bashEx("ls", LeopardLib.pathWin2Unix)).exitCode != 0:
discard bash("mkdir -p", buildDirUnix)
let cmd =
@["cd", buildDirUnix, "&& cmake", leopardDirUnix, LeopardCmakeFlags,
"&& make"]
echo "\nBuilding Leopard-RS: " & cmd.join(" ")
let (output, exitCode) = bashEx cmd
echo output
if exitCode != 0:
discard bash("rm -rf", buildDirUnix)
raise (ref Defect)(msg: "Failed to build Leopard-RS")
else:
if defined(LeopardRebuild): discard gorge "rm -rf " & buildDir
if gorgeEx("ls " & LeopardLib).exitCode != 0:
discard gorge "mkdir -p " & buildDir
let cmd =
"cd " & buildDir & " && cmake " & LeopardDir & " " & LeopardCmakeFlags &
" && make"
echo "\nBuilding Leopard-RS: " & cmd
let (output, exitCode) = gorgeEx cmd
echo output
if exitCode != 0:
discard gorge "rm -rf " & buildDir
raise (ref Defect)(msg: "Failed to build Leopard-RS")
{.passC: LeopardCompilerFlags & " " & LeopardExtraCompilerFlags.}
{.passL: LeopardLinkerFlags & " " & LeopardExtraLinkerFlags.}
{.pragma: leo, cdecl, header: LeopardHeader.}
## -----------------------------------------------------------------------------
## Library version
var LEO_VERSION* {.header: LeopardHeader, importc.}: int
## -----------------------------------------------------------------------------
## Platform/Architecture
# maybe should detect AVX2 and set to 32 if detected, 16 otherwise:
# https://github.com/catid/leopard/blob/master/LeopardCommon.h#L247-L253
# https://github.com/mratsim/Arraymancer/blob/master/src/arraymancer/laser/cpuinfo_x86.nim#L220
const LEO_ALIGN_BYTES* = 16
## -----------------------------------------------------------------------------
## Initialization API
## leoInit()
##
## Perform static initialization for the library, verifying that the platform
## is supported.
##
## Returns 0 on success and other values on failure.
proc leoInit*(): cint {.leo, importcpp: "leo_init".}
## -----------------------------------------------------------------------------
## Shared Constants / Datatypes
## Results
import ./results
export results
## Convert Leopard result to string
func leoResultString*(res: LeopardResult): cstring
{.leo, importc: "leo_result_string".}
## -----------------------------------------------------------------------------
## Encoder API
## leoEncodeWorkCount()
##
## Calculate the number of work data buffers to provide to leoEncode().
##
## The sum of originalCount + recoveryCount must not exceed 65536.
##
## Returns the workCount value to pass into leoEncode().
## Returns 0 on invalid input.
func leoEncodeWorkCount*(originalCount, recoveryCount: cuint): cuint
{.leo, importc: "leo_encode_work_count".}
## leoEncode()
##
## Generate recovery data.
##
## bufferBytes: Number of bytes in each data buffer.
## originalCount: Number of original data buffers provided.
## recoveryCount: Number of desired recovery data buffers.
## workCount: Number of work data buffers, from leoEncodeWorkCount().
## originalData: Array of pointers to original data buffers.
## workData: Array of pointers to work data buffers.
##
## The sum of originalCount + recoveryCount must not exceed 65536.
## The recoveryCount <= originalCount.
##
## The value of bufferBytes must be a multiple of 64.
## Each buffer should have the same number of bytes.
## Even the last piece must be rounded up to the block size.
##
## Returns LeopardSuccess on success.
## The first set of recoveryCount buffers in workData will be the result.
## Returns other values on errors.
proc leoEncode*(
bufferBytes: uint64, ## Number of bytes in each data buffer
originalCount: cuint, ## Number of originalData[] buffer pointers
recoveryCount: cuint, ## Number of recovery data buffer pointers
## (readable post-call from start of workData[])
workCount: cuint, ## Number of workData[] buffer pointers
originalData: ptr pointer, ## Array of pointers to original data buffers
workData: ptr pointer, ## Array of pointers to work data buffers
): LeopardResult {.leo, importc: "leo_encode".}
## -----------------------------------------------------------------------------
## Decoder API
## leoDecodeWorkCount()
##
## Calculate the number of work data buffers to provide to leoDecode().
##
## The sum of originalCount + recoveryCount must not exceed 65536.
##
## Returns the workCount value to pass into leoDecode().
## Returns 0 on invalid input.
func leoDecodeWorkCount*(originalCount, recoveryCount: cuint): cuint
{.leo, importc: "leo_decode_work_count".}
## leoDecode()
##
## Decode original data from recovery data.
##
## bufferBytes: Number of bytes in each data buffer.
## originalCount: Number of original data buffers provided.
## recoveryCount: Number of recovery data buffers provided.
## workCount: Number of work data buffers, from leoDecodeWorkCount().
## originalData: Array of pointers to original data buffers.
## recoveryData: Array of pointers to recovery data buffers.
## workData: Array of pointers to work data buffers.
##
## Lost original/recovery data should be set to NULL.
##
## The sum of recoveryCount + the number of non-NULL original data must be at
## least originalCount in order to perform recovery.
##
## Returns LeopardSuccess on success.
## Returns other values on errors.
proc leoDecode*(
bufferBytes: uint64, ## Number of bytes in each data buffer
originalCount: cuint, ## Number of originalData[] buffer pointers
recoveryCount: cuint, ## Number of recoveryData[] buffer pointers
workCount: cuint, ## Number of workData[] buffer pointers
originalData: ptr pointer, ## Array of pointers to original data buffers
recoveryData: ptr pointer, ## Array of pointers to recovery data buffers
workData: ptr pointer, ## Array of pointers to work data buffers
): LeopardResult {.leo, importc: "leo_decode".}

223
tests/test_leopard.nim
View File

@ -0,0 +1,223 @@
import std/random
import pkg/leopard
import pkg/unittest2
randomize()
type
Data = seq[seq[byte]]
proc genData(outerLen, innerLen: uint): Data =
newSeq(result, outerLen)
for i in 0..<outerLen:
newSeq(result[i], innerLen)
for j in 0..<innerLen:
result[i][j] = rand(255).byte
var
initialized = false
suite "Initialization":
test "leoEncode and leoDecode should fail if Leopard-RS is not initialized":
let
bufferBytes = 64.uint64
originalCount = 5.cuint
recoveryCount = 3.cuint
workCount = leoEncodeWorkCount(originalCount, recoveryCount)
var
dummy = 0
originalData = cast[ptr pointer](addr dummy)
workData = cast[ptr pointer](addr dummy)
let
encodeRes = leoEncode(
bufferBytes,
originalCount,
recoveryCount,
workCount,
originalData,
workData
)
check: encodeRes == LeopardCallInitialize
var
recoveryData = cast[ptr pointer](addr dummy)
let
decodeRes = leoDecode(
bufferBytes,
originalCount,
recoveryCount,
workCount,
originalData,
recoveryData,
workData
)
check: decodeRes == LeopardCallInitialize
test "initialization should succeed":
let init = leoInit()
check: init == 0
if init == 0: initialized = true
suite "Encode + Decode":
proc encodeDecode(decodeShouldFail = false) =
let
# together originalCount = 239+, recoveryCount = 17+, bufferBytes = 64+
# seem to consistently trigger parallel processing with OpenMP
bufferBytesMultiplier = rand(1..8)
bufferBytes = (64 * bufferBytesMultiplier).uint64
originalCount = rand(239..320).cuint
recoveryCount = rand(17..originalCount.int).cuint
losses =
if decodeShouldFail:
recoveryCount.int + 1
else:
rand(1..recoveryCount.int)
recoveryDataHoleCount =
if (losses - 1) == 0: 0 else: rand(1..(losses - 1))
dataHoleCount = losses - recoveryDataHoleCount
check: dataHoleCount + recoveryDataHoleCount == losses
var
originalData = genData(originalCount.uint, bufferBytes.uint)
originalDataAligned = newSeq[pointer](originalCount)
workCount = leoEncodeWorkCount(originalCount, recoveryCount)
workData = newSeq[pointer](workCount)
for i in 0..<originalCount:
originalDataAligned[i] = allocAligned(bufferBytes.int, LEO_ALIGN_BYTES)
for j in 0..<bufferBytes.int:
copyMem(originalDataAligned[i].offset j, addr originalData[i][j], 1)
for i in 0..<workCount:
workData[i] = allocAligned(bufferBytes.int, LEO_ALIGN_BYTES)
let
encodeRes = leoEncode(
bufferBytes,
originalCount,
recoveryCount,
workCount,
addr originalDataAligned[0],
addr workData[0]
)
check: encodeRes == LeopardSuccess
if encodeRes != LeopardSuccess:
for i in 0..<originalCount: freeAligned originalDataAligned[i]
for i in 0..<workCount: freeAligned workData[i]
return
var
recoveryData: Data
recoveryDataAligned = newSeq[pointer](recoveryCount)
newSeq(recoveryData, recoveryCount)
for i in 0..<recoveryCount:
newSeq(recoveryData[i], bufferBytes)
for j in 0..<bufferBytes.int:
copyMem(addr recoveryData[i][j], workData[i].offset j, 1)
for i in 0..<recoveryCount:
recoveryDataAligned[i] = allocAligned(bufferBytes.int, LEO_ALIGN_BYTES)
for j in 0..<bufferBytes.int:
copyMem(recoveryDataAligned[i].offset j, addr recoveryData[i][j], 1)
var
dataHoles: seq[int]
recoveryDataHoles: seq[int]
holeyData = originalDataAligned
holeyRecoveryData = recoveryDataAligned
recoveredData = originalData
for _ in 1..dataHoleCount:
while true:
let
i = rand(originalCount.int - 1)
if dataHoles.find(i) == -1:
dataHoles.add i
break
check: dataHoles.len == dataHoleCount
for i in dataHoles:
holeyData[i] = nil
recoveredData[i] = newSeq[byte](bufferBytes)
for _ in 1..recoveryDataHoleCount:
while true:
let
i = rand(recoveryCount.int - 1)
if recoveryDataHoles.find(i) == -1:
recoveryDataHoles.add i
break
check: recoveryDataHoles.len == recoveryDataHoleCount
for i in recoveryDataHoles:
holeyRecoveryData[i] = nil
for i in 0..<workCount: freeAligned workData[i]
workCount = leoDecodeWorkCount(originalCount, recoveryCount)
workData = newSeq[pointer](workCount)
for i in 0..<workCount:
workData[i] = allocAligned(bufferBytes.int, LEO_ALIGN_BYTES)
let
decodeRes = leoDecode(
bufferBytes,
originalCount,
recoveryCount,
workCount,
addr holeyData[0],
addr holeyRecoveryData[0],
addr workData[0]
)
if decodeShouldFail:
for i in 0..<originalCount: freeAligned originalDataAligned[i]
for i in 0..<recoveryCount: freeAligned recoveryDataAligned[i]
for i in 0..<workCount: freeAligned workData[i]
check: decodeRes == LeopardNeedMoreData
else:
check: decodeRes == LeopardSuccess
if decodeRes != LeopardSuccess:
for i in 0..<originalCount: freeAligned originalDataAligned[i]
for i in 0..<recoveryCount: freeAligned recoveryDataAligned[i]
for i in 0..<workCount: freeAligned workData[i]
return
for i in dataHoles:
for j in 0..<bufferBytes.int:
copyMem(addr recoveredData[i][j], workData[i].offset j, 1)
for i in 0..<originalCount: freeAligned originalDataAligned[i]
for i in 0..<recoveryCount: freeAligned recoveryDataAligned[i]
for i in 0..<workCount: freeAligned workData[i]
check: recoveredData == originalData
test "should fail to recover data when loss count exceeds recovery count":
check: initialized
if not initialized: return
for _ in 1..1000: encodeDecode(decodeShouldFail = true)
test "should recover data otherwise":
check: initialized
if not initialized: return
for _ in 1..1000: encodeDecode()