From 0d2fae23bfdd164c8dca85c4f992e73242ea9634 Mon Sep 17 00:00:00 2001 From: "Michael Bradley, Jr" Date: Mon, 7 Mar 2022 23:57:13 -0600 Subject: [PATCH] initial implementation and tests --- .editorconfig | 5 + .gitattributes | 1 + .github/workflows/test.yml | 154 ++++++++++ .gitignore | 13 + .gitmodules | 2 +- README.md | 1 + leopard.nim | 307 ++++++++++++++++++++ leopard.nimble | 6 +- leopard/wrapper.nim | 293 +++++++++++++++++++ tests/test_leopard.nim | 558 +++++++++++++++++++++++++++++++++++++ 10 files changed, 1337 insertions(+), 3 deletions(-) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..6cc19ff --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +[*] +indent_style = space +insert_final_newline = true +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..24aa972 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,154 @@ +name: Tests + +on: [pull_request, push] + +jobs: + tests: + env: + NPROC: 2 + strategy: + fail-fast: false + matrix: + cache_nonce: [ 1 ] + nim_version: [ 1.2.18, 1.4.8, 1.6.4 ] + platform: + - { + icon: 🐧, + label: Linux, + os: ubuntu, + shell: bash --noprofile --norc -eo pipefail + } + - { + icon: 🍎, + label: macOS, + os: macos, + shell: bash --noprofile --norc -eo pipefail + } + - { + icon: 🏁, + label: Windows, + os: windows, + shell: msys2 + } + name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim_version }} + runs-on: ${{ matrix.platform.os }}-latest + defaults: + run: + shell: ${{ matrix.platform.shell }} {0} + + steps: + # - name: Install tools and libraries via APT (Linux) + # if: matrix.platform.os == 'ubuntu' + # run: | + # sudo apt update + # sudo apt install -y \ + # ... + + - name: Install tools and libraries via Homebrew (macOS) + if: matrix.platform.os == 'macos' + run: | + brew update + brew install \ + findutils \ + libomp + + - name: Install tools and libraries via MSYS2 (Windows) + if: matrix.platform.os == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + install: > + base-devel + git + mingw-w64-ucrt-x86_64-cmake + mingw-w64-ucrt-x86_64-toolchain + + - name: Checkout sources from GitHub + uses: actions/checkout@v2 + with: + submodules: true + + - name: Calculate cache member paths + id: calc-paths + run: | + if [[ ${{ matrix.platform.os }} = windows ]]; then + echo "::set-output name=bash_env::$(cygpath -m "${HOME}")/.bash_env" + echo "::set-output name=choosenim::$(cygpath -m "${USERPROFILE}")/.choosenim" + echo "::set-output name=nimble::$(cygpath -m "${HOME}")/.nimble" + else + echo "::set-output name=bash_env::${HOME}/.bash_env" + echo "::set-output name=choosenim::${HOME}/.choosenim" + echo "::set-output name=nimble::${HOME}/.nimble" + fi + + - name: Restore choosenim and Nim tooling from cache + id: choosenim-nim-tooling-cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.calc-paths.outputs.bash_env }} + ${{ steps.calc-paths.outputs.choosenim }} + ${{ steps.calc-paths.outputs.nimble }}/bin + key: ${{ matrix.platform.os }}-nim_version:${{ matrix.nim_version }}-cache_nonce:${{ matrix.cache_nonce }} + + - name: Install choosenim and Nim tooling + if: steps.choosenim-nim-tooling-cache.outputs.cache-hit != 'true' + run: | + mkdir -p "${HOME}/Downloads" + cd "${HOME}/Downloads" + curl https://nim-lang.org/choosenim/init.sh -sSf -O + chmod +x init.sh + if [[ ${{ matrix.platform.os }} = windows ]]; then + mkdir -p "$(cygpath "${USERPROFILE}")/.nimble/bin" + fi + CHOOSENIM_CHOOSE_VERSION=${{ matrix.nim_version }} ./init.sh -y + if [[ ${{ matrix.platform.os }} = windows ]]; then + mv "$(cygpath "${USERPROFILE}")/.nimble" "${HOME}/" + # intention is to rely only on libs provided by the OS and MSYS2 env + rm -rf "${HOME}/.nimble/bin/"*.dll + rm -rf "${HOME}/.nimble/bin/"*.pem + fi + echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env" + echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env" + + - name: Install project dependencies + run: | + source "${HOME}/.bash_env" + cd "${NIMBLE_DIR}/bin" + # delete broken symlinks, which can arise because e.g. the cache + # restored a symlink that points to an executable within + # ../pkgs/foo-1.2.3/ but the project's .nimble file has been updated + # to install foo-#head. In the case of a broken symlink, nimble's + # auto-overwrite fails (only sometimes? only on macOS?) + if [[ ${{ matrix.platform.os }} = macos ]]; then + gfind . -xtype l -delete + else + find . -xtype l -delete + fi + cd - + nimble --accept install + + - name: Build and run tests + run: | + source "${HOME}/.bash_env" + if [[ ${{ matrix.platform.os }} = windows ]]; then + touch tests/test_leopard.exe + else + touch tests/test_leopard + fi + if [[ ${{ matrix.platform.os }} = macos ]]; then + export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}" + export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib" + nimble test -d:verbose -d:release -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib" + else + nimble test -d:verbose -d:release + fi + if [[ ${{ matrix.platform.os }} = macos ]]; then + echo + echo otool -L tests/test_leopard + otool -L tests/test_leopard + else + echo + echo ldd tests/test_leopard + ldd tests/test_leopard + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d06ac8c --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +* +!*/ +!*.* +*.a +*.dll +*.dylib +*.exe +*.so +.DS_Store +.idea +.vscode +leopard.nims +TODO diff --git a/.gitmodules b/.gitmodules index 5e8fa86..57cc6a9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,5 @@ [submodule "vendor/leopard"] path = vendor/leopard - url = https://github.com/catid/leopard.git + url = https://github.com/status-im/leopard.git ignore = untracked branch = master diff --git a/README.md b/README.md index cb9b139..698093e 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](https://github.com/status-im/nim-leopard#stability) +[![Tests (GitHub Actions)](https://github.com/status-im/nim-leopard/workflows/Tests/badge.svg?branch=initial_impl)](https://github.com/status-im/nim-leopard/actions?query=workflow%3ATests+branch%3Ainitial_impl) Nim wrapper for [Leopard-RS](https://github.com/catid/leopard): a fast library for [Reed-Solomon](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) erasure correction coding. diff --git a/leopard.nim b/leopard.nim index e69de29..77b78bc 100644 --- a/leopard.nim +++ b/leopard.nim @@ -0,0 +1,307 @@ +import pkg/stew/ptrops +import pkg/stew/results +import pkg/upraises + +import ./leopard/wrapper + +export results + +push: {.upraises: [].} + +const + LeopardBadCodeMsg = "Bad RS code" + LeopardInconsistentSizeMsg = + "Buffer sizes must all be the same multiple of 64 bytes" + LeopardNeedLessDataMsg = "Too much recovery data received" + LeopardNotEnoughDataMsg = "Buffer counts are too low" + + MinBufferSize* = 64.uint + +type + Data* = seq[seq[byte]] + + LeopardDefect* = object of Defect + + # It should not be necessary to redefine LeopardResult, but if that's not + # done here then defining LeopardError as `object of CatchableError` will + # cause a mystery crash at compile-time (symbol not found). Can workaround by + # defining as just `object`, but then when trying to work with LeopardResult + # errors in e.g. tests/test_leopard.nim the same mystery crash happens at + # compile-time. The problem may be related to use of importcpp in + # leopard/wrapper.nim, so it could be a compiler bug. By redefining + # LeopardResult in this module (and casting wrapper.LeopardResult values) the + # the problem is avoided. + LeopardResult* = enum + LeopardNotEnoughData = -11.cint # Buffer counts are too low + LeopardNeedLessData = -10.cint # Too much recovery data received + LeopardInconsistentSize = -9.cint # Buffer sizes must all be the same multiple of 64 bytes + LeopardBadCode = -8.cint # Bad RS code + LeopardCallInitialize = wrapper.LeopardCallInitialize + LeopardPlatform = wrapper.LeopardPlatform + LeopardInvalidInput = wrapper.LeopardInvalidInput + LeopardInvalidCounts = wrapper.LeopardInvalidCounts + LeopardInvalidSize = wrapper.LeopardInvalidSize + LeopardTooMuchData = wrapper.LeopardTooMuchData + LeopardNeedMoreData = wrapper.LeopardNeedMoreData + LeopardSuccess = wrapper.LeopardSuccess + + LeopardError* = object of CatchableError + code*: LeopardResult + + ParityData* = Data + + ReedSolomonCode* = tuple[codeword, data, parity: uint] # symbol counts + +# https://github.com/catid/leopard/issues/12 +# https://www.cs.cmu.edu/~guyb/realworld/reedsolomon/reed_solomon_codes.html +# +# RS(255,239) +# --------------------------------- +# codeword symbols = 255 +# data symbols = 239 +# parity symbols = 255 - 239 = 16 + +proc RS*(codeword, data: Positive): ReedSolomonCode = + var + parity = codeword - data + + if parity <= 0: parity = 0 + (codeword: codeword.uint, data: data.uint, parity: parity.uint) + +when (NimMajor, NimMinor, NimPatch) < (1, 4, 0): + const + header = "" + + proc c_malloc(size: csize_t): pointer {.importc: "malloc", header: header.} + proc c_free(p: pointer) {.importc: "free", header: header.} + +proc SIMDSafeAllocate(size: int): pointer {.inline.} = + var + data = + when (NimMajor, NimMinor, NimPatch) < (1, 4, 0): + c_malloc(LEO_ALIGN_BYTES + size.uint) + else: + allocShared(LEO_ALIGN_BYTES + size.uint) + + doffset = cast[uint](data) mod LEO_ALIGN_BYTES + + data = offset(data, (LEO_ALIGN_BYTES + doffset).int) + + var + offsetPtr = cast[pointer](cast[uint](data) - 1) + + moveMem(offsetPtr, addr doffset, sizeof(doffset)) + data + +proc SIMDSafeFree(data: pointer) {.inline.} = + var + data = data + + if not data.isNil: + let + offset = cast[uint](data) - 1 + + if offset >= LEO_ALIGN_BYTES: return + + data = cast[pointer](cast[uint](data) - (LEO_ALIGN_BYTES - offset)) + + when (NimMajor, NimMinor, NimPatch) < (1, 4, 0): + c_free data + else: + deallocShared data + +proc leoInit*() = + if wrapper.leoInit() != 0: + raise (ref LeopardDefect)(msg: "Leopard-RS failed to initialize") + +proc encode*(code: ReedSolomonCode, data: Data): + Result[ParityData, LeopardError] = + if code.parity < 1 or code.parity > code.data: + return err LeopardError(code: LeopardBadCode, msg: LeopardBadCodeMsg) + + var + data = data + + let + symbolBytes = data[0].len + + if data.len < code.data.int: + return err LeopardError(code: LeopardNotEnoughData, + msg: LeopardNotEnoughDataMsg) + + elif data.len > code.data.int: + return err LeopardError(code: LeopardTooMuchData, + msg: $leoResultString(wrapper.LeopardTooMuchData)) + + if symbolBytes < MinBufferSize.int or symbolBytes mod MinBufferSize.int != 0: + return err LeopardError(code: LeopardInvalidSize, + msg: $leoResultString(wrapper.LeopardInvalidSize)) + + var + enData = newSeq[pointer](code.data) + + for i in 0.. code.data: + return err LeopardError(code: LeopardBadCode, msg: LeopardBadCodeMsg) + + var + data = data + parityData = parityData + holes: seq[int] + + if data.len < code.data.int: + return err LeopardError(code: LeopardNotEnoughData, + msg: LeopardNotEnoughDataMsg) + + elif data.len > code.data.int: + return err LeopardError(code: LeopardTooMuchData, + msg: $leoResultString(wrapper.LeopardTooMuchData)) + + if parityData.len < code.parity.int: + return err LeopardError(code: LeopardNeedMoreData, + msg: $leoResultString(wrapper.LeopardNeedMoreData)) + + elif parityData.len > code.parity.int: + return err LeopardError(code: LeopardNeedLessData, + msg: LeopardNeedLessDataMsg) + + if symbolBytes < MinBufferSize or symbolBytes mod MinBufferSize != 0: + return err LeopardError(code: LeopardInvalidSize, + msg: $leoResultString(wrapper.LeopardInvalidSize)) + + var + deData = newSeq[pointer](code.data) + + for i in 0..= 1.2.0", - "stew#head", - "unittest2" + "stew", + "unittest2", + "upraises >= 0.1.0 & < 0.2.0" diff --git a/leopard/wrapper.nim b/leopard/wrapper.nim index e69de29..ea554ac 100644 --- a/leopard/wrapper.nim +++ b/leopard/wrapper.nim @@ -0,0 +1,293 @@ +## Copyright (c) 2017 Christopher A. Taylor. All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted provided that the following conditions are met: +## +## * Redistributions of source code must retain the above copyright notice, +## this list of conditions and the following disclaimer. +## * Redistributions in binary form must reproduce the above copyright notice, +## this list of conditions and the following disclaimer in the documentation +## and/or other materials provided with the distribution. +## * Neither the name of Leopard-RS nor the names of its contributors may be +## used to endorse or promote products derived from this software without +## specific prior written permission. +## +## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +## POSSIBILITY OF SUCH DAMAGE. + + +## Leopard-RS +## MDS Reed-Solomon Erasure Correction Codes for Large Data in C +## +## Algorithms are described in LeopardCommon.h +## +## +## Inspired by discussion with: +## +## Sian-Jhen Lin : Author of {1} {3}, basis for Leopard +## Bulat Ziganshin : Author of FastECC +## Yutaka Sawada : Author of MultiPar +## +## +## References: +## +## {1} S.-J. Lin, T. Y. Al-Naffouri, Y. S. Han, and W.-H. Chung, +## "Novel Polynomial Basis with Fast Fourier Transform +## and Its Application to Reed-Solomon Erasure Codes" +## IEEE Trans. on Information Theory, pp. 6284-6299, November, 2016. +## +## {2} D. G. Cantor, "On arithmetical algorithms over finite fields", +## Journal of Combinatorial Theory, Series A, vol. 50, no. 2, pp. 285-300, 1989. +## +## {3} Sian-Jheng Lin, Wei-Ho Chung, "An Efficient (n, k) Information +## Dispersal Algorithm for High Code Rate System over Fermat Fields," +## IEEE Commun. Lett., vol.16, no.12, pp. 2036-2039, Dec. 2012. +## +## {4} Plank, J. S., Greenan, K. M., Miller, E. L., "Screaming fast Galois Field +## arithmetic using Intel SIMD instructions." In: FAST-2013: 11th Usenix +## Conference on File and Storage Technologies, San Jose, 2013 + + +import upraises +push: {.upraises: [].} + + +## ----------------------------------------------------------------------------- +## Build configuration + +import std/compilesettings +import std/os +import std/strutils + +const + LeopardCmakeFlags {.strdefine.} = + when defined(macosx): + "-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off" + elif defined(windows): + "-G\"MSYS Makefiles\" -DCMAKE_BUILD_TYPE=Release" + else: + "-DCMAKE_BUILD_TYPE=Release" + + LeopardDir {.strdefine.} = + joinPath(currentSourcePath.parentDir.parentDir, "vendor", "leopard") + + buildDir = joinPath(querySetting(nimcacheDir), "vendor_leopard") + + LeopardHeader {.strdefine.} = "leopard.h" + + LeopardLib {.strdefine.} = joinPath(buildDir, "liblibleopard.a") + + LeopardCompilerFlags {.strdefine.} = + when defined(macosx): + "-I" & LeopardDir + else: + "-I" & LeopardDir & " -fopenmp" + + LeopardLinkerFlags {.strdefine.} = + when defined(macosx): + LeopardLib + else: + LeopardLib & " -fopenmp" + + LeopardExtraCompilerFlags {.strdefine.} = "" + + LeopardExtraLinkerFlags {.strdefine.} = "" + +static: + if defined(windows): + func pathUnix2Win(path: string): string = + gorge("cygpath -w " & path.strip).strip + + func pathWin2Unix(path: string): string = + gorge("cygpath " & path.strip).strip + + proc bash(cmd: varargs[string]): string = + gorge(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'") + + proc bashEx(cmd: varargs[string]): tuple[output: string, exitCode: int] = + gorgeEx(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'") + + let + buildDirUnix = buildDir.pathWin2Unix + leopardDirUnix = LeopardDir.pathWin2Unix + if defined(LeopardRebuild): discard bash("rm -rf", buildDirUnix) + if (bashEx("ls", LeopardLib.pathWin2Unix)).exitCode != 0: + discard bash("mkdir -p", buildDirUnix) + let cmd = + @["cd", buildDirUnix, "&& cmake", leopardDirUnix, LeopardCmakeFlags, + "&& make"] + echo "\nBuilding Leopard-RS: " & cmd.join(" ") + let (output, exitCode) = bashEx cmd + echo output + if exitCode != 0: + discard bash("rm -rf", buildDirUnix) + raise (ref Defect)(msg: "Failed to build Leopard-RS") + else: + if defined(LeopardRebuild): discard gorge "rm -rf " & buildDir + if gorgeEx("ls " & LeopardLib).exitCode != 0: + discard gorge "mkdir -p " & buildDir + let cmd = + "cd " & buildDir & " && cmake " & LeopardDir & " " & LeopardCmakeFlags & + " && make" + echo "\nBuilding Leopard-RS: " & cmd + let (output, exitCode) = gorgeEx cmd + echo output + if exitCode != 0: + discard gorge "rm -rf " & buildDir + raise (ref Defect)(msg: "Failed to build Leopard-RS") + +{.passC: LeopardCompilerFlags & " " & LeopardExtraCompilerFlags.} +{.passL: LeopardLinkerFlags & " " & LeopardExtraLinkerFlags.} + +{.pragma: leo, cdecl, header: LeopardHeader.} + + +## ----------------------------------------------------------------------------- +## Library version + +var LEO_VERSION* {.header: LeopardHeader, importc.}: int + + +## ----------------------------------------------------------------------------- +## Platform/Architecture + +# maybe should detect AVX2 and set to 32 if detected, 16 otherwise: +# https://github.com/catid/leopard/blob/master/LeopardCommon.h#L247-L253 +# https://github.com/mratsim/Arraymancer/blob/master/src/arraymancer/laser/cpuinfo_x86.nim#L220 +const LEO_ALIGN_BYTES* = 16 + + +## ----------------------------------------------------------------------------- +## Initialization API + +## leoInit() +## +## Perform static initialization for the library, verifying that the platform +## is supported. +## +## Returns 0 on success and other values on failure. + +proc leoInit*(): cint {.leo, importcpp: "leo_init".} + + +## ----------------------------------------------------------------------------- +## Shared Constants / Datatypes + +## Results +type + LeopardResult* = enum + LeopardCallInitialize = -7.cint ## Call leoInit() first + LeopardPlatform = -6.cint ## Platform is unsupported + LeopardInvalidInput = -5.cint ## A function parameter was invalid + LeopardInvalidCounts = -4.cint ## Invalid counts provided + LeopardInvalidSize = -3.cint ## Buffer size must be multiple of 64 bytes + LeopardTooMuchData = -2.cint ## Buffer counts are too high + LeopardNeedMoreData = -1.cint ## Not enough recovery data received + LeopardSuccess = 0.cint ## Operation succeeded + +## Convert Leopard result to string +func leoResultString*(res: LeopardResult): cstring + {.leo, importc: "leo_result_string".} + + +## ----------------------------------------------------------------------------- +## Encoder API + +## leoEncodeWorkCount() +## +## Calculate the number of work data buffers to provide to leoEncode(). +## +## The sum of originalCount + recoveryCount must not exceed 65536. +## +## Returns the workCount value to pass into leoEncode(). +## Returns 0 on invalid input. + +func leoEncodeWorkCount*(originalCount, recoveryCount: cuint): cuint + {.leo, importc: "leo_encode_work_count".} + +## leoEncode() +## +## Generate recovery data. +## +## bufferBytes: Number of bytes in each data buffer. +## originalCount: Number of original data buffers provided. +## recoveryCount: Number of desired recovery data buffers. +## workCount: Number of work data buffers, from leoEncodeWorkCount(). +## originalData: Array of pointers to original data buffers. +## workData: Array of pointers to work data buffers. +## +## The sum of originalCount + recoveryCount must not exceed 65536. +## The recoveryCount <= originalCount. +## +## The value of bufferBytes must be a multiple of 64. +## Each buffer should have the same number of bytes. +## Even the last piece must be rounded up to the block size. +## +## Returns LeopardSuccess on success. +## The first set of recoveryCount buffers in workData will be the result. +## Returns other values on errors. + +proc leoEncode*( + bufferBytes: uint64, ## Number of bytes in each data buffer + originalCount: cuint, ## Number of originalData[] buffer pointers + recoveryCount: cuint, ## Number of recovery data buffer pointers + ## (readable post-call from start of workData[]) + workCount: cuint, ## Number of workData[] buffer pointers + originalData: pointer, ## Array of pointers to original data buffers + workData: pointer, ## Array of pointers to work data buffers +): LeopardResult {.leo, importc: "leo_encode".} + + +## ----------------------------------------------------------------------------- +## Decoder API + +## leoDecodeWorkCount() +## +## Calculate the number of work data buffers to provide to leoDecode(). +## +## The sum of originalCount + recoveryCount must not exceed 65536. +## +## Returns the workCount value to pass into leoDecode(). +## Returns 0 on invalid input. + +func leoDecodeWorkCount*(originalCount, recoveryCount: cuint): cuint + {.leo, importc: "leo_decode_work_count".} + +## leoDecode() +## +## Decode original data from recovery data. +## +## bufferBytes: Number of bytes in each data buffer. +## originalCount: Number of original data buffers provided. +## recoveryCount: Number of recovery data buffers provided. +## workCount: Number of work data buffers, from leoDecodeWorkCount(). +## originalData: Array of pointers to original data buffers. +## recoveryData: Array of pointers to recovery data buffers. +## workData: Array of pointers to work data buffers. +## +## Lost original/recovery data should be set to NULL. +## +## The sum of recoveryCount + the number of non-NULL original data must be at +## least originalCount in order to perform recovery. +## +## Returns LeopardSuccess on success. +## Returns other values on errors. + +proc leoDecode*( + bufferBytes: uint64, ## Number of bytes in each data buffer + originalCount: cuint, ## Number of originalData[] buffer pointers + recoveryCount: cuint, ## Number of recoveryData[] buffer pointers + workCount: cuint, ## Number of workData[] buffer pointers + originalData: pointer, ## Array of pointers to original data buffers + recoveryData: pointer, ## Array of pointers to recovery data buffers + workData: pointer, ## Array of pointers to work data buffers +): LeopardResult {.leo, importc: "leo_decode".} diff --git a/tests/test_leopard.nim b/tests/test_leopard.nim index e69de29..1e4d496 100644 --- a/tests/test_leopard.nim +++ b/tests/test_leopard.nim @@ -0,0 +1,558 @@ +import std/random + +import pkg/leopard +import pkg/unittest2 + +randomize() + +proc genData(outerLen, innerLen: uint): Data = + var + data = newSeqOfCap[seq[byte]](outerLen) + + for i in 0..