diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..6cc19ff --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +[*] +indent_style = space +insert_final_newline = true +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..d5b68e2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,154 @@ +name: Tests + +on: [pull_request, push] + +jobs: + tests: + env: + NPROC: 2 + strategy: + fail-fast: false + matrix: + cache_nonce: [ 1 ] + nim_version: [ 1.2.18, 1.4.8, 1.6.4 ] + platform: + - { + icon: 🐧, + label: Linux, + os: ubuntu, + shell: bash --noprofile --norc -eo pipefail + } + - { + icon: 🍎, + label: macOS, + os: macos, + shell: bash --noprofile --norc -eo pipefail + } + - { + icon: 🏁, + label: Windows, + os: windows, + shell: msys2 + } + name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim_version }} + runs-on: ${{ matrix.platform.os }}-latest + defaults: + run: + shell: ${{ matrix.platform.shell }} {0} + + steps: + # - name: Install tools and libraries via APT (Linux) + # if: matrix.platform.os == 'ubuntu' + # run: | + # sudo apt update + # sudo apt install -y \ + # ... + + - name: Install tools and libraries via Homebrew (macOS) + if: matrix.platform.os == 'macos' + run: | + brew update + brew install \ + findutils \ + libomp + + - name: Install tools and libraries via MSYS2 (Windows) + if: matrix.platform.os == 'windows' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + install: > + base-devel + git + mingw-w64-ucrt-x86_64-cmake + mingw-w64-ucrt-x86_64-toolchain + + - name: Checkout sources from GitHub + uses: actions/checkout@v2 + with: + submodules: true + + - name: Calculate cache member paths + id: calc-paths + run: | + if [[ ${{ matrix.platform.os }} = windows ]]; then + echo "::set-output name=bash_env::$(cygpath -m "${HOME}")/.bash_env" + echo "::set-output name=choosenim::$(cygpath -m "${USERPROFILE}")/.choosenim" + echo "::set-output name=nimble::$(cygpath -m "${HOME}")/.nimble" + else + echo "::set-output name=bash_env::${HOME}/.bash_env" + echo "::set-output name=choosenim::${HOME}/.choosenim" + echo "::set-output name=nimble::${HOME}/.nimble" + fi + + - name: Restore choosenim and Nim tooling from cache + id: choosenim-nim-tooling-cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.calc-paths.outputs.bash_env }} + ${{ steps.calc-paths.outputs.choosenim }} + ${{ steps.calc-paths.outputs.nimble }}/bin + key: ${{ matrix.platform.os }}-nim_version:${{ matrix.nim_version }}-cache_nonce:${{ matrix.cache_nonce }} + + - name: Install choosenim and Nim tooling + if: steps.choosenim-nim-tooling-cache.outputs.cache-hit != 'true' + run: | + mkdir -p "${HOME}/Downloads" + cd "${HOME}/Downloads" + curl https://nim-lang.org/choosenim/init.sh -sSf -O + chmod +x init.sh + if [[ ${{ matrix.platform.os }} = windows ]]; then + mkdir -p "$(cygpath "${USERPROFILE}")/.nimble/bin" + fi + CHOOSENIM_CHOOSE_VERSION=${{ matrix.nim_version }} ./init.sh -y + if [[ ${{ matrix.platform.os }} = windows ]]; then + mv "$(cygpath "${USERPROFILE}")/.nimble" "${HOME}/" + # intention is to rely only on libs provided by the OS and MSYS2 env + rm -rf "${HOME}/.nimble/bin/"*.dll + rm -rf "${HOME}/.nimble/bin/"*.pem + fi + echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env" + echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env" + + - name: Install project dependencies + run: | + source "${HOME}/.bash_env" + cd "${NIMBLE_DIR}/bin" + # delete broken symlinks, which can arise because e.g. the cache + # restored a symlink that points to an executable within + # ../pkgs/foo-1.2.3/ but the project's .nimble file has been updated + # to install foo-#head; in the case of a broken symlink, nimble's + # auto-overwrite fails + if [[ ${{ matrix.platform.os }} = macos ]]; then + gfind . -xtype l -delete + else + find . -xtype l -delete + fi + cd - + nimble --accept install + + - name: Build and run tests + run: | + source "${HOME}/.bash_env" + if [[ ${{ matrix.platform.os }} = windows ]]; then + touch tests/test_leopard.exe + else + touch tests/test_leopard + fi + if [[ ${{ matrix.platform.os }} = macos ]]; then + export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}" + export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib" + nimble test -d:verbose -d:release -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib" + else + nimble test -d:verbose -d:release + fi + if [[ ${{ matrix.platform.os }} = macos ]]; then + echo + echo otool -L tests/test_leopard + otool -L tests/test_leopard + else + echo + echo ldd tests/test_leopard + ldd tests/test_leopard + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56c7ebe --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +* +!*/ +!*.* +*.a +*.dll +*.dylib +*.exe +*.so +.DS_Store +.idea +.vscode +TODO +leopard.nims diff --git a/.gitmodules b/.gitmodules index 5e8fa86..57cc6a9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,5 +1,5 @@ [submodule "vendor/leopard"] path = vendor/leopard - url = https://github.com/catid/leopard.git + url = https://github.com/status-im/leopard.git ignore = untracked branch = master diff --git a/LICENSE-MIT b/LICENSE-MIT index 8766e65..e6ba156 100644 --- a/LICENSE-MIT +++ b/LICENSE-MIT @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018 Status Research & Development GmbH +Copyright (c) 2022 Status Research & Development GmbH Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index cb9b139..a6dee44 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,92 @@ [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) -[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](https://github.com/status-im/nim-leopard#stability) +[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability) +[![Tests (GitHub Actions)](https://github.com/status-im/nim-leopard/workflows/Tests/badge.svg?branch=main)](https://github.com/status-im/nim-leopard/actions?query=workflow%3ATests+branch%3Amain) Nim wrapper for [Leopard-RS](https://github.com/catid/leopard): a fast library for [Reed-Solomon](https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction) erasure correction coding. +## Requirements + +* Same as Leopard-RS' requirements, e.g. CMake 3.7 or newer. +* Nim 1.2 or newer. + +## Installation + +With [Nimble](https://github.com/nim-lang/nimble) +```text +$ nimble install leopard +``` +In a project's `.nimble` file +```nim +requires "leopard >= 0.0.1 & < 0.0.2" +``` +In a [nimbus-build-system](https://github.com/status-im/nimbus-build-system) project +```text +$ git submodule add https://github.com/status-im/nim-leopard.git vendor/nim-leopard +$ make update +``` + +### Submodule + +#### Init + +[status-im/leopard](https://github.com/status-im/leopard), a fork of [catid/leopard](https://github.com/catid/leopard) (Leopard-RS), is a submodule of nim-leopard. + +When nim-leopard is installed with `nimble install leopard`, or as a dependency in a Nimble project, or vendored in a nimbus-build-system project, submodule init is handled automatically. + +In a standalone `git clone` of nim-leopard, it's necessary to init the submodule before running `nimble develop` or `nimble install` in the root of the clone +```text +$ git submodule update --init +``` + +#### Build + +The submodule is automatically built (in the `nimcache` dir) and statically linked during compilation of any Nim module that has `import leopard` or `import leopard/wrapper`. + +If the `nimcache` dir is set to a custom value, it must be an absolute path. + +For the build to work on Windows, `nimble` or `nim c` must be run from a Bash shell, e.g. Git Bash or an MSYS2 shell, and all needed tools (`cmake`, `make`, compiler, etc.) must be available in and suitable for that environment. + +##### OpenMP + +Leopard-RS' `CMakeLists.txt` checks for [OpenMP](https://en.wikipedia.org/wiki/OpenMP) support. If it is available then it is enabled in the build of `libleopard.a`. + +Build toolchains commonly installed on Linux and Windows come with support for OpenMP. + +The clang compiler that ships with Apple's Xcode does not support OpenMP, but the one installed with `brew install llvm` does support it, though it's also necessary to `brew install libomp`. + +So, on macOS, when running `nimble test` of nim-leopard or compiling a project that imports nim-leopard: +* If libomp is not installed and Xcode clang is used, no extra flags need to be passed to the Nim compiler. OpenMP support will not be enabled in `libleopard.a`. +* If libomp is installed and Xcode clang is used, this flag should be passed to `nim c` + ```text + -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off" + ``` +* If the intent is to use brew-installed clang + libomp, the shell environment should be modified + ```text + $ export PATH="$(brew --prefix)/opt/llvm/bin:${PATH}" + $ export LDFLAGS="-L$(brew --prefix)/opt/libomp/lib -L$(brew --prefix)/opt/llvm/lib -Wl,-rpath,$(brew --prefix)/opt/llvm/lib" + ``` + and these flags should be passed to `nim c` + ```text + -d:LeopardCmakeFlags="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=$(brew --prefix)/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix)/opt/llvm/bin/clang++" -d:LeopardExtraCompilerlags="-fopenmp" -d:LeopardExtraLinkerFlags="-fopenmp -L$(brew --prefix)/opt/libomp/lib" + ``` + ## Usage TODO +### OpenMP + +When OpenMP is enabled, whether or not parallel processing kicks in depends on the symbol and byte counts. `originalCount == 239` and `recoveryCount == 17` with `bufferBytes == 64` seems to be the lower bound for triggering parallel processing on a local machine with a 64-bit Intel processor. + ## Versioning -nim-leopard generally follows the upstream master branch. +nim-leopard generally follows the upstream `master` branch of [status-im/leopard](https://github.com/status-im/leopard) such that changes there will result in a version bump for this project. ## Stability -The API provided by this package is currently marked as experimental. Until it is marked as stable, it may be subject to breaking changes across any version bump. +nim-leopard is currently marked as experimental and may be subject to breaking changes across any version bump until it is marked as stable. ## License diff --git a/config.nims b/config.nims new file mode 100644 index 0000000..bdad7d5 --- /dev/null +++ b/config.nims @@ -0,0 +1,2 @@ +--threads:on +--tlsEmulation:off diff --git a/leopard.nim b/leopard.nim index e69de29..fe98675 100644 --- a/leopard.nim +++ b/leopard.nim @@ -0,0 +1,6 @@ +import pkg/stew/ptrops + +import ./leopard/aligned +import ./leopard/wrapper + +export aligned, ptrops, wrapper diff --git a/leopard.nimble b/leopard.nimble index 2b30660..1beea63 100644 --- a/leopard.nimble +++ b/leopard.nimble @@ -5,7 +5,9 @@ version = "0.0.1" author = "Status Research & Development GmbH" description = "A wrapper for Leopard-RS" license = "Apache License 2.0 or MIT" +installDirs = @["vendor"] requires "nim >= 1.2.0", - "stew#head", - "unittest2" + "stew", + "unittest2", + "upraises >= 0.1.0 & < 0.2.0" diff --git a/leopard/aligned.nim b/leopard/aligned.nim new file mode 100644 index 0000000..92c65e5 --- /dev/null +++ b/leopard/aligned.nim @@ -0,0 +1,49 @@ +# allocAligned, freeAligned, and helpers adapted from mratsim/weave: +# https://github.com/mratsim/weave/blob/master/weave/memory/allocs.nim + +func isPowerOfTwo(n: int): bool {.inline.} = + (n and (n - 1)) == 0 + +func roundNextMultipleOf(x, n: Natural): int {.inline.} = + (x + n - 1) and not (n - 1) + +when defined(windows): + proc aligned_alloc_windows(size, alignment: csize_t): pointer + {.header: "", importc: "_aligned_malloc", sideeffect.} + + proc aligned_free_windows(p: pointer) + {.header: "", importc: "_aligned_free", sideeffect.} + + proc freeAligned*(p: pointer) = + if not p.isNil: + aligned_free_windows(p) + +elif defined(osx): + proc posix_memalign(mem: var pointer, alignment, size: csize_t) + {.header: "", importc, sideeffect.} + + proc aligned_alloc(alignment, size: csize_t): pointer {.inline.} = + posix_memalign(result, alignment, size) + +else: + proc aligned_alloc(alignment, size: csize_t): pointer + {.header: "", importc, sideeffect.} + +when not defined(windows): + proc c_free(p: pointer) {.header: "", importc: "free".} + + proc freeAligned*(p: pointer) {.inline.} = + if not p.isNil: + c_free(p) + +proc allocAligned*(size: int, alignment: static Natural): pointer {.inline.} = + static: + assert alignment.isPowerOfTwo() + + let + requiredMem = size.roundNextMultipleOf(alignment) + + when defined(windows): + aligned_alloc_windows(csize_t requiredMem, csize_t alignment) + else: + aligned_alloc(csize_t alignment, csize_t requiredMem) diff --git a/leopard/results.nim b/leopard/results.nim new file mode 100644 index 0000000..084a8ff --- /dev/null +++ b/leopard/results.nim @@ -0,0 +1,10 @@ +type + LeopardResult* = enum + LeopardCallInitialize = -7.cint ## Call leoInit() first + LeopardPlatform = -6.cint ## Platform is unsupported + LeopardInvalidInput = -5.cint ## A function parameter was invalid + LeopardInvalidCounts = -4.cint ## Invalid counts provided + LeopardInvalidSize = -3.cint ## Buffer size must be multiple of 64 bytes + LeopardTooMuchData = -2.cint ## Buffer counts are too high + LeopardNeedMoreData = -1.cint ## Not enough recovery data received + LeopardSuccess = 0.cint ## Operation succeeded diff --git a/leopard/wrapper.nim b/leopard/wrapper.nim index e69de29..6c1478c 100644 --- a/leopard/wrapper.nim +++ b/leopard/wrapper.nim @@ -0,0 +1,286 @@ +## Copyright (c) 2017 Christopher A. Taylor. All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted provided that the following conditions are met: +## +## * Redistributions of source code must retain the above copyright notice, +## this list of conditions and the following disclaimer. +## * Redistributions in binary form must reproduce the above copyright notice, +## this list of conditions and the following disclaimer in the documentation +## and/or other materials provided with the distribution. +## * Neither the name of Leopard-RS nor the names of its contributors may be +## used to endorse or promote products derived from this software without +## specific prior written permission. +## +## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +## POSSIBILITY OF SUCH DAMAGE. + + +## Leopard-RS +## MDS Reed-Solomon Erasure Correction Codes for Large Data in C +## +## Algorithms are described in LeopardCommon.h +## +## +## Inspired by discussion with: +## +## Sian-Jhen Lin : Author of {1} {3}, basis for Leopard +## Bulat Ziganshin : Author of FastECC +## Yutaka Sawada : Author of MultiPar +## +## +## References: +## +## {1} S.-J. Lin, T. Y. Al-Naffouri, Y. S. Han, and W.-H. Chung, +## "Novel Polynomial Basis with Fast Fourier Transform +## and Its Application to Reed-Solomon Erasure Codes" +## IEEE Trans. on Information Theory, pp. 6284-6299, November, 2016. +## +## {2} D. G. Cantor, "On arithmetical algorithms over finite fields", +## Journal of Combinatorial Theory, Series A, vol. 50, no. 2, pp. 285-300, 1989. +## +## {3} Sian-Jheng Lin, Wei-Ho Chung, "An Efficient (n, k) Information +## Dispersal Algorithm for High Code Rate System over Fermat Fields," +## IEEE Commun. Lett., vol.16, no.12, pp. 2036-2039, Dec. 2012. +## +## {4} Plank, J. S., Greenan, K. M., Miller, E. L., "Screaming fast Galois Field +## arithmetic using Intel SIMD instructions." In: FAST-2013: 11th Usenix +## Conference on File and Storage Technologies, San Jose, 2013 + + +import upraises +push: {.upraises: [].} + + +## ----------------------------------------------------------------------------- +## Build configuration + +import std/compilesettings +import std/os +import std/strutils + +const + LeopardCmakeFlags {.strdefine.} = + when defined(macosx): + "-DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=off" + elif defined(windows): + "-G\"MSYS Makefiles\" -DCMAKE_BUILD_TYPE=Release" + else: + "-DCMAKE_BUILD_TYPE=Release" + + LeopardDir {.strdefine.} = + joinPath(currentSourcePath.parentDir.parentDir, "vendor", "leopard") + + buildDir = joinPath(querySetting(nimcacheDir), "vendor_leopard") + + LeopardHeader {.strdefine.} = "leopard.h" + + LeopardLib {.strdefine.} = joinPath(buildDir, "liblibleopard.a") + + LeopardCompilerFlags {.strdefine.} = + when defined(macosx): + "-I" & LeopardDir + else: + "-I" & LeopardDir & " -fopenmp" + + LeopardLinkerFlags {.strdefine.} = + when defined(macosx): + LeopardLib + else: + LeopardLib & " -fopenmp" + + LeopardExtraCompilerFlags {.strdefine.} = "" + + LeopardExtraLinkerFlags {.strdefine.} = "" + +static: + if defined(windows): + func pathUnix2Win(path: string): string = + gorge("cygpath -w " & path.strip).strip + + func pathWin2Unix(path: string): string = + gorge("cygpath " & path.strip).strip + + proc bash(cmd: varargs[string]): string = + gorge(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'") + + proc bashEx(cmd: varargs[string]): tuple[output: string, exitCode: int] = + gorgeEx(gorge("which bash").pathUnix2Win & " -c '" & cmd.join(" ") & "'") + + let + buildDirUnix = buildDir.pathWin2Unix + leopardDirUnix = LeopardDir.pathWin2Unix + if defined(LeopardRebuild): discard bash("rm -rf", buildDirUnix) + if (bashEx("ls", LeopardLib.pathWin2Unix)).exitCode != 0: + discard bash("mkdir -p", buildDirUnix) + let cmd = + @["cd", buildDirUnix, "&& cmake", leopardDirUnix, LeopardCmakeFlags, + "&& make"] + echo "\nBuilding Leopard-RS: " & cmd.join(" ") + let (output, exitCode) = bashEx cmd + echo output + if exitCode != 0: + discard bash("rm -rf", buildDirUnix) + raise (ref Defect)(msg: "Failed to build Leopard-RS") + else: + if defined(LeopardRebuild): discard gorge "rm -rf " & buildDir + if gorgeEx("ls " & LeopardLib).exitCode != 0: + discard gorge "mkdir -p " & buildDir + let cmd = + "cd " & buildDir & " && cmake " & LeopardDir & " " & LeopardCmakeFlags & + " && make" + echo "\nBuilding Leopard-RS: " & cmd + let (output, exitCode) = gorgeEx cmd + echo output + if exitCode != 0: + discard gorge "rm -rf " & buildDir + raise (ref Defect)(msg: "Failed to build Leopard-RS") + +{.passC: LeopardCompilerFlags & " " & LeopardExtraCompilerFlags.} +{.passL: LeopardLinkerFlags & " " & LeopardExtraLinkerFlags.} + +{.pragma: leo, cdecl, header: LeopardHeader.} + + +## ----------------------------------------------------------------------------- +## Library version + +var LEO_VERSION* {.header: LeopardHeader, importc.}: int + + +## ----------------------------------------------------------------------------- +## Platform/Architecture + +# maybe should detect AVX2 and set to 32 if detected, 16 otherwise: +# https://github.com/catid/leopard/blob/master/LeopardCommon.h#L247-L253 +# https://github.com/mratsim/Arraymancer/blob/master/src/arraymancer/laser/cpuinfo_x86.nim#L220 +const LEO_ALIGN_BYTES* = 16 + + +## ----------------------------------------------------------------------------- +## Initialization API + +## leoInit() +## +## Perform static initialization for the library, verifying that the platform +## is supported. +## +## Returns 0 on success and other values on failure. + +proc leoInit*(): cint {.leo, importcpp: "leo_init".} + + +## ----------------------------------------------------------------------------- +## Shared Constants / Datatypes + +## Results +import ./results + +export results + +## Convert Leopard result to string +func leoResultString*(res: LeopardResult): cstring + {.leo, importc: "leo_result_string".} + + +## ----------------------------------------------------------------------------- +## Encoder API + +## leoEncodeWorkCount() +## +## Calculate the number of work data buffers to provide to leoEncode(). +## +## The sum of originalCount + recoveryCount must not exceed 65536. +## +## Returns the workCount value to pass into leoEncode(). +## Returns 0 on invalid input. + +func leoEncodeWorkCount*(originalCount, recoveryCount: cuint): cuint + {.leo, importc: "leo_encode_work_count".} + +## leoEncode() +## +## Generate recovery data. +## +## bufferBytes: Number of bytes in each data buffer. +## originalCount: Number of original data buffers provided. +## recoveryCount: Number of desired recovery data buffers. +## workCount: Number of work data buffers, from leoEncodeWorkCount(). +## originalData: Array of pointers to original data buffers. +## workData: Array of pointers to work data buffers. +## +## The sum of originalCount + recoveryCount must not exceed 65536. +## The recoveryCount <= originalCount. +## +## The value of bufferBytes must be a multiple of 64. +## Each buffer should have the same number of bytes. +## Even the last piece must be rounded up to the block size. +## +## Returns LeopardSuccess on success. +## The first set of recoveryCount buffers in workData will be the result. +## Returns other values on errors. + +proc leoEncode*( + bufferBytes: uint64, ## Number of bytes in each data buffer + originalCount: cuint, ## Number of originalData[] buffer pointers + recoveryCount: cuint, ## Number of recovery data buffer pointers + ## (readable post-call from start of workData[]) + workCount: cuint, ## Number of workData[] buffer pointers + originalData: ptr pointer, ## Array of pointers to original data buffers + workData: ptr pointer, ## Array of pointers to work data buffers +): LeopardResult {.leo, importc: "leo_encode".} + + +## ----------------------------------------------------------------------------- +## Decoder API + +## leoDecodeWorkCount() +## +## Calculate the number of work data buffers to provide to leoDecode(). +## +## The sum of originalCount + recoveryCount must not exceed 65536. +## +## Returns the workCount value to pass into leoDecode(). +## Returns 0 on invalid input. + +func leoDecodeWorkCount*(originalCount, recoveryCount: cuint): cuint + {.leo, importc: "leo_decode_work_count".} + +## leoDecode() +## +## Decode original data from recovery data. +## +## bufferBytes: Number of bytes in each data buffer. +## originalCount: Number of original data buffers provided. +## recoveryCount: Number of recovery data buffers provided. +## workCount: Number of work data buffers, from leoDecodeWorkCount(). +## originalData: Array of pointers to original data buffers. +## recoveryData: Array of pointers to recovery data buffers. +## workData: Array of pointers to work data buffers. +## +## Lost original/recovery data should be set to NULL. +## +## The sum of recoveryCount + the number of non-NULL original data must be at +## least originalCount in order to perform recovery. +## +## Returns LeopardSuccess on success. +## Returns other values on errors. + +proc leoDecode*( + bufferBytes: uint64, ## Number of bytes in each data buffer + originalCount: cuint, ## Number of originalData[] buffer pointers + recoveryCount: cuint, ## Number of recoveryData[] buffer pointers + workCount: cuint, ## Number of workData[] buffer pointers + originalData: ptr pointer, ## Array of pointers to original data buffers + recoveryData: ptr pointer, ## Array of pointers to recovery data buffers + workData: ptr pointer, ## Array of pointers to work data buffers +): LeopardResult {.leo, importc: "leo_decode".} diff --git a/tests/test_leopard.nim b/tests/test_leopard.nim index e69de29..80c73a3 100644 --- a/tests/test_leopard.nim +++ b/tests/test_leopard.nim @@ -0,0 +1,223 @@ +import std/random + +import pkg/leopard +import pkg/unittest2 + +randomize() + +type + Data = seq[seq[byte]] + +proc genData(outerLen, innerLen: uint): Data = + newSeq(result, outerLen) + for i in 0..