From f1878d6aaa816e6c9d7b80d50d773f51dce1b873 Mon Sep 17 00:00:00 2001 From: Chrysostomos Nanakos Date: Tue, 23 Dec 2025 14:59:51 +0200 Subject: [PATCH] chore: orc support Refactor public API to use `var openArray[seq[byte]]` instead of raw pointers for ORC compatibility. The previous API required callers to extract raw pointers from seqs before calling encode/decode. With refc and shallowCopy, reference counting kept buffers alive. ORC doesn't track raw pointers and can reallocate seq buffers at yield points, leaving pointers dangling. The new openArray API borrows data safely. Also updates CI to Nim 2.2.4 and bumps version to 0.2.0. Part of https://github.com/logos-storage/nim-leopard/issues/24 Signed-off-by: Chrysostomos Nanakos --- .github/workflows/test.yml | 7 +- leopard.nimble | 2 +- leopard/leopard.nim | 144 +++++++++++-------------- tests/helpers.nim | 81 +++++---------- tests/testleopard.nim | 208 +++++++++++++++++-------------------- 5 files changed, 186 insertions(+), 256 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ba7706..e3b5b24 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: runner: windows-latest } # Earliest supported and latest nim - nim: [1.6.18, "stable"] + nim: [binary:2.2.4] name: ${{ matrix.platform.icon }} ${{ matrix.platform.label }} - Nim v${{ matrix.nim }} runs-on: ${{ matrix.platform.runner }} defaults: @@ -33,10 +33,9 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - uses: jiro4989/setup-nim-action@v2 + - uses: iffy/install-nim@v5 with: - nim-version: ${{matrix.nim}} - repo-token: ${{ secrets.GITHUB_TOKEN }} + version: ${{ matrix.nim }} - name: Install run: nimble install -y - name: Build and run tests diff --git a/leopard.nimble b/leopard.nimble index 639cd06..ea81c18 100644 --- a/leopard.nimble +++ b/leopard.nimble @@ -1,7 +1,7 @@ mode = ScriptMode.Verbose packageName = "leopard" -version = "0.1.1" +version = "0.2.0" author = "Status Research & Development GmbH" description = "A wrapper for Leopard-RS" license = "Apache License 2.0 or MIT" diff --git a/leopard/leopard.nim b/leopard/leopard.nim index 26400ca..16d24d2 100644 --- a/leopard/leopard.nim +++ b/leopard/leopard.nim @@ -16,27 +16,27 @@ import ./utils export wrapper, results -const - BuffMultiples* = 64 +const BuffMultiples* = 64 type LeoBufferPtr* = ptr UncheckedArray[byte] LeoCoderKind* {.pure.} = enum - Encoder, + Encoder Decoder Leo* = object of RootObj - bufSize*: int # size of the buffer in multiples of 64 - buffers*: int # total number of data buffers (K) - parity*: int # total number of parity buffers (M) - dataBufferPtr: seq[LeoBufferPtr] # buffer where data is copied before encoding - workBufferCount: int # number of parity work buffers - workBufferPtr: seq[LeoBufferPtr] # buffer where parity data is written during encoding or before decoding + bufSize*: int # size of the buffer in multiples of 64 + buffers*: int # total number of data buffers (K) + parity*: int # total number of parity buffers (M) + dataBufferPtr: seq[LeoBufferPtr] # buffer where data is copied before encoding + workBufferCount: int # number of parity work buffers + workBufferPtr: seq[LeoBufferPtr] + # buffer where parity data is written during encoding or before decoding case kind: LeoCoderKind of LeoCoderKind.Decoder: - decodeBufferCount: int # number of decoding work buffers - decodeBufferPtr: seq[LeoBufferPtr] # work buffer used for decoding + decodeBufferCount: int # number of decoding work buffers + decodeBufferPtr: seq[LeoBufferPtr] # work buffer used for decoding of LeoCoderKind.Encoder: discard @@ -44,9 +44,8 @@ type LeoDecoder* = object of Leo func encode*( - self: var LeoEncoder, - data,parity: ptr UncheckedArray[ptr UncheckedArray[byte]], - dataLen,parityLen: int ): Result[void, cstring] = + self: var LeoEncoder, data, parity: var openArray[seq[byte]] +): Result[void, cstring] = ## Encode a list of buffers in `data` into a number of `bufSize` sized ## `parity` buffers ## @@ -54,43 +53,40 @@ func encode*( ## `parity` - list of parity `buffers` of size `bufSize` ## - if dataLen != self.buffers: + if data.len != self.buffers: return err("Number of data buffers should match!") - if parityLen != self.parity: + if parity.len != self.parity: return err("Number of parity buffers should match!") # zero encode work buffer to avoid corrupting with previous run - for i in 0.. 0: + copyMem(self.dataBufferPtr[i], addr data[i][0], self.bufSize) dataPtr[i] = self.dataBufferPtr[i] else: dataPtr[i] = nil # copy parity into aligned buffer - for i in 0.. 0: copyMem(self.workBufferPtr[i], addr parity[i][0], self.bufSize) parityPtr[i] = self.workBufferPtr[i] else: parityPtr[i] = nil - let - res = leoDecode( - self.bufSize.culonglong, - self.buffers.cuint, - self.parity.cuint, - self.decodeBufferCount.cuint, - cast[LeoDataPtr](addr dataPtr[0]), - cast[LeoDataPtr](addr parityPtr[0]), - cast[ptr pointer](addr self.decodeBufferPtr[0])) + let res = leoDecode( + self.bufSize.culonglong, + self.buffers.cuint, + self.parity.cuint, + self.decodeBufferCount.cuint, + cast[LeoDataPtr](addr dataPtr[0]), + cast[LeoDataPtr](addr parityPtr[0]), + cast[ptr pointer](addr self.decodeBufferPtr[0]), + ) if ord(res) != ord(LeopardSuccess): return err(leoResultString(res.LeopardResult)) @@ -193,11 +189,8 @@ func free*(self: var Leo) = # self.free() proc init[TT: Leo]( - T: type TT, - bufSize, - buffers, - parity: int, - kind: LeoCoderKind): Result[T, cstring] = + T: type TT, bufSize, buffers, parity: int, kind: LeoCoderKind +): Result[T, cstring] = if bufSize mod BuffMultiples != 0: return err("bufSize should be multiples of 64 bytes!") @@ -219,46 +212,33 @@ proc init[TT: Leo]( if (let res = leoInit(); res.ord != LeopardSuccess.ord): return err(leoResultString(res.LeopardResult)) - var - self = T( - kind: kind, - bufSize: bufSize, - buffers: buffers, - parity: parity) + var self = T(kind: kind, bufSize: bufSize, buffers: buffers, parity: parity) - self.workBufferCount = leoEncodeWorkCount( - buffers.cuint, - parity.cuint).int + self.workBufferCount = leoEncodeWorkCount(buffers.cuint, parity.cuint).int # initialize encode work buffers - for _ in 0.. 0: - dropRandomIdx(dataBuf,buffers, dataLosses) + dropRandomIdx(dataBuf, dataLosses) if parityLosses > 0: - dropRandomIdx(parityBuf,parity,parityLosses) + dropRandomIdx(parityBuf, parityLosses) - decoder.decode(dataBuf, parityBuf, recoveredBuf,buffers,parity,buffers).tryGet() + decoder.decode(dataBuf, parityBuf, recoveredBuf).tryGet() - for i in 0..