2026-05-29 13:19:16 +02:00
|
|
|
## Test-only Persistence backend backed by Nim tables. Adapts the
|
|
|
|
|
## snapshot-based `Persistence` interface onto a denormalised
|
|
|
|
|
## `InMemoryStore` shape so test assertions can inspect individual buffers
|
|
|
|
|
## (`store.outgoing`, `store.log`, etc.) directly. The adapter
|
|
|
|
|
## decomposes the meta blob on save and reconstructs it on load.
|
|
|
|
|
##
|
|
|
|
|
## `failingOps` injects backend failures. Op names match the `Persistence`
|
|
|
|
|
## field names: "saveChannelMeta", "updateHistory", "loadChannel",
|
2026-06-02 13:04:58 +02:00
|
|
|
## "dropChannel".
|
2026-05-29 13:19:16 +02:00
|
|
|
|
2026-05-27 17:12:06 +02:00
|
|
|
import std/[tables, sets]
|
feat: make Persistence interface async (#69)
* feat: make Persistence interface async
The 14 Persistence proc fields now return Future[...] with
{.async: (raises: []), gcsafe.}, allowing real I/O backends (SQLite,
encrypted file, network) to suspend rather than block the Chronos event
loop the manager runs on.
Propagates through:
- ReliabilityManager.lock: system.Lock -> chronos.AsyncLock. Acquired
across awaits cleanly; matches the single-threaded Chronos worker the
FFI uses. Multi-OS-thread use is now explicitly the caller's
responsibility.
- sds_utils + sds.nim public API procs (wrapOutgoingMessage,
unwrapReceivedMessage, markDependenciesMet, setCallbacks,
resetReliabilityManager, cleanup, ensureChannel, removeChannel, the
getter snapshots, etc.) are now async.
- FFI request handlers in library/sds_thread/... await the new API.
- Tests converted via an asyncTest template that wraps each test body
in an async proc; setup/teardown use waitFor for their single async
call (ensureChannel / cleanup).
Lock scope is preserved exactly: the same call sites that held the
kernel Lock today hold AsyncLock now -- no new locking added.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
* refactor: drop asyncSpawn, add asyncSetup/asyncTeardown
Three asyncSpawn usages removed:
- sds.nim startPeriodicTasks: stored the periodic-task futures on
ReliabilityManager (new field `periodicTasks: seq[FutureBase]`) so
cleanup can cancel them on shutdown instead of leaking the loops
against a cleared manager.
- library/sds_thread/sds_thread.nim: fireSync moved BEFORE processing,
then `await SdsThreadRequest.process(...)` instead of asyncSpawn'ing
it. Aligns the worker with the SP-channel + lock assumption that
there are no concurrent requests; caller throughput is unchanged
because the caller only waits for receipt (fireSync), not processing.
- tests TestBus repair callback: replaced asyncSpawn(deliverExcept...)
with an explicit pending-delivery queue drained by `bus.drain()`.
Integration tests no longer rely on `sleepAsync(10ms)` to let
spawned deliveries finish — they await drain instead.
Tests also pick up an asyncSetup/asyncTeardown pair (tests/async_unittest.nim)
so suite fixtures can `await` directly. All `waitFor` in setup/teardown
blocks is gone; only the top-level asyncTest wrapper still uses waitFor
(once, to drive the async proc to completion).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
* Correctly propagate error hidden by new async move
* Correctly handle future cancellation exceptions, +some housekeeping
* Apply suggestion from @Ivansete-status
Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com>
* Stylistics, async default implication addressed, nph style run
* Remove leaking CancelledFuture from public facing + as a consequence it is tuneled into handling CatchableError everywhere
---------
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com>
2026-05-25 22:30:15 +02:00
|
|
|
import chronos
|
2026-05-08 03:14:12 +05:30
|
|
|
import sds
|
|
|
|
|
|
|
|
|
|
type InMemoryStore* = ref object
|
|
|
|
|
lamports*: Table[SdsChannelID, int64]
|
|
|
|
|
log*: Table[SdsChannelID, OrderedTable[SdsMessageID, SdsMessage]]
|
|
|
|
|
outgoing*: Table[SdsChannelID, OrderedTable[SdsMessageID, UnacknowledgedMessage]]
|
|
|
|
|
incoming*: Table[SdsChannelID, OrderedTable[SdsMessageID, IncomingMessage]]
|
|
|
|
|
outgoingRepair*: Table[SdsChannelID, OrderedTable[SdsMessageID, OutgoingRepairEntry]]
|
|
|
|
|
incomingRepair*: Table[SdsChannelID, OrderedTable[SdsMessageID, IncomingRepairEntry]]
|
|
|
|
|
dropChannelCalls*: Table[SdsChannelID, int]
|
2026-05-29 13:19:16 +02:00
|
|
|
## Per-channel counter; lets tests assert dropChannel is invoked
|
|
|
|
|
## exactly once per logical drop.
|
2026-06-02 13:04:58 +02:00
|
|
|
failingOps*: HashSet[string] ## Op names that should return an injected backend error.
|
2026-05-08 03:14:12 +05:30
|
|
|
|
|
|
|
|
proc newInMemoryStore*(): InMemoryStore =
|
2026-05-27 17:12:06 +02:00
|
|
|
InMemoryStore(failingOps: initHashSet[string]())
|
|
|
|
|
|
2026-05-08 03:14:12 +05:30
|
|
|
proc newInMemoryPersistence*(store: InMemoryStore): Persistence =
|
|
|
|
|
Persistence(
|
2026-05-29 13:19:16 +02:00
|
|
|
saveChannelMeta: proc(
|
|
|
|
|
channelId: SdsChannelID, meta: ChannelMeta
|
2026-05-27 17:12:06 +02:00
|
|
|
): Future[Result[void, string]] {.async: (raises: []).} =
|
2026-05-29 13:19:16 +02:00
|
|
|
if "saveChannelMeta" in store.failingOps:
|
|
|
|
|
return err("injected backend failure: saveChannelMeta")
|
|
|
|
|
{.cast(raises: []).}:
|
|
|
|
|
# Lamport.
|
|
|
|
|
store.lamports[channelId] = meta.lamportTimestamp
|
|
|
|
|
|
|
|
|
|
# Outgoing buffer — replace existing rows wholesale (snapshot is
|
|
|
|
|
# the complete state, not a delta).
|
|
|
|
|
store.outgoing[channelId] =
|
|
|
|
|
initOrderedTable[SdsMessageID, UnacknowledgedMessage]()
|
|
|
|
|
for u in meta.outgoingBuffer:
|
|
|
|
|
store.outgoing[channelId][u.message.messageId] = u
|
|
|
|
|
|
|
|
|
|
# Incoming buffer.
|
2026-06-02 13:04:58 +02:00
|
|
|
store.incoming[channelId] = initOrderedTable[SdsMessageID, IncomingMessage]()
|
2026-05-29 13:19:16 +02:00
|
|
|
for m in meta.incomingBuffer:
|
|
|
|
|
store.incoming[channelId][m.message.messageId] = m
|
|
|
|
|
|
|
|
|
|
# Repair buffers.
|
|
|
|
|
store.outgoingRepair[channelId] =
|
|
|
|
|
initOrderedTable[SdsMessageID, OutgoingRepairEntry]()
|
|
|
|
|
for kv in meta.outgoingRepairBuffer:
|
|
|
|
|
store.outgoingRepair[channelId][kv.messageId] = kv.entry
|
|
|
|
|
store.incomingRepair[channelId] =
|
|
|
|
|
initOrderedTable[SdsMessageID, IncomingRepairEntry]()
|
|
|
|
|
for kv in meta.incomingRepairBuffer:
|
|
|
|
|
store.incomingRepair[channelId][kv.messageId] = kv.entry
|
2026-05-27 17:12:06 +02:00
|
|
|
ok(),
|
2026-05-29 13:19:16 +02:00
|
|
|
updateHistory: proc(
|
|
|
|
|
channelId: SdsChannelID, update: HistoryUpdate
|
2026-05-27 17:12:06 +02:00
|
|
|
): Future[Result[void, string]] {.async: (raises: []).} =
|
2026-05-29 13:19:16 +02:00
|
|
|
if "updateHistory" in store.failingOps:
|
|
|
|
|
return err("injected backend failure: updateHistory")
|
2026-05-08 03:14:12 +05:30
|
|
|
{.cast(raises: []).}:
|
|
|
|
|
if channelId notin store.log:
|
|
|
|
|
store.log[channelId] = initOrderedTable[SdsMessageID, SdsMessage]()
|
2026-05-29 13:19:16 +02:00
|
|
|
for m in update.append:
|
|
|
|
|
store.log[channelId][m.messageId] = m
|
|
|
|
|
for id in update.evict:
|
|
|
|
|
store.log[channelId].del(id)
|
2026-05-27 17:12:06 +02:00
|
|
|
ok(),
|
2026-05-29 13:19:16 +02:00
|
|
|
loadChannel: proc(
|
|
|
|
|
channelId: SdsChannelID
|
|
|
|
|
): Future[Result[ChannelData, string]] {.async: (raises: []).} =
|
|
|
|
|
if "loadChannel" in store.failingOps:
|
|
|
|
|
return err("injected backend failure: loadChannel")
|
2026-05-08 03:14:12 +05:30
|
|
|
{.cast(raises: []).}:
|
2026-05-29 13:19:16 +02:00
|
|
|
var data = ChannelData.init()
|
|
|
|
|
if channelId in store.lamports:
|
|
|
|
|
data.meta.lamportTimestamp = store.lamports[channelId]
|
2026-05-08 03:14:12 +05:30
|
|
|
if channelId in store.outgoing:
|
2026-05-29 13:19:16 +02:00
|
|
|
for u in store.outgoing[channelId].values:
|
|
|
|
|
data.meta.outgoingBuffer.add(u)
|
2026-05-08 03:14:12 +05:30
|
|
|
if channelId in store.incoming:
|
2026-05-29 13:19:16 +02:00
|
|
|
for m in store.incoming[channelId].values:
|
|
|
|
|
data.meta.incomingBuffer.add(m)
|
2026-05-08 03:14:12 +05:30
|
|
|
if channelId in store.outgoingRepair:
|
2026-05-29 13:19:16 +02:00
|
|
|
for id, e in store.outgoingRepair[channelId].pairs:
|
|
|
|
|
data.meta.outgoingRepairBuffer.add(
|
|
|
|
|
OutgoingRepairKV(messageId: id, entry: e)
|
|
|
|
|
)
|
2026-05-08 03:14:12 +05:30
|
|
|
if channelId in store.incomingRepair:
|
2026-05-29 13:19:16 +02:00
|
|
|
for id, e in store.incomingRepair[channelId].pairs:
|
|
|
|
|
data.meta.incomingRepairBuffer.add(
|
|
|
|
|
IncomingRepairKV(messageId: id, entry: e)
|
|
|
|
|
)
|
|
|
|
|
if channelId in store.log:
|
|
|
|
|
for m in store.log[channelId].values:
|
|
|
|
|
data.messageHistory.add(m)
|
|
|
|
|
return ok(data),
|
2026-05-27 17:12:06 +02:00
|
|
|
dropChannel: proc(
|
|
|
|
|
channelId: SdsChannelID
|
|
|
|
|
): Future[Result[void, string]] {.async: (raises: []).} =
|
2026-05-29 13:19:16 +02:00
|
|
|
if "dropChannel" in store.failingOps:
|
|
|
|
|
return err("injected backend failure: dropChannel")
|
2026-05-08 03:14:12 +05:30
|
|
|
{.cast(raises: []).}:
|
|
|
|
|
store.lamports.del(channelId)
|
|
|
|
|
store.log.del(channelId)
|
|
|
|
|
store.outgoing.del(channelId)
|
|
|
|
|
store.incoming.del(channelId)
|
|
|
|
|
store.outgoingRepair.del(channelId)
|
|
|
|
|
store.incomingRepair.del(channelId)
|
|
|
|
|
store.dropChannelCalls[channelId] =
|
2026-05-27 17:12:06 +02:00
|
|
|
store.dropChannelCalls.getOrDefault(channelId) + 1
|
|
|
|
|
ok(),
|
2026-05-08 03:14:12 +05:30
|
|
|
)
|