mirror of
https://github.com/status-im/nim-dagger.git
synced 2025-02-10 05:44:11 +00:00
70cbdff901
* feat: introduce LRU cache Replace `MemoryStore` with LRU caching mechanism. `lrucache` library was forked to https://github.com/status-im/lrucache.nim. Co-authored-by: Eric Mastro <eric.mastro@gmail.com> # Conflicts: # dagger/dagger.nim # dagger/stores.nim # dagger/stores/manager.nim # tests/dagger/blockexc/testengine.nim # tests/dagger/helpers/nodeutils.nim # tests/dagger/testnode.nim # tests/dagger/teststores.nim * feat: introduce --cache-size CLI option Allow for a value of `0` to disable the cache. # Conflicts: # dagger/dagger.nim * allow dynamic block size in cache allow block size to be variable/dynamic update lrucache to use updated lrucache dep Using removeLru proc, added tests * Refactor CacheStore init block Co-authored-by: Michael Bradley, Jr <michaelsbradleyjr@gmail.com>
142 lines
3.3 KiB
Nim
142 lines
3.3 KiB
Nim
## Nim-Dagger
|
|
## Copyright (c) 2021 Status Research & Development GmbH
|
|
## Licensed under either of
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
## at your option.
|
|
## This file may not be copied, modified, or distributed except according to
|
|
## those terms.
|
|
|
|
# TODO: This is super inneficient and needs a rewrite, but it'll do for now
|
|
|
|
{.push raises: [Defect].}
|
|
|
|
import pkg/chronicles
|
|
import pkg/questionable
|
|
import pkg/questionable/results
|
|
import pkg/chronos
|
|
import pkg/libp2p except shuffle
|
|
|
|
import ./blocktype
|
|
|
|
export blocktype
|
|
|
|
const
|
|
DefaultChunkSize*: Positive = 1024 * 256
|
|
|
|
type
|
|
# default reader type
|
|
ChunkBuffer* = ptr UncheckedArray[byte]
|
|
Reader* =
|
|
proc(data: ChunkBuffer, len: int): Future[int] {.gcsafe, raises: [Defect].}
|
|
|
|
ChunkerType* {.pure.} = enum
|
|
FixedChunker
|
|
RabinChunker
|
|
|
|
Chunker* = object
|
|
reader*: Reader
|
|
case kind*: ChunkerType:
|
|
of FixedChunker:
|
|
chunkSize*: Natural
|
|
pad*: bool # pad last block if less than size
|
|
of RabinChunker:
|
|
discard
|
|
|
|
FileChunker* = Chunker
|
|
LPStreamChunker* = Chunker
|
|
|
|
proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} =
|
|
## returns a chunk of bytes from
|
|
## the instantiated chunker
|
|
##
|
|
|
|
var buff = newSeq[byte](c.chunkSize)
|
|
let read = await c.reader(cast[ChunkBuffer](addr buff[0]), buff.len)
|
|
|
|
if read <= 0:
|
|
return @[]
|
|
|
|
if not c.pad and buff.len > read:
|
|
buff.setLen(read)
|
|
|
|
return buff
|
|
|
|
func new*(
|
|
T: type Chunker,
|
|
kind = ChunkerType.FixedChunker,
|
|
reader: Reader,
|
|
chunkSize = DefaultChunkSize,
|
|
pad = false): T =
|
|
var chunker = Chunker(
|
|
kind: kind,
|
|
reader: reader)
|
|
|
|
if kind == ChunkerType.FixedChunker:
|
|
chunker.pad = pad
|
|
chunker.chunkSize = chunkSize
|
|
|
|
return chunker
|
|
|
|
proc new*(
|
|
T: type LPStreamChunker,
|
|
stream: LPStream,
|
|
kind = ChunkerType.FixedChunker,
|
|
chunkSize = DefaultChunkSize,
|
|
pad = false): T =
|
|
## create the default File chunker
|
|
##
|
|
|
|
proc reader(data: ChunkBuffer, len: int): Future[int]
|
|
{.gcsafe, async, raises: [Defect].} =
|
|
var res = 0
|
|
try:
|
|
while res < len:
|
|
res += await stream.readOnce(addr data[res], len - res)
|
|
except LPStreamEOFError as exc:
|
|
trace "LPStreamChunker stream Eof", exc = exc.msg
|
|
except CatchableError as exc:
|
|
trace "CatchableError exception", exc = exc.msg
|
|
raise newException(Defect, exc.msg)
|
|
|
|
return res
|
|
|
|
Chunker.new(
|
|
kind = ChunkerType.FixedChunker,
|
|
reader = reader,
|
|
pad = pad,
|
|
chunkSize = chunkSize)
|
|
|
|
proc new*(
|
|
T: type FileChunker,
|
|
file: File,
|
|
kind = ChunkerType.FixedChunker,
|
|
chunkSize = DefaultChunkSize,
|
|
pad = false): T =
|
|
## create the default File chunker
|
|
##
|
|
|
|
proc reader(data: ChunkBuffer, len: int): Future[int]
|
|
{.gcsafe, async, raises: [Defect].} =
|
|
var total = 0
|
|
try:
|
|
while total < len:
|
|
let res = file.readBuffer(addr data[total], len - total)
|
|
if res <= 0:
|
|
break
|
|
|
|
total += res
|
|
except IOError as exc:
|
|
trace "Exception reading file", exc = exc.msg
|
|
except CatchableError as exc:
|
|
trace "CatchableError exception", exc = exc.msg
|
|
raise newException(Defect, exc.msg)
|
|
|
|
return total
|
|
|
|
Chunker.new(
|
|
kind = ChunkerType.FixedChunker,
|
|
reader = reader,
|
|
pad = pad,
|
|
chunkSize = chunkSize)
|