nim-codex/dagger/chunker.nim

142 lines
3.3 KiB
Nim

## Nim-Dagger
## Copyright (c) 2021 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# TODO: This is super inneficient and needs a rewrite, but it'll do for now
{.push raises: [Defect].}
import pkg/chronicles
import pkg/questionable
import pkg/questionable/results
import pkg/chronos
import pkg/libp2p except shuffle
import ./blocktype
export blocktype
const
DefaultChunkSize*: int64 = 1024 * 256
type
# default reader type
ChunkBuffer* = ptr UncheckedArray[byte]
Reader* =
proc(data: ChunkBuffer, len: int): Future[int] {.gcsafe, raises: [Defect].}
ChunkerType* {.pure.} = enum
FixedChunker
RabinChunker
Chunker* = object
reader*: Reader
case kind*: ChunkerType:
of FixedChunker:
chunkSize*: Natural
pad*: bool # pad last block if less than size
of RabinChunker:
discard
FileChunker* = Chunker
LPStreamChunker* = Chunker
proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} =
## returns a chunk of bytes from
## the instantiated chunker
##
var buff = newSeq[byte](c.chunkSize)
let read = await c.reader(cast[ChunkBuffer](addr buff[0]), buff.len)
if read <= 0:
return @[]
if not c.pad and buff.len > read:
buff.setLen(read)
return buff
func new*(
T: type Chunker,
kind = ChunkerType.FixedChunker,
reader: Reader,
chunkSize = DefaultChunkSize,
pad = false): T =
var chunker = Chunker(
kind: kind,
reader: reader)
if kind == ChunkerType.FixedChunker:
chunker.pad = pad
chunker.chunkSize = chunkSize
return chunker
proc new*(
T: type LPStreamChunker,
stream: LPStream,
kind = ChunkerType.FixedChunker,
chunkSize = DefaultChunkSize,
pad = false): T =
## create the default File chunker
##
proc reader(data: ChunkBuffer, len: int): Future[int]
{.gcsafe, async, raises: [Defect].} =
var res = 0
try:
while res < len:
res += await stream.readOnce(addr data[res], len - res)
except LPStreamEOFError as exc:
trace "LPStreamChunker stream Eof", exc = exc.msg
except CatchableError as exc:
trace "CatchableError exception", exc = exc.msg
raise newException(Defect, exc.msg)
return res
Chunker.new(
kind = ChunkerType.FixedChunker,
reader = reader,
pad = pad,
chunkSize = chunkSize)
proc new*(
T: type FileChunker,
file: File,
kind = ChunkerType.FixedChunker,
chunkSize = DefaultChunkSize,
pad = false): T =
## create the default File chunker
##
proc reader(data: ChunkBuffer, len: int): Future[int]
{.gcsafe, async, raises: [Defect].} =
var total = 0
try:
while total < len:
let res = file.readBuffer(addr data[total], len - total)
if res <= 0:
break
total += res
except IOError as exc:
trace "Exception reading file", exc = exc.msg
except CatchableError as exc:
trace "CatchableError exception", exc = exc.msg
raise newException(Defect, exc.msg)
return total
Chunker.new(
kind = ChunkerType.FixedChunker,
reader = reader,
pad = pad,
chunkSize = chunkSize)