mirror of
https://github.com/codex-storage/nim-codex.git
synced 2025-01-09 20:45:38 +00:00
de88fd2c53
* implement a logging proxy The logging proxy: - prevents the need to import chronicles (as well as export except toJson), - prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization, - and handles json formatting correctly in chronicles json sinks * Rename logging -> logutils to avoid ambiguity with common names * clean up * add setProperty for JsonRecord, remove nim-json-serialization conflict * Allow specifying textlines and json format separately Not specifying a LogFormat will apply the formatting to both textlines and json sinks. Specifying a LogFormat will apply the formatting to only that sink. * remove unneeded usages of std/json We only need to import utils/json instead of std/json * move serialization from rest/json to utils/json so it can be shared * fix NoColors ambiguity Was causing unit tests to fail on Windows. * Remove nre usage to fix Windows error Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :) * Add logutils module doc * Shorten logutils.formatIt for `NBytes` Both json and textlines formatIt were not needed, and could be combined into one formatIt * remove debug integration test config debug output and logformat of json for integration test logs * Use ## module doc to support docgen * bump nim-poseidon2 to export fromBytes Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening. * fixes to get compiling after rebasing master * Add support for Result types being logged using formatIt
135 lines
3.2 KiB
Nim
135 lines
3.2 KiB
Nim
## Nim-Codex
|
|
## Copyright (c) 2021 Status Research & Development GmbH
|
|
## Licensed under either of
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
## at your option.
|
|
## This file may not be copied, modified, or distributed except according to
|
|
## those terms.
|
|
|
|
# TODO: This is super inneficient and needs a rewrite, but it'll do for now
|
|
|
|
import pkg/upraises
|
|
|
|
push: {.upraises: [].}
|
|
|
|
import pkg/questionable
|
|
import pkg/questionable/results
|
|
import pkg/chronos
|
|
import pkg/libp2p except shuffle
|
|
|
|
import ./blocktype
|
|
import ./logutils
|
|
|
|
export blocktype
|
|
|
|
const
|
|
DefaultChunkSize* = DefaultBlockSize
|
|
|
|
type
|
|
# default reader type
|
|
ChunkBuffer* = ptr UncheckedArray[byte]
|
|
Reader* = proc(data: ChunkBuffer, len: int): Future[int] {.gcsafe, raises: [Defect].}
|
|
|
|
# Reader that splits input data into fixed-size chunks
|
|
Chunker* = ref object
|
|
reader*: Reader # Procedure called to actually read the data
|
|
offset*: int # Bytes read so far (position in the stream)
|
|
chunkSize*: NBytes # Size of each chunk
|
|
pad*: bool # Pad last chunk to chunkSize?
|
|
|
|
FileChunker* = Chunker
|
|
LPStreamChunker* = Chunker
|
|
|
|
proc getBytes*(c: Chunker): Future[seq[byte]] {.async.} =
|
|
## returns a chunk of bytes from
|
|
## the instantiated chunker
|
|
##
|
|
|
|
var buff = newSeq[byte](c.chunkSize.int)
|
|
let read = await c.reader(cast[ChunkBuffer](addr buff[0]), buff.len)
|
|
|
|
if read <= 0:
|
|
return @[]
|
|
|
|
c.offset += read
|
|
|
|
if not c.pad and buff.len > read:
|
|
buff.setLen(read)
|
|
|
|
return move buff
|
|
|
|
proc new*(
|
|
T: type Chunker,
|
|
reader: Reader,
|
|
chunkSize = DefaultChunkSize,
|
|
pad = true
|
|
): Chunker =
|
|
## create a new Chunker instance
|
|
##
|
|
Chunker(
|
|
reader: reader,
|
|
offset: 0,
|
|
chunkSize: chunkSize,
|
|
pad: pad)
|
|
|
|
proc new*(
|
|
T: type LPStreamChunker,
|
|
stream: LPStream,
|
|
chunkSize = DefaultChunkSize,
|
|
pad = true
|
|
): LPStreamChunker =
|
|
## create the default File chunker
|
|
##
|
|
|
|
proc reader(data: ChunkBuffer, len: int): Future[int]
|
|
{.gcsafe, async, raises: [Defect].} =
|
|
var res = 0
|
|
try:
|
|
while res < len:
|
|
res += await stream.readOnce(addr data[res], len - res)
|
|
except LPStreamEOFError as exc:
|
|
trace "LPStreamChunker stream Eof", exc = exc.msg
|
|
except CatchableError as exc:
|
|
trace "CatchableError exception", exc = exc.msg
|
|
raise newException(Defect, exc.msg)
|
|
|
|
return res
|
|
|
|
LPStreamChunker.new(
|
|
reader = reader,
|
|
chunkSize = chunkSize,
|
|
pad = pad)
|
|
|
|
proc new*(
|
|
T: type FileChunker,
|
|
file: File,
|
|
chunkSize = DefaultChunkSize,
|
|
pad = true
|
|
): FileChunker =
|
|
## create the default File chunker
|
|
##
|
|
|
|
proc reader(data: ChunkBuffer, len: int): Future[int]
|
|
{.gcsafe, async, raises: [Defect].} =
|
|
var total = 0
|
|
try:
|
|
while total < len:
|
|
let res = file.readBuffer(addr data[total], len - total)
|
|
if res <= 0:
|
|
break
|
|
|
|
total += res
|
|
except IOError as exc:
|
|
trace "Exception reading file", exc = exc.msg
|
|
except CatchableError as exc:
|
|
trace "CatchableError exception", exc = exc.msg
|
|
raise newException(Defect, exc.msg)
|
|
|
|
return total
|
|
|
|
FileChunker.new(
|
|
reader = reader,
|
|
chunkSize = chunkSize,
|
|
pad = pad)
|