nim-codex/codex/stores/networkstore.nim
Eric de88fd2c53
feat: create logging proxy (#663)
* implement a logging proxy

The logging proxy:
- prevents the need to import chronicles (as well as export except toJson),
- prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization,
- and handles json formatting correctly in chronicles json sinks

* Rename logging -> logutils to avoid ambiguity with common names

* clean up

* add setProperty for JsonRecord, remove nim-json-serialization conflict

* Allow specifying textlines and json format separately

Not specifying a LogFormat will apply the formatting to both textlines and json sinks.

Specifying a LogFormat will apply the formatting to only that sink.

* remove unneeded usages of std/json

We only need to import utils/json instead of std/json

* move serialization from rest/json to utils/json so it can be shared

* fix NoColors ambiguity

Was causing unit tests to fail on Windows.

* Remove nre usage to fix Windows error

Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :)

* Add logutils module doc

* Shorten logutils.formatIt for `NBytes`

Both json and textlines formatIt were not needed, and could be combined into one formatIt

* remove debug integration test config

debug output and logformat of json for integration test logs

* Use ## module doc to support docgen

* bump nim-poseidon2 to export fromBytes

Before the changes in this branch, fromBytes was likely being resolved by nim-stew, or other dependency. With the changes in this branch, that dependency was removed and fromBytes could no longer be resolved. By exporting fromBytes from nim-poseidon, the correct resolution is now happening.

* fixes to get compiling after rebasing master

* Add support for Result types being logged using formatIt
2024-01-22 23:35:03 -08:00

172 lines
4.5 KiB
Nim

## Nim-Codex
## Copyright (c) 2021 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
import pkg/upraises
push: {.upraises: [].}
import pkg/chronos
import pkg/libp2p
import pkg/questionable/results
import ../clock
import ../blocktype
import ../blockexchange
import ../logutils
import ../merkletree
import ../utils/asyncheapqueue
import ../utils/asynciter
import ./blockstore
export blockstore, blockexchange, asyncheapqueue
logScope:
topics = "codex networkstore"
type
NetworkStore* = ref object of BlockStore
engine*: BlockExcEngine # blockexc decision engine
localStore*: BlockStore # local block store
method getBlock*(self: NetworkStore, address: BlockAddress): Future[?!Block] {.async.} =
trace "Getting block from local store or network", address
without blk =? await self.localStore.getBlock(address), error:
if not (error of BlockNotFoundError): return failure error
trace "Block not in local store", address
without newBlock =? (await self.engine.requestBlock(address)).catch, error:
trace "Unable to get block from exchange engine", address
return failure error
return success newBlock
return success blk
method getBlock*(self: NetworkStore, cid: Cid): Future[?!Block] =
## Get a block from the blockstore
##
self.getBlock(BlockAddress.init(cid))
method getBlock*(self: NetworkStore, treeCid: Cid, index: Natural): Future[?!Block] =
## Get a block from the blockstore
##
self.getBlock(BlockAddress.init(treeCid, index))
method putBlock*(
self: NetworkStore,
blk: Block,
ttl = Duration.none): Future[?!void] {.async.} =
## Store block locally and notify the network
##
trace "Putting block into network store", cid = blk.cid
let res = await self.localStore.putBlock(blk, ttl)
if res.isErr:
return res
await self.engine.resolveBlocks(@[blk])
return success()
method putCidAndProof*(
self: NetworkStore,
treeCid: Cid,
index: Natural,
blockCid: Cid,
proof: CodexProof): Future[?!void] =
self.localStore.putCidAndProof(treeCid, index, blockCid, proof)
method getCidAndProof*(
self: NetworkStore,
treeCid: Cid,
index: Natural): Future[?!(Cid, CodexProof)] =
## Get a block proof from the blockstore
##
self.localStore.getCidAndProof(treeCid, index)
method ensureExpiry*(
self: NetworkStore,
cid: Cid,
expiry: SecondsSince1970): Future[?!void] {.async.} =
## Ensure that block's assosicated expiry is at least given timestamp
## If the current expiry is lower then it is updated to the given one, otherwise it is left intact
##
without blockCheck =? await self.localStore.hasBlock(cid), err:
return failure(err)
if blockCheck:
return await self.localStore.ensureExpiry(cid, expiry)
else:
trace "Updating expiry - block not in local store", cid
return success()
method ensureExpiry*(
self: NetworkStore,
treeCid: Cid,
index: Natural,
expiry: SecondsSince1970): Future[?!void] {.async.} =
## Ensure that block's associated expiry is at least given timestamp
## If the current expiry is lower then it is updated to the given one, otherwise it is left intact
##
without blockCheck =? await self.localStore.hasBlock(treeCid, index), err:
return failure(err)
if blockCheck:
return await self.localStore.ensureExpiry(treeCid, index, expiry)
else:
trace "Updating expiry - block not in local store", treeCid, index
return success()
method listBlocks*(
self: NetworkStore,
blockType = BlockType.Manifest): Future[?!AsyncIter[?Cid]] =
self.localStore.listBlocks(blockType)
method delBlock*(self: NetworkStore, cid: Cid): Future[?!void] =
## Delete a block from the blockstore
##
trace "Deleting block from network store", cid
return self.localStore.delBlock(cid)
{.pop.}
method hasBlock*(self: NetworkStore, cid: Cid): Future[?!bool] {.async.} =
## Check if the block exists in the blockstore
##
trace "Checking network store for block existence", cid
return await self.localStore.hasBlock(cid)
method close*(self: NetworkStore): Future[void] {.async.} =
## Close the underlying local blockstore
##
if not self.localStore.isNil:
await self.localStore.close
proc new*(
T: type NetworkStore,
engine: BlockExcEngine,
localStore: BlockStore
): NetworkStore =
## Create new instance of a NetworkStore
##
NetworkStore(
localStore: localStore,
engine: engine)