mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-05 23:13:09 +00:00
4a70312ee9
Implement blockstore manager which executes block storage operations on its block stores, in the order to which they were added to the manager, typically in the order of most local (fastest, eg cache) to least local (slowest, eg filesystem or perhaps a network filesystem). As an example, given a `BlockStoreManager` instantiated with a `@[MemoryStore, FSStore]`, retrieving a block would first attempt to get from the `MemoryStore`, and if not found, attempt to get from the `FSStore`. Remove all dependencies on `BlockStores` (typically in the shape of `localstore`) and instead depend on `BlockStoreManager` via the `BlockExcEngine`. Modify the role of the `BlockExcEngine` to make a “local vs remote” decision on block access/storage. For all operations other than retrieving blocks, this means simply going to the `BlockStoreManager`. For retrieving blocks, however, this means going first to the `BlockStoreManager`, and then if not found, going to the Dagger network (via pending block and want/have lists). Remove `NetworkStore` as its two purposes were to defer block retrieval from a local store first, then go to the block exchange to requeest a block from the Dagger network. `BlockStoreManager` takes care of going to local storage first, and the block exchange engine handles going to Dagger network if retrieval from the store manager fails. ### Notes 1. Future work may want to consider breaking up `BlockExcEngine` further in to three modules: - `BlockExcEngine` (depends on `WantHave`, `DHT`) - `WantHave` - `DHT` (work is in progress) Co-authored-by: Michael Bradley <michaelsbradleyjr@gmail.com>
111 lines
2.5 KiB
Nim
111 lines
2.5 KiB
Nim
## Nim-Dagger
|
|
## Copyright (c) 2021 Status Research & Development GmbH
|
|
## Licensed under either of
|
|
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
|
|
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
|
|
## at your option.
|
|
## This file may not be copied, modified, or distributed except according to
|
|
## those terms.
|
|
|
|
{.push raises: [Defect].}
|
|
|
|
import std/os
|
|
|
|
import pkg/chronos
|
|
import pkg/chronicles
|
|
import pkg/libp2p
|
|
import pkg/questionable
|
|
import pkg/questionable/results
|
|
import pkg/stew/io2
|
|
|
|
import ./memorystore
|
|
import ./blockstore
|
|
import ../blocktype
|
|
|
|
export blockstore
|
|
|
|
logScope:
|
|
topics = "dagger fsstore"
|
|
|
|
type
|
|
FSStore* = ref object of BlockStore
|
|
repoDir: string
|
|
postfixLen*: int
|
|
|
|
template blockPath*(self: FSStore, cid: Cid): string =
|
|
self.repoDir / ($cid)[^self.postfixLen..^1] / $cid
|
|
|
|
method getBlock*(
|
|
self: FSStore,
|
|
cid: Cid): Future[?!Block] {.async.} =
|
|
## Get a block from the stores
|
|
##
|
|
|
|
if cid notin self:
|
|
return Block.failure("Couldn't find block in fs store")
|
|
|
|
var data: seq[byte]
|
|
let path = self.blockPath(cid)
|
|
if (
|
|
let res = io2.readFile(path, data);
|
|
res.isErr):
|
|
let error = io2.ioErrorMsg(res.error)
|
|
trace "Cannot read file from fs store", path , error
|
|
return Block.failure("Cannot read file from fs store")
|
|
|
|
return Block.init(cid, data)
|
|
|
|
method putBlock*(
|
|
self: FSStore,
|
|
blk: Block): Future[bool] {.async.} =
|
|
## Put a block to the blockstore
|
|
##
|
|
|
|
if blk.cid in self:
|
|
return true
|
|
|
|
# if directory exists it wont fail
|
|
if io2.createPath(self.blockPath(blk.cid).parentDir).isErr:
|
|
trace "Unable to create block prefix dir", dir = self.blockPath(blk.cid).parentDir
|
|
return false
|
|
|
|
let path = self.blockPath(blk.cid)
|
|
if (
|
|
let res = io2.writeFile(path, blk.data);
|
|
res.isErr):
|
|
let error = io2.ioErrorMsg(res.error)
|
|
trace "Unable to store block", path, cid = blk.cid, error
|
|
return false
|
|
|
|
return true
|
|
|
|
method delBlock*(
|
|
self: FSStore,
|
|
cid: Cid): Future[bool] {.async.} =
|
|
## Delete a block/s from the block store
|
|
##
|
|
|
|
let path = self.blockPath(cid)
|
|
if (
|
|
let res = io2.removeFile(path);
|
|
res.isErr):
|
|
let error = io2.ioErrorMsg(res.error)
|
|
trace "Unable to delete block", path, cid, error
|
|
return false
|
|
|
|
return true
|
|
|
|
method hasBlock*(self: FSStore, cid: Cid): bool =
|
|
## Check if the block exists in the blockstore
|
|
##
|
|
|
|
self.blockPath(cid).isFile()
|
|
|
|
proc new*(
|
|
T: type FSStore,
|
|
repoDir: string,
|
|
postfixLen = 2): T =
|
|
T(
|
|
postfixLen: postfixLen,
|
|
repoDir: repoDir)
|