From 004d96089fdb38c1070e77d3923cc3651c503b62 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 14 Mar 2023 10:46:33 +0100 Subject: [PATCH] setting up blockstoremanager --- codex/codex.nim | 46 ++++------------- codex/conf.nim | 2 +- codex/consts.nim | 2 + codex/stores/blockstoremanager.nim | 79 ++++++++++++++++++++++++++++++ codex/stores/cachestore.nim | 3 +- codex/stores/memorystore.nim | 3 +- 6 files changed, 94 insertions(+), 41 deletions(-) create mode 100644 codex/consts.nim create mode 100644 codex/stores/blockstoremanager.nim diff --git a/codex/codex.nim b/codex/codex.nim index 9926811c..41b9660f 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -26,7 +26,7 @@ import ./node import ./conf import ./rng import ./rest/api -import ./stores +import ./stores/blockstoremanager import ./blockexchange import ./utils/fileutils import ./erasure @@ -46,8 +46,7 @@ type config: CodexConf restServer: RestServerRef codexNode: CodexNodeRef - repoStore: RepoStore - maintenance: BlockMaintainer + blockStoreManager: BlockStoreManager CodexPrivateKey* = libp2p.PrivateKey # alias @@ -95,12 +94,11 @@ proc bootstrapInteractions(config: CodexConf, repo: RepoStore): Future[Contracts proc start*(s: CodexServer) {.async.} = notice "Starting codex node" - await s.repoStore.start() + await s.blockStoreManager.start() s.restServer.start() s.codexNode.contracts = await bootstrapInteractions(s.config, s.repoStore) await s.codexNode.start() - s.maintenance.start() let # TODO: Can't define these as constants, pity @@ -136,8 +134,7 @@ proc stop*(s: CodexServer) {.async.} = await allFuturesThrowing( s.restServer.stop(), s.codexNode.stop(), - s.repoStore.stop(), - s.maintenance.stop()) + s.blockStoreManager.stop()) s.runHandle.complete() @@ -157,13 +154,6 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): .withTcpTransport({ServerFlags.ReuseAddr}) .build() - var - cache: CacheStore = nil - - if config.cacheSize > 0: - cache = CacheStore.new(cacheSize = config.cacheSize * MiB) - ## Is unused? - let discoveryDir = config.dataDir / CodexDhtNamespace @@ -188,29 +178,14 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): wallet = WalletRef.new(EthPrivateKey.random()) network = BlockExcNetwork.new(switch) - repoData = case config.repoKind - of repoFS: Datastore(FSDatastore.new($config.dataDir, depth = 5) - .expect("Should create repo file data store!")) - of repoSQLite: Datastore(SQLiteDatastore.new($config.dataDir) - .expect("Should create repo SQLite data store!")) - - repoStore = RepoStore.new( - repoDs = repoData, - metaDs = SQLiteDatastore.new(config.dataDir / CodexMetaNamespace) - .expect("Should create meta data store!"), - quotaMaxBytes = config.storageQuota.uint, - blockTtl = config.blockTtlSeconds.seconds) - - maintenance = BlockMaintainer.new( - repoStore, - interval = config.blockMaintenanceIntervalSeconds.seconds, - numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks) + blockStoreManager = BlockStoreManager.new(config) + blockStore = blockStoreManager.getBlockStore() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() - blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks) - engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, peerStore, pendingBlocks) - store = NetworkStore.new(engine, repoStore) + blockDiscovery = DiscoveryEngine.new(blockStore, peerStore, network, discovery, pendingBlocks) + engine = BlockExcEngine.new(blockStore, wallet, network, blockDiscovery, peerStore, pendingBlocks) + store = NetworkStore.new(engine, blockStore) erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider) codexNode = CodexNodeRef.new(switch, store, engine, erasure, discovery) restServer = RestServerRef.new( @@ -225,5 +200,4 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): config: config, codexNode: codexNode, restServer: restServer, - repoStore: repoStore, - maintenance: maintenance) + blockStoreManager: blockStoreManager) diff --git a/codex/conf.nim b/codex/conf.nim index 0fb365bb..75299e51 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -195,7 +195,7 @@ type name: "block-mn" }: int cacheSize* {. - desc: "The size in MiB of the block cache, 0 disables the cache - might help on slow hardrives" + desc: "Size in MiB of the in-RAM block cache, 0 disables the cache - might help on slow hardrives" defaultValue: 0 defaultValueDesc: "0" name: "cache-size" diff --git a/codex/consts.nim b/codex/consts.nim new file mode 100644 index 00000000..304bb91c --- /dev/null +++ b/codex/consts.nim @@ -0,0 +1,2 @@ +const + MiB* = 1024 * 1024 diff --git a/codex/stores/blockstoremanager.nim b/codex/stores/blockstoremanager.nim new file mode 100644 index 00000000..5ab923a5 --- /dev/null +++ b/codex/stores/blockstoremanager.nim @@ -0,0 +1,79 @@ +## Nim-Codex +## Copyright (c) 2023 Status Research & Development GmbH +## Licensed under either of +## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +## * MIT license ([LICENSE-MIT](LICENSE-MIT)) +## at your option. +## This file may not be copied, modified, or distributed except according to +## those terms. + +import pkg/upraises + +push: {.upraises: [].} + +import pkg/chronos +import pkg/chronicles +import pkg/datastore + +import ../node +import ../conf +import ../rng +import ../rest/api +import ../blockexchange +import ../utils/fileutils +import ../erasure +import ../discovery +import ../contracts +import ../utils/addrutils +import ../namespaces + +import ../conf +import ./blockstore +import ./repostore +import ./maintenance + +type + BlockStoreManager* = ref object of RootObj + repoStore: RepoStore + maintenance: BlockMaintainer + blockStore: BlockStore + +proc start*(self: BlockStoreManager): Future[void] {.async.} = + await self.repoStore.start() + self.maintenance.start() + +proc stop*(self: BlockStoreManager): Future[void] {.async.} = + await self.repoStore.stop() + await self.maintenance.stop() + +proc getBlockStore*(self: BlockStoreManager): BlockStore = + self.blockStore + +proc createRepoStore(config: CodexConf): RepoStore = + RepoStore.new( + repoDs = Datastore(FSDatastore.new($config.dataDir, depth = 5).expect("Should create repo data store!")), + metaDs = SQLiteDatastore.new(config.dataDir / CodexMetaNamespace).expect("Should create meta data store!"), + quotaMaxBytes = config.storageQuota.uint, + blockTtl = config.blockTtlSeconds.seconds) + +proc createMaintenance(repoStore: RepoStore, config: CodexConf): BlockMaintainer = + BlockMaintainer.new( + repoStore, + interval = config.blockMaintenanceIntervalSeconds.seconds, + numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks) + +proc getBlockStore(repoStore: RepoStore, config: CodexConf): BlockStore = + if config.cacheSize > 0: + return CacheStore.new(backingStore = repoStore, cacheSize = config.cacheSize * MiB) + return repoStore + +func new*(T: type BlockStoreManager, config: CodexConf): T = + let + repoStore = createRepoStore(config) + maintenance = createMaintenance(repoStore, config) + blockStore = getBlockStore(repoStore, config) + + T( + repoStore: repoStore, + maintenance: maintenance, + blockStore: blockStore) diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index e8d97341..d7251520 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -21,8 +21,8 @@ import pkg/questionable import pkg/questionable/results import ./blockstore +import ../consts import ../chunker -import ../errors import ../manifest export blockstore @@ -36,7 +36,6 @@ type cache: LruCache[Cid, Block] const - MiB* = 1024 * 1024 DefaultCacheSizeMiB* = 5 DefaultCacheSize* = DefaultCacheSizeMiB * MiB diff --git a/codex/stores/memorystore.nim b/codex/stores/memorystore.nim index f1b828bc..f28a3b45 100644 --- a/codex/stores/memorystore.nim +++ b/codex/stores/memorystore.nim @@ -22,8 +22,8 @@ import pkg/questionable import pkg/questionable/results import ./blockstore +import ../consts import ../chunker -import ../errors import ../manifest export blockstore @@ -43,7 +43,6 @@ type list: DoublyLinkedList[MemoryStoreNode] const - MiB* = 1024 * 1024 DefaultMemoryStoreCapacityMiB* = 5 DefaultMemoryStoreCapacity* = DefaultMemoryStoreCapacityMiB * MiB