setting up blockstoremanager

codex/codex.nim

@@ -26,7 +26,7 @@ import ./node
 import ./conf
 import ./rng
 import ./rest/api
-import ./stores
+import ./stores/blockstoremanager
 import ./blockexchange
 import ./utils/fileutils
 import ./erasure
@@ -46,8 +46,7 @@ type
     config: CodexConf
     restServer: RestServerRef
     codexNode: CodexNodeRef
-    repoStore: RepoStore
+    blockStoreManager: BlockStoreManager
-    maintenance: BlockMaintainer
 
   CodexPrivateKey* = libp2p.PrivateKey # alias
 
@@ -95,12 +94,11 @@ proc bootstrapInteractions(config: CodexConf, repo: RepoStore): Future[Contracts
 proc start*(s: CodexServer) {.async.} =
   notice "Starting codex node"
 
-  await s.repoStore.start()
+  await s.blockStoreManager.start()
   s.restServer.start()
 
   s.codexNode.contracts = await bootstrapInteractions(s.config, s.repoStore)
   await s.codexNode.start()
-  s.maintenance.start()
 
   let
     # TODO: Can't define these as constants, pity
@@ -136,8 +134,7 @@ proc stop*(s: CodexServer) {.async.} =
   await allFuturesThrowing(
     s.restServer.stop(),
     s.codexNode.stop(),
-    s.repoStore.stop(),
+    s.blockStoreManager.stop())
-    s.maintenance.stop())
 
   s.runHandle.complete()
 
@@ -157,13 +154,6 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey):
     .withTcpTransport({ServerFlags.ReuseAddr})
     .build()
 
-  var
-    cache: CacheStore = nil
-
-  if config.cacheSize > 0:
-    cache = CacheStore.new(cacheSize = config.cacheSize * MiB)
-    ## Is unused?
-
   let
     discoveryDir = config.dataDir / CodexDhtNamespace
 
@@ -188,29 +178,14 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey):
     wallet = WalletRef.new(EthPrivateKey.random())
     network = BlockExcNetwork.new(switch)
 
-    repoData = case config.repoKind
+    blockStoreManager = BlockStoreManager.new(config)
-                of repoFS: Datastore(FSDatastore.new($config.dataDir, depth = 5)
+    blockStore = blockStoreManager.getBlockStore()
-                  .expect("Should create repo file data store!"))
-                of repoSQLite: Datastore(SQLiteDatastore.new($config.dataDir)
-                  .expect("Should create repo SQLite data store!"))
-
-    repoStore = RepoStore.new(
-      repoDs = repoData,
-      metaDs = SQLiteDatastore.new(config.dataDir / CodexMetaNamespace)
-        .expect("Should create meta data store!"),
-      quotaMaxBytes = config.storageQuota.uint,
-      blockTtl = config.blockTtlSeconds.seconds)
-
-    maintenance = BlockMaintainer.new(
-      repoStore,
-      interval = config.blockMaintenanceIntervalSeconds.seconds,
-      numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks)
 
     peerStore = PeerCtxStore.new()
     pendingBlocks = PendingBlocksManager.new()
-    blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks)
+    blockDiscovery = DiscoveryEngine.new(blockStore, peerStore, network, discovery, pendingBlocks)
-    engine = BlockExcEngine.new(repoStore, wallet, network, blockDiscovery, peerStore, pendingBlocks)
+    engine = BlockExcEngine.new(blockStore, wallet, network, blockDiscovery, peerStore, pendingBlocks)
-    store = NetworkStore.new(engine, repoStore)
+    store = NetworkStore.new(engine, blockStore)
     erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider)
     codexNode = CodexNodeRef.new(switch, store, engine, erasure, discovery)
     restServer = RestServerRef.new(
@@ -225,5 +200,4 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey):
     config: config,
     codexNode: codexNode,
     restServer: restServer,
-    repoStore: repoStore,
+    blockStoreManager: blockStoreManager)
-    maintenance: maintenance)

codex/conf.nim

@@ -195,7 +195,7 @@ type
         name: "block-mn" }: int
 
       cacheSize* {.
-        desc: "The size in MiB of the block cache, 0 disables the cache - might help on slow hardrives"
+        desc: "Size in MiB of the in-RAM block cache, 0 disables the cache - might help on slow hardrives"
         defaultValue: 0
         defaultValueDesc: "0"
         name: "cache-size"

codex/consts.nim

@@ -0,0 +1,2 @@
+const
+  MiB* = 1024 * 1024

codex/stores/blockstoremanager.nim

@@ -0,0 +1,79 @@
+## Nim-Codex
+## Copyright (c) 2023 Status Research & Development GmbH
+## Licensed under either of
+##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+## at your option.
+## This file may not be copied, modified, or distributed except according to
+## those terms.
+
+import pkg/upraises
+
+push: {.upraises: [].}
+
+import pkg/chronos
+import pkg/chronicles
+import pkg/datastore
+
+import ../node
+import ../conf
+import ../rng
+import ../rest/api
+import ../blockexchange
+import ../utils/fileutils
+import ../erasure
+import ../discovery
+import ../contracts
+import ../utils/addrutils
+import ../namespaces
+
+import ../conf
+import ./blockstore
+import ./repostore
+import ./maintenance
+
+type
+  BlockStoreManager* = ref object of RootObj
+    repoStore: RepoStore
+    maintenance: BlockMaintainer
+    blockStore: BlockStore
+
+proc start*(self: BlockStoreManager): Future[void] {.async.} =
+  await self.repoStore.start()
+  self.maintenance.start()
+
+proc stop*(self: BlockStoreManager): Future[void] {.async.} =
+  await self.repoStore.stop()
+  await self.maintenance.stop()
+
+proc getBlockStore*(self: BlockStoreManager): BlockStore =
+  self.blockStore
+
+proc createRepoStore(config: CodexConf): RepoStore =
+  RepoStore.new(
+    repoDs = Datastore(FSDatastore.new($config.dataDir, depth = 5).expect("Should create repo data store!")),
+    metaDs = SQLiteDatastore.new(config.dataDir / CodexMetaNamespace).expect("Should create meta data store!"),
+    quotaMaxBytes = config.storageQuota.uint,
+    blockTtl = config.blockTtlSeconds.seconds)
+
+proc createMaintenance(repoStore: RepoStore, config: CodexConf): BlockMaintainer =
+  BlockMaintainer.new(
+    repoStore,
+    interval = config.blockMaintenanceIntervalSeconds.seconds,
+    numberOfBlocksPerInterval = config.blockMaintenanceNumberOfBlocks)
+
+proc getBlockStore(repoStore: RepoStore, config: CodexConf): BlockStore =
+  if config.cacheSize > 0:
+    return CacheStore.new(backingStore = repoStore, cacheSize = config.cacheSize * MiB)
+  return repoStore
+
+func new*(T: type BlockStoreManager, config: CodexConf): T =
+  let
+    repoStore = createRepoStore(config)
+    maintenance = createMaintenance(repoStore, config)
+    blockStore = getBlockStore(repoStore, config)
+
+  T(
+    repoStore: repoStore,
+    maintenance: maintenance,
+    blockStore: blockStore)

codex/stores/cachestore.nim

@@ -21,8 +21,8 @@ import pkg/questionable
 import pkg/questionable/results
 
 import ./blockstore
+import ../consts
 import ../chunker
-import ../errors
 import ../manifest
 
 export blockstore
@@ -36,7 +36,6 @@ type
     cache: LruCache[Cid, Block]
 
 const
-  MiB* = 1024 * 1024
   DefaultCacheSizeMiB* = 5
   DefaultCacheSize* = DefaultCacheSizeMiB * MiB
 

codex/stores/memorystore.nim

@@ -22,8 +22,8 @@ import pkg/questionable
 import pkg/questionable/results
 
 import ./blockstore
+import ../consts
 import ../chunker
-import ../errors
 import ../manifest
 
 export blockstore
@@ -43,7 +43,6 @@ type
     list: DoublyLinkedList[MemoryStoreNode]
 
 const
-  MiB* = 1024 * 1024
   DefaultMemoryStoreCapacityMiB* = 5
   DefaultMemoryStoreCapacity* = DefaultMemoryStoreCapacityMiB * MiB