diff --git a/benchmarks/repostore_rw_bench.nim b/benchmarks/repostore_rw_bench.nim index d3f3ee87..4502e596 100644 --- a/benchmarks/repostore_rw_bench.nim +++ b/benchmarks/repostore_rw_bench.nim @@ -12,7 +12,7 @@ import pkg/codex/manifest import pkg/codex/stores/[filestore, repostore] const - NBlocks = 81_920 + NBlocks = 163_840 BlockSize = 65_536 DataDir = "" @@ -49,7 +49,7 @@ proc newRepostore(dataDir: string): RepoStore = let repoStore = RepoStore.new( blockStore, LevelDbDatastore.new(dataDir / "meta").expect("Should create metadata store!"), - quotaMaxBytes = 10_000_000_000'nb, + quotaMaxBytes = 50_000_000_000'nb, ) repoStore @@ -67,6 +67,15 @@ proc writeData( (await store.putBlock(manifest.asBlock())).tryGet() +proc readData( + dataset: Dataset, store: RepoStore +): Future[void] {.async: (raises: [CatchableError]).} = + let (blocks, tree, manifest) = dataset + + for i in 0 ..< NBlocks: + let blk = (await store.getBlock(blocks[i].cid)).tryGet() + assert blk.cid == blocks[i].cid + proc writeData( dataset: Dataset, store: FileStore ): Future[void] {.async: (raises: [CatchableError]).} = @@ -77,6 +86,17 @@ proc writeData( for i in 0 ..< NBlocks: (await file.putBlock(i, blocks[i])).tryGet() +proc readData( + dataset: Dataset, store: FileStore +): Future[void] {.async: (raises: [CatchableError]).} = + let (blocks, tree, manifest) = dataset + + let file = store.create(manifest).tryGet() + + for i in 0 ..< NBlocks: + let blk = (await file.getBlock(i)).tryGet() + assert blk.cid == blocks[i].cid + proc runRepostoreBench( dataset: Dataset ): Future[void] {.async: (raises: [CatchableError]).} = @@ -88,7 +108,10 @@ proc runRepostoreBench( defer: removeDir(dir) - benchmark "filestore write data": + benchmark "repostore write data": + await writeData(dataset, store) + + benchmark "repostore read data": await writeData(dataset, store) proc runFilestoreBench( @@ -102,9 +125,12 @@ proc runFilestoreBench( defer: removeDir(dir) - benchmark "repostore write data": + benchmark "filestore write data": await writeData(dataset, store) + benchmark "filestore read data": + await readData(dataset, store) + let dataset = makeDataset(NBlocks, BlockSize).tryGet() waitFor runRepostoreBench(dataset) waitFor runFilestoreBench(dataset) diff --git a/codex/stores/filestore.nim b/codex/stores/filestore.nim index d4f41ab7..bac7e2fc 100644 --- a/codex/stores/filestore.nim +++ b/codex/stores/filestore.nim @@ -1,3 +1,4 @@ +import std/os import std/posix import std/strformat @@ -26,7 +27,7 @@ type File* = object filepath: string fd*: cint -proc allocStorage*(filepath: string, size: int): ?!cint = +proc allocFile*(filepath: string, size: int): ?!cint = let fd = open(filepath, O_CREAT or O_RDWR or O_TRUNC, 0o644) if fd < 0: return failure(&"open failed with error {fd}") @@ -37,15 +38,19 @@ proc allocStorage*(filepath: string, size: int): ?!cint = success(fd) -proc open*(self: var File): ?!void = - self.fd = open(self.filepath, O_RDWR) - if self.fd < 0: - return failure(&"open failed with error {self.fd}") +proc openFile*(filepath: string): ?!cint = + let fd = open(filepath, O_RDWR) + if fd < 0: + return failure(&"open failed with error {fd}") - success() + success(fd) proc initDataset*(filepath: string, manifest: Manifest): ?!File = - let fd = ?allocStorage(filepath, manifest.datasetSize.int) + let fd = + ?( + if fileExists(filepath): openFile(filepath) + else: allocFile(filepath, manifest.datasetSize.int) + ) success(File(manifest: manifest, filepath: filepath, fd: fd)) proc ensureOpen*(self: File): ?!void =