From de20fa4c1e2a22ed7ebc778a5c3cabf5d6dfd33e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 28 Sep 2023 18:32:47 -0700 Subject: [PATCH] reorg --- datastore/fsds.nim | 5 +- datastore/threads/fsbackend.nim | 5 +- tests/datastore/testfsds.nim | 157 ++++++++++---------------------- 3 files changed, 56 insertions(+), 111 deletions(-) diff --git a/datastore/fsds.nim b/datastore/fsds.nim index 40e7c03..7e1ba33 100644 --- a/datastore/fsds.nim +++ b/datastore/fsds.nim @@ -13,7 +13,7 @@ import ./threads/fsbackend import ./threads/threadproxy import ./datastore -export datastore, Taskpool +export datastore, threadproxy, fsbackend, Taskpool push: {.upraises: [].} @@ -21,6 +21,9 @@ type FSDatastore* = ref object of Datastore db: ThreadProxy[FSBackend[KeyId, DataBuffer]] +proc validDepth*(self: FSDatastore, key: Key): bool = + key.len <= self.db.backend.depth + method has*(self: FSDatastore, key: Key): Future[?!bool] {.async.} = await self.db.has(key) diff --git a/datastore/threads/fsbackend.nim b/datastore/threads/fsbackend.nim index 1ad607b..b94c9d5 100644 --- a/datastore/threads/fsbackend.nim +++ b/datastore/threads/fsbackend.nim @@ -8,9 +8,8 @@ from pkg/stew/results as stewResults import get, isErr import pkg/upraises import ./backend -import ./datastore -export datastore, backend +export backend push: {.upraises: [].} @@ -18,7 +17,7 @@ type FSBackend*[K, V] = object root*: DataBuffer ignoreProtected: bool - depth: int + depth*: int proc isRootSubdir*(root, path: string): bool = path.startsWith(root) diff --git a/tests/datastore/testfsds.nim b/tests/datastore/testfsds.nim index f55348d..8f12ea8 100644 --- a/tests/datastore/testfsds.nim +++ b/tests/datastore/testfsds.nim @@ -3,44 +3,15 @@ import std/sequtils import std/os from std/algorithm import sort, reversed -import pkg/unittest2 +import pkg/asynctest import pkg/chronos import pkg/stew/results import pkg/stew/byteutils import pkg/datastore/fsds -import pkg/datastore/key -import pkg/datastore/threads/backend -import ./backendCommonTests - - -suite "Test Basic FSDatastore": - let - path = currentSourcePath() # get this file's name - basePath = "tests_data" - basePathAbs = path.parentDir / basePath - keyFull = Key.init("/a/b").tryGet() - key = KeyId.new keyFull.id() - bytes = DataBuffer.new "some bytes" - otherBytes = DataBuffer.new "some other bytes".toBytes - - var batch: seq[tuple[key: KeyId, data: DataBuffer]] - for k in 0..<100: - let kk = Key.init($keyFull, $k).tryGet().id() - batch.add( (KeyId.new kk, DataBuffer.new @[k.byte]) ) - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - - var - fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() - - testBasicBackend(fsStore, key, bytes, otherBytes, batch) - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) +import ./dscommontests +import ./querycommontests suite "Test Basic FSDatastore": let @@ -51,22 +22,21 @@ suite "Test Basic FSDatastore": bytes = "some bytes".toBytes otherBytes = "some other bytes".toBytes - var batch: seq[tuple[key: Key, data: seq[byte]]] - for k in 0..<100: - let kk = Key.init($key, $k).tryGet() - batch.add( (kk, @[k.byte]) ) - - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) - var - fsStore = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fsStore: FSDatastore - testBasicBackend(fsStore, key, bytes, otherBytes, batch) + setupAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() + + teardownAll: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + basicStoreTests(fsStore, key, bytes, otherBytes) suite "Test Misc FSDatastore": let @@ -86,7 +56,7 @@ suite "Test Misc FSDatastore": test "Test validDepth()": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = "/", depth = 3).tryGet() invalid = Key.init("/a/b/c/d").tryGet() valid = Key.init("/a/b/c").tryGet() @@ -96,40 +66,40 @@ suite "Test Misc FSDatastore": test "Test invalid key (path) depth": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() key = Key.init("/a/b/c/d").tryGet() check: - (fs.put(key, bytes)).isErr - (fs.get(key)).isErr - (fs.delete(key)).isErr - (fs.has(key)).isErr + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr test "Test valid key (path) depth": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() key = Key.init("/a/b/c").tryGet() check: - (fs.put(key, bytes)).isOk - (fs.get(key)).isOk - (fs.delete(key)).isOk - (fs.has(key)).isOk + (await fs.put(key, bytes)).isOk + (await fs.get(key)).isOk + (await fs.delete(key)).isOk + (await fs.has(key)).isOk test "Test key cannot write outside of root": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet() + fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet() key = Key.init("/a/../../c").tryGet() check: - (fs.put(key, bytes)).isErr - (fs.get(key)).isErr - (fs.delete(key)).isErr - (fs.has(key)).isErr + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr test "Test key cannot convert to invalid path": let - fs = newFSDatastore[Key, seq[byte]](root = basePathAbs).tryGet() + fs = FSDatastore.new(root = basePathAbs).tryGet() for c in invalidFilenameChars: if c == ':': continue @@ -139,57 +109,30 @@ suite "Test Misc FSDatastore": key = Key.init("/" & c).tryGet() check: - (fs.put(key, bytes)).isErr - (fs.get(key)).isErr - (fs.delete(key)).isErr - (fs.has(key)).isErr - - -# suite "Test Query": -# let -# path = currentSourcePath() # get this file's name -# basePath = "tests_data" -# basePathAbs = path.parentDir / basePath - -# var -# ds: FSDatastore - -# setup: -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) -# createDir(basePathAbs) - -# ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - -# teardown: - -# removeDir(basePathAbs) -# require(not dirExists(basePathAbs)) - -# queryTests(ds, false) - -suite "queryTests": + (await fs.put(key, bytes)).isErr + (await fs.get(key)).isErr + (await fs.delete(key)).isErr + (await fs.has(key)).isErr +suite "Test Query": let path = currentSourcePath() # get this file's name basePath = "tests_data" basePathAbs = path.parentDir / basePath - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) - createDir(basePathAbs) + var + ds: FSDatastore - let - fsNew = proc(): FSDatastore[KeyId, DataBuffer] = - newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet() - key1 = KeyId.new "/a" - key2 = KeyId.new "/a/b" - key3 = KeyId.new "/a/b/c" - val1 = DataBuffer.new "value for 1" - val2 = DataBuffer.new "value for 2" - val3 = DataBuffer.new "value for 3" + setup: + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + createDir(basePathAbs) - queryTests(fsNew, key1, key2, key3, val1, val2, val3, extended=false) + ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet() - removeDir(basePathAbs) - require(not dirExists(basePathAbs)) + teardown: + + removeDir(basePathAbs) + require(not dirExists(basePathAbs)) + + queryTests(ds, false) \ No newline at end of file