reorg

datastore/fsds.nim

@@ -13,7 +13,7 @@ import ./threads/fsbackend
 import ./threads/threadproxy
 import ./datastore
 
-export datastore, Taskpool
+export datastore, threadproxy, fsbackend, Taskpool
 
 push: {.upraises: [].}
 
@@ -21,6 +21,9 @@ type
   FSDatastore* = ref object of Datastore
     db: ThreadProxy[FSBackend[KeyId, DataBuffer]]
 
+proc validDepth*(self: FSDatastore, key: Key): bool =
+  key.len <= self.db.backend.depth
+
 method has*(self: FSDatastore,
             key: Key): Future[?!bool] {.async.} =
   await self.db.has(key)

datastore/threads/fsbackend.nim

@@ -8,9 +8,8 @@ from pkg/stew/results as stewResults import get, isErr
 import pkg/upraises
 
 import ./backend
-import ./datastore
 
-export datastore, backend
+export backend
 
 push: {.upraises: [].}
 
@@ -18,7 +17,7 @@ type
   FSBackend*[K, V] = object
     root*: DataBuffer
     ignoreProtected: bool
-    depth: int
+    depth*: int
 
 proc isRootSubdir*(root, path: string): bool =
   path.startsWith(root)

tests/datastore/testfsds.nim

@@ -3,44 +3,15 @@ import std/sequtils
 import std/os
 from std/algorithm import sort, reversed
 
-import pkg/unittest2
+import pkg/asynctest
 import pkg/chronos
 import pkg/stew/results
 import pkg/stew/byteutils
 
 import pkg/datastore/fsds
-import pkg/datastore/key
-import pkg/datastore/threads/backend
 
-import ./backendCommonTests
+import ./dscommontests
-
+import ./querycommontests
-
-suite "Test Basic FSDatastore":
-  let
-    path = currentSourcePath() # get this file's name
-    basePath = "tests_data"
-    basePathAbs = path.parentDir / basePath
-    keyFull = Key.init("/a/b").tryGet()
-    key = KeyId.new keyFull.id()
-    bytes = DataBuffer.new "some bytes"
-    otherBytes = DataBuffer.new "some other bytes".toBytes
-
-  var batch: seq[tuple[key: KeyId, data: DataBuffer]]
-  for k in 0..<100:
-    let kk = Key.init($keyFull, $k).tryGet().id()
-    batch.add( (KeyId.new kk, DataBuffer.new @[k.byte]) )
-
-  removeDir(basePathAbs)
-  require(not dirExists(basePathAbs))
-  createDir(basePathAbs)
-
-  var
-    fsStore = newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet()
-
-  testBasicBackend(fsStore, key, bytes, otherBytes, batch)
-
-  removeDir(basePathAbs)
-  require(not dirExists(basePathAbs))
 
 suite "Test Basic FSDatastore":
   let
@@ -51,22 +22,21 @@ suite "Test Basic FSDatastore":
     bytes = "some bytes".toBytes
     otherBytes = "some other bytes".toBytes
 
-  var batch: seq[tuple[key: Key, data: seq[byte]]]
-  for k in 0..<100:
-    let kk = Key.init($key, $k).tryGet()
-    batch.add( (kk, @[k.byte]) )
-
-  removeDir(basePathAbs)
-  require(not dirExists(basePathAbs))
-  createDir(basePathAbs)
-
   var
-    fsStore = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet()
+    fsStore: FSDatastore
 
-  testBasicBackend(fsStore, key, bytes, otherBytes, batch)
+  setupAll:
+    removeDir(basePathAbs)
+    require(not dirExists(basePathAbs))
+    createDir(basePathAbs)
 
-  removeDir(basePathAbs)
+    fsStore = FSDatastore.new(root = basePathAbs, depth = 3).tryGet()
-  require(not dirExists(basePathAbs))
+
+  teardownAll:
+    removeDir(basePathAbs)
+    require(not dirExists(basePathAbs))
+
+  basicStoreTests(fsStore, key, bytes, otherBytes)
 
 suite "Test Misc FSDatastore":
   let
@@ -86,7 +56,7 @@ suite "Test Misc FSDatastore":
 
   test "Test validDepth()":
     let
-      fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet()
+      fs = FSDatastore.new(root = "/", depth = 3).tryGet()
       invalid = Key.init("/a/b/c/d").tryGet()
       valid = Key.init("/a/b/c").tryGet()
 
@@ -96,40 +66,40 @@ suite "Test Misc FSDatastore":
 
   test "Test invalid key (path) depth":
     let
-      fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet()
+      fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet()
       key = Key.init("/a/b/c/d").tryGet()
 
     check:
-      (fs.put(key, bytes)).isErr
+      (await fs.put(key, bytes)).isErr
-      (fs.get(key)).isErr
+      (await fs.get(key)).isErr
-      (fs.delete(key)).isErr
+      (await fs.delete(key)).isErr
-      (fs.has(key)).isErr
+      (await fs.has(key)).isErr
 
   test "Test valid key (path) depth":
     let
-      fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet()
+      fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet()
       key = Key.init("/a/b/c").tryGet()
 
     check:
-      (fs.put(key, bytes)).isOk
+      (await fs.put(key, bytes)).isOk
-      (fs.get(key)).isOk
+      (await fs.get(key)).isOk
-      (fs.delete(key)).isOk
+      (await fs.delete(key)).isOk
-      (fs.has(key)).isOk
+      (await fs.has(key)).isOk
 
   test "Test key cannot write outside of root":
     let
-      fs = newFSDatastore[Key, seq[byte]](root = basePathAbs, depth = 3).tryGet()
+      fs = FSDatastore.new(root = basePathAbs, depth = 3).tryGet()
       key = Key.init("/a/../../c").tryGet()
 
     check:
-      (fs.put(key, bytes)).isErr
+      (await fs.put(key, bytes)).isErr
-      (fs.get(key)).isErr
+      (await fs.get(key)).isErr
-      (fs.delete(key)).isErr
+      (await fs.delete(key)).isErr
-      (fs.has(key)).isErr
+      (await fs.has(key)).isErr
 
   test "Test key cannot convert to invalid path":
     let
-      fs = newFSDatastore[Key, seq[byte]](root = basePathAbs).tryGet()
+      fs = FSDatastore.new(root = basePathAbs).tryGet()
 
     for c in invalidFilenameChars:
       if c == ':': continue
@@ -139,57 +109,30 @@ suite "Test Misc FSDatastore":
         key = Key.init("/" & c).tryGet()
 
       check:
-        (fs.put(key, bytes)).isErr
+        (await fs.put(key, bytes)).isErr
-        (fs.get(key)).isErr
+        (await fs.get(key)).isErr
-        (fs.delete(key)).isErr
+        (await fs.delete(key)).isErr
-        (fs.has(key)).isErr
+        (await fs.has(key)).isErr
-
-
-# suite "Test Query":
-#   let
-#     path = currentSourcePath() # get this file's name
-#     basePath = "tests_data"
-#     basePathAbs = path.parentDir / basePath
-
-#   var
-#     ds: FSDatastore
-
-#   setup:
-#     removeDir(basePathAbs)
-#     require(not dirExists(basePathAbs))
-#     createDir(basePathAbs)
-
-#     ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet()
-
-#   teardown:
-
-#     removeDir(basePathAbs)
-#     require(not dirExists(basePathAbs))
-
-#   queryTests(ds, false)
-
-suite "queryTests":
 
+suite "Test Query":
   let
     path = currentSourcePath() # get this file's name
     basePath = "tests_data"
     basePathAbs = path.parentDir / basePath
 
-  removeDir(basePathAbs)
+  var
-  require(not dirExists(basePathAbs))
+    ds: FSDatastore
-  createDir(basePathAbs)
 
-  let
+  setup:
-    fsNew = proc(): FSDatastore[KeyId, DataBuffer] =
+    removeDir(basePathAbs)
-      newFSDatastore[KeyId, DataBuffer](root = basePathAbs, depth = 3).tryGet()
+    require(not dirExists(basePathAbs))
-    key1 = KeyId.new "/a"
+    createDir(basePathAbs)
-    key2 = KeyId.new "/a/b"
-    key3 = KeyId.new "/a/b/c"
-    val1 = DataBuffer.new "value for 1"
-    val2 = DataBuffer.new "value for 2"
-    val3 = DataBuffer.new "value for 3"
 
-  queryTests(fsNew, key1, key2, key3, val1, val2, val3, extended=false)
+    ds = FSDatastore.new(root = basePathAbs, depth = 5).tryGet()
 
-  removeDir(basePathAbs)
+  teardown:
-  require(not dirExists(basePathAbs))
+
+    removeDir(basePathAbs)
+    require(not dirExists(basePathAbs))
+
+  queryTests(ds, false)