add basic query implementation

datastore/datastore.nim

@@ -4,8 +4,9 @@ import pkg/questionable/results
 import pkg/upraises
 
 import ./key
+import ./query
 
-export key
+export key, query
 
 push: {.upraises: [].}
 
@@ -37,8 +38,8 @@ method put*(
 
   raiseAssert("Not implemented!")
 
-# method query*(
-#   self: Datastore,
-#   query: ...): Future[?!(?...)] {.async, base, locks: "unknown".} =
-#
-#   raiseAssert("Not implemented!")
+iterator query*(
+  self: Datastore,
+  query: Query): Future[QueryResponse] =
+
+  raiseAssert("Not implemented!")

datastore/key.nim

@@ -211,7 +211,7 @@ template `[]`*(
 proc len*(self: Key): int =
   self.namespaces.len
 
-iterator items*(key: Key): Namespace {.inline.} =
+iterator items*(key: Key): Namespace =
   var
     i = 0
 

datastore/null_datastore.nim

@@ -40,8 +40,8 @@ method put*(
 
   return success()
 
-# method query*(
-#   self: NullDatastore,
-#   query: ...): Future[?!(?...)] {.async, locks: "unknown".} =
-#
-#   return success ....none
+iterator query*(
+  self: NullDatastore,
+  query: Query): Future[QueryResponse] =
+
+  discard

datastore/query.nim

@@ -0,0 +1,18 @@
+import ./key
+
+type
+  Query* = object
+    key: QueryKey
+
+  QueryKey* = Key
+
+  QueryResponse* = tuple[key: Key, data: seq[byte]]
+
+proc init*(
+  T: type Query,
+  key: QueryKey): T =
+
+  T(key: key)
+
+proc key*(self: Query): QueryKey =
+  self.key

datastore/sqlite.nim

@@ -30,6 +30,13 @@ type
 
   SQLiteStmt*[Params, Res] = distinct RawStmtPtr
 
+  # see https://github.com/arnetheduck/nim-sqlite3-abi/issues/4
+  sqlite3_destructor_type_gcsafe =
+    proc (a1: pointer) {.cdecl, gcsafe, upraises: [].}
+
+const
+  SQLITE_TRANSIENT_GCSAFE* = cast[sqlite3_destructor_type_gcsafe](-1)
+
 proc bindParam(
   s: RawStmtPtr,
   n: int,
@@ -248,8 +255,8 @@ proc sqlite3_column_text_not_null*(
     # https://www.sqlite.org/c3ref/column_blob.html
     # a null pointer here implies an out-of-memory error
     let
-      code = sqlite3_errcode(sqlite3_db_handle(s))
+      v = sqlite3_errcode(sqlite3_db_handle(s))
 
-    raise (ref Defect)(msg: $sqlite3_errstr(code))
+    raise (ref Defect)(msg: $sqlite3_errstr(v))
 
   text

datastore/sqlite_datastore.nim

@@ -6,7 +6,7 @@ import pkg/questionable
 import pkg/questionable/results
 import pkg/sqlite3_abi
 import pkg/stew/byteutils
-from pkg/stew/results as stewResults import get, isErr
+from pkg/stew/results as stewResults import isErr
 import pkg/upraises
 
 import ./datastore
@@ -34,6 +34,8 @@ type
 
   PutStmt = SQLiteStmt[(string, seq[byte], int64), void]
 
+  QueryStmt = SQLiteStmt[(string), void]
+
   SQLiteDatastore* = ref object of Datastore
     dbPath: string
     containsStmt: ContainsStmt
@@ -97,6 +99,14 @@ const
     ) VALUES (?, ?, ?);
   """
 
+  queryStmtStr = """
+    SELECT """ & idColName & """, """ & dataColName & """ FROM """ & tableName &
+    """ WHERE """ & idColName & """ GLOB ?;
+  """
+
+  queryStmtIdCol = 0
+  queryStmtDataCol = 1
+
 proc checkColMetadata(s: RawStmtPtr, i: int, expectedName: string) =
   let
     colName = sqlite3_column_origin_name(s, i.cint)
@@ -140,10 +150,10 @@ proc dataCol*(
     # result code is an error code
     if blob.isNil:
       let
-        code = sqlite3_errcode(sqlite3_db_handle(s))
+        v = sqlite3_errcode(sqlite3_db_handle(s))
 
-      if not (code in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]):
-        raise (ref Defect)(msg: $sqlite3_errstr(code))
+      if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]):
+        raise (ref Defect)(msg: $sqlite3_errstr(v))
 
     let
       dataLen = sqlite3_column_bytes(s, i)
@@ -339,8 +349,64 @@ method put*(
 
   return await self.put(key, data, timestamp())
 
-# method query*(
-#   self: SQLiteDatastore,
-#   query: ...): Future[?!(?...)] {.async, locks: "unknown".} =
-#
-#   return success ....some
+iterator query*(
+  self: SQLiteDatastore,
+  query: Query): Future[QueryResponse] =
+
+  let
+    queryStmt = QueryStmt.prepare(
+      self.env, queryStmtStr).expect("should not fail")
+
+    s = RawStmtPtr(queryStmt)
+
+  defer:
+    discard sqlite3_reset(s)
+    discard sqlite3_clear_bindings(s)
+    s.dispose
+
+  let
+    v = sqlite3_bind_text(s, 1.cint, query.key.id.cstring, -1.cint,
+      SQLITE_TRANSIENT_GCSAFE)
+
+  if not (v == SQLITE_OK):
+    raise (ref Defect)(msg: $sqlite3_errstr(v))
+
+  while true:
+    let
+      v = sqlite3_step(s)
+
+    case v
+    of SQLITE_ROW:
+      let
+        key = Key.init($sqlite3_column_text_not_null(
+          s, queryStmtIdCol)).expect("should not fail")
+
+        blob = sqlite3_column_blob(s, queryStmtDataCol)
+
+      # detect out-of-memory error
+      # see the conversion table and final paragraph of:
+      # https://www.sqlite.org/c3ref/column_blob.html
+      # see also https://www.sqlite.org/rescode.html
+
+      # the "data" column can be NULL so in order to detect an out-of-memory
+      # error it is necessary to check that the result is a null pointer and
+      # that the result code is an error code
+      if blob.isNil:
+        let
+          v = sqlite3_errcode(sqlite3_db_handle(s))
+
+        if not (v in [SQLITE_OK, SQLITE_ROW, SQLITE_DONE]):
+          raise (ref Defect)(msg: $sqlite3_errstr(v))
+
+      let
+        dataLen = sqlite3_column_bytes(s, queryStmtDataCol)
+        dataBytes = cast[ptr UncheckedArray[byte]](blob)
+        data = @(toOpenArray(dataBytes, 0, dataLen - 1))
+        fut = newFuture[QueryResponse]()
+
+      fut.complete((key, data))
+      yield fut
+    of SQLITE_DONE:
+      break
+    else:
+      raise (ref Defect)(msg: $sqlite3_errstr(v))

tests/datastore/test_datastore.nim

@@ -11,10 +11,9 @@ suite "Datastore (base)":
   let
     key = Key.init("a").get
     ds = Datastore()
-    oneByte = @[1.byte]
 
   asyncTest "put":
-    expect Defect: discard ds.put(key, oneByte)
+    expect Defect: discard ds.put(key, @[1.byte])
 
   asyncTest "delete":
     expect Defect: discard ds.delete(key)
@@ -25,5 +24,6 @@ suite "Datastore (base)":
   asyncTest "get":
     expect Defect: discard ds.get(key)
 
-  # asyncTest "query":
-  #   expect Defect: discard ds.query(...)
+  asyncTest "query":
+    expect Defect:
+      for n in ds.query(Query.init(key)): discard

tests/datastore/test_null_datastore.nim

@@ -31,7 +31,14 @@ suite "NullDatastore":
       (await ds.get(key)).isOk
       (await ds.get(key)).get.isNone
 
-  # asyncTest "query":
-  #   check:
-  #     (await ds.query(...)).isOk
-  #     (await ds.query(...)).get.isNone
+  asyncTest "query":
+    var
+      x = true
+
+    for n in ds.query(Query.init(key)):
+      # `iterator query` for NullDatastore never yields so the following lines
+      # are not run (else the test would hang)
+      x = false
+      discard (await n)
+
+    check: x

tests/datastore/test_sqlite_datastore.nim

@@ -1,3 +1,4 @@
+import std/algorithm
 import std/options
 import std/os
 
@@ -331,6 +332,261 @@ suite "SQLiteDatastore":
 
     check: getOpt.isNone
 
-  # asyncTest "query":
-  #   check:
-  #     true
+  asyncTest "query":
+    ds = SQLiteDatastore.new(basePathAbs, filename).get
+
+    var
+      key1 = Key.init("a").get
+      key2 = Key.init("a/b").get
+      key3 = Key.init("a/b:c").get
+      key4 = Key.init("a:b").get
+      key5 = Key.init("a:b/c").get
+      key6 = Key.init("a:b/c:d").get
+      key7 = Key.init("A").get
+      key8 = Key.init("A/B").get
+      key9 = Key.init("A/B:C").get
+      key10 = Key.init("A:B").get
+      key11 = Key.init("A:B/C").get
+      key12 = Key.init("A:B/C:D").get
+
+      bytes1  = @[1.byte, 2.byte, 3.byte]
+      bytes2  = @[4.byte, 5.byte, 6.byte]
+      bytes3: seq[byte] = @[]
+      bytes4  = bytes1
+      bytes5  = bytes2
+      bytes6  = bytes3
+      bytes7  = bytes1
+      bytes8  = bytes2
+      bytes9  = bytes3
+      bytes10  = bytes1
+      bytes11  = bytes2
+      bytes12  = bytes3
+
+      queryKey = Key.init("*").get
+
+    var
+      putRes = await ds.put(key1, bytes1)
+
+    assert putRes.isOk
+    putRes = await ds.put(key2, bytes2)
+    assert putRes.isOk
+    putRes = await ds.put(key3, bytes3)
+    assert putRes.isOk
+    putRes = await ds.put(key4, bytes4)
+    assert putRes.isOk
+    putRes = await ds.put(key5, bytes5)
+    assert putRes.isOk
+    putRes = await ds.put(key6, bytes6)
+    assert putRes.isOk
+    putRes = await ds.put(key7, bytes7)
+    assert putRes.isOk
+    putRes = await ds.put(key8, bytes8)
+    assert putRes.isOk
+    putRes = await ds.put(key9, bytes9)
+    assert putRes.isOk
+    putRes = await ds.put(key10, bytes10)
+    assert putRes.isOk
+    putRes = await ds.put(key11, bytes11)
+    assert putRes.isOk
+    putRes = await ds.put(key12, bytes12)
+    assert putRes.isOk
+
+    var
+      kds: seq[QueryResponse]
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    # see https://sqlite.org/lang_select.html#the_order_by_clause
+    # If a SELECT statement that returns more than one row does not have an
+    # ORDER BY clause, the order in which the rows are returned is undefined.
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key1, data: bytes1),
+      (key: key2, data: bytes2),
+      (key: key3, data: bytes3),
+      (key: key4, data: bytes4),
+      (key: key5, data: bytes5),
+      (key: key6, data: bytes6),
+      (key: key7, data: bytes7),
+      (key: key8, data: bytes8),
+      (key: key9, data: bytes9),
+      (key: key10, data: bytes10),
+      (key: key11, data: bytes11),
+      (key: key12, data: bytes12)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("a*").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key1, data: bytes1),
+      (key: key2, data: bytes2),
+      (key: key3, data: bytes3),
+      (key: key4, data: bytes4),
+      (key: key5, data: bytes5),
+      (key: key6, data: bytes6)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("A*").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key7, data: bytes7),
+      (key: key8, data: bytes8),
+      (key: key9, data: bytes9),
+      (key: key10, data: bytes10),
+      (key: key11, data: bytes11),
+      (key: key12, data: bytes12)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("a/?").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key2, data: bytes2)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("A/?").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key8, data: bytes8)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("*/?").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key2, data: bytes2),
+      (key: key5, data: bytes5),
+      (key: key8, data: bytes8),
+      (key: key11, data: bytes11)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("[Aa]/?").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key2, data: bytes2),
+      (key: key8, data: bytes8)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    # SQLite's GLOB operator, akin to Unix file globbing syntax, is greedy re:
+    # wildcard "*". So a pattern such as "a:*[^/]" will not restrict results to
+    # "/a:b", i.e. it will match on "/a:b", "/a:b/c" and "/a:b/c:d".
+
+    queryKey = Key.init("a:*[^/]").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key4, data: bytes4),
+      (key: key5, data: bytes5),
+      (key: key6, data: bytes6)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    queryKey = Key.init("a:*[Bb]").get
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds.sortedByIt(it.key.id) == @[
+      (key: key4, data: bytes4)
+    ].sortedByIt(it.key.id)
+
+    kds = @[]
+
+    var
+      deleteRes = await ds.delete(key1)
+
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key2)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key3)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key4)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key5)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key6)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key7)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key8)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key9)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key10)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key11)
+    assert deleteRes.isOk
+    deleteRes = await ds.delete(key12)
+    assert deleteRes.isOk
+
+    let
+      emptyKds: seq[QueryResponse] = @[]
+
+    for kd in ds.query(Query.init(queryKey)):
+      let
+        (key, data) = await kd
+
+      kds.add (key, data)
+
+    check: kds == emptyKds