logos-storage/nim-datastore
1
0
Fork 0
mirror of https://github.com/logos-storage/nim-datastore.git synced 2026-08-02 04:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

introduce method query in the base API and refactor accordingly

Browse Source
This commit is contained in:
Michael Bradley, Jr 2022-09-12 00:34:07 -05:00
parent 2546740ed0
commit 1065c95ce7
No known key found for this signature in database
GPG Key ID: D0307DBCF21A9A58
10 changed files with 159 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
datastore/datastore.nim
View File

@ -13,6 +13,10 @@ push: {.upraises: [].}
type
Datastore* = ref object of RootObj
QueryIterator* = iterator (
datastore: Datastore,
query: Query): Future[QueryResponse] {.closure, gcsafe.}
method close*(self: Datastore): Future[void] {.base, locks: "unknown".} =
raiseAssert("Not implemented!")
@ -41,8 +45,5 @@ method put*(
raiseAssert("Not implemented!")
iterator query*(
self: Datastore,
query: Query): Future[QueryResponse] =
method query*(self: Datastore): QueryIterator {.base, locks: "unknown".} =
raiseAssert("Not implemented!")

6
datastore/filesystem_datastore.nim
View File

@ -157,9 +157,3 @@ method put*(
except OSError as e:
return failure e
# method query*(
# self: FileSystemDatastore,
# query: ...): Future[?!(?...)] {.async, locks: "unknown".} =
#
# return success ....some

9
datastore/null_datastore.nim
View File

@ -43,8 +43,11 @@ method put*(
return success()
iterator query*(
self: NullDatastore,
query: Query): Future[QueryResponse] =
iterator queryImpl(
datastore: Datastore,
query: Query): Future[QueryResponse] {.closure.} =
discard
method query*(self: NullDatastore): QueryIterator {.locks: "unknown".} =
queryImpl

12
datastore/sqlite_datastore.nim
View File

@ -367,13 +367,14 @@ method put*(
return await self.put(key, data, timestamp())
iterator query*(
self: SQLiteDatastore,
query: Query): Future[QueryResponse] =
iterator queryImpl(
datastore: Datastore,
query: Query): Future[QueryResponse] {.closure.} =
let
datastore = SQLiteDatastore(datastore)
queryStmt = QueryStmt.prepare(
self.env, queryStmtStr).expect("should not fail")
datastore.env, queryStmtStr).expect("should not fail")
s = RawStmtPtr(queryStmt)
@ -428,3 +429,6 @@ iterator query*(
break
else:
raise (ref Defect)(msg: $sqlite3_errstr(v))
method query*(self: SQLiteDatastore): QueryIterator {.locks: "unknown".} =
queryImpl

31
datastore/tiered_datastore.nim
View File

@ -53,7 +53,10 @@ method delete*(
pending = await allFinished(self.stores.mapIt(it.delete(key)))
for fut in pending:
if fut.read().isErr: return fut.read()
let
delRes = await fut
if delRes.isErr: return delRes
return success()
@ -94,12 +97,26 @@ method put*(
pending = await allFinished(self.stores.mapIt(it.put(key, data)))
for fut in pending:
if fut.read().isErr: return fut.read()
let
putRes = await fut
if putRes.isErr: return putRes
return success()
# method query*(
# self: TieredDatastore,
# query: ...): Future[?!(?...)] {.async, locks: "unknown".} =
#
# return success ....some
iterator queryImpl(
datastore: Datastore,
query: Query): Future[QueryResponse] {.closure.} =
let
datastore = TieredDatastore(datastore)
# https://github.com/datastore/datastore/blob/7ccf0cd4748001d3dbf5e6dda369b0f63e0269d3/datastore/core/basic.py#L1027-L1035
bottom = datastore.stores[^1]
try:
let q = bottom.query(); for kv in q(bottom, query): yield kv
except Exception as e:
raise (ref Defect)(msg: e.msg)
method query*(self: TieredDatastore): QueryIterator {.locks: "unknown".} =
queryImpl

3
tests/datastore/test_datastore.nim
View File

@ -25,5 +25,4 @@ suite "Datastore (base)":
expect Defect: discard ds.get(key)
asyncTest "query":
expect Defect:
for n in ds.query(Query.init(key)): discard
expect Defect: discard ds.query

4
tests/datastore/test_filesystem_datastore.nim
View File

@ -190,7 +190,3 @@ suite "FileSystemDatastore":
getOpt = getRes.get
check: getOpt.isNone
# asyncTest "query":
# check:
# true

2
tests/datastore/test_null_datastore.nim
View File

@ -35,7 +35,7 @@ suite "NullDatastore":
var
x = true
for n in ds.query(Query.init(key)):
let q = ds.query; for n in q(ds, Query.init(key)):
# `iterator query` for NullDatastore never yields so the following lines
# are not run (else the test would hang)
x = false

102
tests/datastore/test_sqlite_datastore.nim
View File

@ -387,19 +387,19 @@ suite "SQLiteDatastore":
assert putRes.isOk
var
kds: seq[QueryResponse]
kvs: seq[QueryResponse]
for kd in ds.query(Query.init(queryKey)):
var q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(rkey, data) = await kv
kds.add (key, data)
kvs.add (rkey, data)
# see https://sqlite.org/lang_select.html#the_order_by_clause
# If a SELECT statement that returns more than one row does not have an
# ORDER BY clause, the order in which the rows are returned is undefined.
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key1, data: bytes1),
(key: key2, data: bytes2),
(key: key3, data: bytes3),
@ -414,17 +414,17 @@ suite "SQLiteDatastore":
(key: key12, data: bytes12)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("a*").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(rkey, data) = await kv
kds.add (key, data)
kvs.add (rkey, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key1, data: bytes1),
(key: key2, data: bytes2),
(key: key3, data: bytes3),
@ -433,17 +433,17 @@ suite "SQLiteDatastore":
(key: key6, data: bytes6)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("A*").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(rkey, data) = await kv
kds.add (key, data)
kvs.add (rkey, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key7, data: bytes7),
(key: key8, data: bytes8),
(key: key9, data: bytes9),
@ -452,67 +452,67 @@ suite "SQLiteDatastore":
(key: key12, data: bytes12)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("a/?").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(rkey, data) = await kv
kds.add (key, data)
kvs.add (rkey, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key2, data: bytes2)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("A/?").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(rkey, data) = await kv
kds.add (key, data)
kvs.add (rkey, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key8, data: bytes8)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("*/?").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(rkey, data) = await kv
kds.add (key, data)
kvs.add (rkey, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key2, data: bytes2),
(key: key5, data: bytes5),
(key: key8, data: bytes8),
(key: key11, data: bytes11)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("[Aa]/?").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(key, data) = await kv
kds.add (key, data)
kvs.add (key, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key2, data: bytes2),
(key: key8, data: bytes8)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
# SQLite's GLOB operator, akin to Unix file globbing syntax, is greedy re:
# wildcard "*". So a pattern such as "a:*[^/]" will not restrict results to
@ -520,33 +520,33 @@ suite "SQLiteDatastore":
queryKey = Key.init("a:*[^/]").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(key, data) = await kv
kds.add (key, data)
kvs.add (key, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key4, data: bytes4),
(key: key5, data: bytes5),
(key: key6, data: bytes6)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
queryKey = Key.init("a:*[Bb]").get
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(key, data) = await kv
kds.add (key, data)
kvs.add (key, data)
check: kds.sortedByIt(it.key.id) == @[
check: kvs.sortedByIt(it.key.id) == @[
(key: key4, data: bytes4)
].sortedByIt(it.key.id)
kds = @[]
kvs = @[]
var
deleteRes = await ds.delete(key1)
@ -576,12 +576,12 @@ suite "SQLiteDatastore":
assert deleteRes.isOk
let
emptyKds: seq[QueryResponse] = @[]
emptyKvs: seq[QueryResponse] = @[]
for kd in ds.query(Query.init(queryKey)):
q = ds.query; for kv in q(ds, Query.init(queryKey)):
let
(key, data) = await kd
(key, data) = await kv
kds.add (key, data)
kvs.add (key, data)
check: kds == emptyKds
check: kvs == emptyKvs

74
tests/datastore/test_tiered_datastore.nim
View File

@ -1,3 +1,4 @@
import std/algorithm
import std/options
import std/os
@ -19,21 +20,21 @@ suite "TieredDatastore":
rootAbs = getCurrentDir() / root
var
ds1: SQLiteDatastore
ds2: FileSystemDatastore
ds1: FileSystemDatastore
ds2: SQLiteDatastore
setup:
removeDir(rootAbs)
require(not dirExists(rootAbs))
createDir(rootAbs)
ds1 = SQLiteDatastore.new(memory).get
ds2 = FileSystemDatastore.new(rootAbs).get
ds1 = FileSystemDatastore.new(rootAbs).get
ds2 = SQLiteDatastore.new(memory).get
teardown:
if not ds1.isNil: await ds1.close
ds1 = nil
removeDir(rootAbs)
require(not dirExists(rootAbs))
if not ds2.isNil: await ds2.close
ds2 = nil
asyncTest "new":
check:
@ -132,8 +133,10 @@ suite "TieredDatastore":
(await ds1.get(key)).get.get == bytes
(await ds2.get(key)).get.get == bytes
await ds1.close
ds1 = SQLiteDatastore.new(memory).get
removeDir(rootAbs)
assert (not dirExists(rootAbs))
createDir(rootAbs)
ds1 = FileSystemDatastore.new(rootAbs).get
ds = TieredDatastore.new(ds1, ds2).get
assert (await ds1.get(key)).get.isNone
@ -149,6 +152,55 @@ suite "TieredDatastore":
(await ds1.get(key)).get.isSome
(await ds1.get(key)).get.get == bytes
# asyncTest "query":
# check:
# true
asyncTest "query":
let
ds = TieredDatastore.new(ds1, ds2).get
key1 = Key.init("a/b").get
key2 = Key.init("a/b:c").get
key3 = Key.init("a/b:c/d").get
bytes1 = @[1.byte, 2.byte, 3.byte]
bytes2 = @[4.byte, 5.byte, 6.byte]
bytes3: seq[byte] = @[]
queryKey1 = Key.init("a/*").get
queryKey2 = Key.init("b/*").get
var
putRes = await ds.put(key1, bytes1)
assert putRes.isOk
putRes = await ds.put(key2, bytes2)
assert putRes.isOk
putRes = await ds.put(key3, bytes3)
assert putRes.isOk
var
kvs: seq[QueryResponse]
var q = ds.query(); for kv in q(ds, Query.init(queryKey1)):
let
(key, data) = await kv
kvs.add (key, data)
check: kvs.sortedByIt(it.key.id) == @[
(key: key1, data: bytes1),
(key: key2, data: bytes2),
(key: key3, data: bytes3)
].sortedByIt(it.key.id)
kvs = @[]
q = ds.query()
let
emptyKvs: seq[QueryResponse] = @[]
for kv in q(ds, Query.init(queryKey2)):
let
(key, data) = await kv
kvs.add (key, data)
check: kvs == emptyKvs