From 1065c95ce77c517af9f82364e9cf47a364c9a19f Mon Sep 17 00:00:00 2001 From: "Michael Bradley, Jr" Date: Mon, 12 Sep 2022 00:34:07 -0500 Subject: [PATCH] introduce `method query` in the base API and refactor accordingly --- datastore/datastore.nim | 9 +- datastore/filesystem_datastore.nim | 6 -- datastore/null_datastore.nim | 9 +- datastore/sqlite_datastore.nim | 12 ++- datastore/tiered_datastore.nim | 31 ++++-- tests/datastore/test_datastore.nim | 3 +- tests/datastore/test_filesystem_datastore.nim | 4 - tests/datastore/test_null_datastore.nim | 2 +- tests/datastore/test_sqlite_datastore.nim | 102 +++++++++--------- tests/datastore/test_tiered_datastore.nim | 74 +++++++++++-- 10 files changed, 159 insertions(+), 93 deletions(-) diff --git a/datastore/datastore.nim b/datastore/datastore.nim index 2965469..a15a305 100644 --- a/datastore/datastore.nim +++ b/datastore/datastore.nim @@ -13,6 +13,10 @@ push: {.upraises: [].} type Datastore* = ref object of RootObj + QueryIterator* = iterator ( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure, gcsafe.} + method close*(self: Datastore): Future[void] {.base, locks: "unknown".} = raiseAssert("Not implemented!") @@ -41,8 +45,5 @@ method put*( raiseAssert("Not implemented!") -iterator query*( - self: Datastore, - query: Query): Future[QueryResponse] = - +method query*(self: Datastore): QueryIterator {.base, locks: "unknown".} = raiseAssert("Not implemented!") diff --git a/datastore/filesystem_datastore.nim b/datastore/filesystem_datastore.nim index 0fc39d6..27bc2dc 100644 --- a/datastore/filesystem_datastore.nim +++ b/datastore/filesystem_datastore.nim @@ -157,9 +157,3 @@ method put*( except OSError as e: return failure e - -# method query*( -# self: FileSystemDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some diff --git a/datastore/null_datastore.nim b/datastore/null_datastore.nim index 6df4c23..b5e6ad5 100644 --- a/datastore/null_datastore.nim +++ b/datastore/null_datastore.nim @@ -43,8 +43,11 @@ method put*( return success() -iterator query*( - self: NullDatastore, - query: Query): Future[QueryResponse] = +iterator queryImpl( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure.} = discard + +method query*(self: NullDatastore): QueryIterator {.locks: "unknown".} = + queryImpl diff --git a/datastore/sqlite_datastore.nim b/datastore/sqlite_datastore.nim index e8c2e4d..3e5dafc 100644 --- a/datastore/sqlite_datastore.nim +++ b/datastore/sqlite_datastore.nim @@ -367,13 +367,14 @@ method put*( return await self.put(key, data, timestamp()) -iterator query*( - self: SQLiteDatastore, - query: Query): Future[QueryResponse] = +iterator queryImpl( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure.} = let + datastore = SQLiteDatastore(datastore) queryStmt = QueryStmt.prepare( - self.env, queryStmtStr).expect("should not fail") + datastore.env, queryStmtStr).expect("should not fail") s = RawStmtPtr(queryStmt) @@ -428,3 +429,6 @@ iterator query*( break else: raise (ref Defect)(msg: $sqlite3_errstr(v)) + +method query*(self: SQLiteDatastore): QueryIterator {.locks: "unknown".} = + queryImpl diff --git a/datastore/tiered_datastore.nim b/datastore/tiered_datastore.nim index 0d3ffdd..cca814f 100644 --- a/datastore/tiered_datastore.nim +++ b/datastore/tiered_datastore.nim @@ -53,7 +53,10 @@ method delete*( pending = await allFinished(self.stores.mapIt(it.delete(key))) for fut in pending: - if fut.read().isErr: return fut.read() + let + delRes = await fut + + if delRes.isErr: return delRes return success() @@ -94,12 +97,26 @@ method put*( pending = await allFinished(self.stores.mapIt(it.put(key, data))) for fut in pending: - if fut.read().isErr: return fut.read() + let + putRes = await fut + + if putRes.isErr: return putRes return success() -# method query*( -# self: TieredDatastore, -# query: ...): Future[?!(?...)] {.async, locks: "unknown".} = -# -# return success ....some +iterator queryImpl( + datastore: Datastore, + query: Query): Future[QueryResponse] {.closure.} = + + let + datastore = TieredDatastore(datastore) + # https://github.com/datastore/datastore/blob/7ccf0cd4748001d3dbf5e6dda369b0f63e0269d3/datastore/core/basic.py#L1027-L1035 + bottom = datastore.stores[^1] + + try: + let q = bottom.query(); for kv in q(bottom, query): yield kv + except Exception as e: + raise (ref Defect)(msg: e.msg) + +method query*(self: TieredDatastore): QueryIterator {.locks: "unknown".} = + queryImpl diff --git a/tests/datastore/test_datastore.nim b/tests/datastore/test_datastore.nim index ccf4d75..ffda20d 100644 --- a/tests/datastore/test_datastore.nim +++ b/tests/datastore/test_datastore.nim @@ -25,5 +25,4 @@ suite "Datastore (base)": expect Defect: discard ds.get(key) asyncTest "query": - expect Defect: - for n in ds.query(Query.init(key)): discard + expect Defect: discard ds.query diff --git a/tests/datastore/test_filesystem_datastore.nim b/tests/datastore/test_filesystem_datastore.nim index 3c93244..ab90ae4 100644 --- a/tests/datastore/test_filesystem_datastore.nim +++ b/tests/datastore/test_filesystem_datastore.nim @@ -190,7 +190,3 @@ suite "FileSystemDatastore": getOpt = getRes.get check: getOpt.isNone - - # asyncTest "query": - # check: - # true diff --git a/tests/datastore/test_null_datastore.nim b/tests/datastore/test_null_datastore.nim index c37d4f3..af4551a 100644 --- a/tests/datastore/test_null_datastore.nim +++ b/tests/datastore/test_null_datastore.nim @@ -35,7 +35,7 @@ suite "NullDatastore": var x = true - for n in ds.query(Query.init(key)): + let q = ds.query; for n in q(ds, Query.init(key)): # `iterator query` for NullDatastore never yields so the following lines # are not run (else the test would hang) x = false diff --git a/tests/datastore/test_sqlite_datastore.nim b/tests/datastore/test_sqlite_datastore.nim index 633153f..272ee9d 100644 --- a/tests/datastore/test_sqlite_datastore.nim +++ b/tests/datastore/test_sqlite_datastore.nim @@ -387,19 +387,19 @@ suite "SQLiteDatastore": assert putRes.isOk var - kds: seq[QueryResponse] + kvs: seq[QueryResponse] - for kd in ds.query(Query.init(queryKey)): + var q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) # see https://sqlite.org/lang_select.html#the_order_by_clause # If a SELECT statement that returns more than one row does not have an # ORDER BY clause, the order in which the rows are returned is undefined. - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key1, data: bytes1), (key: key2, data: bytes2), (key: key3, data: bytes3), @@ -414,17 +414,17 @@ suite "SQLiteDatastore": (key: key12, data: bytes12) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("a*").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key1, data: bytes1), (key: key2, data: bytes2), (key: key3, data: bytes3), @@ -433,17 +433,17 @@ suite "SQLiteDatastore": (key: key6, data: bytes6) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("A*").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key7, data: bytes7), (key: key8, data: bytes8), (key: key9, data: bytes9), @@ -452,67 +452,67 @@ suite "SQLiteDatastore": (key: key12, data: bytes12) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("a/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key2, data: bytes2) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("A/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key8, data: bytes8) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("*/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (rkey, data) = await kv - kds.add (key, data) + kvs.add (rkey, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key2, data: bytes2), (key: key5, data: bytes5), (key: key8, data: bytes8), (key: key11, data: bytes11) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("[Aa]/?").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key2, data: bytes2), (key: key8, data: bytes8) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] # SQLite's GLOB operator, akin to Unix file globbing syntax, is greedy re: # wildcard "*". So a pattern such as "a:*[^/]" will not restrict results to @@ -520,33 +520,33 @@ suite "SQLiteDatastore": queryKey = Key.init("a:*[^/]").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key4, data: bytes4), (key: key5, data: bytes5), (key: key6, data: bytes6) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] queryKey = Key.init("a:*[Bb]").get - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds.sortedByIt(it.key.id) == @[ + check: kvs.sortedByIt(it.key.id) == @[ (key: key4, data: bytes4) ].sortedByIt(it.key.id) - kds = @[] + kvs = @[] var deleteRes = await ds.delete(key1) @@ -576,12 +576,12 @@ suite "SQLiteDatastore": assert deleteRes.isOk let - emptyKds: seq[QueryResponse] = @[] + emptyKvs: seq[QueryResponse] = @[] - for kd in ds.query(Query.init(queryKey)): + q = ds.query; for kv in q(ds, Query.init(queryKey)): let - (key, data) = await kd + (key, data) = await kv - kds.add (key, data) + kvs.add (key, data) - check: kds == emptyKds + check: kvs == emptyKvs diff --git a/tests/datastore/test_tiered_datastore.nim b/tests/datastore/test_tiered_datastore.nim index 49ba928..91b50fe 100644 --- a/tests/datastore/test_tiered_datastore.nim +++ b/tests/datastore/test_tiered_datastore.nim @@ -1,3 +1,4 @@ +import std/algorithm import std/options import std/os @@ -19,21 +20,21 @@ suite "TieredDatastore": rootAbs = getCurrentDir() / root var - ds1: SQLiteDatastore - ds2: FileSystemDatastore + ds1: FileSystemDatastore + ds2: SQLiteDatastore setup: removeDir(rootAbs) require(not dirExists(rootAbs)) createDir(rootAbs) - ds1 = SQLiteDatastore.new(memory).get - ds2 = FileSystemDatastore.new(rootAbs).get + ds1 = FileSystemDatastore.new(rootAbs).get + ds2 = SQLiteDatastore.new(memory).get teardown: - if not ds1.isNil: await ds1.close - ds1 = nil removeDir(rootAbs) require(not dirExists(rootAbs)) + if not ds2.isNil: await ds2.close + ds2 = nil asyncTest "new": check: @@ -132,8 +133,10 @@ suite "TieredDatastore": (await ds1.get(key)).get.get == bytes (await ds2.get(key)).get.get == bytes - await ds1.close - ds1 = SQLiteDatastore.new(memory).get + removeDir(rootAbs) + assert (not dirExists(rootAbs)) + createDir(rootAbs) + ds1 = FileSystemDatastore.new(rootAbs).get ds = TieredDatastore.new(ds1, ds2).get assert (await ds1.get(key)).get.isNone @@ -149,6 +152,55 @@ suite "TieredDatastore": (await ds1.get(key)).get.isSome (await ds1.get(key)).get.get == bytes - # asyncTest "query": - # check: - # true + asyncTest "query": + let + ds = TieredDatastore.new(ds1, ds2).get + + key1 = Key.init("a/b").get + key2 = Key.init("a/b:c").get + key3 = Key.init("a/b:c/d").get + + bytes1 = @[1.byte, 2.byte, 3.byte] + bytes2 = @[4.byte, 5.byte, 6.byte] + bytes3: seq[byte] = @[] + + queryKey1 = Key.init("a/*").get + queryKey2 = Key.init("b/*").get + + var + putRes = await ds.put(key1, bytes1) + + assert putRes.isOk + putRes = await ds.put(key2, bytes2) + assert putRes.isOk + putRes = await ds.put(key3, bytes3) + assert putRes.isOk + + var + kvs: seq[QueryResponse] + + var q = ds.query(); for kv in q(ds, Query.init(queryKey1)): + let + (key, data) = await kv + + kvs.add (key, data) + + check: kvs.sortedByIt(it.key.id) == @[ + (key: key1, data: bytes1), + (key: key2, data: bytes2), + (key: key3, data: bytes3) + ].sortedByIt(it.key.id) + + kvs = @[] + q = ds.query() + + let + emptyKvs: seq[QueryResponse] = @[] + + for kv in q(ds, Query.init(queryKey2)): + let + (key, data) = await kv + + kvs.add (key, data) + + check: kvs == emptyKvs