From 6a4e460e58002448286830c1b544b1e3270fcd3d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:41:27 -0700 Subject: [PATCH 1/3] queryLocks can be done at FsDs backend level --- datastore/threads/threadproxyds.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index 772dc72..dcc1680 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -376,8 +376,5 @@ proc new*[DB](self: type ThreadDatastore, success ThreadDatastore( tp: tp, backend: backend, - # TODO: are these needed anymore?? - # withLocks: withLocks, - # queryLock: newAsyncLock(), semaphore: AsyncSemaphore.new(tp.numThreads - 1) ) From b4b534bf67d8ac1928492a70fece524759148ef2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:54:22 -0700 Subject: [PATCH 2/3] loop tests --- datastore/threads/threadproxyds.nim | 2 + tests/datastore/testthreadproxyds.nim | 112 ++++++++++---------------- 2 files changed, 44 insertions(+), 70 deletions(-) diff --git a/datastore/threads/threadproxyds.nim b/datastore/threads/threadproxyds.nim index dcc1680..432eb6f 100644 --- a/datastore/threads/threadproxyds.nim +++ b/datastore/threads/threadproxyds.nim @@ -348,6 +348,7 @@ method query*(self: ThreadDatastore, trace "Cancelling thread future!", exc = exc.msg ctx.setCancelled() discard ctx[].signal.close() + echo "nextSignal:CLOSE!" discard nextSignal.close() self.semaphore.release() raise exc @@ -357,6 +358,7 @@ method query*(self: ThreadDatastore, except CancelledError as exc: trace "Cancelling thread future!", exc = exc.msg discard signal.close() + echo "nextSignal:CLOSE!" discard nextSignal.close() self.semaphore.release() raise exc diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 14a41b6..27c8e82 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -22,88 +22,60 @@ import pkg/datastore/threads/threadproxyds {.all.} import ./dscommontests import ./querycommontests -const NumThreads = 20 # IO threads aren't attached to CPU count +const + NumThreads = 20 # IO threads aren't attached to CPU count + N = 100 -suite "Test Basic ThreadProxyDatastore": - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a").tryGet() - data = "some bytes".toBytes +for i in 1..N: + suite "Test Basic ThreadDatastore with SQLite": - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes - teardownAll: - echo "teardown done" + setupAll: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - test "check put": - echo "\n\n=== put ===" - let res1 = await ds.put(key, data) - echo "res1: ", res1.repr - check res1.isOk + teardown: + GC_fullCollect() - test "check get": - echo "\n\n=== get ===" - echo "get send key: ", key.repr - let res2 = await ds.get(key) - echo "get key post: ", key.repr - echo "get res2: ", res2.repr - echo res2.get() == data - var val = "" - for c in res2.get(): - val &= char(c) - echo "get res2: ", $val + teardownAll: + (await ds.close()).tryGet() + taskPool.shutdown() -suite "Test Basic ThreadDatastore with SQLite": + basicStoreTests(ds, key, bytes, otherBytes) + GC_fullCollect() - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes +for i in 1..N: + suite "Test Query ThreadDatastore with SQLite": - setupAll: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() + var + sqlStore: SQLiteBackend[KeyId,DataBuffer] + ds: ThreadDatastore + taskPool: Taskpool + key = Key.init("/a/b").tryGet() + bytes = "some bytes".toBytes + otherBytes = "some other bytes".toBytes - teardown: - GC_fullCollect() + setup: + sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() + taskPool = Taskpool.new(NumThreads) + ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - teardownAll: - (await ds.close()).tryGet() - taskPool.shutdown() + teardown: + GC_fullCollect() - basicStoreTests(ds, key, bytes, otherBytes) + (await ds.close()).tryGet() + taskPool.shutdown() -suite "Test Query ThreadDatastore with SQLite": - - var - sqlStore: SQLiteBackend[KeyId,DataBuffer] - ds: ThreadDatastore - taskPool: Taskpool - key = Key.init("/a/b").tryGet() - bytes = "some bytes".toBytes - otherBytes = "some other bytes".toBytes - - setup: - sqlStore = newSQLiteBackend[KeyId, DataBuffer](Memory).tryGet() - taskPool = Taskpool.new(NumThreads) - ds = ThreadDatastore.new(sqlStore, tp = taskPool).tryGet() - - teardown: - GC_fullCollect() - - (await ds.close()).tryGet() - taskPool.shutdown() - - queryTests(ds, true) + queryTests(ds, true) + GC_fullCollect() # suite "Test Basic ThreadDatastore with fsds": # let From 030084186ef68472f4e0c7f60d962f95aee9ad93 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 27 Sep 2023 12:55:48 -0700 Subject: [PATCH 3/3] loop tests --- tests/datastore/testthreadproxyds.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/datastore/testthreadproxyds.nim b/tests/datastore/testthreadproxyds.nim index 27c8e82..b42cbb0 100644 --- a/tests/datastore/testthreadproxyds.nim +++ b/tests/datastore/testthreadproxyds.nim @@ -24,7 +24,8 @@ import ./querycommontests const NumThreads = 20 # IO threads aren't attached to CPU count - N = 100 + ThreadTestLoops {.intdefine.} = 10 + N = ThreadTestLoops for i in 1..N: suite "Test Basic ThreadDatastore with SQLite":