From 6ddcc5d4f863fc3ed5c612ddf05aa83099e20139 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 19:51:43 -0700 Subject: [PATCH 01/43] initial setup --- tests/exCursor.nim | 1 + tests/exFailure.nim | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/exCursor.nim create mode 100644 tests/exFailure.nim diff --git a/tests/exCursor.nim b/tests/exCursor.nim new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/exCursor.nim @@ -0,0 +1 @@ + diff --git a/tests/exFailure.nim b/tests/exFailure.nim new file mode 100644 index 0000000..8cea4ac --- /dev/null +++ b/tests/exFailure.nim @@ -0,0 +1,31 @@ +import std/os +import std/sequtils + +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +import apatheia/queues + +proc worker(data: seq[char], queue: SignalQueue[int]) = + os.sleep(50) + echo "worker: ", data + discard queue.send(data.len()) + +suite "async tests": + + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + var queue = newSignalQueue[int]() + + asyncTest "test": + + ## init + var data = "hello world!".toSeq + tp.spawn worker(data, queue) + + let res = await wait(queue).wait(1500.milliseconds) + + check res.get() == 12 + + From a8ebb17dc82541ef87feda0c68becc7014d60810 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 19:53:04 -0700 Subject: [PATCH 02/43] initial setup --- tests/exFailure.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 8cea4ac..14559fe 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -24,8 +24,7 @@ suite "async tests": var data = "hello world!".toSeq tp.spawn worker(data, queue) - let res = await wait(queue).wait(1500.milliseconds) - + let res = await wait(queue) check res.get() == 12 From 3995c73974fa3b231f76fce5e0d7bfdf93c10cba Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 20:07:24 -0700 Subject: [PATCH 03/43] initial setup --- tests/exFailure.nim | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 14559fe..d889890 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -8,11 +8,30 @@ import taskpools import apatheia/queues +type + DataObj = ref object + data: seq[char] + proc worker(data: seq[char], queue: SignalQueue[int]) = - os.sleep(50) + os.sleep(5000) echo "worker: ", data discard queue.send(data.len()) +proc finalizer(obj: DataObj) = + echo "FINALIZE!!" + +proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = + ## init + var data = "hello world!".toSeq + var obj: DataObj + new(obj, finalizer) + + echo "spawn worker" + tp.spawn worker(data, queue) + + let res = await wait(queue) + check res.get() == 12 + suite "async tests": var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. @@ -20,11 +39,8 @@ suite "async tests": asyncTest "test": - ## init - var data = "hello world!".toSeq - tp.spawn worker(data, queue) - - let res = await wait(queue) - check res.get() == 12 + await runTest(tp, queue) + GC_fullCollect() + From b53cced09e70c0f51a4f7464d5fd16914a0115aa Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 20:26:08 -0700 Subject: [PATCH 04/43] initial setup --- tests/exFailure.nim | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index d889890..97f3c14 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -12,8 +12,8 @@ type DataObj = ref object data: seq[char] -proc worker(data: seq[char], queue: SignalQueue[int]) = - os.sleep(5000) +proc worker(data: OpenArrayHolder[char], queue: SignalQueue[int]) = + os.sleep(1_000) echo "worker: ", data discard queue.send(data.len()) @@ -22,14 +22,15 @@ proc finalizer(obj: DataObj) = proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = ## init - var data = "hello world!".toSeq var obj: DataObj new(obj, finalizer) + obj.data = "hello world!".toSeq echo "spawn worker" - tp.spawn worker(data, queue) + tp.spawn worker(obj.data, queue) - let res = await wait(queue) + let res = + await wait(queue).wait(100.milliseconds) check res.get() == 12 suite "async tests": @@ -39,8 +40,13 @@ suite "async tests": asyncTest "test": - await runTest(tp, queue) - GC_fullCollect() + try: + await runTest(tp, queue) + except AsyncTimeoutError as err: + echo "Run GC" + GC_fullCollect() + os.sleep(2_000) + echo "Done" From 1e75c8458e79a7ddf585644cada1e4be0d1dbcc4 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 21:59:19 -0700 Subject: [PATCH 05/43] updates --- src/apatheia/jobs.nim | 13 +++++++------ tests/exFailure.nim | 7 ++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/apatheia/jobs.nim b/src/apatheia/jobs.nim index 74a677e..92050e5 100644 --- a/src/apatheia/jobs.nim +++ b/src/apatheia/jobs.nim @@ -50,6 +50,11 @@ type template toOpenArray*[T](arr: OpenArrayHolder[T]): auto = system.toOpenArray(arr.data, 0, arr.size) +proc toArrayHolder*[T](data: seq[T]): OpenArrayHolder[T] = + OpenArrayHolder[T]( + data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() + ) + func jobId*[T](fut: Future[T]): JobId = JobId fut.id() @@ -93,9 +98,7 @@ template checkJobArgs*[T](exp: seq[T], fut: untyped): OpenArrayHolder[T] = when T is SupportedSeqTypes: let rval = SeqRetainer[T](data: exp) retainMemory(fut.jobId(), rval) - let expPtr = OpenArrayHolder[T]( - data: cast[ptr UncheckedArray[T]](unsafeAddr(rval.data[0])), size: rval.data.len() - ) + let expPtr = toArrayHolder(rval.data) expPtr else: {.error: "unsupported sequence type for job argument: " & $typeof(seq[T]).} @@ -103,9 +106,7 @@ template checkJobArgs*[T](exp: seq[T], fut: untyped): OpenArrayHolder[T] = template checkJobArgs*(exp: string, fut: untyped): OpenArrayHolder[char] = let rval = StrRetainer(data: exp) retainMemory(fut.jobId(), rval) - let expPtr = OpenArrayHolder[char]( - data: cast[ptr UncheckedArray[char]](unsafeAddr(rval.data[0])), size: rval.data.len() - ) + let expPtr = toArrayHolder(rval.data) expPtr template checkJobArgs*(exp: typed, fut: untyped): auto = diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 97f3c14..6d9ea66 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -7,6 +7,7 @@ import chronos/unittest2/asynctests import taskpools import apatheia/queues +import apatheia/jobs type DataObj = ref object @@ -14,8 +15,8 @@ type proc worker(data: OpenArrayHolder[char], queue: SignalQueue[int]) = os.sleep(1_000) - echo "worker: ", data - discard queue.send(data.len()) + echo "worker: ", data.toOpenArray() + discard queue.send(data.toOpenArray().len()) proc finalizer(obj: DataObj) = echo "FINALIZE!!" @@ -27,7 +28,7 @@ proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = obj.data = "hello world!".toSeq echo "spawn worker" - tp.spawn worker(obj.data, queue) + tp.spawn worker(toArrayHolder(obj.data), queue) let res = await wait(queue).wait(100.milliseconds) From ceace3e7b6dcb584b5da6a678a295b751ff806bc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:04:43 -0700 Subject: [PATCH 06/43] segfault --- tests/exFailure.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 6d9ea66..54b16b4 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -11,24 +11,26 @@ import apatheia/jobs type DataObj = ref object - data: seq[char] + holder: OpenArrayHolder[char] -proc worker(data: OpenArrayHolder[char], queue: SignalQueue[int]) = +proc worker(data: ptr OpenArrayHolder[char], queue: SignalQueue[int]) = os.sleep(1_000) - echo "worker: ", data.toOpenArray() - discard queue.send(data.toOpenArray().len()) + echo "worker: ", data[].toOpenArray() + discard queue.send(data[].toOpenArray().len()) proc finalizer(obj: DataObj) = echo "FINALIZE!!" + obj.holder.data = nil proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = ## init var obj: DataObj new(obj, finalizer) - obj.data = "hello world!".toSeq + let data = "hello world!".toSeq + obj.holder = data.toArrayHolder() echo "spawn worker" - tp.spawn worker(toArrayHolder(obj.data), queue) + tp.spawn worker(addr obj.holder, queue) let res = await wait(queue).wait(100.milliseconds) From 891e492775dcffa2234e15dafc6c954aedc420db Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:08:12 -0700 Subject: [PATCH 07/43] segfault --- tests/exFailure.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 54b16b4..9bf07a8 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -20,14 +20,18 @@ proc worker(data: ptr OpenArrayHolder[char], queue: SignalQueue[int]) = proc finalizer(obj: DataObj) = echo "FINALIZE!!" - obj.holder.data = nil + obj.holder.data.dealloc() + # obj.holder.data = nil proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = ## init var obj: DataObj new(obj, finalizer) - let data = "hello world!".toSeq - obj.holder = data.toArrayHolder() + + obj.holder.data = cast[ptr UncheckedArray[char]](alloc0(13)) + for i, c in "hello world!": + obj.holder.data[i] = c + obj.holder.size = 12 echo "spawn worker" tp.spawn worker(addr obj.holder, queue) From f7801cfad65367d0bbcd3f10609320af0c95d98e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:09:34 -0700 Subject: [PATCH 08/43] segfault --- tests/exFailure.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 9bf07a8..d81deb7 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -15,13 +15,14 @@ type proc worker(data: ptr OpenArrayHolder[char], queue: SignalQueue[int]) = os.sleep(1_000) + assert data[].data != nil echo "worker: ", data[].toOpenArray() discard queue.send(data[].toOpenArray().len()) proc finalizer(obj: DataObj) = echo "FINALIZE!!" obj.holder.data.dealloc() - # obj.holder.data = nil + obj.holder.data = nil proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = ## init @@ -49,7 +50,7 @@ suite "async tests": try: await runTest(tp, queue) - except AsyncTimeoutError as err: + except AsyncTimeoutError: echo "Run GC" GC_fullCollect() os.sleep(2_000) From 776e7e6905bc3a16480fb77f6737340c04d33fad Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:11:50 -0700 Subject: [PATCH 09/43] segfault --- tests/exFailure.nim | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index d81deb7..6e88acc 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -7,12 +7,22 @@ import chronos/unittest2/asynctests import taskpools import apatheia/queues -import apatheia/jobs type + OpenArrayHolder*[T] = object + data*: ptr UncheckedArray[T] + size*: int DataObj = ref object holder: OpenArrayHolder[char] +template toOpenArray*[T](arr: OpenArrayHolder[T]): auto = + system.toOpenArray(arr.data, 0, arr.size) + +proc toArrayHolder*[T](data: seq[T]): OpenArrayHolder[T] = + OpenArrayHolder[T]( + data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() + ) + proc worker(data: ptr OpenArrayHolder[char], queue: SignalQueue[int]) = os.sleep(1_000) assert data[].data != nil From 0a5a1f1f5ee32fc138cf88831e93cfce27f4515f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:12:55 -0700 Subject: [PATCH 10/43] segfault --- tests/exFailure.nim | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 6e88acc..77cb426 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -9,21 +9,22 @@ import taskpools import apatheia/queues type - OpenArrayHolder*[T] = object + Seq*[T] = object data*: ptr UncheckedArray[T] size*: int - DataObj = ref object - holder: OpenArrayHolder[char] -template toOpenArray*[T](arr: OpenArrayHolder[T]): auto = + DataObj = ref object + holder: Seq[char] + +template toOpenArray*[T](arr: Seq[T]): auto = system.toOpenArray(arr.data, 0, arr.size) -proc toArrayHolder*[T](data: seq[T]): OpenArrayHolder[T] = - OpenArrayHolder[T]( +proc toArrayHolder*[T](data: seq[T]): Seq[T] = + Seq[T]( data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() ) -proc worker(data: ptr OpenArrayHolder[char], queue: SignalQueue[int]) = +proc worker(data: ptr Seq[char], queue: SignalQueue[int]) = os.sleep(1_000) assert data[].data != nil echo "worker: ", data[].toOpenArray() From 04d1a1e6cc82b7fb699a9a1f0a788946ed2dc4e5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:13:13 -0700 Subject: [PATCH 11/43] segfault --- tests/exFailure.nim | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 77cb426..92267a0 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -19,11 +19,6 @@ type template toOpenArray*[T](arr: Seq[T]): auto = system.toOpenArray(arr.data, 0, arr.size) -proc toArrayHolder*[T](data: seq[T]): Seq[T] = - Seq[T]( - data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() - ) - proc worker(data: ptr Seq[char], queue: SignalQueue[int]) = os.sleep(1_000) assert data[].data != nil From 13e1f1d63b5e5299fd665c6fc93bb221348e7790 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:13:45 -0700 Subject: [PATCH 12/43] segfault --- tests/exFailure.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 92267a0..e361d86 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -21,6 +21,7 @@ template toOpenArray*[T](arr: Seq[T]): auto = proc worker(data: ptr Seq[char], queue: SignalQueue[int]) = os.sleep(1_000) + echo "running worker: " assert data[].data != nil echo "worker: ", data[].toOpenArray() discard queue.send(data[].toOpenArray().len()) From 417b3f7afc9a81c6e786305628ad5ebda82c881e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:15:54 -0700 Subject: [PATCH 13/43] segfault --- tests/exFailure.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index e361d86..f6a4d2b 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -33,7 +33,7 @@ proc finalizer(obj: DataObj) = proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = ## init - var obj: DataObj + var obj: DataObj new(obj, finalizer) obj.holder.data = cast[ptr UncheckedArray[char]](alloc0(13)) From f554457c67cb71ceca21eedb52c5e444eed1d5b1 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:19:00 -0700 Subject: [PATCH 14/43] segfault --- tests/exFailure.nim | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index f6a4d2b..c594a9d 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -14,7 +14,7 @@ type size*: int DataObj = ref object - holder: Seq[char] + mockSeq: Seq[char] template toOpenArray*[T](arr: Seq[T]): auto = system.toOpenArray(arr.data, 0, arr.size) @@ -28,21 +28,24 @@ proc worker(data: ptr Seq[char], queue: SignalQueue[int]) = proc finalizer(obj: DataObj) = echo "FINALIZE!!" - obj.holder.data.dealloc() - obj.holder.data = nil + obj.mockSeq.data.dealloc() + obj.mockSeq.data = nil + +proc initMockSeq(msg: string): Seq[char] = + result.data = cast[ptr UncheckedArray[char]](alloc0(13)) + for i, c in msg: + result.data[i] = c + result.size = 12 proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = ## init var obj: DataObj new(obj, finalizer) - obj.holder.data = cast[ptr UncheckedArray[char]](alloc0(13)) - for i, c in "hello world!": - obj.holder.data[i] = c - obj.holder.size = 12 + obj.mockSeq = initMockSeq("hello world!") echo "spawn worker" - tp.spawn worker(addr obj.holder, queue) + tp.spawn worker(addr obj.mockSeq, queue) let res = await wait(queue).wait(100.milliseconds) From e8a7972ed42644c3f3089cd69665ea16e3f5e6f2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:24:16 -0700 Subject: [PATCH 15/43] segfault --- tests/exFailure.nim | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index c594a9d..a4eee75 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -19,12 +19,12 @@ type template toOpenArray*[T](arr: Seq[T]): auto = system.toOpenArray(arr.data, 0, arr.size) -proc worker(data: ptr Seq[char], queue: SignalQueue[int]) = +proc worker(data: ptr Seq[char], sig: ThreadSignalPtr) = os.sleep(1_000) echo "running worker: " assert data[].data != nil echo "worker: ", data[].toOpenArray() - discard queue.send(data[].toOpenArray().len()) + discard sig.fireSync() proc finalizer(obj: DataObj) = echo "FINALIZE!!" @@ -37,29 +37,26 @@ proc initMockSeq(msg: string): Seq[char] = result.data[i] = c result.size = 12 -proc runTest(tp: TaskPool, queue: SignalQueue[int]) {.async.} = +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init var obj: DataObj - new(obj, finalizer) - + obj.new(finalizer) obj.mockSeq = initMockSeq("hello world!") echo "spawn worker" - tp.spawn worker(addr obj.mockSeq, queue) + tp.spawn worker(addr obj.mockSeq, sig) - let res = - await wait(queue).wait(100.milliseconds) - check res.get() == 12 + await wait(sig).wait(100.milliseconds) suite "async tests": var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. - var queue = newSignalQueue[int]() + let sig = ThreadSignalPtr.new().get() asyncTest "test": try: - await runTest(tp, queue) + await runTest(tp, sig) except AsyncTimeoutError: echo "Run GC" GC_fullCollect() From ddfb8a6f699d8e12b0c875a7966044dba5e16b2f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:26:40 -0700 Subject: [PATCH 16/43] segfault --- tests/exFailure.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index a4eee75..365191c 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -27,7 +27,7 @@ proc worker(data: ptr Seq[char], sig: ThreadSignalPtr) = discard sig.fireSync() proc finalizer(obj: DataObj) = - echo "FINALIZE!!" + echo "finalize DataObj and freeing mockSeq" obj.mockSeq.data.dealloc() obj.mockSeq.data = nil @@ -46,7 +46,9 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = echo "spawn worker" tp.spawn worker(addr obj.mockSeq, sig) + ## adding fut.wait(100.milliseconds) creates memory issue await wait(sig).wait(100.milliseconds) + # await wait(sig) suite "async tests": From 760f13a7d0a27d67689045c192bb6f3e0d919c8e Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:30:19 -0700 Subject: [PATCH 17/43] format --- tests/exFailure.nim | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 365191c..e6e4209 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -51,12 +51,10 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = # await wait(sig) suite "async tests": - var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. let sig = ThreadSignalPtr.new().get() asyncTest "test": - try: await runTest(tp, sig) except AsyncTimeoutError: @@ -64,6 +62,3 @@ suite "async tests": GC_fullCollect() os.sleep(2_000) echo "Done" - - - From 074b573668a3019808fc7a6ba18b5bbeb54c474f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:31:02 -0700 Subject: [PATCH 18/43] format --- tests/exFailure.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index e6e4209..948975f 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -6,8 +6,6 @@ import chronos/threadsync import chronos/unittest2/asynctests import taskpools -import apatheia/queues - type Seq*[T] = object data*: ptr UncheckedArray[T] From 9ff041dd6e0bd5861dc4c3b068f18e01bfd34ac3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:31:10 -0700 Subject: [PATCH 19/43] format --- tests/exFailure.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 948975f..47bdf9d 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -1,6 +1,4 @@ import std/os -import std/sequtils - import chronos import chronos/threadsync import chronos/unittest2/asynctests From 61bd7aa5675b6b2f9d49c6a1e86a94032d1e65f2 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:33:01 -0700 Subject: [PATCH 20/43] format --- tests/exFailure.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/exFailure.nim b/tests/exFailure.nim index 47bdf9d..4f45c7a 100644 --- a/tests/exFailure.nim +++ b/tests/exFailure.nim @@ -44,6 +44,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## adding fut.wait(100.milliseconds) creates memory issue await wait(sig).wait(100.milliseconds) + ## just doing the wait is fine: # await wait(sig) suite "async tests": From 5a3d098557de4306626ef4da503bd4fd471cce99 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:39:22 -0700 Subject: [PATCH 21/43] more examples --- tests/exFailureNoGcCollect.nim | 63 ++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/exFailureNoGcCollect.nim diff --git a/tests/exFailureNoGcCollect.nim b/tests/exFailureNoGcCollect.nim new file mode 100644 index 0000000..9622ec2 --- /dev/null +++ b/tests/exFailureNoGcCollect.nim @@ -0,0 +1,63 @@ +import std/os +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +type + Seq*[T] = object + data*: ptr UncheckedArray[T] + size*: int + + DataObj = ref object + mockSeq: Seq[char] + +template toOpenArray*[T](arr: Seq[T]): auto = + system.toOpenArray(arr.data, 0, arr.size) + +proc worker(data: ptr Seq[char], sig: ThreadSignalPtr) = + os.sleep(2_000) + echo "running worker: " + assert data[].data != nil + echo "worker: ", data[].toOpenArray() + discard sig.fireSync() + +proc finalizer(obj: DataObj) = + echo "finalize DataObj and freeing mockSeq" + obj.mockSeq.data.dealloc() + obj.mockSeq.data = nil + +proc initMockSeq(msg: string): Seq[char] = + result.data = cast[ptr UncheckedArray[char]](alloc0(13)) + for i, c in msg: + result.data[i] = c + result.size = 12 + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj: DataObj + obj.new(finalizer) + obj.mockSeq = initMockSeq("hello world!") + + echo "spawn worker" + tp.spawn worker(addr obj.mockSeq, sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + await wait(sig).wait(10.milliseconds) + ## just doing the wait is fine: + # await wait(sig) + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + for i in 1..2_000: + try: + await runTest(tp, sig) + os.sleep(200) + except AsyncTimeoutError: + echo "Run GC" + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig) From 555b134643a00d6888c4cacf39000fc1d57409e5 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:41:43 -0700 Subject: [PATCH 22/43] more examples --- tests/exFailureNoGcCollect.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exFailureNoGcCollect.nim b/tests/exFailureNoGcCollect.nim index 9622ec2..b8a0354 100644 --- a/tests/exFailureNoGcCollect.nim +++ b/tests/exFailureNoGcCollect.nim @@ -16,7 +16,7 @@ template toOpenArray*[T](arr: Seq[T]): auto = system.toOpenArray(arr.data, 0, arr.size) proc worker(data: ptr Seq[char], sig: ThreadSignalPtr) = - os.sleep(2_000) + os.sleep(4_000) echo "running worker: " assert data[].data != nil echo "worker: ", data[].toOpenArray() @@ -43,7 +43,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = tp.spawn worker(addr obj.mockSeq, sig) ## adding fut.wait(100.milliseconds) creates memory issue - await wait(sig).wait(10.milliseconds) + await wait(sig).wait(100.milliseconds) ## just doing the wait is fine: # await wait(sig) @@ -53,7 +53,7 @@ proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = await runTest(tp, sig) os.sleep(200) except AsyncTimeoutError: - echo "Run GC" + echo "looping..." suite "async tests": var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. From 7dd789412209114fc1a9d0010186cc6646136778 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:49:22 -0700 Subject: [PATCH 23/43] more examples --- tests/{ => exampleGcFailures}/exFailure.nim | 0 .../exFailureNoGcCollect.nim | 0 tests/exampleGcFailures/exFailureSeq.nim | 56 +++++++++++++++++++ 3 files changed, 56 insertions(+) rename tests/{ => exampleGcFailures}/exFailure.nim (100%) rename tests/{ => exampleGcFailures}/exFailureNoGcCollect.nim (100%) create mode 100644 tests/exampleGcFailures/exFailureSeq.nim diff --git a/tests/exFailure.nim b/tests/exampleGcFailures/exFailure.nim similarity index 100% rename from tests/exFailure.nim rename to tests/exampleGcFailures/exFailure.nim diff --git a/tests/exFailureNoGcCollect.nim b/tests/exampleGcFailures/exFailureNoGcCollect.nim similarity index 100% rename from tests/exFailureNoGcCollect.nim rename to tests/exampleGcFailures/exFailureNoGcCollect.nim diff --git a/tests/exampleGcFailures/exFailureSeq.nim b/tests/exampleGcFailures/exFailureSeq.nim new file mode 100644 index 0000000..eadc91a --- /dev/null +++ b/tests/exampleGcFailures/exFailureSeq.nim @@ -0,0 +1,56 @@ +import std/os +import std/sequtils +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +type + Seq*[T] = object + data*: ptr UncheckedArray[T] + size*: int + + +template toOpenArray*[T](arr: Seq[T]): auto = + system.toOpenArray(arr.data, 0, arr.size) + +proc toArrayHolder*[T](data: seq[T]): Seq[T] = + Seq[T]( + data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() + ) + +proc worker(data: Seq[char], sig: ThreadSignalPtr) = + os.sleep(100) + echo "running worker: " + echo "worker: ", data.toOpenArray() + for i, c in data.toOpenArray(): + data.data[i] = char(c.uint8 + 10) + discard sig.fireSync() + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj = "hello world!".toSeq() + + echo "spawn worker" + tp.spawn worker(obj.toArrayHolder(), sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + await wait(sig).wait(10.milliseconds) + ## just doing the wait is fine: + # await wait(sig) + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + for i in 1..10_000: + try: + await runTest(tp, sig) + os.sleep(200) + except AsyncTimeoutError: + echo "looping..." + GC_fullCollect() + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig) From fde10adac58e79c7d10ab82fa5a4c7ec2b38bf57 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:56:17 -0700 Subject: [PATCH 24/43] more examples --- tests/exampleGcFailures/config.nims | 2 ++ tests/exampleGcFailures/exFailure.nim | 8 ++++++++ tests/exampleGcFailures/exFailureSeq.nim | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/exampleGcFailures/config.nims diff --git a/tests/exampleGcFailures/config.nims b/tests/exampleGcFailures/config.nims new file mode 100644 index 0000000..1e8aee5 --- /dev/null +++ b/tests/exampleGcFailures/config.nims @@ -0,0 +1,2 @@ +--threads:on +--mm:refc diff --git a/tests/exampleGcFailures/exFailure.nim b/tests/exampleGcFailures/exFailure.nim index 4f45c7a..6f07597 100644 --- a/tests/exampleGcFailures/exFailure.nim +++ b/tests/exampleGcFailures/exFailure.nim @@ -4,6 +4,14 @@ import chronos/threadsync import chronos/unittest2/asynctests import taskpools +## This example mocks up a sequence and uses +## a finalizer and GC_fullCollect to more +## deterministically create a memory error. +## +## see `exFailureSeq.nim` for a probablisitc based +## example using a real seq object. +## + type Seq*[T] = object data*: ptr UncheckedArray[T] diff --git a/tests/exampleGcFailures/exFailureSeq.nim b/tests/exampleGcFailures/exFailureSeq.nim index eadc91a..4afb79e 100644 --- a/tests/exampleGcFailures/exFailureSeq.nim +++ b/tests/exampleGcFailures/exFailureSeq.nim @@ -5,12 +5,22 @@ import chronos/threadsync import chronos/unittest2/asynctests import taskpools +## create a probablistically likely failure of +## using sequence memory from another thread +## with refc. +## +## However, unlike `exFailure.nim`, this can take +## a while to run. +## +## It may not always produce an error either, but +## generally does so in a few seconds of running. +## + type Seq*[T] = object data*: ptr UncheckedArray[T] size*: int - template toOpenArray*[T](arr: Seq[T]): auto = system.toOpenArray(arr.data, 0, arr.size) @@ -20,7 +30,7 @@ proc toArrayHolder*[T](data: seq[T]): Seq[T] = ) proc worker(data: Seq[char], sig: ThreadSignalPtr) = - os.sleep(100) + os.sleep(300) echo "running worker: " echo "worker: ", data.toOpenArray() for i, c in data.toOpenArray(): From 4729c7db6eb813684d6d13696b8016369cc77147 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 22:57:43 -0700 Subject: [PATCH 25/43] more examples --- tests/exampleGcFailures/exFailureSeq.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeq.nim b/tests/exampleGcFailures/exFailureSeq.nim index 4afb79e..6aa08c9 100644 --- a/tests/exampleGcFailures/exFailureSeq.nim +++ b/tests/exampleGcFailures/exFailureSeq.nim @@ -17,19 +17,19 @@ import taskpools ## type - Seq*[T] = object + SeqDataPtr*[T] = object data*: ptr UncheckedArray[T] size*: int -template toOpenArray*[T](arr: Seq[T]): auto = +template toOpenArray*[T](arr: SeqDataPtr[T]): auto = system.toOpenArray(arr.data, 0, arr.size) -proc toArrayHolder*[T](data: seq[T]): Seq[T] = - Seq[T]( +proc toArrayHolder*[T](data: seq[T]): SeqDataPtr[T] = + SeqDataPtr[T]( data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() ) -proc worker(data: Seq[char], sig: ThreadSignalPtr) = +proc worker(data: SeqDataPtr[char], sig: ThreadSignalPtr) = os.sleep(300) echo "running worker: " echo "worker: ", data.toOpenArray() @@ -39,7 +39,7 @@ proc worker(data: Seq[char], sig: ThreadSignalPtr) = proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init - var obj = "hello world!".toSeq() + var obj = "hello world!".toSeqDataPtr() echo "spawn worker" tp.spawn worker(obj.toArrayHolder(), sig) From 7fcc8dba1bd064260f783d5bd6e3ad8eec56b9bd Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 23:00:03 -0700 Subject: [PATCH 26/43] more examples --- tests/exampleGcFailures/exFailureSeq.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeq.nim b/tests/exampleGcFailures/exFailureSeq.nim index 6aa08c9..9fbe848 100644 --- a/tests/exampleGcFailures/exFailureSeq.nim +++ b/tests/exampleGcFailures/exFailureSeq.nim @@ -24,7 +24,7 @@ type template toOpenArray*[T](arr: SeqDataPtr[T]): auto = system.toOpenArray(arr.data, 0, arr.size) -proc toArrayHolder*[T](data: seq[T]): SeqDataPtr[T] = +proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = SeqDataPtr[T]( data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() ) @@ -39,10 +39,10 @@ proc worker(data: SeqDataPtr[char], sig: ThreadSignalPtr) = proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init - var obj = "hello world!".toSeqDataPtr() + var obj = "hello world!".toSeq() echo "spawn worker" - tp.spawn worker(obj.toArrayHolder(), sig) + tp.spawn worker(obj.toSeqDataPtr(), sig) ## adding fut.wait(100.milliseconds) creates memory issue await wait(sig).wait(10.milliseconds) From 81f304f87943429f86b108434f7e72af1f2894b6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 23:41:11 -0700 Subject: [PATCH 27/43] more examples --- tests/exampleGcFailures/exFailureSeq.nim | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeq.nim b/tests/exampleGcFailures/exFailureSeq.nim index 9fbe848..66620a4 100644 --- a/tests/exampleGcFailures/exFailureSeq.nim +++ b/tests/exampleGcFailures/exFailureSeq.nim @@ -30,7 +30,7 @@ proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = ) proc worker(data: SeqDataPtr[char], sig: ThreadSignalPtr) = - os.sleep(300) + os.sleep(1_0) echo "running worker: " echo "worker: ", data.toOpenArray() for i, c in data.toOpenArray(): @@ -41,22 +41,23 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init var obj = "hello world!".toSeq() - echo "spawn worker" + # echo "spawn worker" tp.spawn worker(obj.toSeqDataPtr(), sig) ## adding fut.wait(100.milliseconds) creates memory issue - await wait(sig).wait(10.milliseconds) + await wait(sig).wait(1.milliseconds) ## just doing the wait is fine: # await wait(sig) proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = - for i in 1..10_000: + for i in 1..30_000: try: await runTest(tp, sig) - os.sleep(200) except AsyncTimeoutError: - echo "looping..." - GC_fullCollect() + # os.sleep(1) + # echo "looping..." + # GC_fullCollect() + discard suite "async tests": var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. @@ -64,3 +65,4 @@ suite "async tests": asyncTest "test": await runTests(tp, sig) + os.sleep(10_000) From 7853fa9904a0b6a74bcdace8663de89c7fc44c44 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 21 Feb 2024 23:42:49 -0700 Subject: [PATCH 28/43] more examples --- tests/exampleGcFailures/exFailureSeq.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exampleGcFailures/exFailureSeq.nim b/tests/exampleGcFailures/exFailureSeq.nim index 66620a4..f00ba37 100644 --- a/tests/exampleGcFailures/exFailureSeq.nim +++ b/tests/exampleGcFailures/exFailureSeq.nim @@ -30,7 +30,7 @@ proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = ) proc worker(data: SeqDataPtr[char], sig: ThreadSignalPtr) = - os.sleep(1_0) + os.sleep(10) echo "running worker: " echo "worker: ", data.toOpenArray() for i, c in data.toOpenArray(): From bfd1078d3c45c61fd8e3661b6d1a646213ffb153 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sat, 24 Feb 2024 15:08:02 -0700 Subject: [PATCH 29/43] updates --- tests/exampleGcFailures/exFailureSeqSeq.nim | 71 +++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/exampleGcFailures/exFailureSeqSeq.nim diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim new file mode 100644 index 0000000..beffd08 --- /dev/null +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -0,0 +1,71 @@ +import std/os +import std/sequtils +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +## create a probablistically likely failure of +## using sequence memory from another thread +## with refc. +## +## However, unlike `exFailure.nim`, this can take +## a while to run. +## +## It may not always produce an error either, but +## generally does so in a few seconds of running. +## + +type + SeqDataPtr*[T] = object + data*: ptr UncheckedArray[T] + size*: int + +template toOpenArray*[T](arr: SeqDataPtr[T]): auto = + system.toOpenArray(arr.data, 0, arr.size) + +proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = + SeqDataPtr[T]( + data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() + ) + +proc worker(data: ptr seq[seq[char]], sig: ThreadSignalPtr) = + os.sleep(10) + echo "running worker: " + echo "worker: ", data[] + for i, d in data[]: + for j, c in d: + data[i][j] = char(c.uint8 + 10) + discard sig.fireSync() + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj1 = "hello world!".toSeq() + var obj2 = "goodbye denver!".toSeq() + var data = @[obj1, obj2] + + # echo "spawn worker" + tp.spawn worker(addr data, sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + await wait(sig).wait(1.milliseconds) + ## just doing the wait is fine: + # await wait(sig) + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + for i in 1..30_000: + try: + await runTest(tp, sig) + except AsyncTimeoutError: + # os.sleep(1) + # echo "looping..." + # GC_fullCollect() + discard + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig) + os.sleep(10_000) From 3de128b26a4ad8a79112e919c40dcc531deee406 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 26 Feb 2024 16:57:42 -0700 Subject: [PATCH 30/43] updates --- tests/exampleGcFailures/exFailureGcRef.nim | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/exampleGcFailures/exFailureGcRef.nim diff --git a/tests/exampleGcFailures/exFailureGcRef.nim b/tests/exampleGcFailures/exFailureGcRef.nim new file mode 100644 index 0000000..4d2eab4 --- /dev/null +++ b/tests/exampleGcFailures/exFailureGcRef.nim @@ -0,0 +1,78 @@ +import std/os +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +## This example mocks up a sequence and uses +## a finalizer and GC_fullCollect to more +## deterministically create a memory error. +## +## This variant tries using `allFutures` to prevent +## cancelation of the thread-future. +## +## It keeps the future from finishing before +## the task runs. +## + +type + Seq*[T] = object + data*: ptr UncheckedArray[T] + size*: int + + DataObj = ref object + mockSeq: Seq[char] + +template toOpenArray*[T](arr: Seq[T]): auto = + system.toOpenArray(arr.data, 0, arr.size) + +proc worker(data: ptr Seq[char], sig: ThreadSignalPtr) = + os.sleep(1_000) + echo "running worker: " + assert data[].data != nil + echo "worker: ", data[].toOpenArray() + discard sig.fireSync() + +proc finalizer(obj: DataObj) = + echo "finalize DataObj and freeing mockSeq" + obj.mockSeq.data.dealloc() + obj.mockSeq.data = nil + +proc initMockSeq(msg: string): Seq[char] = + result.data = cast[ptr UncheckedArray[char]](alloc0(13)) + for i, c in msg: + result.data[i] = c + result.size = 12 + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj: DataObj + obj.new(finalizer) + obj.mockSeq = initMockSeq("hello world!") + + echo "spawn worker" + tp.spawn worker(addr obj.mockSeq, sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + try: + await wait(sig) + finally: + echo "finishing runTest" + ## just doing the wait is fine: + # await wait(sig) + +proc runTestWrap(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + await runTest(tp, sig).allFutures().wait(100.milliseconds) + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + try: + await runTestWrap(tp, sig) + except AsyncTimeoutError: + echo "Run GC" + GC_fullCollect() + os.sleep(2_000) + echo "Done" From 4837738b4a88ce216a75efb007946d15839549c0 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 26 Feb 2024 17:10:19 -0700 Subject: [PATCH 31/43] updates --- tests/exampleGcFailures/exFailureGcRef.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/exampleGcFailures/exFailureGcRef.nim b/tests/exampleGcFailures/exFailureGcRef.nim index 4d2eab4..3b8aace 100644 --- a/tests/exampleGcFailures/exFailureGcRef.nim +++ b/tests/exampleGcFailures/exFailureGcRef.nim @@ -12,7 +12,8 @@ import taskpools ## cancelation of the thread-future. ## ## It keeps the future from finishing before -## the task runs. +## the task runs. However, it doesn't appear to be +## triggering the finalizer for the MockSeq. ## type @@ -76,3 +77,8 @@ suite "async tests": GC_fullCollect() os.sleep(2_000) echo "Done" + os.sleep(10_000) + GC_fullCollect() + + teardown: + GC_fullCollect() From 71f294450fa8d96bb7e3be85d1edef30668596fc Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 26 Feb 2024 17:23:45 -0700 Subject: [PATCH 32/43] updates --- tests/exampleGcFailures/exFailure.nim | 4 +- tests/exampleGcFailures/exFailure2.nim | 81 +++++++++++++++++++ ...lureGcRef.nim => exFailureWrapSuccess.nim} | 0 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 tests/exampleGcFailures/exFailure2.nim rename tests/exampleGcFailures/{exFailureGcRef.nim => exFailureWrapSuccess.nim} (100%) diff --git a/tests/exampleGcFailures/exFailure.nim b/tests/exampleGcFailures/exFailure.nim index 6f07597..3382a29 100644 --- a/tests/exampleGcFailures/exFailure.nim +++ b/tests/exampleGcFailures/exFailure.nim @@ -51,7 +51,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = tp.spawn worker(addr obj.mockSeq, sig) ## adding fut.wait(100.milliseconds) creates memory issue - await wait(sig).wait(100.milliseconds) + await wait(sig) ## just doing the wait is fine: # await wait(sig) @@ -61,7 +61,7 @@ suite "async tests": asyncTest "test": try: - await runTest(tp, sig) + await runTest(tp, sig).wait(100.milliseconds) except AsyncTimeoutError: echo "Run GC" GC_fullCollect() diff --git a/tests/exampleGcFailures/exFailure2.nim b/tests/exampleGcFailures/exFailure2.nim new file mode 100644 index 0000000..a5aa743 --- /dev/null +++ b/tests/exampleGcFailures/exFailure2.nim @@ -0,0 +1,81 @@ +import std/os +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +## This example mocks up a sequence and uses +## a finalizer and GC_fullCollect to more +## deterministically create a memory error. +## +## see `exFailureSeq.nim` for a probablisitc based +## example using a real seq object. +## + +type + Seq*[T] = object + data*: ptr UncheckedArray[T] + size*: int + + DataObj = ref object + mockSeq: Seq[char] + +template toOpenArray*[T](arr: Seq[T]): auto = + system.toOpenArray(arr.data, 0, arr.size) + +proc worker(data: ptr Seq[char], sig: ThreadSignalPtr) = + os.sleep(5_000) + echo "running worker: " + assert data[].data != nil + echo "worker: ", data[].toOpenArray() + discard sig.fireSync() + +proc finalizer(obj: DataObj) = + echo "finalize DataObj and freeing mockSeq" + obj.mockSeq.data.dealloc() + obj.mockSeq.data = nil + +proc initMockSeq(msg: string): Seq[char] = + result.data = cast[ptr UncheckedArray[char]](alloc0(13)) + for i, c in msg: + result.data[i] = c + result.size = 12 + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj: DataObj + obj.new(finalizer) + obj.mockSeq = initMockSeq("hello world!") + + echo "spawn worker" + tp.spawn worker(addr obj.mockSeq, sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + try: + await wait(sig) + echo "runTest got sig" + finally: + echo "runTest done" + ## just doing the wait is fine: + # await wait(sig) + +proc doFail() {.async.} = + await sleepAsync(10.milliseconds) + raise newException(CatchableError, "error") + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + try: + await allFutures( + runTest(tp, sig), + doFail(), + ) + except Defect: + echo "Errored out" + finally: + echo "Run GC" + GC_fullCollect() + os.sleep(2_000) diff --git a/tests/exampleGcFailures/exFailureGcRef.nim b/tests/exampleGcFailures/exFailureWrapSuccess.nim similarity index 100% rename from tests/exampleGcFailures/exFailureGcRef.nim rename to tests/exampleGcFailures/exFailureWrapSuccess.nim From 1a2d4dd1336767f8f0f3390d728ec2f3b6fcca2f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 22:07:14 -0700 Subject: [PATCH 33/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 26 ++++---- .../exFailureSeqSeqCursor.nim | 63 +++++++++++++++++++ tests/exampleGcFailures/test.nim | 25 ++++++++ 3 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 tests/exampleGcFailures/exFailureSeqSeqCursor.nim create mode 100644 tests/exampleGcFailures/test.nim diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index beffd08..599bc81 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -29,13 +29,14 @@ proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() ) -proc worker(data: ptr seq[seq[char]], sig: ThreadSignalPtr) = - os.sleep(10) +proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = + # os.sleep(10) echo "running worker: " - echo "worker: ", data[] - for i, d in data[]: - for j, c in d: - data[i][j] = char(c.uint8 + 10) + echo "worker: ", data + # for i, d in data: + # for j, c in d: + # d[j] = char(c.uint8 + 10) + GC_fullCollect() discard sig.fireSync() proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = @@ -45,27 +46,28 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = var data = @[obj1, obj2] # echo "spawn worker" - tp.spawn worker(addr data, sig) + tp.spawn worker(data, sig) ## adding fut.wait(100.milliseconds) creates memory issue - await wait(sig).wait(1.milliseconds) + await wait(sig) ## just doing the wait is fine: # await wait(sig) + echo "data: ", data proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = - for i in 1..30_000: + for i in 1..10_000: try: await runTest(tp, sig) except AsyncTimeoutError: # os.sleep(1) # echo "looping..." - # GC_fullCollect() discard + GC_fullCollect() suite "async tests": - var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + var tp = Taskpool.new(num_threads = 8) # Default to the number of hardware threads. let sig = ThreadSignalPtr.new().get() asyncTest "test": await runTests(tp, sig) - os.sleep(10_000) + # os.sleep(10_000) diff --git a/tests/exampleGcFailures/exFailureSeqSeqCursor.nim b/tests/exampleGcFailures/exFailureSeqSeqCursor.nim new file mode 100644 index 0000000..8b21d99 --- /dev/null +++ b/tests/exampleGcFailures/exFailureSeqSeqCursor.nim @@ -0,0 +1,63 @@ +import std/os +import std/sequtils +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +## create a probablistically likely failure of +## using sequence memory from another thread +## with refc. +## +## However, unlike `exFailure.nim`, this can take +## a while to run. +## +## It may not always produce an error either, but +## generally does so in a few seconds of running. +## + +type + SeqCursor* = object + data* {.cursor.}: seq[seq[char]] + +proc worker(data: SeqCursor, sig: ThreadSignalPtr) = + os.sleep(10) + echo "running worker: " + echo "worker: ", data.data + # for i, d in data.data: + # for j, c in d: + # data.data[i][j] = char(c.uint8 + 10) + discard sig.fireSync() + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj1 = "hello world!".toSeq() + var obj2 = "goodbye denver!".toSeq() + var data = @[obj1, obj2] + var cur = SeqCursor(data: data) + + # echo "spawn worker" + tp.spawn worker(cur, sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + await wait(sig) + ## just doing the wait is fine: + # await wait(sig) + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + for i in 1..3_000: + try: + await runTest(tp, sig) + except AsyncTimeoutError: + # os.sleep(1) + # echo "looping..." + # GC_fullCollect() + discard + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig) + os.sleep(10_000) diff --git a/tests/exampleGcFailures/test.nim b/tests/exampleGcFailures/test.nim new file mode 100644 index 0000000..91ab044 --- /dev/null +++ b/tests/exampleGcFailures/test.nim @@ -0,0 +1,25 @@ + +when false: + type AnObject* = object of RootObj + value*: int + + proc mutate(a: sink AnObject) = + a.value = 1 + + var obj = AnObject(value: 42) + mutate(obj) + doAssert obj.value == 42 + +else: + type AnObject = object of RootObj + value*: int + + proc `=destroy`(x: var AnObject) = + echo "DEST" + + proc mutate(a: sink AnObject) = + a.value = 1 + + var obj = AnObject(value: 42) + mutate(obj) + doAssert obj.value == 42 From 2f42f513c0afa43e1cf4c57e9e1da59c5fd1d30a Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 22:08:05 -0700 Subject: [PATCH 34/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index 599bc81..258ad61 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -30,14 +30,14 @@ proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = ) proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = - # os.sleep(10) + os.sleep(100) echo "running worker: " echo "worker: ", data # for i, d in data: # for j, c in d: # d[j] = char(c.uint8 + 10) GC_fullCollect() - discard sig.fireSync() + # discard sig.fireSync() proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init @@ -49,7 +49,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = tp.spawn worker(data, sig) ## adding fut.wait(100.milliseconds) creates memory issue - await wait(sig) + # await wait(sig) ## just doing the wait is fine: # await wait(sig) echo "data: ", data From 6885d4410f8c241707617714c0f80d844811794f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 22:10:30 -0700 Subject: [PATCH 35/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index 258ad61..8ae85f6 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -37,7 +37,7 @@ proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = # for j, c in d: # d[j] = char(c.uint8 + 10) GC_fullCollect() - # discard sig.fireSync() + discard sig.fireSync() proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init @@ -48,9 +48,6 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = # echo "spawn worker" tp.spawn worker(data, sig) - ## adding fut.wait(100.milliseconds) creates memory issue - # await wait(sig) - ## just doing the wait is fine: # await wait(sig) echo "data: ", data From 989dc8963bc098b8438bf86acd9c1f49bdcfef30 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 23:32:10 -0700 Subject: [PATCH 36/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index 8ae85f6..eb01c15 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -30,7 +30,7 @@ proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = ) proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = - os.sleep(100) + # os.sleep(100) echo "running worker: " echo "worker: ", data # for i, d in data: @@ -48,7 +48,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = # echo "spawn worker" tp.spawn worker(data, sig) - # await wait(sig) + await wait(sig) echo "data: ", data proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = From cde8ba09d56093aed42e2978b8a8b5432803e4ca Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 23:32:59 -0700 Subject: [PATCH 37/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index eb01c15..d16b0f2 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -16,19 +16,6 @@ import taskpools ## generally does so in a few seconds of running. ## -type - SeqDataPtr*[T] = object - data*: ptr UncheckedArray[T] - size*: int - -template toOpenArray*[T](arr: SeqDataPtr[T]): auto = - system.toOpenArray(arr.data, 0, arr.size) - -proc toSeqDataPtr*[T](data: seq[T]): SeqDataPtr[T] = - SeqDataPtr[T]( - data: cast[ptr UncheckedArray[T]](unsafeAddr(data[0])), size: data.len() - ) - proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = # os.sleep(100) echo "running worker: " @@ -62,7 +49,7 @@ proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = GC_fullCollect() suite "async tests": - var tp = Taskpool.new(num_threads = 8) # Default to the number of hardware threads. + var tp = Taskpool.new(num_threads = 4) # Default to the number of hardware threads. let sig = ThreadSignalPtr.new().get() asyncTest "test": From ce858b6f383faa40f0bf763dd061f2d53f80b670 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 23:35:28 -0700 Subject: [PATCH 38/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index d16b0f2..5f05864 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -19,6 +19,7 @@ import taskpools proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = # os.sleep(100) echo "running worker: " + echo "worker: ", data.unsafeAddr.pointer.repr echo "worker: ", data # for i, d in data: # for j, c in d: @@ -26,9 +27,9 @@ proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = GC_fullCollect() discard sig.fireSync() -proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = +proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = ## init - var obj1 = "hello world!".toSeq() + var obj1 = ("hello world! " & $i).toSeq() var obj2 = "goodbye denver!".toSeq() var data = @[obj1, obj2] @@ -36,12 +37,13 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = tp.spawn worker(data, sig) await wait(sig) + echo "data: ", data.unsafeAddr.pointer.repr echo "data: ", data proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = - for i in 1..10_000: + for i in 1..2: try: - await runTest(tp, sig) + await runTest(tp, sig, i) except AsyncTimeoutError: # os.sleep(1) # echo "looping..." From 44df47de2a55a3a88de871f4899ec2294ff0a994 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 27 Feb 2024 23:51:53 -0700 Subject: [PATCH 39/43] more experiments --- tests/exampleGcFailures/exFailureSeqSeq.nim | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exFailureSeqSeq.nim index 5f05864..16c9f51 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exFailureSeqSeq.nim @@ -23,12 +23,13 @@ proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = echo "worker: ", data # for i, d in data: # for j, c in d: - # d[j] = char(c.uint8 + 10) + # data[i][j] = char(c.uint8 + 10) GC_fullCollect() discard sig.fireSync() proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = ## init + # await sleepAsync(10.milliseconds) var obj1 = ("hello world! " & $i).toSeq() var obj2 = "goodbye denver!".toSeq() var data = @[obj1, obj2] @@ -37,18 +38,18 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = tp.spawn worker(data, sig) await wait(sig) - echo "data: ", data.unsafeAddr.pointer.repr + echo "data: ", data.addr.pointer.repr echo "data: ", data + echo "" proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = - for i in 1..2: - try: - await runTest(tp, sig, i) - except AsyncTimeoutError: - # os.sleep(1) - # echo "looping..." - discard + var futs = newSeq[Future[void]]() + for i in 1..10_000: + let f = runTest(tp, sig, i) + # futs.add f + await f GC_fullCollect() + await allFutures(futs) suite "async tests": var tp = Taskpool.new(num_threads = 4) # Default to the number of hardware threads. From 5fd1e789bbbf6822b989bd4c33d580d39fad0609 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 19 Mar 2024 13:53:29 +0200 Subject: [PATCH 40/43] add ptr seq-seq example --- .../exNoFailurePtrSeqSeq.nim | 60 +++++++++++++++++++ ...ailureSeqSeq.nim => exNoFailureSeqSeq.nim} | 8 +-- 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim rename tests/exampleGcFailures/{exFailureSeqSeq.nim => exNoFailureSeqSeq.nim} (89%) diff --git a/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim b/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim new file mode 100644 index 0000000..5043af2 --- /dev/null +++ b/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim @@ -0,0 +1,60 @@ +import std/os +import std/sequtils +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +## create a probablistically likely failure of +## using sequence memory from another thread +## with refc. +## +## However, unlike `exFailure.nim`, this can take +## a while to run. +## +## It may not always produce an error either, but +## generally does so in a few seconds of running. +## + +proc worker(data: ptr seq[seq[char]], sig: ThreadSignalPtr) = + # os.sleep(100) + echo "running worker: " + echo "worker: ", data.pointer.repr + echo "worker: ", data[] + # for i, d in data: + # for j, c in d: + # data[i][j] = char(c.uint8 + 10) + GC_fullCollect() + discard sig.fireSync() + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = + ## init + # await sleepAsync(10.milliseconds) + var obj1 = ("hello world! " & $i).toSeq() + var obj2 = "goodbye denver!".toSeq() + var data = @[obj1, obj2] + + # echo "spawn worker" + tp.spawn worker(addr data, sig) + + await wait(sig) + echo "data: ", data.addr.pointer.repr + echo "data: ", data + echo "" + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + var futs = newSeq[Future[void]]() + for i in 1..40_000: + let f = runTest(tp, sig, i) + # futs.add f + await f + GC_fullCollect() + await allFutures(futs) + +suite "async tests": + var tp = Taskpool.new(num_threads = 4) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig) + # os.sleep(10_000) diff --git a/tests/exampleGcFailures/exFailureSeqSeq.nim b/tests/exampleGcFailures/exNoFailureSeqSeq.nim similarity index 89% rename from tests/exampleGcFailures/exFailureSeqSeq.nim rename to tests/exampleGcFailures/exNoFailureSeqSeq.nim index 16c9f51..f723391 100644 --- a/tests/exampleGcFailures/exFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exNoFailureSeqSeq.nim @@ -16,11 +16,11 @@ import taskpools ## generally does so in a few seconds of running. ## -proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = +proc worker(data: ptr seq[seq[char]], sig: ThreadSignalPtr) = # os.sleep(100) echo "running worker: " - echo "worker: ", data.unsafeAddr.pointer.repr - echo "worker: ", data + echo "worker: ", data.pointer.repr + echo "worker: ", data[] # for i, d in data: # for j, c in d: # data[i][j] = char(c.uint8 + 10) @@ -35,7 +35,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = var data = @[obj1, obj2] # echo "spawn worker" - tp.spawn worker(data, sig) + tp.spawn worker(addr data, sig) await wait(sig) echo "data: ", data.addr.pointer.repr From 1156b0b63a6e5692eab40c79f41648da4d27c405 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 19 Mar 2024 14:01:43 +0200 Subject: [PATCH 41/43] updates --- tests/exampleGcFailures/exFailureSeqSeqCursor.nim | 7 +++++-- tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim | 2 +- tests/exampleGcFailures/exNoFailureSeqSeq.nim | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/exampleGcFailures/exFailureSeqSeqCursor.nim b/tests/exampleGcFailures/exFailureSeqSeqCursor.nim index 8b21d99..9b3544f 100644 --- a/tests/exampleGcFailures/exFailureSeqSeqCursor.nim +++ b/tests/exampleGcFailures/exFailureSeqSeqCursor.nim @@ -24,6 +24,7 @@ proc worker(data: SeqCursor, sig: ThreadSignalPtr) = os.sleep(10) echo "running worker: " echo "worker: ", data.data + echo "worker:addr: ", cast[pointer](data.data).repr # for i, d in data.data: # for j, c in d: # data.data[i][j] = char(c.uint8 + 10) @@ -41,11 +42,13 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## adding fut.wait(100.milliseconds) creates memory issue await wait(sig) + echo "data:addr: ", data.addr.pointer.repr + echo "data:cursor:addr: ", cast[pointer](cur.data).repr ## just doing the wait is fine: # await wait(sig) proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = - for i in 1..3_000: + for i in 1..1: try: await runTest(tp, sig) except AsyncTimeoutError: @@ -60,4 +63,4 @@ suite "async tests": asyncTest "test": await runTests(tp, sig) - os.sleep(10_000) + # os.sleep(10_000) diff --git a/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim b/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim index 5043af2..d12cd54 100644 --- a/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim +++ b/tests/exampleGcFailures/exNoFailurePtrSeqSeq.nim @@ -44,7 +44,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = var futs = newSeq[Future[void]]() - for i in 1..40_000: + for i in 1..1: let f = runTest(tp, sig, i) # futs.add f await f diff --git a/tests/exampleGcFailures/exNoFailureSeqSeq.nim b/tests/exampleGcFailures/exNoFailureSeqSeq.nim index f723391..16c9f51 100644 --- a/tests/exampleGcFailures/exNoFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exNoFailureSeqSeq.nim @@ -16,11 +16,11 @@ import taskpools ## generally does so in a few seconds of running. ## -proc worker(data: ptr seq[seq[char]], sig: ThreadSignalPtr) = +proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = # os.sleep(100) echo "running worker: " - echo "worker: ", data.pointer.repr - echo "worker: ", data[] + echo "worker: ", data.unsafeAddr.pointer.repr + echo "worker: ", data # for i, d in data: # for j, c in d: # data[i][j] = char(c.uint8 + 10) @@ -35,7 +35,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = var data = @[obj1, obj2] # echo "spawn worker" - tp.spawn worker(addr data, sig) + tp.spawn worker(data, sig) await wait(sig) echo "data: ", data.addr.pointer.repr From bc4d8d309578f4c5b9ea1e36fef9cb26925936bb Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Mar 2024 13:46:53 +0200 Subject: [PATCH 42/43] updates --- .../exNoFailureNoGcCollectSeq.nim | 57 +++++ tests/exampleGcFailures/exNoFailureSeqSeq.nim | 4 +- tests/exampleGcFailures/example.nim | 214 ++++++++++++++++++ 3 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim create mode 100644 tests/exampleGcFailures/example.nim diff --git a/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim b/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim new file mode 100644 index 0000000..7fa4d0d --- /dev/null +++ b/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim @@ -0,0 +1,57 @@ +import std/os +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +type + Seq*[T] = object + data*: ptr UncheckedArray[T] + size*: int + + DataObj = ref object + mockSeq: Seq[char] + +proc worker(data: seq[char], sig: ThreadSignalPtr) = + os.sleep(4_000) + echo "running worker: " + echo "worker: ", data + discard sig.fireSync() + +proc finalizer(obj: DataObj) = + echo "finalize DataObj and freeing mockSeq" + obj.mockSeq.data.dealloc() + obj.mockSeq.data = nil + +proc initMockSeq(msg: string): Seq[char] = + result.data = cast[ptr UncheckedArray[char]](alloc0(13)) + for i, c in msg: + result.data[i] = c + result.size = 12 + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj = @"hello world!" + + echo "spawn worker" + tp.spawn worker(obj, sig) + + ## adding fut.wait(100.milliseconds) creates memory issue + await wait(sig).wait(100.milliseconds) + ## just doing the wait is fine: + # await wait(sig) + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + for i in 1..2_000: + try: + await runTest(tp, sig) + os.sleep(200) + except AsyncTimeoutError: + echo "looping..." + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig) diff --git a/tests/exampleGcFailures/exNoFailureSeqSeq.nim b/tests/exampleGcFailures/exNoFailureSeqSeq.nim index 16c9f51..f2acabf 100644 --- a/tests/exampleGcFailures/exNoFailureSeqSeq.nim +++ b/tests/exampleGcFailures/exNoFailureSeqSeq.nim @@ -17,7 +17,7 @@ import taskpools ## proc worker(data: seq[seq[char]], sig: ThreadSignalPtr) = - # os.sleep(100) + os.sleep(1_000) echo "running worker: " echo "worker: ", data.unsafeAddr.pointer.repr echo "worker: ", data @@ -45,7 +45,7 @@ proc runTest(tp: TaskPool, sig: ThreadSignalPtr, i: int) {.async.} = proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = var futs = newSeq[Future[void]]() for i in 1..10_000: - let f = runTest(tp, sig, i) + let f = runTest(tp, sig, i).wait(100.milliseconds) # futs.add f await f GC_fullCollect() diff --git a/tests/exampleGcFailures/example.nim b/tests/exampleGcFailures/example.nim new file mode 100644 index 0000000..ab8b7d9 --- /dev/null +++ b/tests/exampleGcFailures/example.nim @@ -0,0 +1,214 @@ +while true: + block :stateLoop: + try: + var + :tmpD + :tmpD_1 + :tmpD_2 + :tmpD_3 + closureIterSetupExc(:envP.`:curExc1`) + goto :envP.`:state`6 + state 0: + template result(): auto {.used.} = + {.fatal: "You should not reference the `result` variable inside" & + " a void async proc".} + + var :envP.closureSucceeded25 = true + :envP.`:state` = 1 + break :stateLoop + state 1: + var :envP.obj117 = + type + OutType`gensym13 = typeof(items("hello world! " & $i)) + block :tmp: + :tmpD = + let :envP.`:tmp5` = `&`("hello world! ", $:envP.`:up`.i1) + template s2_838861096(): untyped = + :tmp_1 + + var :envP.i`gensym1310 = 0 + var :envP.result`gensym139 = newSeq( + chckRange(len(:envP.`:tmp5`), 0, 9223372036854775807)) + block :tmp_2: + var :envP.it`gensym138 + var :envP.i6 = 0 + let :envP.L7 = len(:envP.`:tmp5`) + block :tmp_3: + while :envP.i6 < :envP.L7: + :envP.it`gensym138 = :envP.`:tmp5`[:envP.i6] + :envP.result`gensym139[:envP.i`gensym1310] = :envP.it`gensym138 + :envP.i`gensym1310 += 1 + inc(:envP.i6, 1) + const + loc`gensym22 = (filename: "/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim", + line: 258, column: 10) + ploc`gensym22 = "/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim(258, 11)" + bind instantiationInfo + mixin failedAssertImpl + {.line: (filename: "/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim", + line: 258, column: 10).}: + if not (len(:envP.`:tmp5`) == :envP.L7): + failedAssertImpl("/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim(258, 11) `len(a) == L` the length of the string changed while iterating over it") + :envP.result`gensym139 + :tmpD + var :envP.obj218 = + type + OutType`gensym20 = typeof(items("goodbye denver!")) + block :tmp_4: + :tmpD_1 = + let :envP.`:tmp11` = "goodbye denver!" + template s2_838861153(): untyped = + :tmp_5 + + var :envP.i`gensym2016 = 0 + var :envP.result`gensym2015 = newSeq( + chckRange(len(:envP.`:tmp11`), 0, 9223372036854775807)) + block :tmp_6: + var :envP.it`gensym2014 + var :envP.i12 = 0 + let :envP.L13 = len(:envP.`:tmp11`) + block :tmp_7: + while :envP.i12 < :envP.L13: + :envP.it`gensym2014 = :envP.`:tmp11`[:envP.i12] + :envP.result`gensym2015[:envP.i`gensym2016] = :envP.it`gensym2014 + :envP.i`gensym2016 += 1 + inc(:envP.i12, 1) + const + loc`gensym22 = (filename: "/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim", + line: 258, column: 10) + ploc`gensym22 = "/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim(258, 11)" + bind instantiationInfo + mixin failedAssertImpl + {.line: (filename: "/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim", + line: 258, column: 10).}: + if not (len(:envP.`:tmp11`) == :envP.L13): + failedAssertImpl("/Users/elcritch/.asdf/installs/nim/1.6.18/lib/system/iterators.nim(258, 11) `len(a) == L` the length of the string changed while iterating over it") + :envP.result`gensym2015 + :tmpD_1 + var :envP.data20 = @[ + :tmpD_2 = :envP.obj117 + :tmpD_2, + :tmpD_3 = :envP.obj218 + :tmpD_3] + block :tmp_8: + var + :tmpD_4 + :tmpD_5 + :tmpD_6 + :tmpD_7 + let :envP.taskNode23 = new(TaskNode, workerContext.currentTask) do: + type + ScratchObj = object + data: seq[seq[char]] + sig: ThreadSignalPtr + + let :envP.scratch19 = cast[ptr ScratchObj](c_calloc(1'u, 16'u)) + if isNil(:envP.scratch19): + raise + (ref OutOfMemDefect)(msg: "Could not allocate memory", parent: nil) + block :tmp_9: + var + :tmpD_8 + :tmpD_9 + `=sink`(:envP.isoTemp21, isolate do: + :tmpD_8 = :envP.data20 + :tmpD_8) + :envP.scratch19.data = extract(:envP.isoTemp21) + `=sink_1`(:envP.isoTemp22, isolate do: + :tmpD_9 = :envP.`:up`.sig2 + :tmpD_9) + :envP.scratch19.sig = extract_1(:envP.isoTemp22) + proc worker_838861197(args`gensym27: pointer) {.gcsafe, nimcall, + raises: [].} = + let objTemp = cast[ptr ScratchObj](args`gensym27) + let data_1 = objTemp.data + let sig_1 = objTemp.sig + worker(data_1, sig_1) + + proc destroyScratch_838861198(args`gensym27_1: pointer) {.gcsafe, + nimcall, raises: [].} = + let obj = cast[ptr ScratchObj](args`gensym27_1) + `=destroy`(obj[]) + + Task(callback: + :tmpD_4 = worker_1 + :tmpD_4, args: + :tmpD_5 = :envP.scratch19 + :tmpD_5, destroy: + :tmpD_6 = destroyScratch + :tmpD_6) + schedule(workerContext): + :tmpD_7 = :envP.taskNode23 + :tmpD_7 + chronosInternalRetFuture.internalChild = FutureBase(wait(:envP.`:up`.sig2)) + :envP.`:state` = 4 + return chronosInternalRetFuture.internalChild + state 2: + :envP.`:curExc1` = nil + if of(getCurrentException(), CancelledError): + :envP.closureSucceeded25 = false + cancelAndSchedule(FutureBase(cast[Future[void]](chronosInternalRetFuture)), + srcLocImpl("", "exNoFailureSeqSeq.nim", 43)) + elif of(getCurrentException(), CatchableError): + let :envP.exc26 = getCurrentException() + :envP.closureSucceeded25 = false + failImpl(FutureBase(cast[Future[void]](chronosInternalRetFuture)), + :envP.exc26, srcLocImpl("", "exNoFailureSeqSeq.nim", 43)) + elif of(getCurrentException(), Defect): + var :tmpD_10 + let :envP.exc27 = getCurrentException() + :envP.closureSucceeded25 = false + raise + :tmpD_10 = :envP.exc27 + :tmpD_10 + else: + :envP.`:unrollFinally3` = true + :envP.`:curExc1` = getCurrentException() + :envP.`:state` = 3 + break :stateLoop + :envP.`:state` = 3 + break :stateLoop + state 3: + if :envP.closureSucceeded25: + complete(cast[Future[void]](chronosInternalRetFuture), + srcLocImpl("", "exNoFailureSeqSeq.nim", 43)) + if :envP.`:unrollFinally3`: + if `==`(:envP.`:curExc1`, nil): + :envP.`:state` = -1 + return result = :envP.`:tmpResult2` + else: + var :tmpD_11 + closureIterSetupExc(nil) + raise + :tmpD_11 = :envP.`:curExc1` + :tmpD_11 + :envP.`:state` = 6 + break :stateLoop + state 4: + {.cast(raises: [AsyncError, CancelledError]).}: + if isNil(cast[type(wait(sig_2))](chronosInternalRetFuture.internalChild).internalError): + discard + else: + var :tmpD_12 + raise + :tmpD_12 = cast[type(wait(sig_2))](chronosInternalRetFuture.internalChild).internalError + :tmpD_12 + :envP.`:state` = 5 + break :stateLoop + state 5: + echo ["data: ", repr(pointer(addr(:envP.data20)))] + echo ["data: ", `$_1`(:envP.data20)] + echo [""] + :envP.`:state` = 3 + break :stateLoop + state 6: + :envP.`:state` = -1 + break :stateLoop + except: + :envP.`:state` = [0, -2, 3, 0, -2, -2, 0][:envP.`:state`] + if `==`(:envP.`:state`, 0): + raise + :envP.`:unrollFinally3` = `<`(0, :envP.`:state`) + if `<`(:envP.`:state`, 0): + :envP.`:state` = `-`(:envP.`:state`) + :envP.`:curExc1` = getCurrentException() \ No newline at end of file From eff04d8efa57f37a9ba156e0ffe3b6baf5cbe552 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 20 Mar 2024 14:04:18 +0200 Subject: [PATCH 43/43] add seq destroy example --- .../exNoFailureNoGcCollectSeq.nim | 24 ++-------- .../exNoFailureNoGcCollectSeqDestroy.nim | 48 +++++++++++++++++++ 2 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 tests/exampleGcFailures/exNoFailureNoGcCollectSeqDestroy.nim diff --git a/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim b/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim index 7fa4d0d..973d6ab 100644 --- a/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim +++ b/tests/exampleGcFailures/exNoFailureNoGcCollectSeq.nim @@ -4,34 +4,16 @@ import chronos/threadsync import chronos/unittest2/asynctests import taskpools -type - Seq*[T] = object - data*: ptr UncheckedArray[T] - size*: int - - DataObj = ref object - mockSeq: Seq[char] - -proc worker(data: seq[char], sig: ThreadSignalPtr) = +proc worker(data: string, sig: ThreadSignalPtr) = os.sleep(4_000) echo "running worker: " echo "worker: ", data discard sig.fireSync() -proc finalizer(obj: DataObj) = - echo "finalize DataObj and freeing mockSeq" - obj.mockSeq.data.dealloc() - obj.mockSeq.data = nil - -proc initMockSeq(msg: string): Seq[char] = - result.data = cast[ptr UncheckedArray[char]](alloc0(13)) - for i, c in msg: - result.data[i] = c - result.size = 12 - proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = ## init - var obj = @"hello world!" + var obj = "hello world!" + # obj.shallow() echo "spawn worker" tp.spawn worker(obj, sig) diff --git a/tests/exampleGcFailures/exNoFailureNoGcCollectSeqDestroy.nim b/tests/exampleGcFailures/exNoFailureNoGcCollectSeqDestroy.nim new file mode 100644 index 0000000..7eeee5a --- /dev/null +++ b/tests/exampleGcFailures/exNoFailureNoGcCollectSeqDestroy.nim @@ -0,0 +1,48 @@ +import std/os +import chronos +import chronos/threadsync +import chronos/unittest2/asynctests +import taskpools + +type + Test* = object + count*: int + +proc `=destroy`*(obj: var Test) = + echo "destroy count: ", obj.count, " thread: ", getThreadId() + +proc worker(data: seq[Test], sig: ThreadSignalPtr) = + os.sleep(40) + echo "running worker: ", getThreadId() + echo "worker: ", data + discard sig.fireSync() + +proc runTest(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + ## init + var obj = @[Test(count: 1), Test(count: 2)] + + echo "spawn worker" + tp.spawn worker(obj, sig) + + obj[0].count = 10 + obj[1].count = 20 + ## adding fut.wait(100.milliseconds) creates memory issue + await wait(sig) + ## just doing the wait is fine: + # await wait(sig) + +proc runTests(tp: TaskPool, sig: ThreadSignalPtr) {.async.} = + for i in 1..1: + try: + echo "\n\nrunning main: ", getThreadId() + await runTest(tp, sig) + os.sleep(200) + except AsyncTimeoutError: + echo "looping..." + +suite "async tests": + var tp = Taskpool.new(num_threads = 2) # Default to the number of hardware threads. + let sig = ThreadSignalPtr.new().get() + + asyncTest "test": + await runTests(tp, sig)