Minimized example

codex/erasure/asyncbackend.nim

@@ -10,6 +10,7 @@
 import std/sequtils
 import std/sugar
 import std/macros
+import std/importutils
 
 import pkg/taskpools
 import pkg/taskpools/flowvars
@@ -38,13 +39,14 @@ import
   pkg/taskpools/flowvars,
   pkg/taskpools/ast_utils
 
-
 when (NimMajor,NimMinor,NimPatch) >= (1,6,0):
   import std/[isolation, tasks]
   export isolation
 else:
   import pkg/taskpools/shims_pre_1_6/tasks
 
+privateAccess Task
+
 import
   std/[cpuinfo, atomics, macros]
 
@@ -212,14 +214,14 @@ proc decodeTask(args: DecodeTaskArgs, odata: seq[seq[byte]], oparity: seq[seq[by
 
   var ptrsParity: seq[pointer]
   for i in 0..<oparity.len:
-    # if (unsafeAddr oparity[i]).isNil:
+    if (unsafeAddr oparity[i]).isNil:
-    #   echo "oparity is Nil " & $i
+      # echo "oparity is Nil " & $i
-    #   ptrsParity.add(unsafeAddr oparity)
-    # else:
-    if oparity[i].len > 0:
-      ptrsParity.add(unsafeAddr oparity[i][0])
-    else:
       ptrsParity.add(unsafeAddr oparity)
+    else:
+      if oparity[i].len > 0:
+        ptrsParity.add(unsafeAddr oparity[i][0])
+      else:
+        ptrsParity.add(unsafeAddr oparity)
 
   echo "bef unsafe hash of data " & unsafeHashOf(ptrsData, odata.mapIt(it.len))
   echo "bef unsafe hash of parity " & unsafeHashOf(ptrsParity, oparity.mapIt(it.len))

config.nims

@@ -65,7 +65,7 @@ else:
 
 --tlsEmulation:off
 --threads:on
---opt:speed
+--opt:none
 --excessiveStackTrace:on
 # enable metric collection
 --define:metrics

tests/codex/extract_gc_bug.nim

@@ -0,0 +1,73 @@
+import std/[isolation, macros, sequtils,tasks]
+import std/importutils
+import system/ansi_c
+
+privateAccess Task
+
+const size = 10000
+
+proc someTask(input: seq[seq[int]]) =
+  var sum = 0.int
+
+  for i in 0..<len(input):
+    for j in 0..<len(input[i]):
+      sum = sum + input[i][j]
+
+  echo $sum
+
+proc main() =
+  # 100 numbers of value 1
+  let input = newSeqWith[seq[int]](size, newSeqWith[int](size, 1.int))
+
+  # expandMacros: # expanded version of is in `expandedMain`
+  let task = toTask(someTask(input))
+
+  echo "sum is:"
+  task.invoke()
+
+  echo "sum should be:"
+  someTask(input)
+
+proc expandedMain() =
+  # 100 numbers of value 1
+  let input = newSeqWith[seq[int]](size, newSeqWith[int](size, 1.int))
+
+  type
+    ScratchObj_838860902 = object
+      input: seq[seq[int]]
+
+  let scratch_838860896 = cast[ptr ScratchObj_838860902](c_calloc(csize_t(1),
+      csize_t(8)))
+  if isNil(scratch_838860896):
+    raise
+      (ref OutOfMemDefect)(msg: "Could not allocate memory", parent: nil)
+  
+  var isoTemp_838860900 = isolate(input)
+  scratch_838860896.input = extract(isoTemp_838860900)
+
+  # discard GC_getStatistics() # GC stats - shouldn't change anything
+
+  GC_fullCollect() # GC - shouldn't change anything
+  discard newSeqWith[seq[int]](size, newSeqWith[int](size, 2.int)) # Allocation - shouldn't change anything
+
+  proc someTask_838860903(argsgensym8: pointer) {.gcsafe, nimcall.} =
+    let objTemp_838860899 = cast[ptr ScratchObj_838860902](argsgensym8)
+    let input_838860901 = objTemp_838860899.input
+    someTask(input_838860901)
+
+  proc destroyScratch_838860904(argsgensym8: pointer) {.gcsafe, nimcall.} =
+    let obj_838860905 = cast[ptr ScratchObj_838860902](argsgensym8)
+    `=destroy`(obj_838860905[])
+
+  let task = Task(callback: someTask_838860903, args: scratch_838860896,
+    destroy: destroyScratch_838860904)
+  
+  echo "sum is:"
+  task.invoke()
+
+  echo "sum should be:"
+  someTask(input)
+
+main()
+echo "---"
+expandedMain()

tests/codex/macro.nim

@@ -0,0 +1,52 @@
+let fut = newFlowVar(typeof(DecodeTaskResult))
+proc taskpool_decodeTask(args: DecodeTaskArgs; odata: seq[seq[byte]];
+                        oparity: seq[seq[byte]]; debug: bool;
+                        fut: Flowvar[DecodeTaskResult]) {.nimcall.} =
+let res`gensym115 = decodeTask(args, odata, oparity, debug)
+readyWith(fut, res`gensym115)
+
+let task =
+type
+    ScratchObj_11005855178 = object
+    args: DecodeTaskArgs
+    odata: seq[seq[byte]]
+    oparity: seq[seq[byte]]
+    debug: bool
+    fut: Flowvar[EncodeTaskResult]
+
+let scratch_11005855162 = cast[ptr ScratchObj_11005855178](c_calloc(
+    csize_t(1), csize_t(64)))
+if isNil(scratch_11005855162):
+    raise
+    (ref OutOfMemDefect)(msg: "Could not allocate memory", parent: nil)
+block:
+    var isoTemp_11005855168 = isolate(args)
+    scratch_11005855162.args = extract(isoTemp_11005855168)
+    var isoTemp_11005855170 = isolate(data[])
+    scratch_11005855162.odata = extract(isoTemp_11005855170)
+    var isoTemp_11005855172 = isolate(parity[])
+    scratch_11005855162.oparity = extract(isoTemp_11005855172)
+    var isoTemp_11005855174 = isolate(debug)
+    scratch_11005855162.debug = extract(isoTemp_11005855174)
+    var isoTemp_11005855176 = isolate(fut)
+    scratch_11005855162.fut = extract(isoTemp_11005855176)
+proc taskpool_decodeTask_11005855179(args`gensym120: pointer) {.gcsafe,
+    nimcall.} =
+    let objTemp_11005855167 = cast[ptr ScratchObj_11005855178](args`gensym120)
+    let args_11005855169 = objTemp_11005855167.args
+    let odata_11005855171 = objTemp_11005855167.odata
+    let oparity_11005855173 = objTemp_11005855167.oparity
+    let debug_11005855175 = objTemp_11005855167.debug
+    let fut_11005855177 = objTemp_11005855167.fut
+    taskpool_decodeTask(args_11005855169, odata_11005855171, oparity_11005855173,
+                        debug_11005855175, fut_11005855177)
+
+proc destroyScratch_11005855180(args`gensym120: pointer) {.gcsafe, nimcall.} =
+    let obj_11005855181 = cast[ptr ScratchObj_11005855178](args`gensym120)
+    `=destroy`(obj_11005855181[])
+
+Task(callback: taskpool_decodeTask_11005855179, args: scratch_11005855162,
+        destroy: destroyScratch_11005855180)
+let taskNode = new(TaskNode, workerContext.currentTask, task)
+schedule(workerContext, taskNode)
+fut

tests/codex/takspool_gc_bug.nim

@@ -0,0 +1,96 @@
+import std/unittest
+import std/sugar
+import std/sequtils
+import std/macros
+import std/times
+import std/os
+
+import pkg/taskpools
+import pkg/taskpools/flowvars
+
+import
+  system/ansi_c,
+  std/[random, cpuinfo, atomics, macros],
+  pkg/taskpools/channels_spsc_single,
+  pkg/taskpools/chase_lev_deques,
+  pkg/taskpools/event_notifiers,
+  pkg/taskpools/primitives/[barriers, allocs],
+  pkg/taskpools/instrumentation/[contracts, loggers],
+  pkg/taskpools/sparsesets,
+  pkg/taskpools/flowvars,
+  pkg/taskpools/ast_utils
+
+when (NimMajor,NimMinor,NimPatch) >= (1,6,0):
+  import std/[isolation, tasks]
+  export isolation
+else:
+  import pkg/taskpools/shims_pre_1_6/tasks
+
+
+proc someTask(input: seq[seq[byte]]): byte =
+  var sum = 0.byte
+
+  for i in 0..<len(input):
+    for j in 0..<len(input[i]):
+      sum = sum + input[i][j]
+
+  return sum
+
+proc main() =
+  let input = newSeqWith[seq[byte]](10, newSeqWith[byte](10, 1.byte))
+
+  var tp = Taskpool.new(numThreads = 4)
+
+  # ========================== 
+  # START OF MACRO EXPANSION
+  #
+  # expandMacros:
+  #   let f1 = tp.spawn someTask(input)
+
+  let fut = newFlowVar(typeof(byte))
+  proc taskpool_someTask(input: seq[seq[byte]]; fut: Flowvar[byte]) {.nimcall.} =
+    let resgensym25 = someTask(input)
+    readyWith(fut, resgensym25)
+  type
+      ScratchObj_838861353 = object
+        input: seq[seq[byte]]
+        fut: Flowvar[byte]
+
+  let scratch_838861345 = cast[ptr ScratchObj_838861353](c_calloc(csize_t(1),
+      csize_t(16)))
+  if isNil(scratch_838861345):
+    raise
+      (ref OutOfMemDefect)(msg: "Could not allocate memory", parent: nil)
+
+
+  var isoTemp_838861349 = isolate(input)
+  scratch_838861345.input = extract(isoTemp_838861349)
+
+  GC_fullCollect()
+  discard newSeqWith[seq[byte]](10, newSeqWith[byte](10, 2.byte)) # Additional allocation - shouldn't change anything (but it will)
+
+  var isoTemp_838861351 = isolate(fut)
+  scratch_838861345.fut = extract(isoTemp_838861351)
+
+  proc taskpool_someTask_838861354(argsgensym49: pointer) {.gcsafe, nimcall.} =
+    let objTemp_838861348 = cast[ptr ScratchObj_838861353](argsgensym49)
+    let input_838861350 = objTemp_838861348.input
+    let fut_838861352 = objTemp_838861348.fut
+    taskpool_someTask(input_838861350, fut_838861352)
+
+  proc destroyScratch_838861355(argsgensym49: pointer) {.gcsafe, nimcall.} =
+    let obj_838861356 = cast[ptr ScratchObj_838861353](argsgensym49)
+    `=destroy`(obj_838861356[])
+
+  let task = Task(callback: taskpool_someTask_838861354, args: scratch_838861345,
+        destroy: destroyScratch_838861355)
+  let taskNode = new(TaskNode, workerContext.currentTask, task)
+  schedule(workerContext, taskNode)
+
+  # END OF MACRO EXPANSION
+  # ========================== 
+
+  echo "result is " & $(sync(fut)) & " while it should be " & $(someTask(input))
+  tp.shutdown()
+
+main()

tests/codex/tp_example.nim

@@ -0,0 +1,42 @@
+import pkg/taskpools
+
+block: # Async without result
+
+  proc displayInt(x: int) =
+    stdout.write(x)
+    stdout.write(" - SUCCESS\n")
+
+  proc main() =
+    echo "\nSanity check 1: Printing 123456 654321 in parallel"
+
+    var tp = Taskpool.new(numThreads = 4)
+    tp.spawn displayInt(123456)
+    tp.spawn displayInt(654321)
+    tp.shutdown()
+
+  main()
+
+block: # Async/Await
+
+  var tp: Taskpool
+
+
+  proc asyncFib(n: int): int {.gcsafe.} =
+    if n < 2:
+      return n
+
+    let x = tp.spawn asyncFib(n-1)
+    let y = asyncFib(n-2)
+
+    result = sync(x) + y
+
+  proc main2() =
+    echo "\nSanity check 2: fib(20)"
+
+    tp = Taskpool.new()
+    let f = asyncFib(20)
+    tp.shutdown()
+
+    doAssert f == 6765
+
+  main2()