Merge branch 'stable' into gc

This commit is contained in:
Jacek Sieka 2023-07-05 12:01:59 +02:00 committed by GitHub
commit cd6a2ad372
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 82 additions and 228 deletions

View File

@ -27,11 +27,11 @@ jobs:
cpu: amd64
#- os: windows
#cpu: i386
branch: [version-1-2, version-1-4, version-1-6, devel]
branch: [version-1-6, version-2-0, devel]
include:
- target:
os: linux
builder: ubuntu-18.04
builder: ubuntu-20.04
shell: bash
- target:
os: macos

View File

@ -65,7 +65,7 @@ proc bpc_consume_nopoll(usec: int32) =
dummy_cpt()
proc bpc_produce(n, d: int32) {.gcsafe.} =
proc bpc_produce(n, d: int32) {.gcsafe, raises: [].} =
if d > 0:
# Create producer task
tp.spawn bpc_produce(n, d-1)

View File

@ -10,14 +10,14 @@ import
system/ansi_c, strformat, os, strutils, cpuinfo,
# Library
../../taskpools
when not defined(windows):
# bench
import ../wtime
var tp: Taskpool
proc dfs(depth, breadth: int): uint32 {.gcsafe.} =
proc dfs(depth, breadth: int): uint32 {.gcsafe, raises: [].} =
if depth == 0:
return 1

View File

@ -5,6 +5,8 @@
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
# From fibril
#
# Original license
@ -251,18 +253,22 @@ proc verify() =
me /= nx * ny
if mae > 1e-12:
echo &"Local maximal absolute error {mae:1.3e}"
quit 1
if mre > 1e-12:
echo &"Local maximal relative error {mre:1.3e}"
quit 1
if me > 1e-12:
echo &"Global mean absolute error {me:1.3e}"
quit 1
try:
if mae > 1e-12:
echo &"Local maximal absolute error {mae:1.3e}"
quit 1
if mre > 1e-12:
echo &"Local maximal relative error {mre:1.3e}"
quit 1
if me > 1e-12:
echo &"Global mean absolute error {me:1.3e}"
quit 1
except ValueError: raiseAssert "format strings"
echo "Verification successful"
{.pop.}
proc main() =
var nthreads: int
if existsEnv"TASKPOOL_NUM_THREADS":

View File

@ -88,10 +88,10 @@ func isValid(n: int32, a: CharArray): bool =
## Returns true if none of the queens conflict and 0 otherwise.
for i in 0'i32 ..< n:
let p = cast[int32](a[i])
let p = int32(a[i])
for j in i+1 ..< n:
let q = cast[int32](a[j])
let q = int32(a[j])
if q == p or q == p - (j-i) or q == p + (j-i):
return false
return true
@ -111,7 +111,7 @@ proc nqueens_ser(n, j: int32, a: CharArray): int32 =
if isValid(j+1, a):
result += nqueens_ser(n, j+1, a)
proc nqueens_par(n, j: int32, a: CharArray): int32 {.gcsafe.} =
proc nqueens_par(n, j: int32, a: CharArray): int32 {.gcsafe, raises: [].} =
if n == j:
# Good solution, count it

View File

@ -3,8 +3,11 @@ import ../taskpools
block: # Async without result
proc displayInt(x: int) =
stdout.write(x)
stdout.write(" - SUCCESS\n")
try:
stdout.write(x)
stdout.write(" - SUCCESS\n")
except IOError:
quit 1 # can't do anything productive
proc main() =
echo "\nSanity check 1: Printing 123456 654321 in parallel"
@ -21,7 +24,7 @@ block: # Async/Await
var tp: Taskpool
proc asyncFib(n: int): int {.gcsafe.} =
proc asyncFib(n: int): int {.gcsafe, raises: [].} =
if n < 2:
return n

View File

@ -1,7 +1,7 @@
# Demo of API using a very inefficient π approcimation algorithm.
import
std/[strutils, math, cpuinfo],
std/[strutils, cpuinfo],
../taskpools
# From https://github.com/nim-lang/Nim/blob/v1.6.2/tests/parallel/tpi.nim

View File

@ -7,16 +7,15 @@ description = "lightweight, energy-efficient, easily auditable threadpool"
license = "MIT"
skipDirs = @["tests"]
requires "nim >= 1.2.12"
requires "nim >= 1.6.0"
let nimc = getEnv("NIMC", "nim") # Which nim compiler to use
let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js)
let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler
let verbose = getEnv("V", "") notin ["", "0"]
let styleCheckStyle = if (NimMajor, NimMinor) < (1, 6): "hint" else: "error"
let cfg =
" --styleCheck:usages --styleCheck:" & styleCheckStyle &
" --styleCheck:usages --styleCheck:error" &
(if verbose: "" else: " --verbosity:0 --hints:off") &
" --skipParentCfg --skipUserCfg --outdir:build --nimcache:build/nimcache -f" &
" --stacktrace:on --linetrace:on" &

View File

@ -5,6 +5,8 @@
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
std/atomics,
./instrumentation/[contracts, loggers]
@ -28,11 +30,6 @@ type
itemSize*: uint8
buffer*{.align: 8.}: UncheckedArray[byte]
when (NimMajor,NimMinor,NimPatch) <= (1,4,0):
type AssertionDefect = AssertionError
{.push raises: [AssertionDefect].} # Ensure no exceptions can happen
proc `=`(
dest: var ChannelSPSCSingle,
source: ChannelSPSCSingle
@ -81,7 +78,7 @@ func trySend*[T](chan: var ChannelSPSCSingle, src: sink T): bool {.inline.} =
chan.full.store(true, moRelease)
return true
{.pop.} # raises: [AssertionDefect]
{.pop.} # raises: []
# Sanity checks
# ------------------------------------------------------------------------------

View File

@ -35,6 +35,8 @@
# To reduce contention, stealing is done on the opposite end from push/pop
# so that there is a race only for the very last task.
{.push raises: [].} # Ensure no exceptions can happen
import
system/ansi_c,
std/atomics,
@ -63,10 +65,6 @@ type
buf: Atomic[ptr Buf[T]]
garbage: ptr Buf[T]
when (NimMajor,NimMinor,NimPatch) <= (1,4,0):
type AssertionDefect = AssertionError
{.push raises: [AssertionDefect].} # Ensure no exceptions can happen
{.push overflowChecks: off.} # We don't want exceptions (for Defect) in a multithreaded context
# but we don't to deal with underflow of unsigned int either
# say "if a < b - c" with c > b
@ -201,4 +199,4 @@ proc steal*[T](deque: var ChaseLevDeque[T]): T =
return default(T)
{.pop.} # overflowChecks
{.pop.} # raises: [AssertionDefect]
{.pop.} # raises: []

View File

@ -1,5 +1,5 @@
# Nim-Taskpools
# Copyright (c) 2021 Status Research & Development GmbH
# Copyright (c) 2021-2023 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
@ -22,6 +22,8 @@
# but requires the threadpool to be message-passing based.
# https://github.com/mratsim/weave/blob/a230cce98a8524b2680011e496ec17de3c1039f2/weave/cross_thread_com/event_notifiers.nim
{.push raises: [].} # Ensure no exceptions can happen
import
std/locks,
./instrumentation/contracts
@ -36,10 +38,6 @@ type
parked: int
signals: int
when (NimMajor,NimMinor,NimPatch) <= (1,4,0):
type AssertionDefect = AssertionError
{.push raises: [AssertionDefect].} # Ensure no exceptions can happen
{.push overflowChecks: off.} # We don't want exceptions (for Defect) in a multithreaded context
# but we don't to deal with underflow of unsigned int either
# say "if a < b - c" with c > b

View File

@ -5,8 +5,9 @@
# * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
std/os,
./instrumentation/contracts,
./channels_spsc_single,
./primitives/allocs

View File

@ -1,12 +0,0 @@
# Versions
## std/tasks
- https://github.com/nim-lang/Nim/blob/3619a5a2aa1c7387ec7df01b195bc683943654ff/lib/std/tasks.nim
We don't support aborting if there is a closure as this requires [#17501](https://github.com/nim-lang/Nim/pull/17501/files)
## std/isolation
- https://github.com/nim-lang/Nim/blob/603af22b7ca46ac566f8c7c15402028f3f976a4e/lib/std/isolation.nim
## std/effecttraits
- https://github.com/nim-lang/Nim/blob/603af22b7ca46ac566f8c7c15402028f3f976a4e/lib/std/effecttraits.nim

View File

@ -1,54 +0,0 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2020 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## This module provides access to the inferred .raises effects
## for Nim's macro system.
## **Since**: Version 1.4.
##
## One can test for the existance of this standard module
## via `defined(nimHasEffectTraitsModule)`.
import macros
proc getRaisesListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim"
proc getTagsListImpl(n: NimNode): NimNode = discard "see compiler/vmops.nim"
proc isGcSafeImpl(n: NimNode): bool = discard "see compiler/vmops.nim"
proc hasNoSideEffectsImpl(n: NimNode): bool = discard "see compiler/vmops.nim"
proc getRaisesList*(fn: NimNode): NimNode =
## Extracts the `.raises` list of the func/proc/etc `fn`.
## `fn` has to be a resolved symbol of kind `nnkSym`. This
## implies that the macro that calls this proc should accept `typed`
## arguments and not `untyped` arguments.
expectKind fn, nnkSym
result = getRaisesListImpl(fn)
proc getTagsList*(fn: NimNode): NimNode =
## Extracts the `.tags` list of the func/proc/etc `fn`.
## `fn` has to be a resolved symbol of kind `nnkSym`. This
## implies that the macro that calls this proc should accept `typed`
## arguments and not `untyped` arguments.
expectKind fn, nnkSym
result = getTagsListImpl(fn)
proc isGcSafe*(fn: NimNode): bool =
## Return true if the func/proc/etc `fn` is `gcsafe`.
## `fn` has to be a resolved symbol of kind `nnkSym`. This
## implies that the macro that calls this proc should accept `typed`
## arguments and not `untyped` arguments.
expectKind fn, nnkSym
result = isGcSafeImpl(fn)
proc hasNoSideEffects*(fn: NimNode): bool =
## Return true if the func/proc/etc `fn` has `noSideEffect`.
## `fn` has to be a resolved symbol of kind `nnkSym`. This
## implies that the macro that calls this proc should accept `typed`
## arguments and not `untyped` arguments.
expectKind fn, nnkSym
result = hasNoSideEffectsImpl(fn)

View File

@ -1,50 +0,0 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2020 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## This module implements the `Isolated[T]` type for
## safe construction of isolated subgraphs that can be
## passed efficiently to different channels and threads.
##
## .. warning:: This module is experimental and its interface may change.
##
type
Isolated*[T] = object ## Isolated data can only be moved, not copied.
value: T
proc `=copy`*[T](dest: var Isolated[T]; src: Isolated[T]) {.error.}
proc `=sink`*[T](dest: var Isolated[T]; src: Isolated[T]) {.inline.} =
# delegate to value's sink operation
`=sink`(dest.value, src.value)
proc `=destroy`*[T](dest: var Isolated[T]) {.inline.} =
# delegate to value's destroy operation
`=destroy`(dest.value)
# XXX: removed the {.magic: "Isolate".}
func isolate*[T](value: sink T): Isolated[T] =
## Creates an isolated subgraph from the expression `value`.
## Isolation is checked at compile time.
##
## Please read https://github.com/nim-lang/RFCs/issues/244
## for more details.
Isolated[T](value: value)
func unsafeIsolate*[T](value: sink T): Isolated[T] =
## Creates an isolated subgraph from the expression `value`.
##
## .. warning:: The proc doesn't check whether `value` is isolated.
##
Isolated[T](value: value)
func extract*[T](src: var Isolated[T]): T =
## Returns the internal value of `src`.
## The value is moved from `src`.
result = move(src.value)

View File

@ -35,10 +35,7 @@
# In case a thread is blocked for IO, other threads can steal pending tasks in that thread.
# If all threads are pending for IO, the threadpool will not make any progress and be soft-locked.
when (NimMajor,NimMinor,NimPatch) <= (1,4,0):
type AssertionDefect = AssertionError
{.push raises: [AssertionDefect].} # Ensure no exceptions can happen
{.push raises: [].} # Ensure no exceptions can happen
import
system/ansi_c,
@ -50,17 +47,13 @@ import
./instrumentation/[contracts, loggers],
./sparsesets,
./flowvars,
./ast_utils
./ast_utils,
./tasks
export
# flowvars
Flowvar, isSpawned, isReady, sync
Flowvar, isSpawned, isReady, sync, tasks
when (NimMajor,NimMinor,NimPatch) >= (1,6,0):
import std/[isolation, tasks]
export isolation
else:
import ./shims_pre_1_6/tasks
type
WorkerID = int32
@ -188,14 +181,11 @@ proc new(T: type TaskNode, parent: TaskNode, task: sink Task): T =
tn.task = task
return tn
proc runTask(tn: var TaskNode) {.raises:[Exception], inline.} =
proc runTask(tn: var TaskNode) {.raises:[], inline.} =
## Run a task and consumes the taskNode
tn.task.invoke()
when (NimMajor,NimMinor,NimPatch) >= (1,6,0):
{.gcsafe.}: # Upstream missing tagging `=destroy` as gcsafe
tn.task.`=destroy`()
else:
tn.task.shim_destroy()
{.gcsafe.}: # Upstream missing tagging `=destroy` as gcsafe
tn.task.`=destroy`()
tn.c_free()
proc schedule(ctx: WorkerContext, tn: sink TaskNode) {.inline.} =
@ -254,7 +244,7 @@ const RootTask = default(Task) # TODO: sentinel value different from null task
template isRootTask(task: Task): bool =
task == RootTask
proc forceFuture*[T](fv: Flowvar[T], parentResult: var T) {.raises:[Exception].} =
proc forceFuture*[T](fv: Flowvar[T], parentResult: var T) {.raises:[].} =
## Eagerly complete an awaited FlowVar
template ctx: untyped = workerContext
@ -445,11 +435,9 @@ macro spawn*(tp: Taskpool, fnCall: typed): untyped =
# Package in a task
let taskNode = ident("taskNode")
let task = ident("task")
if not needFuture:
result.add quote do:
let `task` = toTask(`fnCall`)
let `taskNode` = TaskNode.new(workerContext.currentTask, `task`)
let `taskNode` = TaskNode.new(workerContext.currentTask, toTask(`fnCall`))
schedule(workerContext, `taskNode`)
else:
@ -512,8 +500,7 @@ macro spawn*(tp: Taskpool, fnCall: typed): untyped =
asyncCall.add fut
result.add quote do:
let `task` = toTask(`asyncCall`)
let `taskNode` = TaskNode.new(workerContext.currentTask, `task`)
let `taskNode` = TaskNode.new(workerContext.currentTask, toTask(`asyncCall`))
schedule(workerContext, `taskNode`)
# Return the future / flowvar

View File

@ -1,23 +1,20 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2021 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# Copyright (c) 2023- Status Research & Development GmbH
## This module provides basic primitives for creating parallel programs.
## A `Task` should be only owned by a single Thread, it cannot be shared by threads.
##
## The module was forked from std/tasks in Nim 1.6 to add new functionality and
## tune to the taskpools use case.
import std/[macros, typetraits]
import std/[macros, isolation, typetraits]
import system/ansi_c
import ./isolation
export isolation
when compileOption("threads"):
from ./effecttraits import isGcSafe
from std/effecttraits import isGcSafe
#
@ -56,18 +53,18 @@ when compileOption("threads"):
# let t = Task(callback: hello_369098781, args: scratch_369098762, destroy: destroyScratch_369098782)
#
{.push raises: [].}
type
Task* = object ## `Task` contains the callback and its arguments.
callback: proc (args: pointer) {.nimcall, gcsafe.}
callback: proc (args: pointer) {.nimcall, gcsafe, raises: [].}
args: pointer
destroy: proc (args: pointer) {.nimcall, gcsafe.}
destroy: proc (args: pointer) {.nimcall, gcsafe, raises: [].}
# XXX: ⚠️ No destructors for 1.2 due to unreliable codegen
# proc `=copy`*(x: var Task, y: Task) {.error.}
proc `=copy`*(x: var Task, y: Task) {.error.}
proc shim_destroy*(t: var Task) {.inline, gcsafe.} =
proc `=destroy`*(t: var Task) {.inline, gcsafe.} =
## Frees the resources allocated for a `Task`.
if t.args != nil:
if t.destroy != nil:
@ -86,14 +83,9 @@ template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr
# var isoTempB = isolate(literal)
# scratch.b = extract(isolateB)
let isolatedTemp = genSym(nskTemp, "isoTemp")
# XXX: Fix sym bindings
# scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam))
# scratchAssignList.add newAssignment(scratchDotExpr,
# newCall(newIdentNode("extract"), isolatedTemp))
scratchAssignList.add newVarStmt(isolatedTemp, newCall(bindSym("isolate"), procParam))
scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam))
scratchAssignList.add newAssignment(scratchDotExpr,
newCall(bindSym("extract"), isolatedTemp))
newCall(newIdentNode("extract"), isolatedTemp))
template addAllNode(assignParam: NimNode, procParam: NimNode) =
let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0])
@ -115,16 +107,12 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
doAssert getTypeInst(e).typeKind == ntyVoid
# requires 1.6
# when compileOption("threads"):
# if not isGcSafe(e[0]):
# error("'toTask' takes a GC safe call expression")
when compileOption("threads"):
if not isGcSafe(e[0]):
error("'toTask' takes a GC safe call expression", e)
# TODO
# https://github.com/nim-lang/Nim/pull/17501/files
#
# if hasClosure(e[0]):
# error("closure call is not allowed")
if hasClosure(e[0]):
error("closure call is not allowed", e)
if e.len > 1:
let scratchIdent = genSym(kind = nskTemp, ident = "scratch")
@ -151,17 +139,17 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
param = param[0]
if param.typeKind in {ntyExpr, ntyStmt}:
error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter")
error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e)
case param.kind
of nnkVarTy:
error("'toTask'ed function cannot have a 'var' parameter")
error("'toTask'ed function cannot have a 'var' parameter", e)
of nnkBracketExpr:
if param[0].typeKind == ntyTypeDesc:
callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i])
elif param[0].typeKind in {ntyVarargs, ntyOpenArray}:
if param[1].typeKind in {ntyExpr, ntyStmt}:
error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter")
error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e)
let
seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1])
seqCallNode = newCall("@", e[i])
@ -177,7 +165,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
of nnkCharLit..nnkNilLit:
callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i])
else:
error("not supported type kinds")
error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e)
let scratchObjType = genSym(kind = nskType, ident = "ScratchObj")
let scratchObj = nnkTypeSection.newTree(
@ -203,9 +191,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
let scratchCheck = quote do:
if `scratchIdent`.isNil:
# Renamed in 1.4
# raise newException(OutOfMemDefect, "Could not allocate memory")
raise newException(OutOfMemError, "Could not allocate memory")
raise newException(OutOfMemDefect, "Could not allocate memory")
var stmtList = newStmtList()
stmtList.add(scratchObj)
@ -222,21 +208,16 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
let destroyName = genSym(nskProc, "destroyScratch")
let objTemp2 = genSym(ident = "obj")
let tempNode = quote("@") do:
# XXX:
# We avoid destructors for Nim 1.2 due to bad codegen
# For taskpool there are no destructor to run.
# We ensure that by checking that we only operate on plain old data
static: doAssert supportsCopyMem(@scratchObjType)
# `=destroy`(@objTemp2[])
`=destroy`(@objTemp2[])
result = quote do:
`stmtList`
proc `funcName`(args: pointer) {.gcsafe, nimcall.} =
proc `funcName`(args: pointer) {.gcsafe, nimcall, raises: [].} =
let `objTemp` = cast[ptr `scratchObjType`](args)
`functionStmtList`
proc `destroyName`(args: pointer) {.nimcall.} =
proc `destroyName`(args: pointer) {.gcsafe, nimcall, raises: [].} =
let `objTemp2` = cast[ptr `scratchObjType`](args)
`tempNode`
@ -246,7 +227,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
let funcName = genSym(nskProc, e[0].strVal)
result = quote do:
proc `funcName`(args: pointer) {.gcsafe, nimcall.} =
proc `funcName`(args: pointer) {.gcsafe, nimcall, raises: [].} =
`funcCall`
Task(callback: `funcName`, args: nil)
@ -254,7 +235,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC
when defined(nimTasksDebug):
echo result.repr
runnableExamples("--gc:orc"):
when isMainModule:
block:
var num = 0
proc hello(a: int) = inc num, a