From f91ac169dc43c015b8631f33f1eb51623581b251 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Sun, 23 Jul 2023 19:40:57 +0300 Subject: [PATCH 01/50] Fix `NoVerifyServerName` do not actually disables SNI extension. (#423) Fix HTTP client SSL/TLS error information is now part of connection error exception. --- chronos/apps/http/httpclient.nim | 15 ++++++++++++--- chronos/streams/tlsstream.nim | 33 ++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 6e9ea0c..9c79889 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -523,7 +523,7 @@ proc connect(session: HttpSessionRef, ha: HttpAddress): Future[HttpClientConnectionRef] {.async.} = ## Establish new connection with remote server using ``url`` and ``flags``. ## On success returns ``HttpClientConnectionRef`` object. - + var lastError = "" # Here we trying to connect to every possible remote host address we got after # DNS resolution. for address in ha.addresses: @@ -547,9 +547,14 @@ proc connect(session: HttpSessionRef, except CancelledError as exc: await res.closeWait() raise exc - except AsyncStreamError: + except TLSStreamProtocolError as exc: await res.closeWait() res.state = HttpClientConnectionState.Error + lastError = $exc.msg + except AsyncStreamError as exc: + await res.closeWait() + res.state = HttpClientConnectionState.Error + lastError = $exc.msg of HttpClientScheme.Nonsecure: res.state = HttpClientConnectionState.Ready res @@ -557,7 +562,11 @@ proc connect(session: HttpSessionRef, return conn # If all attempts to connect to the remote host have failed. - raiseHttpConnectionError("Could not connect to remote host") + if len(lastError) > 0: + raiseHttpConnectionError("Could not connect to remote host, reason: " & + lastError) + else: + raiseHttpConnectionError("Could not connect to remote host") proc removeConnection(session: HttpSessionRef, conn: HttpClientConnectionRef) {.async.} = diff --git a/chronos/streams/tlsstream.nim b/chronos/streams/tlsstream.nim index ceacaff..2999f7a 100644 --- a/chronos/streams/tlsstream.nim +++ b/chronos/streams/tlsstream.nim @@ -95,6 +95,7 @@ type trustAnchors: TrustAnchorStore SomeTLSStreamType* = TLSStreamReader|TLSStreamWriter|TLSAsyncStream + SomeTrustAnchorType* = TrustAnchorStore | openArray[X509TrustAnchor] TLSStreamError* = object of AsyncStreamError TLSStreamHandshakeError* = object of TLSStreamError @@ -139,12 +140,14 @@ proc newTLSStreamProtocolError[T](message: T): ref TLSStreamProtocolError = proc raiseTLSStreamProtocolError[T](message: T) {.noreturn, noinline.} = raise newTLSStreamProtocolImpl(message) -proc new*(T: typedesc[TrustAnchorStore], anchors: openArray[X509TrustAnchor]): TrustAnchorStore = +proc new*(T: typedesc[TrustAnchorStore], + anchors: openArray[X509TrustAnchor]): TrustAnchorStore = var res: seq[X509TrustAnchor] for anchor in anchors: res.add(anchor) - doAssert(unsafeAddr(anchor) != unsafeAddr(res[^1]), "Anchors should be copied") - return TrustAnchorStore(anchors: res) + doAssert(unsafeAddr(anchor) != unsafeAddr(res[^1]), + "Anchors should be copied") + TrustAnchorStore(anchors: res) proc tlsWriteRec(engine: ptr SslEngineContext, writer: TLSStreamWriter): Future[TLSResult] {.async.} = @@ -453,15 +456,16 @@ proc getSignerAlgo(xc: X509Certificate): int = else: int(x509DecoderGetSignerKeyType(dc)) -proc newTLSClientAsyncStream*(rsource: AsyncStreamReader, - wsource: AsyncStreamWriter, - serverName: string, - bufferSize = SSL_BUFSIZE_BIDI, - minVersion = TLSVersion.TLS12, - maxVersion = TLSVersion.TLS12, - flags: set[TLSFlags] = {}, - trustAnchors: TrustAnchorStore | openArray[X509TrustAnchor] = MozillaTrustAnchors - ): TLSAsyncStream = +proc newTLSClientAsyncStream*( + rsource: AsyncStreamReader, + wsource: AsyncStreamWriter, + serverName: string, + bufferSize = SSL_BUFSIZE_BIDI, + minVersion = TLSVersion.TLS12, + maxVersion = TLSVersion.TLS12, + flags: set[TLSFlags] = {}, + trustAnchors: SomeTrustAnchorType = MozillaTrustAnchors + ): TLSAsyncStream = ## Create new TLS asynchronous stream for outbound (client) connections ## using reading stream ``rsource`` and writing stream ``wsource``. ## @@ -484,7 +488,8 @@ proc newTLSClientAsyncStream*(rsource: AsyncStreamReader, ## a ``TrustAnchorStore`` you should reuse the same instance for ## every call to avoid making a copy of the trust anchors per call. when trustAnchors is TrustAnchorStore: - doAssert(len(trustAnchors.anchors) > 0, "Empty trust anchor list is invalid") + doAssert(len(trustAnchors.anchors) > 0, + "Empty trust anchor list is invalid") else: doAssert(len(trustAnchors) > 0, "Empty trust anchor list is invalid") var res = TLSAsyncStream() @@ -524,7 +529,7 @@ proc newTLSClientAsyncStream*(rsource: AsyncStreamReader, uint16(maxVersion)) if TLSFlags.NoVerifyServerName in flags: - let err = sslClientReset(res.ccontext, "", 0) + let err = sslClientReset(res.ccontext, nil, 0) if err == 0: raise newException(TLSStreamInitError, "Could not initialize TLS layer") else: From 53e9f75735464ea196cfc2b92b8bb98fe59ca693 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Fri, 28 Jul 2023 11:54:53 +0300 Subject: [PATCH 02/50] Add some helpers for asyncproc. (#424) * Initial commit. * Adjust posix tests. * Fix compilation issue. * Attempt to fix flaky addProcess() test. --- chronos/asyncproc.nim | 102 +++++++++++++++++++++++++++++++++++++----- tests/testproc.bat | 6 +++ tests/testproc.nim | 52 ++++++++++++++++++++- tests/testproc.sh | 8 ++++ 4 files changed, 157 insertions(+), 11 deletions(-) diff --git a/chronos/asyncproc.nim b/chronos/asyncproc.nim index 8d0cdb7..8df8e33 100644 --- a/chronos/asyncproc.nim +++ b/chronos/asyncproc.nim @@ -24,7 +24,8 @@ const ## AsyncProcess leaks tracker name type - AsyncProcessError* = object of CatchableError + AsyncProcessError* = object of AsyncError + AsyncProcessTimeoutError* = object of AsyncProcessError AsyncProcessResult*[T] = Result[T, OSErrorCode] @@ -107,6 +108,9 @@ type stdError*: string status*: int + WaitOperation {.pure.} = enum + Kill, Terminate + template Pipe*(t: typedesc[AsyncProcess]): ProcessStreamHandle = ProcessStreamHandle(kind: ProcessStreamHandleKind.Auto) @@ -294,6 +298,11 @@ proc raiseAsyncProcessError(msg: string, exc: ref CatchableError = nil) {. msg & " ([" & $exc.name & "]: " & $exc.msg & ")" raise newException(AsyncProcessError, message) +proc raiseAsyncProcessTimeoutError() {. + noreturn, noinit, noinline, raises: [AsyncProcessTimeoutError].} = + let message = "Operation timed out" + raise newException(AsyncProcessTimeoutError, message) + proc raiseAsyncProcessError(msg: string, error: OSErrorCode|cint) {. noreturn, noinit, noinline, raises: [AsyncProcessError].} = when error is OSErrorCode: @@ -1189,6 +1198,45 @@ proc closeProcessStreams(pipes: AsyncProcessPipes, res allFutures(pending) +proc opAndWaitForExit(p: AsyncProcessRef, op: WaitOperation, + timeout = InfiniteDuration): Future[int] {.async.} = + let timerFut = + if timeout == InfiniteDuration: + newFuture[void]("chronos.killAndwaitForExit") + else: + sleepAsync(timeout) + + while true: + if p.running().get(true): + # We ignore operation errors because we going to repeat calling + # operation until process will not exit. + case op + of WaitOperation.Kill: + discard p.kill() + of WaitOperation.Terminate: + discard p.terminate() + else: + let exitCode = p.peekExitCode().valueOr: + raiseAsyncProcessError("Unable to peek process exit code", error) + if not(timerFut.finished()): + await cancelAndWait(timerFut) + return exitCode + + let waitFut = p.waitForExit().wait(100.milliseconds) + discard await race(FutureBase(waitFut), FutureBase(timerFut)) + + if waitFut.finished() and not(waitFut.failed()): + let res = p.peekExitCode() + if res.isOk(): + if not(timerFut.finished()): + await cancelAndWait(timerFut) + return res.get() + + if timerFut.finished(): + if not(waitFut.finished()): + await waitFut.cancelAndWait() + raiseAsyncProcessTimeoutError() + proc closeWait*(p: AsyncProcessRef) {.async.} = # Here we ignore all possible errrors, because we do not want to raise # exceptions. @@ -1216,14 +1264,15 @@ proc execCommand*(command: string, options = {AsyncProcessOption.EvalCommand}, timeout = InfiniteDuration ): Future[int] {.async.} = - let poptions = options + {AsyncProcessOption.EvalCommand} - let process = await startProcess(command, options = poptions) - let res = - try: - await process.waitForExit(timeout) - finally: - await process.closeWait() - return res + let + poptions = options + {AsyncProcessOption.EvalCommand} + process = await startProcess(command, options = poptions) + res = + try: + await process.waitForExit(timeout) + finally: + await process.closeWait() + res proc execCommandEx*(command: string, options = {AsyncProcessOption.EvalCommand}, @@ -1256,10 +1305,43 @@ proc execCommandEx*(command: string, finally: await process.closeWait() - return res + res proc pid*(p: AsyncProcessRef): int = ## Returns process ``p`` identifier. int(p.processId) template processId*(p: AsyncProcessRef): int = pid(p) + +proc killAndWaitForExit*(p: AsyncProcessRef, + timeout = InfiniteDuration): Future[int] = + ## Perform continuous attempts to kill the ``p`` process for specified period + ## of time ``timeout``. + ## + ## On Posix systems, killing means sending ``SIGKILL`` to the process ``p``, + ## On Windows, it uses ``TerminateProcess`` to kill the process ``p``. + ## + ## If the process ``p`` fails to be killed within the ``timeout`` time, it + ## will raise ``AsyncProcessTimeoutError``. + ## + ## In case of error this it will raise ``AsyncProcessError``. + ## + ## Returns process ``p`` exit code. + opAndWaitForExit(p, WaitOperation.Kill, timeout) + +proc terminateAndWaitForExit*(p: AsyncProcessRef, + timeout = InfiniteDuration): Future[int] = + ## Perform continuous attempts to terminate the ``p`` process for specified + ## period of time ``timeout``. + ## + ## On Posix systems, terminating means sending ``SIGTERM`` to the process + ## ``p``, on Windows, it uses ``TerminateProcess`` to terminate the process + ## ``p``. + ## + ## If the process ``p`` fails to be terminated within the ``timeout`` time, it + ## will raise ``AsyncProcessTimeoutError``. + ## + ## In case of error this it will raise ``AsyncProcessError``. + ## + ## Returns process ``p`` exit code. + opAndWaitForExit(p, WaitOperation.Terminate, timeout) diff --git a/tests/testproc.bat b/tests/testproc.bat index 314bea7..11b4047 100644 --- a/tests/testproc.bat +++ b/tests/testproc.bat @@ -2,6 +2,8 @@ IF /I "%1" == "STDIN" ( GOTO :STDINTEST +) ELSE IF /I "%1" == "TIMEOUT1" ( + GOTO :TIMEOUTTEST1 ) ELSE IF /I "%1" == "TIMEOUT2" ( GOTO :TIMEOUTTEST2 ) ELSE IF /I "%1" == "TIMEOUT10" ( @@ -19,6 +21,10 @@ SET /P "INPUTDATA=" ECHO STDIN DATA: %INPUTDATA% EXIT 0 +:TIMEOUTTEST1 +ping -n 1 127.0.0.1 > NUL +EXIT 1 + :TIMEOUTTEST2 ping -n 2 127.0.0.1 > NUL EXIT 2 diff --git a/tests/testproc.nim b/tests/testproc.nim index b038325..cfcafe6 100644 --- a/tests/testproc.nim +++ b/tests/testproc.nim @@ -96,7 +96,11 @@ suite "Asynchronous process management test suite": let options = {AsyncProcessOption.EvalCommand} - command = "exit 1" + command = + when defined(windows): + "tests\\testproc.bat timeout1" + else: + "tests/testproc.sh timeout1" process = await startProcess(command, options = options) @@ -407,6 +411,52 @@ suite "Asynchronous process management test suite": finally: await process.closeWait() + asyncTest "killAndWaitForExit() test": + let command = + when defined(windows): + ("tests\\testproc.bat", "timeout10", 0) + else: + ("tests/testproc.sh", "timeout10", 128 + int(SIGKILL)) + let process = await startProcess(command[0], arguments = @[command[1]]) + try: + let exitCode = await process.killAndWaitForExit(10.seconds) + check exitCode == command[2] + finally: + await process.closeWait() + + asyncTest "terminateAndWaitForExit() test": + let command = + when defined(windows): + ("tests\\testproc.bat", "timeout10", 0) + else: + ("tests/testproc.sh", "timeout10", 128 + int(SIGTERM)) + let process = await startProcess(command[0], arguments = @[command[1]]) + try: + let exitCode = await process.terminateAndWaitForExit(10.seconds) + check exitCode == command[2] + finally: + await process.closeWait() + + asyncTest "terminateAndWaitForExit() timeout test": + when defined(windows): + skip() + else: + let + command = ("tests/testproc.sh", "noterm", 128 + int(SIGKILL)) + process = await startProcess(command[0], arguments = @[command[1]]) + # We should wait here to allow `bash` execute `trap` command, otherwise + # our test script will be killed with SIGTERM. Increase this timeout + # if test become flaky. + await sleepAsync(1.seconds) + try: + expect AsyncProcessTimeoutError: + let exitCode {.used.} = + await process.terminateAndWaitForExit(1.seconds) + let exitCode = await process.killAndWaitForExit(10.seconds) + check exitCode == command[2] + finally: + await process.closeWait() + test "File descriptors leaks test": when defined(windows): skip() diff --git a/tests/testproc.sh b/tests/testproc.sh index 1725d49..c5e7e0a 100755 --- a/tests/testproc.sh +++ b/tests/testproc.sh @@ -3,6 +3,9 @@ if [ "$1" == "stdin" ]; then read -r inputdata echo "STDIN DATA: $inputdata" +elif [ "$1" == "timeout1" ]; then + sleep 1 + exit 1 elif [ "$1" == "timeout2" ]; then sleep 2 exit 2 @@ -15,6 +18,11 @@ elif [ "$1" == "bigdata" ]; then done elif [ "$1" == "envtest" ]; then echo "$CHRONOSASYNC" +elif [ "$1" == "noterm" ]; then + trap -- '' SIGTERM + while true; do + sleep 1 + done else echo "arguments missing" fi From 926956bcbee5f4f49124f1f2215530a2d93bac96 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Sun, 30 Jul 2023 12:43:25 +0300 Subject: [PATCH 03/50] Add time used to establish HTTP client connection. (#427) --- chronos/apps/http/httpclient.nim | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 9c79889..63ffc37 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -108,6 +108,7 @@ type remoteHostname*: string flags*: set[HttpClientConnectionFlag] timestamp*: Moment + duration*: Duration HttpClientConnectionRef* = ref HttpClientConnection @@ -233,6 +234,12 @@ template setDuration( reqresp.duration = timestamp - reqresp.timestamp reqresp.connection.setTimestamp(timestamp) +template setDuration(conn: HttpClientConnectionRef): untyped = + if not(isNil(conn)): + let timestamp = Moment.now() + conn.duration = timestamp - conn.timestamp + conn.setTimestamp(timestamp) + template isReady(conn: HttpClientConnectionRef): bool = (conn.state == HttpClientConnectionState.Ready) and (HttpClientConnectionFlag.KeepAlive in conn.flags) and @@ -596,9 +603,9 @@ proc acquireConnection( ): Future[HttpClientConnectionRef] {.async.} = ## Obtain connection from ``session`` or establish a new one. var default: seq[HttpClientConnectionRef] + let timestamp = Moment.now() if session.connectionPoolEnabled(flags): # Trying to reuse existing connection from our connection's pool. - let timestamp = Moment.now() # We looking for non-idle connection at `Ready` state, all idle connections # will be freed by sessionWatcher(). for connection in session.connections.getOrDefault(ha.id): @@ -615,6 +622,8 @@ proc acquireConnection( connection.state = HttpClientConnectionState.Acquired session.connections.mgetOrPut(ha.id, default).add(connection) inc(session.connectionsCount) + connection.setTimestamp(timestamp) + connection.setDuration() return connection proc releaseConnection(session: HttpSessionRef, From d214bcfb4f4995bcb9a66b17893a6d32a5e138ca Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 31 Jul 2023 22:40:00 +0300 Subject: [PATCH 04/50] Increase backlog defaults to maximum possible values. (#428) --- chronos/apps/http/httpserver.nim | 2 +- chronos/apps/http/shttpserver.nim | 2 +- chronos/transports/stream.nim | 34 +++++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index b86c0b3..c1e45c0 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -187,7 +187,7 @@ proc new*(htype: typedesc[HttpServerRef], serverIdent = "", maxConnections: int = -1, bufferSize: int = 4096, - backlogSize: int = 100, + backlogSize: int = DefaultBacklogSize, httpHeadersTimeout = 10.seconds, maxHeadersSize: int = 8192, maxRequestBodySize: int = 1_048_576): HttpResult[HttpServerRef] {. diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index 927ca62..b993cb5 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -90,7 +90,7 @@ proc new*(htype: typedesc[SecureHttpServerRef], secureFlags: set[TLSFlags] = {}, maxConnections: int = -1, bufferSize: int = 4096, - backlogSize: int = 100, + backlogSize: int = DefaultBacklogSize, httpHeadersTimeout = 10.seconds, maxHeadersSize: int = 8192, maxRequestBodySize: int = 1_048_576 diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 257c475..45e4054 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -61,6 +61,7 @@ type const StreamTransportTrackerName* = "stream.transport" StreamServerTrackerName* = "stream.server" + DefaultBacklogSize* = high(int32) when defined(windows): type @@ -1819,11 +1820,32 @@ proc closeWait*(server: StreamServer): Future[void] = server.close() server.join() +proc getBacklogSize(backlog: int): cint = + doAssert(backlog >= 0 and backlog <= high(int32)) + when defined(windows): + # The maximum length of the queue of pending connections. If set to + # SOMAXCONN, the underlying service provider responsible for + # socket s will set the backlog to a maximum reasonable value. If set to + # SOMAXCONN_HINT(N) (where N is a number), the backlog value will be N, + # adjusted to be within the range (200, 65535). Note that SOMAXCONN_HINT + # can be used to set the backlog to a larger value than possible with + # SOMAXCONN. + # + # Microsoft SDK values are + # #define SOMAXCONN 0x7fffffff + # #define SOMAXCONN_HINT(b) (-(b)) + if backlog != high(int32): + cint(-backlog) + else: + cint(backlog) + else: + cint(backlog) + proc createStreamServer*(host: TransportAddress, cbproc: StreamCallback, flags: set[ServerFlags] = {}, sock: AsyncFD = asyncInvalidSocket, - backlog: int = 100, + backlog: int = DefaultBacklogSize, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil, @@ -1906,7 +1928,7 @@ proc createStreamServer*(host: TransportAddress, raiseTransportOsError(err) fromSAddr(addr saddr, slen, localAddress) - if listen(SocketHandle(serverSocket), cint(backlog)) != 0: + if listen(SocketHandle(serverSocket), getBacklogSize(backlog)) != 0: let err = osLastError() if sock == asyncInvalidSocket: discard closeFd(SocketHandle(serverSocket)) @@ -1980,7 +2002,7 @@ proc createStreamServer*(host: TransportAddress, raiseTransportOsError(err) fromSAddr(addr saddr, slen, localAddress) - if listen(SocketHandle(serverSocket), cint(backlog)) != 0: + if listen(SocketHandle(serverSocket), getBacklogSize(backlog)) != 0: let err = osLastError() if sock == asyncInvalidSocket: discard unregisterAndCloseFd(serverSocket) @@ -2031,7 +2053,7 @@ proc createStreamServer*(host: TransportAddress, proc createStreamServer*(host: TransportAddress, flags: set[ServerFlags] = {}, sock: AsyncFD = asyncInvalidSocket, - backlog: int = 100, + backlog: int = DefaultBacklogSize, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil, @@ -2045,7 +2067,7 @@ proc createStreamServer*[T](host: TransportAddress, flags: set[ServerFlags] = {}, udata: ref T, sock: AsyncFD = asyncInvalidSocket, - backlog: int = 100, + backlog: int = DefaultBacklogSize, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil): StreamServer {. @@ -2059,7 +2081,7 @@ proc createStreamServer*[T](host: TransportAddress, flags: set[ServerFlags] = {}, udata: ref T, sock: AsyncFD = asyncInvalidSocket, - backlog: int = 100, + backlog: int = DefaultBacklogSize, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil): StreamServer {. From 5c39bf47bea3c7f98c11c707475ebfdb52438cef Mon Sep 17 00:00:00 2001 From: rockcavera Date: Mon, 31 Jul 2023 19:28:34 -0300 Subject: [PATCH 05/50] fixing unfreed memory leak with `freeAddrInfo()` (#425) * fixing unfreed memory leak with `freeAddrInfo()` * `freeaddrinfo` to `freeAddrInfo()` --- chronos/osdefs.nim | 14 +++++++------- chronos/transports/common.nim | 4 ++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/chronos/osdefs.nim b/chronos/osdefs.nim index ecf770b..bf5c060 100644 --- a/chronos/osdefs.nim +++ b/chronos/osdefs.nim @@ -708,7 +708,7 @@ when defined(windows): res: var ptr AddrInfo): cint {. stdcall, dynlib: "ws2_32", importc: "getaddrinfo", sideEffect.} - proc freeaddrinfo*(ai: ptr AddrInfo) {. + proc freeAddrInfo*(ai: ptr AddrInfo) {. stdcall, dynlib: "ws2_32", importc: "freeaddrinfo", sideEffect.} proc createIoCompletionPort*(fileHandle: HANDLE, @@ -880,7 +880,7 @@ elif defined(macos) or defined(macosx): sigemptyset, sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, + socketpair, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, @@ -905,7 +905,7 @@ elif defined(macos) or defined(macosx): sigemptyset, sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, + socketpair, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, @@ -947,7 +947,7 @@ elif defined(linux): unlink, listen, sendmsg, recvmsg, getpid, fcntl, pthread_sigmask, sigprocmask, clock_gettime, signal, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, + socketpair, freeAddrInfo, ClockId, Itimerspec, Timespec, Sigset, Time, Pid, Mode, SigInfo, Id, Tmsghdr, IOVec, RLimit, Timeval, TFdSet, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, @@ -974,7 +974,7 @@ elif defined(linux): unlink, listen, sendmsg, recvmsg, getpid, fcntl, pthread_sigmask, sigprocmask, clock_gettime, signal, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, + socketpair, freeAddrInfo, ClockId, Itimerspec, Timespec, Sigset, Time, Pid, Mode, SigInfo, Id, Tmsghdr, IOVec, RLimit, TFdSet, Timeval, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, @@ -1097,7 +1097,7 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, clock_gettime, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, + socketpair, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, @@ -1123,7 +1123,7 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, clock_gettime, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, + socketpair, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, diff --git a/chronos/transports/common.nim b/chronos/transports/common.nim index 5a9072c..4b4be7d 100644 --- a/chronos/transports/common.nim +++ b/chronos/transports/common.nim @@ -298,6 +298,9 @@ proc getAddrInfo(address: string, port: Port, domain: Domain, raises: [TransportAddressError].} = ## We have this one copy of ``getAddrInfo()`` because of AI_V4MAPPED in ## ``net.nim:getAddrInfo()``, which is not cross-platform. + ## + ## Warning: `ptr AddrInfo` returned by `getAddrInfo()` needs to be freed by + ## calling `freeAddrInfo()`. var hints: AddrInfo var res: ptr AddrInfo = nil hints.ai_family = toInt(domain) @@ -420,6 +423,7 @@ proc resolveTAddress*(address: string, port: Port, if ta notin res: res.add(ta) it = it.ai_next + freeAddrInfo(aiList) res proc resolveTAddress*(address: string, domain: Domain): seq[TransportAddress] {. From 6b4f5a1d23b1583b2b0ccee409e2e7c6dc6fff93 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Tue, 1 Aug 2023 12:56:08 +0300 Subject: [PATCH 06/50] Recover `poll` engine and add tests. (#421) * Initial commit. * Fix one more place with deprecated constant. * Fix testall and nimble file. * Fix poll issue. * Workaround Nim's faulty declaration of `poll()` and types on MacOS. * Fix syntax errors. * Fix MacOS post-rebase issue. * Add more conditionals. * Address review comments. * Fix Nim 1.2 configuration defaults. --- chronos.nim | 5 ++- chronos.nimble | 23 +++++++++---- chronos/asyncloop.nim | 13 +++----- chronos/config.nim | 39 ++++++++++++++++++++++ chronos/ioselects/ioselectors_epoll.nim | 8 ++--- chronos/ioselects/ioselectors_kqueue.nim | 8 ++--- chronos/ioselects/ioselectors_poll.nim | 14 ++++---- chronos/osdefs.nim | 42 ++++++++++++++++++------ chronos/selectors2.nim | 34 +++++-------------- chronos/sendfile.nim | 6 +++- tests/testall.nim | 24 ++++++++++---- tests/testproc.nim | 1 + tests/teststream.nim | 5 ++- 13 files changed, 146 insertions(+), 76 deletions(-) diff --git a/chronos.nim b/chronos.nim index 6801b28..8295924 100644 --- a/chronos.nim +++ b/chronos.nim @@ -5,6 +5,5 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import chronos/[asyncloop, asyncsync, handles, transport, timer, - asyncproc, debugutils] -export asyncloop, asyncsync, handles, transport, timer, asyncproc, debugutils +import chronos/[asyncloop, asyncsync, handles, transport, timer, debugutils] +export asyncloop, asyncsync, handles, transport, timer, debugutils diff --git a/chronos.nimble b/chronos.nimble index 6b4ac58..e9c1b11 100644 --- a/chronos.nimble +++ b/chronos.nimble @@ -17,6 +17,22 @@ let nimc = getEnv("NIMC", "nim") # Which nim compiler to use let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js) let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler let verbose = getEnv("V", "") notin ["", "0"] +let testArguments = + when defined(windows): + [ + "-d:debug -d:chronosDebug -d:useSysAssert -d:useGcAssert", + "-d:debug -d:chronosPreviewV4", + "-d:release", + "-d:release -d:chronosPreviewV4" + ] + else: + [ + "-d:debug -d:chronosDebug -d:useSysAssert -d:useGcAssert", + "-d:debug -d:chronosPreviewV4", + "-d:debug -d:chronosDebug -d:chronosEventEngine=poll -d:useSysAssert -d:useGcAssert", + "-d:release", + "-d:release -d:chronosPreviewV4" + ] let styleCheckStyle = if (NimMajor, NimMinor) < (1, 6): "hint" else: "error" let cfg = @@ -31,12 +47,7 @@ proc run(args, path: string) = build args & " -r", path task test, "Run all tests": - for args in [ - "-d:debug -d:chronosDebug", - "-d:debug -d:chronosPreviewV4", - "-d:debug -d:chronosDebug -d:useSysAssert -d:useGcAssert", - "-d:release", - "-d:release -d:chronosPreviewV4"]: + for args in testArguments: run args, "tests/testall" if (NimMajor, NimMinor) > (1, 6): run args & " --mm:refc", "tests/testall" diff --git a/chronos/asyncloop.nim b/chronos/asyncloop.nim index 9d5ac23..a644b77 100644 --- a/chronos/asyncloop.nim +++ b/chronos/asyncloop.nim @@ -825,9 +825,9 @@ elif defined(macosx) or defined(freebsd) or defined(netbsd) or var res = PDispatcher( selector: selector, timers: initHeapQueue[TimerCallback](), - callbacks: initDeque[AsyncCallback](asyncEventsCount), + callbacks: initDeque[AsyncCallback](chronosEventsCount), idlers: initDeque[AsyncCallback](), - keys: newSeq[ReadyKey](asyncEventsCount), + keys: newSeq[ReadyKey](chronosEventsCount), trackers: initTable[string, TrackerBase](), counters: initTable[string, TrackerCounter]() ) @@ -1009,7 +1009,7 @@ elif defined(macosx) or defined(freebsd) or defined(netbsd) or ## You can execute ``aftercb`` before actual socket close operation. closeSocket(fd, aftercb) - when asyncEventEngine in ["epoll", "kqueue"]: + when chronosEventEngine in ["epoll", "kqueue"]: type ProcessHandle* = distinct int SignalHandle* = distinct int @@ -1123,7 +1123,7 @@ elif defined(macosx) or defined(freebsd) or defined(netbsd) or if not isNil(adata.reader.function): loop.callbacks.addLast(adata.reader) - when asyncEventEngine in ["epoll", "kqueue"]: + when chronosEventEngine in ["epoll", "kqueue"]: let customSet = {Event.Timer, Event.Signal, Event.Process, Event.Vnode} if customSet * events != {}: @@ -1257,10 +1257,7 @@ proc callIdle*(cbproc: CallbackFunc) = include asyncfutures2 - -when defined(macosx) or defined(macos) or defined(freebsd) or - defined(netbsd) or defined(openbsd) or defined(dragonfly) or - defined(linux) or defined(windows): +when (chronosEventEngine in ["epoll", "kqueue"]) or defined(windows): proc waitSignal*(signal: int): Future[void] {.raises: [].} = var retFuture = newFuture[void]("chronos.waitSignal()") diff --git a/chronos/config.nim b/chronos/config.nim index 0a439a1..bd6c2b9 100644 --- a/chronos/config.nim +++ b/chronos/config.nim @@ -49,6 +49,27 @@ when (NimMajor, NimMinor) >= (1, 4): ## using `AsyncProcessOption.EvalCommand` and API calls such as ## ``execCommand(command)`` and ``execCommandEx(command)``. + chronosEventsCount* {.intdefine.} = 64 + ## Number of OS poll events retrieved by syscall (epoll, kqueue, poll). + + chronosInitialSize* {.intdefine.} = 64 + ## Initial size of Selector[T]'s array of file descriptors. + + chronosEventEngine* {.strdefine.}: string = + when defined(linux) and not(defined(android) or defined(emscripten)): + "epoll" + elif defined(macosx) or defined(macos) or defined(ios) or + defined(freebsd) or defined(netbsd) or defined(openbsd) or + defined(dragonfly): + "kqueue" + elif defined(android) or defined(emscripten): + "poll" + elif defined(posix): + "poll" + else: + "" + ## OS polling engine type which is going to be used by chronos. + else: # 1.2 doesn't support `booldefine` in `when` properly const @@ -69,6 +90,21 @@ else: "/system/bin/sh" else: "/bin/sh" + chronosEventsCount*: int = 64 + chronosInitialSize*: int = 64 + chronosEventEngine* {.strdefine.}: string = + when defined(linux) and not(defined(android) or defined(emscripten)): + "epoll" + elif defined(macosx) or defined(macos) or defined(ios) or + defined(freebsd) or defined(netbsd) or defined(openbsd) or + defined(dragonfly): + "kqueue" + elif defined(android) or defined(emscripten): + "poll" + elif defined(posix): + "poll" + else: + "" when defined(debug) or defined(chronosConfig): import std/macros @@ -83,3 +119,6 @@ when defined(debug) or defined(chronosConfig): printOption("chronosFutureTracking", chronosFutureTracking) printOption("chronosDumpAsync", chronosDumpAsync) printOption("chronosProcShell", chronosProcShell) + printOption("chronosEventEngine", chronosEventEngine) + printOption("chronosEventsCount", chronosEventsCount) + printOption("chronosInitialSize", chronosInitialSize) diff --git a/chronos/ioselects/ioselectors_epoll.nim b/chronos/ioselects/ioselectors_epoll.nim index d438bac..161a5df 100644 --- a/chronos/ioselects/ioselectors_epoll.nim +++ b/chronos/ioselects/ioselectors_epoll.nim @@ -97,12 +97,12 @@ proc new*(t: typedesc[Selector], T: typedesc): SelectResult[Selector[T]] = var nmask: Sigset if sigemptyset(nmask) < 0: return err(osLastError()) - let epollFd = epoll_create(asyncEventsCount) + let epollFd = epoll_create(chronosEventsCount) if epollFd < 0: return err(osLastError()) let selector = Selector[T]( epollFd: epollFd, - fds: initTable[int32, SelectorKey[T]](asyncInitialSize), + fds: initTable[int32, SelectorKey[T]](chronosInitialSize), signalMask: nmask, virtualId: -1'i32, # Should start with -1, because `InvalidIdent` == -1 childrenExited: false, @@ -627,7 +627,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, readyKeys: var openArray[ReadyKey] ): SelectResult[int] = var - queueEvents: array[asyncEventsCount, EpollEvent] + queueEvents: array[chronosEventsCount, EpollEvent] k: int = 0 verifySelectParams(timeout, -1, int(high(cint))) @@ -668,7 +668,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, ok(k) proc select2*[T](s: Selector[T], timeout: int): SelectResult[seq[ReadyKey]] = - var res = newSeq[ReadyKey](asyncEventsCount) + var res = newSeq[ReadyKey](chronosEventsCount) let count = ? selectInto2(s, timeout, res) res.setLen(count) ok(res) diff --git a/chronos/ioselects/ioselectors_kqueue.nim b/chronos/ioselects/ioselectors_kqueue.nim index 9f0627a..e39f968 100644 --- a/chronos/ioselects/ioselectors_kqueue.nim +++ b/chronos/ioselects/ioselectors_kqueue.nim @@ -110,7 +110,7 @@ proc new*(t: typedesc[Selector], T: typedesc): SelectResult[Selector[T]] = let selector = Selector[T]( kqFd: kqFd, - fds: initTable[int32, SelectorKey[T]](asyncInitialSize), + fds: initTable[int32, SelectorKey[T]](chronosInitialSize), virtualId: -1'i32, # Should start with -1, because `InvalidIdent` == -1 virtualHoles: initDeque[int32]() ) @@ -559,7 +559,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, ): SelectResult[int] = var tv: Timespec - queueEvents: array[asyncEventsCount, KEvent] + queueEvents: array[chronosEventsCount, KEvent] verifySelectParams(timeout, -1, high(int)) @@ -575,7 +575,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, addr tv else: nil - maxEventsCount = cint(min(asyncEventsCount, len(readyKeys))) + maxEventsCount = cint(min(chronosEventsCount, len(readyKeys))) eventsCount = block: var res = 0 @@ -601,7 +601,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, proc select2*[T](s: Selector[T], timeout: int): Result[seq[ReadyKey], OSErrorCode] = - var res = newSeq[ReadyKey](asyncEventsCount) + var res = newSeq[ReadyKey](chronosEventsCount) let count = ? selectInto2(s, timeout, res) res.setLen(count) ok(res) diff --git a/chronos/ioselects/ioselectors_poll.nim b/chronos/ioselects/ioselectors_poll.nim index d0d533c..25cc035 100644 --- a/chronos/ioselects/ioselectors_poll.nim +++ b/chronos/ioselects/ioselectors_poll.nim @@ -16,7 +16,7 @@ import stew/base10 type SelectorImpl[T] = object fds: Table[int32, SelectorKey[T]] - pollfds: seq[TPollFd] + pollfds: seq[TPollfd] Selector*[T] = ref SelectorImpl[T] type @@ -50,7 +50,7 @@ proc freeKey[T](s: Selector[T], key: int32) = proc new*(t: typedesc[Selector], T: typedesc): SelectResult[Selector[T]] = let selector = Selector[T]( - fds: initTable[int32, SelectorKey[T]](asyncInitialSize) + fds: initTable[int32, SelectorKey[T]](chronosInitialSize) ) ok(selector) @@ -72,7 +72,7 @@ proc trigger2*(event: SelectEvent): SelectResult[void] = if res == -1: err(osLastError()) elif res != sizeof(uint64): - err(OSErrorCode(osdefs.EINVAL)) + err(osdefs.EINVAL) else: ok() @@ -98,13 +98,14 @@ template toPollEvents(events: set[Event]): cshort = res template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = - s.pollfds.add(TPollFd(fd: sock, events: toPollEvents(events), revents: 0)) + s.pollfds.add(TPollfd(fd: sock, events: toPollEvents(events), revents: 0)) template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = var updated = false for mitem in s.pollfds.mitems(): if mitem.fd == sock: mitem.events = toPollEvents(events) + updated = true break if not(updated): raiseAssert "Descriptor [" & $sock & "] is not registered in the queue!" @@ -177,7 +178,6 @@ proc unregister2*[T](s: Selector[T], event: SelectEvent): SelectResult[void] = proc prepareKey[T](s: Selector[T], event: var TPollfd): Opt[ReadyKey] = let - defaultKey = SelectorKey[T](ident: InvalidIdent) fdi32 = int32(event.fd) revents = event.revents @@ -224,7 +224,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, eventsCount = if maxEventsCount > 0: let res = handleEintr(poll(addr(s.pollfds[0]), Tnfds(maxEventsCount), - timeout)) + cint(timeout))) if res < 0: return err(osLastError()) res @@ -241,7 +241,7 @@ proc selectInto2*[T](s: Selector[T], timeout: int, ok(k) proc select2*[T](s: Selector[T], timeout: int): SelectResult[seq[ReadyKey]] = - var res = newSeq[ReadyKey](asyncEventsCount) + var res = newSeq[ReadyKey](chronosEventsCount) let count = ? selectInto2(s, timeout, res) res.setLen(count) ok(res) diff --git a/chronos/osdefs.nim b/chronos/osdefs.nim index bf5c060..75ceb67 100644 --- a/chronos/osdefs.nim +++ b/chronos/osdefs.nim @@ -880,7 +880,7 @@ elif defined(macos) or defined(macosx): sigemptyset, sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, freeAddrInfo, + socketpair, poll, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, @@ -905,7 +905,7 @@ elif defined(macos) or defined(macosx): sigemptyset, sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, freeAddrInfo, + socketpair, poll, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, @@ -929,6 +929,21 @@ elif defined(macos) or defined(macosx): numer*: uint32 denom*: uint32 + TPollfd* {.importc: "struct pollfd", pure, final, + header: "".} = object + fd*: cint + events*: cshort + revents*: cshort + + Tnfds* {.importc: "nfds_t", header: "".} = cuint + + const + POLLIN* = 0x0001 + POLLOUT* = 0x0004 + POLLERR* = 0x0008 + POLLHUP* = 0x0010 + POLLNVAL* = 0x0020 + proc posix_gettimeofday*(tp: var Timeval, unused: pointer = nil) {. importc: "gettimeofday", header: "".} @@ -938,6 +953,9 @@ elif defined(macos) or defined(macosx): proc mach_absolute_time*(): uint64 {. importc, header: "".} + proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: cint): cint {. + importc, header: "", sideEffect.} + elif defined(linux): from std/posix import close, shutdown, sigemptyset, sigaddset, sigismember, sigdelset, write, read, waitid, getaddrinfo, @@ -947,12 +965,12 @@ elif defined(linux): unlink, listen, sendmsg, recvmsg, getpid, fcntl, pthread_sigmask, sigprocmask, clock_gettime, signal, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, freeAddrInfo, + socketpair, poll, freeAddrInfo, ClockId, Itimerspec, Timespec, Sigset, Time, Pid, Mode, SigInfo, Id, Tmsghdr, IOVec, RLimit, Timeval, TFdSet, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, AddrInfo, SocketHandle, - Suseconds, + Suseconds, TPollfd, Tnfds, FD_CLR, FD_ISSET, FD_SET, FD_ZERO, CLOCK_MONOTONIC, F_GETFL, F_SETFL, F_GETFD, F_SETFD, FD_CLOEXEC, O_NONBLOCK, SIG_BLOCK, SIG_UNBLOCK, @@ -961,6 +979,7 @@ elif defined(linux): AF_INET, AF_INET6, AF_UNIX, SO_REUSEADDR, SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, IPV6_MULTICAST_HOPS, SOCK_DGRAM, SOCK_STREAM, SHUT_RD, SHUT_WR, SHUT_RDWR, + POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGKILL, SIGUSR1, SIGSEGV, SIGUSR2, SIGPIPE, SIGALRM, SIGTERM, SIGPIPE, SIGCHLD, SIGSTOP, @@ -974,12 +993,12 @@ elif defined(linux): unlink, listen, sendmsg, recvmsg, getpid, fcntl, pthread_sigmask, sigprocmask, clock_gettime, signal, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, freeAddrInfo, + socketpair, poll, freeAddrInfo, ClockId, Itimerspec, Timespec, Sigset, Time, Pid, Mode, SigInfo, Id, Tmsghdr, IOVec, RLimit, TFdSet, Timeval, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, AddrInfo, SocketHandle, - Suseconds, + Suseconds, TPollfd, Tnfds, FD_CLR, FD_ISSET, FD_SET, FD_ZERO, CLOCK_MONOTONIC, F_GETFL, F_SETFL, F_GETFD, F_SETFD, FD_CLOEXEC, O_NONBLOCK, SIG_BLOCK, SIG_UNBLOCK, @@ -988,6 +1007,7 @@ elif defined(linux): AF_INET, AF_INET6, AF_UNIX, SO_REUSEADDR, SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, IPV6_MULTICAST_HOPS, SOCK_DGRAM, SOCK_STREAM, SHUT_RD, SHUT_WR, SHUT_RDWR, + POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGKILL, SIGUSR1, SIGSEGV, SIGUSR2, SIGPIPE, SIGALRM, SIGTERM, SIGPIPE, SIGCHLD, SIGSTOP, @@ -1097,11 +1117,11 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, clock_gettime, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, freeAddrInfo, + socketpair, poll, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, - Suseconds, + Suseconds, TPollfd, Tnfds, FD_CLR, FD_ISSET, FD_SET, FD_ZERO, F_GETFL, F_SETFL, F_GETFD, F_SETFD, FD_CLOEXEC, O_NONBLOCK, SOL_SOCKET, SOCK_RAW, SOCK_DGRAM, @@ -1111,6 +1131,7 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or IPV6_MULTICAST_HOPS, SOCK_DGRAM, RLIMIT_NOFILE, SIG_BLOCK, SIG_UNBLOCK, CLOCK_MONOTONIC, SHUT_RD, SHUT_WR, SHUT_RDWR, + POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGKILL, SIGUSR1, SIGSEGV, SIGUSR2, SIGPIPE, SIGALRM, SIGTERM, SIGPIPE, SIGCHLD, SIGSTOP, @@ -1123,11 +1144,11 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or sigaddset, sigismember, fcntl, accept, pipe, write, signal, read, setsockopt, getsockopt, clock_gettime, getcwd, chdir, waitpid, kill, select, pselect, - socketpair, freeAddrInfo, + socketpair, poll, freeAddrInfo, Timeval, Timespec, Pid, Mode, Time, Sigset, SockAddr, SockLen, Sockaddr_storage, Sockaddr_in, Sockaddr_in6, Sockaddr_un, SocketHandle, AddrInfo, RLimit, TFdSet, - Suseconds, + Suseconds, TPollfd, Tnfds, FD_CLR, FD_ISSET, FD_SET, FD_ZERO, F_GETFL, F_SETFL, F_GETFD, F_SETFD, FD_CLOEXEC, O_NONBLOCK, SOL_SOCKET, SOCK_RAW, SOCK_DGRAM, @@ -1137,6 +1158,7 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or IPV6_MULTICAST_HOPS, SOCK_DGRAM, RLIMIT_NOFILE, SIG_BLOCK, SIG_UNBLOCK, CLOCK_MONOTONIC, SHUT_RD, SHUT_WR, SHUT_RDWR, + POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGKILL, SIGUSR1, SIGSEGV, SIGUSR2, SIGPIPE, SIGALRM, SIGTERM, SIGPIPE, SIGCHLD, SIGSTOP, diff --git a/chronos/selectors2.nim b/chronos/selectors2.nim index 45c4533..c5918fd 100644 --- a/chronos/selectors2.nim +++ b/chronos/selectors2.nim @@ -32,29 +32,9 @@ # backwards-compatible. import stew/results -import osdefs, osutils, oserrno +import config, osdefs, osutils, oserrno export results, oserrno -const - asyncEventsCount* {.intdefine.} = 64 - ## Number of epoll events retrieved by syscall. - asyncInitialSize* {.intdefine.} = 64 - ## Initial size of Selector[T]'s array of file descriptors. - asyncEventEngine* {.strdefine.} = - when defined(linux): - "epoll" - elif defined(macosx) or defined(macos) or defined(ios) or - defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly): - "kqueue" - elif defined(posix): - "poll" - else: - "" - ## Engine type which is going to be used by module. - - hasThreadSupport = compileOption("threads") - when defined(nimdoc): type @@ -281,7 +261,9 @@ else: var err = newException(IOSelectorsException, msg) raise err - when asyncEventEngine in ["epoll", "kqueue"]: + when chronosEventEngine in ["epoll", "kqueue"]: + const hasThreadSupport = compileOption("threads") + proc blockSignals(newmask: Sigset, oldmask: var Sigset): Result[void, OSErrorCode] = var nmask = newmask @@ -324,11 +306,11 @@ else: doAssert((timeout >= min) and (timeout <= max), "Cannot select with incorrect timeout value, got " & $timeout) -when asyncEventEngine == "epoll": +when chronosEventEngine == "epoll": include ./ioselects/ioselectors_epoll -elif asyncEventEngine == "kqueue": +elif chronosEventEngine == "kqueue": include ./ioselects/ioselectors_kqueue -elif asyncEventEngine == "poll": +elif chronosEventEngine == "poll": include ./ioselects/ioselectors_poll else: - {.fatal: "Event engine `" & asyncEventEngine & "` is not supported!".} + {.fatal: "Event engine `" & chronosEventEngine & "` is not supported!".} diff --git a/chronos/sendfile.nim b/chronos/sendfile.nim index 8cba9e8..7afcb73 100644 --- a/chronos/sendfile.nim +++ b/chronos/sendfile.nim @@ -38,8 +38,12 @@ when defined(nimdoc): ## be prepared to retry the call if there were unsent bytes. ## ## On error, ``-1`` is returned. +elif defined(emscripten): -elif defined(linux) or defined(android): + proc sendfile*(outfd, infd: int, offset: int, count: var int): int = + raiseAssert "sendfile() is not implemented yet" + +elif (defined(linux) or defined(android)) and not(defined(emscripten)): proc osSendFile*(outfd, infd: cint, offset: ptr int, count: int): int {.importc: "sendfile", header: "".} diff --git a/tests/testall.nim b/tests/testall.nim index 4861a85..6419f98 100644 --- a/tests/testall.nim +++ b/tests/testall.nim @@ -5,10 +5,22 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import testmacro, testsync, testsoon, testtime, testfut, testsignal, - testaddress, testdatagram, teststream, testserver, testbugs, testnet, - testasyncstream, testhttpserver, testshttpserver, testhttpclient, - testproc, testratelimit, testfutures, testthreadsync +import ".."/chronos/config -# Must be imported last to check for Pending futures -import testutils +when (chronosEventEngine in ["epoll", "kqueue"]) or defined(windows): + import testmacro, testsync, testsoon, testtime, testfut, testsignal, + testaddress, testdatagram, teststream, testserver, testbugs, testnet, + testasyncstream, testhttpserver, testshttpserver, testhttpclient, + testproc, testratelimit, testfutures, testthreadsync + + # Must be imported last to check for Pending futures + import testutils +elif chronosEventEngine == "poll": + # `poll` engine do not support signals and processes + import testmacro, testsync, testsoon, testtime, testfut, testaddress, + testdatagram, teststream, testserver, testbugs, testnet, + testasyncstream, testhttpserver, testshttpserver, testhttpclient, + testratelimit, testfutures, testthreadsync + + # Must be imported last to check for Pending futures + import testutils diff --git a/tests/testproc.nim b/tests/testproc.nim index cfcafe6..288ec18 100644 --- a/tests/testproc.nim +++ b/tests/testproc.nim @@ -8,6 +8,7 @@ import std/os import stew/[base10, byteutils] import ".."/chronos/unittest2/asynctests +import ".."/chronos/asyncproc when defined(posix): from ".."/chronos/osdefs import SIGKILL diff --git a/tests/teststream.nim b/tests/teststream.nim index f6bc99b..8c3c77e 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -1339,7 +1339,10 @@ suite "Stream Transport test suite": else: skip() else: - check waitFor(testSendFile(addresses[i])) == FilesCount + if defined(emscripten): + skip() + else: + check waitFor(testSendFile(addresses[i])) == FilesCount test prefixes[i] & "Connection refused test": var address: TransportAddress if addresses[i].family == AddressFamily.Unix: From c546a4329cad426fc1deb32f3a7bca6ead4c2b2c Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Wed, 2 Aug 2023 21:04:30 +0200 Subject: [PATCH 07/50] Use random ports (#429) --- tests/teststream.nim | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/teststream.nim b/tests/teststream.nim index 8c3c77e..4b5cb15 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -1259,46 +1259,44 @@ suite "Stream Transport test suite": return buffer == message proc testConnectBindLocalAddress() {.async.} = - let dst1 = initTAddress("127.0.0.1:33335") - let dst2 = initTAddress("127.0.0.1:33336") - let dst3 = initTAddress("127.0.0.1:33337") proc client(server: StreamServer, transp: StreamTransport) {.async.} = await transp.closeWait() - # We use ReuseAddr here only to be able to reuse the same IP/Port when there's a TIME_WAIT socket. It's useful when - # running the test multiple times or if a test ran previously used the same port. - let servers = - [createStreamServer(dst1, client, {ReuseAddr}), - createStreamServer(dst2, client, {ReuseAddr}), - createStreamServer(dst3, client, {ReusePort})] + let server1 = createStreamServer(initTAddress("127.0.0.1:0"), client) + let server2 = createStreamServer(initTAddress("127.0.0.1:0"), client) + let server3 = createStreamServer(initTAddress("127.0.0.1:0"), client, {ReusePort}) - for server in servers: - server.start() - - let ta = initTAddress("0.0.0.0:35000") + server1.start() + server2.start() + server3.start() # It works cause there's no active listening socket bound to ta and we are using ReuseAddr - var transp1 = await connect(dst1, localAddress = ta, flags={SocketFlags.ReuseAddr}) - var transp2 = await connect(dst2, localAddress = ta, flags={SocketFlags.ReuseAddr}) + var transp1 = await connect(server1.local, flags={SocketFlags.ReuseAddr}) + let ta = transp1.localAddress + var transp2 = await connect(server2.local, localAddress = ta, flags={SocketFlags.ReuseAddr}) - # It works cause even thought there's an active listening socket bound to dst3, we are using ReusePort - var transp3 = await connect(dst2, localAddress = dst3, flags={SocketFlags.ReusePort}) + # It works cause even though there's an active listening socket bound to dst3, we are using ReusePort + var transp3 = await connect(server2.local, localAddress = server3.local, flags={SocketFlags.ReusePort}) expect(TransportOsError): - var transp2 {.used.} = await connect(dst3, localAddress = ta) + var transp2 {.used.} = await connect(server3.local, localAddress = ta) expect(TransportOsError): - var transp3 {.used.} = - await connect(dst3, localAddress = initTAddress(":::35000")) + var transp3 {.used.} = await connect(server3.local, localAddress = initTAddress("::", transp1.localAddress.port)) await transp1.closeWait() await transp2.closeWait() await transp3.closeWait() - for server in servers: - server.stop() - await server.closeWait() + server1.stop() + await server1.closeWait() + + server2.stop() + await server2.closeWait() + + server3.stop() + await server3.closeWait() markFD = getCurrentFD() From a1eb30360b10b850b3105dac1daae64c73de5a03 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 4 Aug 2023 08:08:34 +0200 Subject: [PATCH 08/50] fix invalid protocol casts (#430) --- chronos/transports/datagram.nim | 10 +++++----- chronos/transports/stream.nim | 20 +++++++++----------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/chronos/transports/datagram.nim b/chronos/transports/datagram.nim index 3e10f76..665bc0e 100644 --- a/chronos/transports/datagram.nim +++ b/chronos/transports/datagram.nim @@ -466,11 +466,11 @@ else: var res = if isNil(child): DatagramTransport() else: child if sock == asyncInvalidSocket: - var proto = Protocol.IPPROTO_UDP - if local.family == AddressFamily.Unix: - # `Protocol` enum is missing `0` value, so we making here cast, until - # `Protocol` enum will not support IPPROTO_IP == 0. - proto = cast[Protocol](0) + let proto = + if local.family == AddressFamily.Unix: + Protocol.IPPROTO_IP + else: + Protocol.IPPROTO_UDP localSock = createAsyncSocket(local.getDomain(), SockType.SOCK_DGRAM, proto) if localSock == asyncInvalidSocket: diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 45e4054..44a39b2 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -1475,14 +1475,13 @@ else: var saddr: Sockaddr_storage slen: SockLen - proto: Protocol var retFuture = newFuture[StreamTransport]("stream.transport.connect") address.toSAddr(saddr, slen) - proto = Protocol.IPPROTO_TCP - if address.family == AddressFamily.Unix: - # `Protocol` enum is missing `0` value, so we making here cast, until - # `Protocol` enum will not support IPPROTO_IP == 0. - proto = cast[Protocol](0) + let proto = + if address.family == AddressFamily.Unix: + Protocol.IPPROTO_IP + else: + Protocol.IPPROTO_TCP let sock = createAsyncSocket(address.getDomain(), SockType.SOCK_STREAM, proto) @@ -1938,11 +1937,10 @@ proc createStreamServer*(host: TransportAddress, else: # Posix if sock == asyncInvalidSocket: - var proto = Protocol.IPPROTO_TCP - if host.family == AddressFamily.Unix: - # `Protocol` enum is missing `0` value, so we making here cast, until - # `Protocol` enum will not support IPPROTO_IP == 0. - proto = cast[Protocol](0) + let proto = if host.family == AddressFamily.Unix: + Protocol.IPPROTO_IP + else: + Protocol.IPPROTO_TCP serverSocket = createAsyncSocket(host.getDomain(), SockType.SOCK_STREAM, proto) From 38c31e21d392c8a0924867bdebc764172065770e Mon Sep 17 00:00:00 2001 From: andri lim Date: Fri, 4 Aug 2023 14:27:01 +0700 Subject: [PATCH 09/50] fix type mismatch error in asyncstream join (#433) --- chronos/streams/asyncstream.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chronos/streams/asyncstream.nim b/chronos/streams/asyncstream.nim index 7e6e5d2..191b36a 100644 --- a/chronos/streams/asyncstream.nim +++ b/chronos/streams/asyncstream.nim @@ -873,10 +873,10 @@ proc join*(rw: AsyncStreamRW): Future[void] = else: var retFuture = newFuture[void]("async.stream.writer.join") - proc continuation(udata: pointer) {.gcsafe.} = + proc continuation(udata: pointer) {.gcsafe, raises:[].} = retFuture.complete() - proc cancellation(udata: pointer) {.gcsafe.} = + proc cancellation(udata: pointer) {.gcsafe, raises:[].} = rw.future.removeCallback(continuation, cast[pointer](retFuture)) if not(rw.future.finished()): From c4b066a2c4faeedd564e8467a9416920bfb4b9a9 Mon Sep 17 00:00:00 2001 From: andri lim Date: Fri, 4 Aug 2023 14:32:12 +0700 Subject: [PATCH 10/50] ci: upgrade github actions/cache to v3 (#434) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b78f2a1..b7aa0fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: - name: Restore Nim DLLs dependencies (Windows) from cache if: runner.os == 'Windows' id: windows-dlls-cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: external/dlls-${{ matrix.target.cpu }} key: 'dlls-${{ matrix.target.cpu }}' From 194226a0e06e60bee9de503bc4a2d219c7dd93d2 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Tue, 8 Aug 2023 02:10:28 +0200 Subject: [PATCH 11/50] Remove hard-coded ports when non-windows (#437) --- tests/teststream.nim | 66 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tests/teststream.nim b/tests/teststream.nim index 4b5cb15..73d34c6 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -34,7 +34,7 @@ suite "Stream Transport test suite": ] else: let addresses = [ - initTAddress("127.0.0.1:33335"), + initTAddress("127.0.0.1:0"), initTAddress(r"/tmp/testpipe") ] @@ -43,7 +43,7 @@ suite "Stream Transport test suite": var markFD: int proc getCurrentFD(): int = - let local = initTAddress("127.0.0.1:33334") + let local = initTAddress("127.0.0.1:0") let sock = createAsyncSocket(local.getDomain(), SockType.SOCK_DGRAM, Protocol.IPPROTO_UDP) closeSocket(sock) @@ -348,7 +348,7 @@ suite "Stream Transport test suite": proc test1(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient1, {ReuseAddr}) server.start() - result = await swarmManager1(address) + result = await swarmManager1(server.local) server.stop() server.close() await server.join() @@ -356,7 +356,7 @@ suite "Stream Transport test suite": proc test2(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient2, {ReuseAddr}) server.start() - result = await swarmManager2(address) + result = await swarmManager2(server.local) server.stop() server.close() await server.join() @@ -364,7 +364,7 @@ suite "Stream Transport test suite": proc test3(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient3, {ReuseAddr}) server.start() - result = await swarmManager3(address) + result = await swarmManager3(server.local) server.stop() server.close() await server.join() @@ -372,7 +372,7 @@ suite "Stream Transport test suite": proc testSendFile(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient4, {ReuseAddr}) server.start() - result = await swarmManager4(address) + result = await swarmManager4(server.local) server.stop() server.close() await server.join() @@ -414,7 +414,7 @@ suite "Stream Transport test suite": var server = createStreamServer(address, serveClient, {ReuseAddr}) server.start() - result = await swarmManager(address) + result = await swarmManager(server.local) await server.join() proc testWCR(address: TransportAddress): Future[int] {.async.} = @@ -456,13 +456,13 @@ suite "Stream Transport test suite": var server = createStreamServer(address, serveClient, {ReuseAddr}) server.start() - result = await swarmManager(address) + result = await swarmManager(server.local) await server.join() proc test7(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient7, {ReuseAddr}) server.start() - result = await swarmWorker7(address) + result = await swarmWorker7(server.local) server.stop() server.close() await server.join() @@ -470,7 +470,7 @@ suite "Stream Transport test suite": proc test8(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient8, {ReuseAddr}) server.start() - result = await swarmWorker8(address) + result = await swarmWorker8(server.local) await server.join() # proc serveClient9(server: StreamServer, transp: StreamTransport) {.async.} = @@ -553,7 +553,7 @@ suite "Stream Transport test suite": proc test11(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient11, {ReuseAddr}) server.start() - result = await swarmWorker11(address) + result = await swarmWorker11(server.local) server.stop() server.close() await server.join() @@ -579,7 +579,7 @@ suite "Stream Transport test suite": proc test12(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient12, {ReuseAddr}) server.start() - result = await swarmWorker12(address) + result = await swarmWorker12(server.local) server.stop() server.close() await server.join() @@ -601,7 +601,7 @@ suite "Stream Transport test suite": proc test13(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient13, {ReuseAddr}) server.start() - result = await swarmWorker13(address) + result = await swarmWorker13(server.local) server.stop() server.close() await server.join() @@ -621,7 +621,7 @@ suite "Stream Transport test suite": subres = 0 server.start() - var transp = await connect(address) + var transp = await connect(server.local) var fut = swarmWorker(transp) # We perfrom shutdown(SHUT_RD/SD_RECEIVE) for the socket, in such way its # possible to emulate socket's EOF. @@ -674,7 +674,7 @@ suite "Stream Transport test suite": proc test16(address: TransportAddress): Future[int] {.async.} = var server = createStreamServer(address, serveClient16, {ReuseAddr}) server.start() - result = await swarmWorker16(address) + result = await swarmWorker16(server.local) server.stop() server.close() await server.join() @@ -701,7 +701,7 @@ suite "Stream Transport test suite": var server = createStreamServer(address, client, {ReuseAddr}) server.start() var msg = "HELLO" - var ntransp = await connect(address) + var ntransp = await connect(server.local) await syncFut while true: var res = await ntransp.write(msg) @@ -763,7 +763,7 @@ suite "Stream Transport test suite": var transp: StreamTransport try: - transp = await connect(address) + transp = await connect(server.local) flag = true except CatchableError: server.stop() @@ -796,31 +796,31 @@ suite "Stream Transport test suite": server.start() try: var r1, r2, r3, r4, r5: string - var t1 = await connect(address) + var t1 = await connect(server.local) try: r1 = await t1.readLine(4) finally: await t1.closeWait() - var t2 = await connect(address) + var t2 = await connect(server.local) try: r2 = await t2.readLine(6) finally: await t2.closeWait() - var t3 = await connect(address) + var t3 = await connect(server.local) try: r3 = await t3.readLine(8) finally: await t3.closeWait() - var t4 = await connect(address) + var t4 = await connect(server.local) try: r4 = await t4.readLine(8) finally: await t4.closeWait() - var t5 = await connect(address) + var t5 = await connect(server.local) try: r5 = await t5.readLine() finally: @@ -945,7 +945,7 @@ suite "Stream Transport test suite": var server = createStreamServer(address, serveClient, {ReuseAddr}) server.start() - var t1 = await connect(address) + var t1 = await connect(server.local) try: discard await t1.readLV(2000) except TransportIncompleteError: @@ -959,7 +959,7 @@ suite "Stream Transport test suite": await server.join() return false - var t2 = await connect(address) + var t2 = await connect(server.local) try: var r2 = await t2.readLV(2000) c2 = (r2 == @[]) @@ -972,7 +972,7 @@ suite "Stream Transport test suite": await server.join() return false - var t3 = await connect(address) + var t3 = await connect(server.local) try: discard await t3.readLV(2000) except TransportIncompleteError: @@ -986,7 +986,7 @@ suite "Stream Transport test suite": await server.join() return false - var t4 = await connect(address) + var t4 = await connect(server.local) try: discard await t4.readLV(2000) except TransportIncompleteError: @@ -1000,7 +1000,7 @@ suite "Stream Transport test suite": await server.join() return false - var t5 = await connect(address) + var t5 = await connect(server.local) try: discard await t5.readLV(1000) except ValueError: @@ -1014,7 +1014,7 @@ suite "Stream Transport test suite": await server.join() return false - var t6 = await connect(address) + var t6 = await connect(server.local) try: var expectMsg = createMessage(1024) var r6 = await t6.readLV(2000) @@ -1029,7 +1029,7 @@ suite "Stream Transport test suite": await server.join() return false - var t7 = await connect(address) + var t7 = await connect(server.local) try: var expectMsg = createMessage(1024) var expectDone = "DONE" @@ -1062,7 +1062,7 @@ suite "Stream Transport test suite": try: for i in 0 ..< TestsCount: - transp = await connect(address) + transp = await connect(server.local) await sleepAsync(10.milliseconds) await transp.closeWait() inc(connected) @@ -1117,7 +1117,7 @@ suite "Stream Transport test suite": try: for i in 0 ..< 3: try: - let transp = await connect(address) + let transp = await connect(server.local) await sleepAsync(10.milliseconds) await transp.closeWait() except TransportTooManyError: @@ -1166,7 +1166,7 @@ suite "Stream Transport test suite": await server.closeWait() var acceptFut = acceptTask(server) - var transp = await connect(address) + var transp = await connect(server.local) await server.join() await transp.closeWait() await acceptFut @@ -1187,7 +1187,7 @@ suite "Stream Transport test suite": await server.closeWait() var acceptFut = acceptTask(server) - var transp = await connect(address) + var transp = await connect(server.local) await server.join() await transp.closeWait() await acceptFut From 466241aa958af4ee4d07c5e0019d40bdaa9f6a36 Mon Sep 17 00:00:00 2001 From: diegomrsantos Date: Tue, 8 Aug 2023 02:11:35 +0200 Subject: [PATCH 12/50] Remove reuseaddr (#438) * Remove hard-coded ports when non-windows * Remove ReuseAddr from test --- tests/teststream.nim | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/teststream.nim b/tests/teststream.nim index 73d34c6..9e1ce55 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -1271,23 +1271,18 @@ suite "Stream Transport test suite": server2.start() server3.start() - # It works cause there's no active listening socket bound to ta and we are using ReuseAddr - var transp1 = await connect(server1.local, flags={SocketFlags.ReuseAddr}) - let ta = transp1.localAddress - var transp2 = await connect(server2.local, localAddress = ta, flags={SocketFlags.ReuseAddr}) - # It works cause even though there's an active listening socket bound to dst3, we are using ReusePort - var transp3 = await connect(server2.local, localAddress = server3.local, flags={SocketFlags.ReusePort}) + var transp1 = await connect(server1.local, localAddress = server3.local, flags={SocketFlags.ReusePort}) + var transp2 = await connect(server2.local, localAddress = server3.local, flags={SocketFlags.ReusePort}) expect(TransportOsError): - var transp2 {.used.} = await connect(server3.local, localAddress = ta) + var transp2 {.used.} = await connect(server2.local, localAddress = server3.local) expect(TransportOsError): - var transp3 {.used.} = await connect(server3.local, localAddress = initTAddress("::", transp1.localAddress.port)) + var transp3 {.used.} = await connect(server2.local, localAddress = initTAddress("::", server3.local.port)) await transp1.closeWait() await transp2.closeWait() - await transp3.closeWait() server1.stop() await server1.closeWait() From 6c2ea675123ed0bf5c5d76c92ed4985bacd1a9ec Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 9 Aug 2023 10:57:49 +0300 Subject: [PATCH 13/50] Unroll `defer`s and remove `break`s. (#440) * Unpack `finally/defer` blocks and introduce explicit cleaning of objects. Add request query to debug information. * Unroll one more loop to avoid `break`. Add test for query debug string. * Fix cancellation behavior. * Address review comments. --- chronos/apps/http/httpdebug.nim | 9 ++++ chronos/apps/http/httpserver.nim | 88 +++++++++++++++++-------------- chronos/apps/http/httptable.nim | 4 ++ chronos/apps/http/shttpserver.nim | 12 ++++- tests/testhttpserver.nim | 8 +-- tests/testshttpserver.nim | 3 +- 6 files changed, 77 insertions(+), 47 deletions(-) diff --git a/chronos/apps/http/httpdebug.nim b/chronos/apps/http/httpdebug.nim index 2f40674..a1dc022 100644 --- a/chronos/apps/http/httpdebug.nim +++ b/chronos/apps/http/httpdebug.nim @@ -29,6 +29,7 @@ type handle*: SocketHandle connectionType*: ConnectionType connectionState*: ConnectionState + query*: Opt[string] remoteAddress*: Opt[TransportAddress] localAddress*: Opt[TransportAddress] acceptMoment*: Moment @@ -85,6 +86,12 @@ proc getConnectionState*(holder: HttpConnectionHolderRef): ConnectionState = else: ConnectionState.Accepted +proc getQueryString*(holder: HttpConnectionHolderRef): Opt[string] = + if not(isNil(holder.connection)): + holder.connection.currentRawQuery + else: + Opt.none(string) + proc init*(t: typedesc[ServerConnectionInfo], holder: HttpConnectionHolderRef): ServerConnectionInfo = let @@ -98,6 +105,7 @@ proc init*(t: typedesc[ServerConnectionInfo], Opt.some(holder.transp.remoteAddress()) except CatchableError: Opt.none(TransportAddress) + queryString = holder.getQueryString() ServerConnectionInfo( handle: SocketHandle(holder.transp.fd), @@ -106,6 +114,7 @@ proc init*(t: typedesc[ServerConnectionInfo], remoteAddress: remoteAddress, localAddress: localAddress, acceptMoment: holder.acceptMoment, + query: queryString, createMoment: if not(isNil(holder.connection)): Opt.some(holder.connection.createMoment) diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index c1e45c0..eafa27c 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -148,6 +148,7 @@ type writer*: AsyncStreamWriter closeCb*: HttpCloseConnectionCallback createMoment*: Moment + currentRawQuery*: Opt[string] buffer: seq[byte] HttpConnectionRef* = ref HttpConnection @@ -813,6 +814,7 @@ proc closeUnsecureConnection(conn: HttpConnectionRef) {.async.} = except CancelledError: await allFutures(pending) untrackCounter(HttpServerUnsecureConnectionTrackerName) + reset(conn[]) conn.state = HttpState.Closed proc new(ht: typedesc[HttpConnectionRef], server: HttpServerRef, @@ -844,7 +846,9 @@ proc closeWait*(req: HttpRequestRef) {.async.} = await writer except CancelledError: await writer + reset(resp[]) untrackCounter(HttpServerRequestTrackerName) + reset(req[]) req.state = HttpState.Closed proc createConnection(server: HttpServerRef, @@ -931,6 +935,7 @@ proc getRequestFence*(server: HttpServerRef, await connection.getRequest() else: await connection.getRequest().wait(server.headersTimeout) + connection.currentRawQuery = Opt.some(res.rawPath) RequestFence.ok(res) except CancelledError: RequestFence.err(HttpProcessError.init(HttpServerError.InterruptError)) @@ -962,13 +967,17 @@ proc getConnectionFence*(server: HttpServerRef, let res = await server.createConnCallback(server, transp) ConnectionFence.ok(res) except CancelledError: - await transp.closeWait() ConnectionFence.err(HttpProcessError.init(HttpServerError.InterruptError)) except HttpCriticalError as exc: - await transp.closeWait() - let address = transp.getRemoteAddress() + # On error `transp` will be closed by `createConnCallback()` call. + let address = Opt.none(TransportAddress) ConnectionFence.err(HttpProcessError.init( HttpServerError.CriticalError, exc, address, exc.code)) + except CatchableError as exc: + # On error `transp` will be closed by `createConnCallback()` call. + let address = Opt.none(TransportAddress) + ConnectionFence.err(HttpProcessError.init( + HttpServerError.CriticalError, exc, address, Http503)) proc processRequest(server: HttpServerRef, connection: HttpConnectionRef, @@ -984,19 +993,23 @@ proc processRequest(server: HttpServerRef, else: discard - defer: - if requestFence.isOk(): - await requestFence.get().closeWait() - let responseFence = await getResponseFence(connection, requestFence) if responseFence.isErr() and (responseFence.error.kind == HttpServerError.InterruptError): + if requestFence.isOk(): + await requestFence.get().closeWait() return HttpProcessExitType.Immediate - if responseFence.isErr(): - await connection.sendErrorResponse(requestFence, responseFence.error) - else: - await connection.sendDefaultResponse(requestFence, responseFence.get()) + let res = + if responseFence.isErr(): + await connection.sendErrorResponse(requestFence, responseFence.error) + else: + await connection.sendDefaultResponse(requestFence, responseFence.get()) + + if requestFence.isOk(): + await requestFence.get().closeWait() + + res proc processLoop(holder: HttpConnectionHolderRef) {.async.} = let @@ -1016,23 +1029,27 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async.} = holder.connection = connection var runLoop = HttpProcessExitType.KeepAlive - - defer: - server.connections.del(connectionId) - case runLoop - of HttpProcessExitType.KeepAlive: - # This could happened only on CancelledError. - await connection.closeWait() - of HttpProcessExitType.Immediate: - await connection.closeWait() - of HttpProcessExitType.Graceful: - await connection.gracefulCloseWait() - while runLoop == HttpProcessExitType.KeepAlive: - runLoop = await server.processRequest(connection, connectionId) + runLoop = + try: + await server.processRequest(connection, connectionId) + except CancelledError: + HttpProcessExitType.Immediate + except CatchableError as exc: + raiseAssert "Unexpected error [" & $exc.name & "] happens: " & $exc.msg + + server.connections.del(connectionId) + case runLoop + of HttpProcessExitType.KeepAlive: + await connection.closeWait() + of HttpProcessExitType.Immediate: + await connection.closeWait() + of HttpProcessExitType.Graceful: + await connection.gracefulCloseWait() proc acceptClientLoop(server: HttpServerRef) {.async.} = - while true: + var runLoop = true + while runLoop: try: # if server.maxConnections > 0: # await server.semaphore.acquire() @@ -1042,27 +1059,18 @@ proc acceptClientLoop(server: HttpServerRef) {.async.} = # We are unable to identify remote peer, it means that remote peer # disconnected before identification. await transp.closeWait() - break + runLoop = false else: let connId = resId.get() let holder = HttpConnectionHolderRef.new(server, transp, resId.get()) server.connections[connId] = holder holder.future = processLoop(holder) - except CancelledError: - # Server was stopped - break - except TransportOsError: - # This is some critical unrecoverable error. - break - except TransportTooManyError: - # Non critical error + except TransportTooManyError, TransportAbortedError: + # Non-critical error discard - except TransportAbortedError: - # Non critical error - discard - except CatchableError: - # Unexpected error - break + except CancelledError, TransportOsError, CatchableError: + # Critical, cancellation or unexpected error + runLoop = false proc state*(server: HttpServerRef): HttpServerState {.raises: [].} = ## Returns current HTTP server's state. diff --git a/chronos/apps/http/httptable.nim b/chronos/apps/http/httptable.nim index 86060de..f44765a 100644 --- a/chronos/apps/http/httptable.nim +++ b/chronos/apps/http/httptable.nim @@ -197,3 +197,7 @@ proc toList*(ht: HttpTables, normKey = false): auto = for key, value in ht.stringItems(normKey): res.add((key, value)) res + +proc clear*(ht: var HttpTables) = + ## Resets the HtppTable so that it is empty. + ht.table.clear() diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index b993cb5..bc5c3fb 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -43,6 +43,7 @@ proc closeSecConnection(conn: HttpConnectionRef) {.async.} = await allFutures(pending) except CancelledError: await allFutures(pending) + reset(cast[SecureHttpConnectionRef](conn)[]) untrackCounter(HttpServerSecureConnectionTrackerName) conn.state = HttpState.Closed @@ -74,9 +75,16 @@ proc createSecConnection(server: HttpServerRef, except CancelledError as exc: await HttpConnectionRef(sconn).closeWait() raise exc - except TLSStreamError: + except TLSStreamError as exc: await HttpConnectionRef(sconn).closeWait() - raiseHttpCriticalError("Unable to establish secure connection") + let msg = "Unable to establish secure connection, reason [" & + $exc.msg & "]" + raiseHttpCriticalError(msg) + except CatchableError as exc: + await HttpConnectionRef(sconn).closeWait() + let msg = "Unexpected error while trying to establish secure connection, " & + "reason [" & $exc.msg & "]" + raiseHttpCriticalError(msg) proc new*(htype: typedesc[SecureHttpServerRef], address: TransportAddress, diff --git a/tests/testhttpserver.nim b/tests/testhttpserver.nim index 83372ea..0ecc9aa 100644 --- a/tests/testhttpserver.nim +++ b/tests/testhttpserver.nim @@ -7,9 +7,8 @@ # MIT license (LICENSE-MIT) import std/[strutils, algorithm] import ".."/chronos/unittest2/asynctests, - ".."/chronos, ".."/chronos/apps/http/httpserver, - ".."/chronos/apps/http/httpcommon, - ".."/chronos/apps/http/httpdebug + ".."/chronos, + ".."/chronos/apps/http/[httpserver, httpcommon, httpdebug] import stew/base10 {.used.} @@ -1357,7 +1356,7 @@ suite "HTTP server testing suite": asyncTest "HTTP debug tests": const TestsCount = 10 - TestRequest = "GET / HTTP/1.1\r\nConnection: keep-alive\r\n\r\n" + TestRequest = "GET /httpdebug HTTP/1.1\r\nConnection: keep-alive\r\n\r\n" proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = if r.isOk(): @@ -1417,6 +1416,7 @@ suite "HTTP server testing suite": connection.localAddress.get() == transp.remoteAddress() connection.connectionType == ConnectionType.NonSecure connection.connectionState == ConnectionState.Alive + connection.query.get("") == "/httpdebug" (currentTime - connection.createMoment.get()) != ZeroDuration (currentTime - connection.acceptMoment) != ZeroDuration var pending: seq[Future[void]] diff --git a/tests/testshttpserver.nim b/tests/testshttpserver.nim index a83d0b2..8aacb8e 100644 --- a/tests/testshttpserver.nim +++ b/tests/testshttpserver.nim @@ -7,7 +7,8 @@ # MIT license (LICENSE-MIT) import std/strutils import ".."/chronos/unittest2/asynctests -import ".."/chronos, ".."/chronos/apps/http/shttpserver +import ".."/chronos, + ".."/chronos/apps/http/shttpserver import stew/base10 {.used.} From a7f708bea897ab81ee57aab66628c62f12aa213a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 9 Aug 2023 16:27:17 +0200 Subject: [PATCH 14/50] futures: lentify (#413) sometimes avoid copies when reading from `Future` --- chronos/asyncfutures2.nim | 18 ++++++++++++------ chronos/asyncmacro2.nim | 2 +- chronos/futures.nim | 17 ++++++++++++++--- tests/testfut.nim | 2 ++ 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/chronos/asyncfutures2.nim b/chronos/asyncfutures2.nim index d170f08..d3954ba 100644 --- a/chronos/asyncfutures2.nim +++ b/chronos/asyncfutures2.nim @@ -451,19 +451,25 @@ proc internalCheckComplete*(fut: FutureBase) {.raises: [CatchableError].} = injectStacktrace(fut.internalError) raise fut.internalError -proc internalRead*[T](fut: Future[T]): T {.inline.} = - # For internal use only. Used in asyncmacro - when T isnot void: - return fut.internalValue +proc read*[T: not void](future: Future[T] ): lent T {.raises: [CatchableError].} = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if not future.finished(): + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") -proc read*[T](future: Future[T] ): T {.raises: [CatchableError].} = + internalCheckComplete(future) + future.internalValue + +proc read*(future: Future[void] ) {.raises: [CatchableError].} = ## Retrieves the value of ``future``. Future must be finished otherwise ## this function will fail with a ``ValueError`` exception. ## ## If the result of the future is an error then that error will be raised. if future.finished(): internalCheckComplete(future) - internalRead(future) else: # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") diff --git a/chronos/asyncmacro2.nim b/chronos/asyncmacro2.nim index 45146a3..8e74073 100644 --- a/chronos/asyncmacro2.nim +++ b/chronos/asyncmacro2.nim @@ -309,7 +309,7 @@ template await*[T](f: Future[T]): untyped = # `child` released by `futureContinue` chronosInternalRetFuture.internalChild.internalCheckComplete() when T isnot void: - cast[type(f)](chronosInternalRetFuture.internalChild).internalRead() + cast[type(f)](chronosInternalRetFuture.internalChild).value() else: unsupported "await is only available within {.async.}" diff --git a/chronos/futures.nim b/chronos/futures.nim index edfae32..9b2667b 100644 --- a/chronos/futures.nim +++ b/chronos/futures.nim @@ -184,7 +184,7 @@ func completed*(future: FutureBase): bool {.inline.} = func location*(future: FutureBase): array[LocationKind, ptr SrcLoc] = future.internalLocation -func value*[T](future: Future[T]): T = +func value*[T: not void](future: Future[T]): lent T = ## Return the value in a completed future - raises Defect when ## `fut.completed()` is `false`. ## @@ -196,8 +196,19 @@ func value*[T](future: Future[T]): T = msg: "Future not completed while accessing value", cause: future) - when T isnot void: - future.internalValue + future.internalValue + +func value*(future: Future[void]) = + ## Return the value in a completed future - raises Defect when + ## `fut.completed()` is `false`. + ## + ## See `read` for a version that raises an catchable error when future + ## has not completed. + when chronosStrictFutureAccess: + if not future.completed(): + raise (ref FutureDefect)( + msg: "Future not completed while accessing value", + cause: future) func error*(future: FutureBase): ref CatchableError = ## Return the error of `future`, or `nil` if future did not fail. diff --git a/tests/testfut.nim b/tests/testfut.nim index af92354..a9fba05 100644 --- a/tests/testfut.nim +++ b/tests/testfut.nim @@ -1237,12 +1237,14 @@ suite "Future[T] behavior test suite": fut2.complete() # LINE POSITION 4 fut3.complete() # LINE POSITION 6 + {.push warning[Deprecated]: off.} # testing backwards compatibility interface let loc10 = fut1.location[0] let loc11 = fut1.location[1] let loc20 = fut2.location[0] let loc21 = fut2.location[1] let loc30 = fut3.location[0] let loc31 = fut3.location[1] + {.pop.} proc chk(loc: ptr SrcLoc, file: string, line: int, procedure: string): bool = From 60e6fc55bf93895f71816046284d55c1ec42a6ac Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Fri, 11 Aug 2023 00:31:47 +0300 Subject: [PATCH 15/50] Fix #431. (#441) --- chronos/asyncloop.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chronos/asyncloop.nim b/chronos/asyncloop.nim index a644b77..c6d69fd 100644 --- a/chronos/asyncloop.nim +++ b/chronos/asyncloop.nim @@ -1548,7 +1548,7 @@ proc isCounterLeaked*(name: string): bool {.noinit.} = ## number of `closed` requests. let tracker = TrackerCounter(opened: 0'u64, closed: 0'u64) let res = getThreadDispatcher().counters.getOrDefault(name, tracker) - res.opened == res.closed + res.opened != res.closed iterator trackerCounters*( loop: PDispatcher From 300fbaaf09cf8cc8d3798daa328d90d015906623 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 4 Sep 2023 21:49:45 +0300 Subject: [PATCH 16/50] HttpAddress errors should be not only critical. (#446) * Distinguish between resolve errors and check errors. * Fix issues and add test for getHttpAddress() procedure. * Address review comments. --- chronos/apps/http/httpclient.nim | 85 ++++++++++++++++++++++++++++++++ chronos/apps/http/httpcommon.nim | 42 ++++++++++++++++ tests/testhttpclient.nim | 83 +++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 63ffc37..b4b3202 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -195,6 +195,8 @@ type name*: string data*: string + HttpAddressResult* = Result[HttpAddress, HttpAddressErrorType] + # HttpClientRequestRef valid states are: # Ready -> Open -> (Finished, Error) -> (Closing, Closed) # @@ -298,6 +300,89 @@ proc getTLSFlags(flags: HttpClientFlags): set[TLSFlags] {.raises: [] .} = res.incl(TLSFlags.NoVerifyServerName) res +proc getHttpAddress*( + url: Uri, + flags: HttpClientFlags = {} + ): HttpAddressResult {.raises: [].} = + let + scheme = + if len(url.scheme) == 0: + HttpClientScheme.NonSecure + else: + case toLowerAscii(url.scheme) + of "http": + HttpClientScheme.NonSecure + of "https": + HttpClientScheme.Secure + else: + return err(HttpAddressErrorType.InvalidUrlScheme) + port = + if len(url.port) == 0: + case scheme + of HttpClientScheme.NonSecure: + 80'u16 + of HttpClientScheme.Secure: + 443'u16 + else: + Base10.decode(uint16, url.port).valueOr: + return err(HttpAddressErrorType.InvalidPortNumber) + hostname = + block: + if len(url.hostname) == 0: + return err(HttpAddressErrorType.MissingHostname) + url.hostname + id = hostname & ":" & Base10.toString(port) + addresses = + if (HttpClientFlag.NoInet4Resolution in flags) and + (HttpClientFlag.NoInet6Resolution in flags): + # DNS resolution is disabled. + try: + @[initTAddress(hostname, Port(port))] + except TransportAddressError: + return err(HttpAddressErrorType.InvalidIpHostname) + else: + try: + if (HttpClientFlag.NoInet4Resolution notin flags) and + (HttpClientFlag.NoInet6Resolution notin flags): + # DNS resolution for both IPv4 and IPv6 addresses. + resolveTAddress(hostname, Port(port)) + else: + if HttpClientFlag.NoInet6Resolution in flags: + # DNS resolution only for IPv4 addresses. + resolveTAddress(hostname, Port(port), AddressFamily.IPv4) + else: + # DNS resolution only for IPv6 addresses + resolveTAddress(hostname, Port(port), AddressFamily.IPv6) + except TransportAddressError: + return err(HttpAddressErrorType.NameLookupFailed) + + if len(addresses) == 0: + return err(HttpAddressErrorType.NoAddressResolved) + + ok(HttpAddress(id: id, scheme: scheme, hostname: hostname, port: port, + path: url.path, query: url.query, anchor: url.anchor, + username: url.username, password: url.password, + addresses: addresses)) + +proc getHttpAddress*( + url: string, + flags: HttpClientFlags = {} + ): HttpAddressResult {.raises: [].} = + getHttpAddress(parseUri(url), flags) + +proc getHttpAddress*( + session: HttpSessionRef, + url: Uri + ): HttpAddressResult {.raises: [].} = + getHttpAddress(url, session.flags) + +proc getHttpAddress*( + session: HttpSessionRef, + url: string + ): HttpAddressResult {.raises: [].} = + ## Create new HTTP address using URL string ``url`` and . + getHttpAddress(parseUri(url), session.flags) + proc getAddress*(session: HttpSessionRef, url: Uri): HttpResult[HttpAddress] {. raises: [] .} = let scheme = diff --git a/chronos/apps/http/httpcommon.nim b/chronos/apps/http/httpcommon.nim index 5a4a628..c01c1c3 100644 --- a/chronos/apps/http/httpcommon.nim +++ b/chronos/apps/http/httpcommon.nim @@ -82,6 +82,48 @@ type HttpState* {.pure.} = enum Alive, Closing, Closed + HttpAddressErrorType* {.pure.} = enum + InvalidUrlScheme, + InvalidPortNumber, + MissingHostname, + InvalidIpHostname, + NameLookupFailed, + NoAddressResolved + +const + CriticalHttpAddressError* = { + HttpAddressErrorType.InvalidUrlScheme, + HttpAddressErrorType.InvalidPortNumber, + HttpAddressErrorType.MissingHostname, + HttpAddressErrorType.InvalidIpHostname + } + + RecoverableHttpAddressError* = { + HttpAddressErrorType.NameLookupFailed, + HttpAddressErrorType.NoAddressResolved + } + +func isCriticalError*(error: HttpAddressErrorType): bool = + error in CriticalHttpAddressError + +func isRecoverableError*(error: HttpAddressErrorType): bool = + error in RecoverableHttpAddressError + +func toString*(error: HttpAddressErrorType): string = + case error + of HttpAddressErrorType.InvalidUrlScheme: + "URL scheme not supported" + of HttpAddressErrorType.InvalidPortNumber: + "Invalid URL port number" + of HttpAddressErrorType.MissingHostname: + "Missing URL hostname" + of HttpAddressErrorType.InvalidIpHostname: + "Invalid IPv4/IPv6 address in hostname" + of HttpAddressErrorType.NameLookupFailed: + "Could not resolve remote address" + of HttpAddressErrorType.NoAddressResolved: + "No address has been resolved" + proc raiseHttpCriticalError*(msg: string, code = Http400) {.noinline, noreturn.} = raise (ref HttpCriticalError)(code: code, msg: msg) diff --git a/tests/testhttpclient.nim b/tests/testhttpclient.nim index 1eacc21..4daaf87 100644 --- a/tests/testhttpclient.nim +++ b/tests/testhttpclient.nim @@ -1262,5 +1262,88 @@ suite "HTTP client testing suite": test "HTTP client server-sent events test": check waitFor(testServerSentEvents(false)) == true + test "HTTP getHttpAddress() test": + block: + # HTTP client supports only `http` and `https` schemes in URL. + let res = getHttpAddress("ftp://ftp.scene.org") + check: + res.isErr() + res.error == HttpAddressErrorType.InvalidUrlScheme + res.error.isCriticalError() + block: + # HTTP URL default ports and custom ports test + let + res1 = getHttpAddress("http://www.google.com") + res2 = getHttpAddress("https://www.google.com") + res3 = getHttpAddress("http://www.google.com:35000") + res4 = getHttpAddress("https://www.google.com:25000") + check: + res1.isOk() + res2.isOk() + res3.isOk() + res4.isOk() + res1.get().port == 80 + res2.get().port == 443 + res3.get().port == 35000 + res4.get().port == 25000 + block: + # HTTP URL invalid port values test + let + res1 = getHttpAddress("http://www.google.com:-80") + res2 = getHttpAddress("http://www.google.com:0") + res3 = getHttpAddress("http://www.google.com:65536") + res4 = getHttpAddress("http://www.google.com:65537") + res5 = getHttpAddress("https://www.google.com:-443") + res6 = getHttpAddress("https://www.google.com:0") + res7 = getHttpAddress("https://www.google.com:65536") + res8 = getHttpAddress("https://www.google.com:65537") + check: + res1.isErr() and res1.error == HttpAddressErrorType.InvalidPortNumber + res1.error.isCriticalError() + res2.isOk() + res2.get().port == 0 + res3.isErr() and res3.error == HttpAddressErrorType.InvalidPortNumber + res3.error.isCriticalError() + res4.isErr() and res4.error == HttpAddressErrorType.InvalidPortNumber + res4.error.isCriticalError() + res5.isErr() and res5.error == HttpAddressErrorType.InvalidPortNumber + res5.error.isCriticalError() + res6.isOk() + res6.get().port == 0 + res7.isErr() and res7.error == HttpAddressErrorType.InvalidPortNumber + res7.error.isCriticalError() + res8.isErr() and res8.error == HttpAddressErrorType.InvalidPortNumber + res8.error.isCriticalError() + block: + # HTTP URL missing hostname + let + res1 = getHttpAddress("http://") + res2 = getHttpAddress("https://") + check: + res1.isErr() and res1.error == HttpAddressErrorType.MissingHostname + res1.error.isCriticalError() + res2.isErr() and res2.error == HttpAddressErrorType.MissingHostname + res2.error.isCriticalError() + block: + # No resolution flags and incorrect URL + let + flags = {HttpClientFlag.NoInet4Resolution, + HttpClientFlag.NoInet6Resolution} + res1 = getHttpAddress("http://256.256.256.256", flags) + res2 = getHttpAddress( + "http://[FFFFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF]", flags) + check: + res1.isErr() and res1.error == HttpAddressErrorType.InvalidIpHostname + res1.error.isCriticalError() + res2.isErr() and res2.error == HttpAddressErrorType.InvalidIpHostname + res2.error.isCriticalError() + block: + # Resolution of non-existent hostname + let res = getHttpAddress("http://eYr6bdBo.com") + check: + res.isErr() and res.error == HttpAddressErrorType.NameLookupFailed + res.error.isRecoverableError() + not(res.error.isCriticalError()) + test "Leaks test": checkLeaks() From e706167a532cbb0ba4346e3dde7fd3f7f5c16f4f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 5 Sep 2023 12:41:52 +0200 Subject: [PATCH 17/50] add connect cancellation test (#444) --- chronos/unittest2/asynctests.nim | 6 +- tests/testserver.nim | 101 ++++++++++++++++++------------- 2 files changed, 62 insertions(+), 45 deletions(-) diff --git a/chronos/unittest2/asynctests.nim b/chronos/unittest2/asynctests.nim index bc703b7..758e0a6 100644 --- a/chronos/unittest2/asynctests.nim +++ b/chronos/unittest2/asynctests.nim @@ -21,9 +21,9 @@ template asyncTest*(name: string, body: untyped): untyped = template checkLeaks*(name: string): untyped = let counter = getTrackerCounter(name) - if counter.opened != counter.closed: - echo "[" & name & "] opened = ", counter.opened, - ", closed = ", counter.closed + checkpoint: + "[" & name & "] opened = " & $counter.opened & + ", closed = " & $ counter.closed check counter.opened == counter.closed template checkLeaks*(): untyped = diff --git a/tests/testserver.nim b/tests/testserver.nim index e7e834e..a63c9df 100644 --- a/tests/testserver.nim +++ b/tests/testserver.nim @@ -5,8 +5,8 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest2 -import ../chronos + +import ../chronos/unittest2/asynctests {.used.} @@ -23,6 +23,9 @@ suite "Server's test suite": CustomData = ref object test: string + teardown: + checkLeaks() + proc serveStreamClient(server: StreamServer, transp: StreamTransport) {.async.} = discard @@ -54,37 +57,47 @@ suite "Server's test suite": transp.test = "CUSTOM" result = cast[StreamTransport](transp) - proc test1(): bool = + asyncTest "Stream Server start/stop test": var ta = initTAddress("127.0.0.1:31354") var server1 = createStreamServer(ta, serveStreamClient, {ReuseAddr}) server1.start() server1.stop() server1.close() - waitFor server1.join() + await server1.join() + var server2 = createStreamServer(ta, serveStreamClient, {ReuseAddr}) server2.start() server2.stop() server2.close() - waitFor server2.join() - result = true + await server2.join() - proc test5(): bool = - var ta = initTAddress("127.0.0.1:31354") + asyncTest "Stream Server stop without start test": + var ta = initTAddress("127.0.0.1:0") var server1 = createStreamServer(ta, serveStreamClient, {ReuseAddr}) + ta = server1.localAddress() server1.stop() server1.close() - waitFor server1.join() + + await server1.join() var server2 = createStreamServer(ta, serveStreamClient, {ReuseAddr}) server2.stop() server2.close() - waitFor server2.join() - result = true + await server2.join() + + asyncTest "Stream Server inherited object test": + var server = CustomServer() + server.test1 = "TEST" + var ta = initTAddress("127.0.0.1:0") + var pserver = createStreamServer(ta, serveCustomStreamClient, {ReuseAddr}, + child = server, + init = customServerTransport) + check: + pserver == server - proc client1(server: CustomServer, ta: TransportAddress) {.async.} = var transp = CustomTransport() transp.test = "CLIENT" server.start() - var ptransp = await connect(ta, child = transp) + var ptransp = await connect(server.localAddress(), child = transp) var etransp = cast[CustomTransport](ptransp) doAssert(etransp.test == "CLIENT") var msg = "TEST\r\n" @@ -96,44 +109,48 @@ suite "Server's test suite": server.close() await server.join() - proc client2(server: StreamServer, - ta: TransportAddress): Future[bool] {.async.} = + check: + server.test1 == "CONNECTION" + server.test2 == "CUSTOM" + + asyncTest "StreamServer[T] test": + var co = CustomData() + co.test = "CUSTOMDATA" + var ta = initTAddress("127.0.0.1:0") + var server = createStreamServer(ta, serveUdataStreamClient, {ReuseAddr}, + udata = co) + server.start() - var transp = await connect(ta) + var transp = await connect(server.localAddress()) var msg = "TEST\r\n" discard await transp.write(msg) var line = await transp.readLine() - result = (line == "TESTCUSTOMDATA") + check: + line == "TESTCUSTOMDATA" transp.close() server.stop() server.close() await server.join() - proc test3(): bool = - var server = CustomServer() - server.test1 = "TEST" - var ta = initTAddress("127.0.0.1:31354") - var pserver = createStreamServer(ta, serveCustomStreamClient, {ReuseAddr}, - child = cast[StreamServer](server), - init = customServerTransport) - doAssert(not isNil(pserver)) - waitFor client1(server, ta) - result = (server.test1 == "CONNECTION") and (server.test2 == "CUSTOM") + asyncTest "Backlog and connect cancellation": + var ta = initTAddress("127.0.0.1:0") + var server1 = createStreamServer(ta, serveStreamClient, {ReuseAddr}, backlog = 1) + ta = server1.localAddress() - proc test4(): bool = - var co = CustomData() - co.test = "CUSTOMDATA" - var ta = initTAddress("127.0.0.1:31354") - var server = createStreamServer(ta, serveUdataStreamClient, {ReuseAddr}, - udata = co) - result = waitFor client2(server, ta) + var clients: seq[Future[StreamTransport]] + for i in 0..<10: + clients.add(connect(server1.localAddress)) + # Check for leaks in cancellation / connect when server is not accepting + for c in clients: + if not c.finished: + await c.cancelAndWait() + else: + # The backlog connection "should" end up here + try: + await c.read().closeWait() + except CatchableError: + discard - test "Stream Server start/stop test": - check test1() == true - test "Stream Server stop without start test": - check test5() == true - test "Stream Server inherited object test": - check test3() == true - test "StreamServer[T] test": - check test4() == true + server1.close() + await server1.join() From db6410f835c51676f78002a9aa786630e16fbb08 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Tue, 5 Sep 2023 13:48:09 +0300 Subject: [PATCH 18/50] Fix CI badge status. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c0cc230..3772c12 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Chronos - An efficient library for asynchronous programming -[![Github action](https://github.com/status-im/nim-chronos/workflows/nim-chronos%20CI/badge.svg)](https://github.com/status-im/nim-chronos/actions/workflows/ci.yml) +[![Github action](https://github.com/status-im/nim-chronos/workflows/CI/badge.svg)](https://github.com/status-im/nim-chronos/actions/workflows/ci.yml) [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg) From 00614476c68f0553432b4bb505e24d6ad5586ae4 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Thu, 7 Sep 2023 16:25:25 +0300 Subject: [PATCH 19/50] Address issue #443. (#447) * Address issue #443. * Address review comments. --- chronos/apps/http/httpclient.nim | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index b4b3202..1815d28 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -13,7 +13,7 @@ import ../../streams/[asyncstream, tlsstream, chunkstream, boundstream] import httptable, httpcommon, httpagent, httpbodyrw, multipart export results, asyncloop, asyncsync, asyncstream, tlsstream, chunkstream, boundstream, httptable, httpcommon, httpagent, httpbodyrw, multipart, - httputils + httputils, uri export SocketFlags const @@ -1420,8 +1420,13 @@ proc redirect*(request: HttpClientRequestRef, if redirectCount > request.session.maxRedirections: err("Maximum number of redirects exceeded") else: + let headers = + block: + var res = request.headers + res.set(HostHeader, ha.hostname) + res var res = HttpClientRequestRef.new(request.session, ha, request.meth, - request.version, request.flags, request.headers.toList(), request.buffer) + request.version, request.flags, headers.toList(), request.buffer) res.redirectCount = redirectCount ok(res) @@ -1438,8 +1443,14 @@ proc redirect*(request: HttpClientRequestRef, err("Maximum number of redirects exceeded") else: let address = ? request.session.redirect(request.address, uri) + # Update Host header to redirected URL hostname + let headers = + block: + var res = request.headers + res.set(HostHeader, address.hostname) + res var res = HttpClientRequestRef.new(request.session, address, request.meth, - request.version, request.flags, request.headers.toList(), request.buffer) + request.version, request.flags, headers.toList(), request.buffer) res.redirectCount = redirectCount ok(res) From 2e8551b0d973cfbebfab3be7f3329e11b9049007 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Fri, 15 Sep 2023 19:38:39 +0300 Subject: [PATCH 20/50] Cancellation fixes and tests. (#445) * Add callTick and stream cancellation tests. * Fix stepsAsync() test. * Cancellation changes. * Update and add more cancellation tests. * Fix Posix shutdown call to handle ENOTCONN error. * With new changes to to cancellation its now possible. * Refactor testsoon.nim to not produce artifacts after tests are finished. * Debugging MacOS issue. * Adjust flaky test times. * Fix issue. * Add test for issue #334 which was also addressed in this PR. Avoid `break` in problematic test. * Add noCancelWait() call which prohibits cancellation. Fix closeWait() calls to use noCancelWait() predicate. Adding sleep to flaky MacOS test. * Remove all debugging echoes. * Fix cancelAndWait() which now could perform multiple attempts to cancel target Future (mustCancel behavior). * Fix issues revealed by switch to different cancelAndWait(). * Address review comments. * Fix testutils compilation warning. * Rename callTick() to internalCallTick(). * Add some documentation comments. * Disable flaky ratelimit test. * Rename noCancelWait() to noCancel(). Address review comments. --- chronos/apps/http/httpbodyrw.nim | 6 +- chronos/apps/http/httpclient.nim | 31 +- chronos/apps/http/httpserver.nim | 18 +- chronos/apps/http/shttpserver.nim | 17 +- chronos/asyncfutures2.nim | 201 ++++++++---- chronos/asyncloop.nim | 175 ++++++----- chronos/asyncmacro2.nim | 7 - chronos/asyncproc.nim | 2 +- chronos/asyncsync.nim | 12 +- chronos/futures.nim | 41 ++- chronos/ratelimit.nim | 4 +- chronos/streams/asyncstream.nim | 30 +- chronos/streams/tlsstream.nim | 18 +- chronos/transports/datagram.nim | 22 +- chronos/transports/stream.nim | 32 +- tests/testbugs.nim | 2 +- tests/testfut.nim | 496 +++++++++++++++++++++++++++++- tests/testhttpclient.nim | 113 +++++++ tests/testhttpserver.nim | 95 +++--- tests/testratelimit.nim | 19 +- tests/testsoon.nim | 146 ++++----- tests/teststream.nim | 95 +++++- tests/testsync.nim | 4 +- tests/testtime.nim | 24 +- tests/testutils.nim | 2 +- 25 files changed, 1243 insertions(+), 369 deletions(-) diff --git a/chronos/apps/http/httpbodyrw.nim b/chronos/apps/http/httpbodyrw.nim index b948fbd..bb28ea6 100644 --- a/chronos/apps/http/httpbodyrw.nim +++ b/chronos/apps/http/httpbodyrw.nim @@ -45,8 +45,8 @@ proc closeWait*(bstream: HttpBodyReader) {.async.} = # data from stream at position [1]. for index in countdown((len(bstream.streams) - 1), 0): res.add(bstream.streams[index].closeWait()) - await allFutures(res) - await procCall(closeWait(AsyncStreamReader(bstream))) + res.add(procCall(closeWait(AsyncStreamReader(bstream)))) + await noCancel(allFutures(res)) bstream.bstate = HttpState.Closed untrackCounter(HttpBodyReaderTrackerName) @@ -68,7 +68,7 @@ proc closeWait*(bstream: HttpBodyWriter) {.async.} = var res = newSeq[Future[void]]() for index in countdown(len(bstream.streams) - 1, 0): res.add(bstream.streams[index].closeWait()) - await allFutures(res) + await noCancel(allFutures(res)) await procCall(closeWait(AsyncStreamWriter(bstream))) bstream.bstate = HttpState.Closed untrackCounter(HttpBodyWriterTrackerName) diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 1815d28..01e2bab 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -600,14 +600,12 @@ proc closeWait(conn: HttpClientConnectionRef) {.async.} = res.add(conn.reader.closeWait()) if not(isNil(conn.writer)) and not(conn.writer.closed()): res.add(conn.writer.closeWait()) + if conn.kind == HttpClientScheme.Secure: + res.add(conn.treader.closeWait()) + res.add(conn.twriter.closeWait()) + res.add(conn.transp.closeWait()) res - if len(pending) > 0: await allFutures(pending) - case conn.kind - of HttpClientScheme.Secure: - await allFutures(conn.treader.closeWait(), conn.twriter.closeWait()) - of HttpClientScheme.NonSecure: - discard - await conn.transp.closeWait() + if len(pending) > 0: await noCancel(allFutures(pending)) conn.state = HttpClientConnectionState.Closed untrackCounter(HttpClientConnectionTrackerName) @@ -631,8 +629,7 @@ proc connect(session: HttpSessionRef, let conn = block: let res = HttpClientConnectionRef.new(session, ha, transp) - case res.kind - of HttpClientScheme.Secure: + if res.kind == HttpClientScheme.Secure: try: await res.tls.handshake() res.state = HttpClientConnectionState.Ready @@ -647,7 +644,7 @@ proc connect(session: HttpSessionRef, await res.closeWait() res.state = HttpClientConnectionState.Error lastError = $exc.msg - of HttpClientScheme.Nonsecure: + else: res.state = HttpClientConnectionState.Ready res if conn.state == HttpClientConnectionState.Ready: @@ -785,7 +782,7 @@ proc closeWait*(session: HttpSessionRef) {.async.} = for connections in session.connections.values(): for conn in connections: pending.add(closeWait(conn)) - await allFutures(pending) + await noCancel(allFutures(pending)) proc sessionWatcher(session: HttpSessionRef) {.async.} = while true: @@ -830,26 +827,30 @@ proc sessionWatcher(session: HttpSessionRef) {.async.} = break proc closeWait*(request: HttpClientRequestRef) {.async.} = + var pending: seq[FutureBase] if request.state notin {HttpReqRespState.Closing, HttpReqRespState.Closed}: request.state = HttpReqRespState.Closing if not(isNil(request.writer)): if not(request.writer.closed()): - await request.writer.closeWait() + pending.add(FutureBase(request.writer.closeWait())) request.writer = nil - await request.releaseConnection() + pending.add(FutureBase(request.releaseConnection())) + await noCancel(allFutures(pending)) request.session = nil request.error = nil request.state = HttpReqRespState.Closed untrackCounter(HttpClientRequestTrackerName) proc closeWait*(response: HttpClientResponseRef) {.async.} = + var pending: seq[FutureBase] if response.state notin {HttpReqRespState.Closing, HttpReqRespState.Closed}: response.state = HttpReqRespState.Closing if not(isNil(response.reader)): if not(response.reader.closed()): - await response.reader.closeWait() + pending.add(FutureBase(response.reader.closeWait())) response.reader = nil - await response.releaseConnection() + pending.add(FutureBase(response.releaseConnection())) + await noCancel(allFutures(pending)) response.session = nil response.error = nil response.state = HttpReqRespState.Closed diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index eafa27c..f0788e2 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -809,10 +809,7 @@ proc closeUnsecureConnection(conn: HttpConnectionRef) {.async.} = pending.add(conn.mainReader.closeWait()) pending.add(conn.mainWriter.closeWait()) pending.add(conn.transp.closeWait()) - try: - await allFutures(pending) - except CancelledError: - await allFutures(pending) + await noCancel(allFutures(pending)) untrackCounter(HttpServerUnsecureConnectionTrackerName) reset(conn[]) conn.state = HttpState.Closed @@ -829,7 +826,7 @@ proc new(ht: typedesc[HttpConnectionRef], server: HttpServerRef, res proc gracefulCloseWait*(conn: HttpConnectionRef) {.async.} = - await conn.transp.shutdownWait() + await noCancel(conn.transp.shutdownWait()) await conn.closeCb(conn) proc closeWait*(conn: HttpConnectionRef): Future[void] = @@ -841,11 +838,7 @@ proc closeWait*(req: HttpRequestRef) {.async.} = req.state = HttpState.Closing let resp = req.response.get() if (HttpResponseFlags.Stream in resp.flags) and not(isNil(resp.writer)): - var writer = resp.writer.closeWait() - try: - await writer - except CancelledError: - await writer + await closeWait(resp.writer) reset(resp[]) untrackCounter(HttpServerRequestTrackerName) reset(req[]) @@ -1038,7 +1031,6 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async.} = except CatchableError as exc: raiseAssert "Unexpected error [" & $exc.name & "] happens: " & $exc.msg - server.connections.del(connectionId) case runLoop of HttpProcessExitType.KeepAlive: await connection.closeWait() @@ -1047,6 +1039,8 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async.} = of HttpProcessExitType.Graceful: await connection.gracefulCloseWait() + server.connections.del(connectionId) + proc acceptClientLoop(server: HttpServerRef) {.async.} = var runLoop = true while runLoop: @@ -1102,7 +1096,7 @@ proc drop*(server: HttpServerRef) {.async.} = for holder in server.connections.values(): if not(isNil(holder.future)) and not(holder.future.finished()): pending.add(holder.future.cancelAndWait()) - await allFutures(pending) + await noCancel(allFutures(pending)) server.connections.clear() proc closeWait*(server: HttpServerRef) {.async.} = diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index bc5c3fb..6d321a0 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -30,19 +30,10 @@ proc closeSecConnection(conn: HttpConnectionRef) {.async.} = var pending: seq[Future[void]] pending.add(conn.writer.closeWait()) pending.add(conn.reader.closeWait()) - try: - await allFutures(pending) - except CancelledError: - await allFutures(pending) - # After we going to close everything else. - pending.setLen(3) - pending[0] = conn.mainReader.closeWait() - pending[1] = conn.mainWriter.closeWait() - pending[2] = conn.transp.closeWait() - try: - await allFutures(pending) - except CancelledError: - await allFutures(pending) + pending.add(conn.mainReader.closeWait()) + pending.add(conn.mainWriter.closeWait()) + pending.add(conn.transp.closeWait()) + await noCancel(allFutures(pending)) reset(cast[SecureHttpConnectionRef](conn)[]) untrackCounter(HttpServerSecureConnectionTrackerName) conn.state = HttpState.Closed diff --git a/chronos/asyncfutures2.nim b/chronos/asyncfutures2.nim index d3954ba..ee6e8e0 100644 --- a/chronos/asyncfutures2.nim +++ b/chronos/asyncfutures2.nim @@ -12,6 +12,7 @@ import std/sequtils import stew/base10 when chronosStackTrace: + import std/strutils when defined(nimHasStacktracesModule): import system/stacktraces else: @@ -26,7 +27,8 @@ template LocFinishIndex*: auto {.deprecated: "LocationKind.Finish".} = template LocCompleteIndex*: untyped {.deprecated: "LocationKind.Finish".} = LocationKind.Finish -func `[]`*(loc: array[LocationKind, ptr SrcLoc], v: int): ptr SrcLoc {.deprecated: "use LocationKind".} = +func `[]`*(loc: array[LocationKind, ptr SrcLoc], v: int): ptr SrcLoc {. + deprecated: "use LocationKind".} = case v of 0: loc[LocationKind.Create] of 1: loc[LocationKind.Finish] @@ -43,29 +45,37 @@ type # Backwards compatibility for old FutureState name template Finished* {.deprecated: "Use Completed instead".} = Completed -template Finished*(T: type FutureState): FutureState {.deprecated: "Use FutureState.Completed instead".} = FutureState.Completed +template Finished*(T: type FutureState): FutureState {. + deprecated: "Use FutureState.Completed instead".} = + FutureState.Completed proc newFutureImpl[T](loc: ptr SrcLoc): Future[T] = let fut = Future[T]() - internalInitFutureBase(fut, loc, FutureState.Pending) + internalInitFutureBase(fut, loc, FutureState.Pending, {}) + fut + +proc newFutureImpl[T](loc: ptr SrcLoc, flags: FutureFlags): Future[T] = + let fut = Future[T]() + internalInitFutureBase(fut, loc, FutureState.Pending, flags) fut proc newFutureSeqImpl[A, B](loc: ptr SrcLoc): FutureSeq[A, B] = let fut = FutureSeq[A, B]() - internalInitFutureBase(fut, loc, FutureState.Pending) + internalInitFutureBase(fut, loc, FutureState.Pending, {}) fut proc newFutureStrImpl[T](loc: ptr SrcLoc): FutureStr[T] = let fut = FutureStr[T]() - internalInitFutureBase(fut, loc, FutureState.Pending) + internalInitFutureBase(fut, loc, FutureState.Pending, {}) fut -template newFuture*[T](fromProc: static[string] = ""): Future[T] = +template newFuture*[T](fromProc: static[string] = "", + flags: static[FutureFlags] = {}): Future[T] = ## Creates a new future. ## ## Specifying ``fromProc``, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. - newFutureImpl[T](getSrcLocation(fromProc)) + newFutureImpl[T](getSrcLocation(fromProc), flags) template newFutureSeq*[A, B](fromProc: static[string] = ""): FutureSeq[A, B] = ## Create a new future which can hold/preserve GC sequence until future will @@ -132,8 +142,6 @@ proc finish(fut: FutureBase, state: FutureState) = # 1. `finish()` is a private procedure and `state` is under our control. # 2. `fut.state` is checked by `checkFinished()`. fut.internalState = state - when chronosStrictFutureAccess: - doAssert fut.internalCancelcb == nil or state != FutureState.Cancelled fut.internalCancelcb = nil # release cancellation callback memory for item in fut.internalCallbacks.mitems(): if not(isNil(item.function)): @@ -194,21 +202,23 @@ proc cancelAndSchedule(future: FutureBase, loc: ptr SrcLoc) = template cancelAndSchedule*(future: FutureBase) = cancelAndSchedule(future, getSrcLocation()) -proc cancel(future: FutureBase, loc: ptr SrcLoc): bool = - ## Request that Future ``future`` cancel itself. +proc tryCancel(future: FutureBase, loc: ptr SrcLoc): bool = + ## Perform an attempt to cancel ``future``. ## - ## This arranges for a `CancelledError` to be thrown into procedure which - ## waits for ``future`` on the next cycle through the event loop. - ## The procedure then has a chance to clean up or even deny the request - ## using `try/except/finally`. + ## NOTE: This procedure does not guarantee that cancellation will actually + ## happened. ## - ## This call do not guarantee that the ``future`` will be cancelled: the - ## exception might be caught and acted upon, delaying cancellation of the - ## ``future`` or preventing cancellation completely. The ``future`` may also - ## return value or raise different exception. + ## Cancellation is the process which starts from the last ``future`` + ## descendent and moves step by step to the parent ``future``. To initiate + ## this process procedure iterates through all non-finished ``future`` + ## descendents and tries to find the last one. If last descendent is still + ## pending it will become cancelled and process will be initiated. In such + ## case this procedure returns ``true``. ## - ## Immediately after this procedure is called, ``future.cancelled()`` will - ## not return ``true`` (unless the Future was already cancelled). + ## If last descendent future is not pending, this procedure will be unable to + ## initiate cancellation process and so it returns ``false``. + if future.cancelled(): + return true if future.finished(): return false @@ -217,23 +227,18 @@ proc cancel(future: FutureBase, loc: ptr SrcLoc): bool = # mechanism and/or use a regular `addCallback` when chronosStrictFutureAccess: doAssert future.internalCancelcb.isNil, - "futures returned from `{.async.}` functions must not use `cancelCallback`" - - if cancel(future.internalChild, getSrcLocation()): - return true - + "futures returned from `{.async.}` functions must not use " & + "`cancelCallback`" + tryCancel(future.internalChild, loc) else: if not(isNil(future.internalCancelcb)): future.internalCancelcb(cast[pointer](future)) - future.internalCancelcb = nil - cancelAndSchedule(future, getSrcLocation()) + if FutureFlag.OwnCancelSchedule notin future.internalFlags: + cancelAndSchedule(future, loc) + future.cancelled() - future.internalMustCancel = true - return true - -template cancel*(future: FutureBase) = - ## Cancel ``future``. - discard cancel(future, getSrcLocation()) +template tryCancel*(future: FutureBase): bool = + tryCancel(future, getSrcLocation()) proc clearCallbacks(future: FutureBase) = future.internalCallbacks = default(seq[AsyncCallback]) @@ -778,27 +783,117 @@ proc oneValue*[T](futs: varargs[Future[T]]): Future[T] {. return retFuture -proc cancelAndWait*(fut: FutureBase): Future[void] = - ## Initiate cancellation process for Future ``fut`` and wait until ``fut`` is - ## done e.g. changes its state (become completed, failed or cancelled). +proc cancelSoon(future: FutureBase, aftercb: CallbackFunc, udata: pointer, + loc: ptr SrcLoc) = + ## Perform cancellation ``future`` and call ``aftercb`` callback when + ## ``future`` become finished (completed with value, failed or cancelled). ## - ## If ``fut`` is already finished (completed, failed or cancelled) result - ## Future[void] object will be returned complete. - var retFuture = newFuture[void]("chronos.cancelAndWait(T)") - proc continuation(udata: pointer) = - if not(retFuture.finished()): - retFuture.complete() - proc cancellation(udata: pointer) = - if not(fut.finished()): - fut.removeCallback(continuation) - if fut.finished(): + ## NOTE: Compared to the `tryCancel()` call, this procedure call guarantees + ## that ``future``will be finished (completed with value, failed or cancelled) + ## as quickly as possible. + proc checktick(udata: pointer) {.gcsafe.} = + # We trying to cancel Future on more time, and if `cancel()` succeeds we + # return early. + if tryCancel(future, loc): + return + # Cancellation signal was not delivered, so we trying to deliver it one + # more time after one tick. But we need to check situation when child + # future was finished but our completion callback is not yet invoked. + if not(future.finished()): + internalCallTick(checktick) + + proc continuation(udata: pointer) {.gcsafe.} = + # We do not use `callSoon` here because we was just scheduled from `poll()`. + if not(isNil(aftercb)): + aftercb(udata) + + if future.finished(): + # We could not schedule callback directly otherwise we could fall into + # recursion problem. + if not(isNil(aftercb)): + let loop = getThreadDispatcher() + loop.callbacks.addLast(AsyncCallback(function: aftercb, udata: udata)) + return + + future.addCallback(continuation) + # Initiate cancellation process. + if not(tryCancel(future, loc)): + # Cancellation signal was not delivered, so we trying to deliver it one + # more time after async tick. But we need to check case, when future was + # finished but our completion callback is not yet invoked. + if not(future.finished()): + internalCallTick(checktick) + +template cancelSoon*(fut: FutureBase, cb: CallbackFunc, udata: pointer) = + cancelSoon(fut, cb, udata, getSrcLocation()) + +template cancelSoon*(fut: FutureBase, cb: CallbackFunc) = + cancelSoon(fut, cb, nil, getSrcLocation()) + +template cancelSoon*(fut: FutureBase, acb: AsyncCallback) = + cancelSoon(fut, acb.function, acb.udata, getSrcLocation()) + +template cancelSoon*(fut: FutureBase) = + cancelSoon(fut, nil, nil, getSrcLocation()) + +template cancel*(future: FutureBase) {. + deprecated: "Please use cancelSoon() or cancelAndWait() instead".} = + ## Cancel ``future``. + cancelSoon(future, nil, nil, getSrcLocation()) + +proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] = + ## Perform cancellation ``future`` return Future which will be completed when + ## ``future`` become finished (completed with value, failed or cancelled). + ## + ## NOTE: Compared to the `tryCancel()` call, this procedure call guarantees + ## that ``future``will be finished (completed with value, failed or cancelled) + ## as quickly as possible. + let retFuture = newFuture[void]("chronos.cancelAndWait(FutureBase)", + {FutureFlag.OwnCancelSchedule}) + + proc continuation(udata: pointer) {.gcsafe.} = + retFuture.complete() + + if future.finished(): retFuture.complete() else: - fut.addCallback(continuation) - retFuture.cancelCallback = cancellation - # Initiate cancellation process. - fut.cancel() - return retFuture + cancelSoon(future, continuation, cast[pointer](retFuture), loc) + + retFuture + +template cancelAndWait*(future: FutureBase): Future[void] = + ## Cancel ``future``. + cancelAndWait(future, getSrcLocation()) + +proc noCancel*[T](future: Future[T]): Future[T] = + ## Prevent cancellation requests from propagating to ``future`` while + ## forwarding its value or error when it finishes. + ## + ## This procedure should be used when you need to perform operations which + ## should not be cancelled at all cost, for example closing sockets, pipes, + ## connections or servers. Usually it become useful in exception or finally + ## blocks. + let retFuture = newFuture[T]("chronos.noCancel(T)", + {FutureFlag.OwnCancelSchedule}) + template completeFuture() = + if future.completed(): + when T is void: + retFuture.complete() + else: + retFuture.complete(future.value) + elif future.failed(): + retFuture.fail(future.error) + else: + raiseAssert("Unexpected future state [" & $future.state & "]") + + proc continuation(udata: pointer) {.gcsafe.} = + completeFuture() + + if future.finished(): + completeFuture() + else: + future.addCallback(continuation) + retFuture proc allFutures*(futs: varargs[FutureBase]): Future[void] = ## Returns a future which will complete only when all futures in ``futs`` @@ -836,7 +931,7 @@ proc allFutures*(futs: varargs[FutureBase]): Future[void] = if len(nfuts) == 0 or len(nfuts) == finishedFutures: retFuture.complete() - return retFuture + retFuture proc allFutures*[T](futs: varargs[Future[T]]): Future[void] = ## Returns a future which will complete only when all futures in ``futs`` diff --git a/chronos/asyncloop.nim b/chronos/asyncloop.nim index c6d69fd..fecec39 100644 --- a/chronos/asyncloop.nim +++ b/chronos/asyncloop.nim @@ -11,7 +11,7 @@ {.push raises: [].} from nativesockets import Port -import std/[tables, strutils, heapqueue, deques] +import std/[tables, heapqueue, deques] import stew/results import "."/[config, futures, osdefs, oserrno, osutils, timer] @@ -179,10 +179,11 @@ type timers*: HeapQueue[TimerCallback] callbacks*: Deque[AsyncCallback] idlers*: Deque[AsyncCallback] + ticks*: Deque[AsyncCallback] trackers*: Table[string, TrackerBase] counters*: Table[string, TrackerCounter] -proc sentinelCallbackImpl(arg: pointer) {.gcsafe.} = +proc sentinelCallbackImpl(arg: pointer) {.gcsafe, noreturn.} = raiseAssert "Sentinel callback MUST not be scheduled" const @@ -254,6 +255,10 @@ template processIdlers(loop: untyped) = if len(loop.idlers) > 0: loop.callbacks.addLast(loop.idlers.popFirst()) +template processTicks(loop: untyped) = + while len(loop.ticks) > 0: + loop.callbacks.addLast(loop.ticks.popFirst()) + template processCallbacks(loop: untyped) = while true: let callable = loop.callbacks.popFirst() # len must be > 0 due to sentinel @@ -417,6 +422,7 @@ when defined(windows): timers: initHeapQueue[TimerCallback](), callbacks: initDeque[AsyncCallback](64), idlers: initDeque[AsyncCallback](), + ticks: initDeque[AsyncCallback](), trackers: initTable[string, TrackerBase](), counters: initTable[string, TrackerCounter]() ) @@ -746,6 +752,9 @@ when defined(windows): if networkEventsCount == 0: loop.processIdlers() + # We move tick callbacks to `loop.callbacks` always. + processTicks(loop) + # All callbacks which will be added during `processCallbacks` will be # scheduled after the sentinel and are processed on next `poll()` call. loop.callbacks.addLast(SentinelCallback) @@ -1138,6 +1147,9 @@ elif defined(macosx) or defined(freebsd) or defined(netbsd) or if count == 0: loop.processIdlers() + # We move tick callbacks to `loop.callbacks` always. + processTicks(loop) + # All callbacks which will be added during `processCallbacks` will be # scheduled after the sentinel and are processed on next `poll()` call. loop.callbacks.addLast(SentinelCallback) @@ -1255,6 +1267,20 @@ proc callIdle*(cbproc: CallbackFunc, data: pointer) = proc callIdle*(cbproc: CallbackFunc) = callIdle(cbproc, nil) +proc internalCallTick*(acb: AsyncCallback) = + ## Schedule ``cbproc`` to be called after all scheduled callbacks, but only + ## when OS system queue finished processing events. + getThreadDispatcher().ticks.addLast(acb) + +proc internalCallTick*(cbproc: CallbackFunc, data: pointer) = + ## Schedule ``cbproc`` to be called after all scheduled callbacks when + ## OS system queue processing is done. + doAssert(not isNil(cbproc)) + internalCallTick(AsyncCallback(function: cbproc, udata: data)) + +proc internalCallTick*(cbproc: CallbackFunc) = + internalCallTick(AsyncCallback(function: cbproc, udata: nil)) + include asyncfutures2 when (chronosEventEngine in ["epoll", "kqueue"]) or defined(windows): @@ -1322,30 +1348,24 @@ proc stepsAsync*(number: int): Future[void] = ## ## This primitive can be useful when you need to create more deterministic ## tests and cases. - ## - ## WARNING! Do not use this primitive to perform switch between tasks, because - ## this can lead to 100% CPU load in the moments when there are no I/O - ## events. Usually when there no I/O events CPU consumption should be near 0%. - var retFuture = newFuture[void]("chronos.stepsAsync(int)") - var counter = 0 + doAssert(number > 0, "Number should be positive integer") + var + retFuture = newFuture[void]("chronos.stepsAsync(int)") + counter = 0 + continuation: proc(data: pointer) {.gcsafe, raises: [].} - var continuation: proc(data: pointer) {.gcsafe, raises: [].} continuation = proc(data: pointer) {.gcsafe, raises: [].} = if not(retFuture.finished()): inc(counter) if counter < number: - callSoon(continuation, nil) + internalCallTick(continuation) else: retFuture.complete() - proc cancellation(udata: pointer) = - discard - if number <= 0: retFuture.complete() else: - retFuture.cancelCallback = cancellation - callSoon(continuation, nil) + internalCallTick(continuation) retFuture @@ -1374,37 +1394,46 @@ proc withTimeout*[T](fut: Future[T], timeout: Duration): Future[bool] = ## If ``fut`` completes first the returned future will hold true, ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned ## future will hold false. - var retFuture = newFuture[bool]("chronos.`withTimeout`") - var moment: Moment - var timer: TimerCallback - var cancelling = false + var + retFuture = newFuture[bool]("chronos.withTimeout", + {FutureFlag.OwnCancelSchedule}) + moment: Moment + timer: TimerCallback + timeouted = false + + template completeFuture(fut: untyped): untyped = + if fut.failed() or fut.completed(): + retFuture.complete(true) + else: + retFuture.cancelAndSchedule() # TODO: raises annotation shouldn't be needed, but likely similar issue as # https://github.com/nim-lang/Nim/issues/17369 proc continuation(udata: pointer) {.gcsafe, raises: [].} = if not(retFuture.finished()): - if not(cancelling): - if not(fut.finished()): - # Timer exceeded first, we going to cancel `fut` and wait until it - # not completes. - cancelling = true - fut.cancel() - else: - # Future `fut` completed/failed/cancelled first. - if not(isNil(timer)): - clearTimer(timer) - retFuture.complete(true) - else: + if timeouted: retFuture.complete(false) + return + if not(fut.finished()): + # Timer exceeded first, we going to cancel `fut` and wait until it + # not completes. + timeouted = true + fut.cancelSoon() + else: + # Future `fut` completed/failed/cancelled first. + if not(isNil(timer)): + clearTimer(timer) + fut.completeFuture() # TODO: raises annotation shouldn't be needed, but likely similar issue as # https://github.com/nim-lang/Nim/issues/17369 proc cancellation(udata: pointer) {.gcsafe, raises: [].} = - if not isNil(timer): - clearTimer(timer) if not(fut.finished()): - fut.removeCallback(continuation) - fut.cancel() + if not isNil(timer): + clearTimer(timer) + fut.cancelSoon() + else: + fut.completeFuture() if fut.finished(): retFuture.complete(true) @@ -1420,11 +1449,11 @@ proc withTimeout*[T](fut: Future[T], timeout: Duration): Future[bool] = timer = setTimer(moment, continuation, nil) fut.addCallback(continuation) - return retFuture + retFuture proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] {. inline, deprecated: "Use withTimeout(Future[T], Duration)".} = - result = withTimeout(fut, timeout.milliseconds()) + withTimeout(fut, timeout.milliseconds()) proc wait*[T](fut: Future[T], timeout = InfiniteDuration): Future[T] = ## Returns a future which will complete once future ``fut`` completes @@ -1435,49 +1464,49 @@ proc wait*[T](fut: Future[T], timeout = InfiniteDuration): Future[T] = ## ## TODO: In case when ``fut`` got cancelled, what result Future[T] ## should return, because it can't be cancelled too. - var retFuture = newFuture[T]("chronos.wait()") - var moment: Moment - var timer: TimerCallback - var cancelling = false + var + retFuture = newFuture[T]("chronos.wait()", {FutureFlag.OwnCancelSchedule}) + moment: Moment + timer: TimerCallback + timeouted = false - proc continuation(udata: pointer) {.raises: [].} = - if not(retFuture.finished()): - if not(cancelling): - if not(fut.finished()): - # Timer exceeded first. - cancelling = true - fut.cancel() - else: - # Future `fut` completed/failed/cancelled first. - if not isNil(timer): - clearTimer(timer) - - if fut.failed(): - retFuture.fail(fut.error) - else: - when T is void: - retFuture.complete() - else: - retFuture.complete(fut.value) - else: - retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) - - var cancellation: proc(udata: pointer) {.gcsafe, raises: [].} - cancellation = proc(udata: pointer) {.gcsafe, raises: [].} = - if not isNil(timer): - clearTimer(timer) - if not(fut.finished()): - fut.removeCallback(continuation) - fut.cancel() - - if fut.finished(): + template completeFuture(fut: untyped): untyped = if fut.failed(): retFuture.fail(fut.error) + elif fut.cancelled(): + retFuture.cancelAndSchedule() else: when T is void: retFuture.complete() else: retFuture.complete(fut.value) + + proc continuation(udata: pointer) {.raises: [].} = + if not(retFuture.finished()): + if timeouted: + retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) + return + if not(fut.finished()): + # Timer exceeded first. + timeouted = true + fut.cancelSoon() + else: + # Future `fut` completed/failed/cancelled first. + if not(isNil(timer)): + clearTimer(timer) + fut.completeFuture() + + var cancellation: proc(udata: pointer) {.gcsafe, raises: [].} + cancellation = proc(udata: pointer) {.gcsafe, raises: [].} = + if not(fut.finished()): + if not(isNil(timer)): + clearTimer(timer) + fut.cancelSoon() + else: + fut.completeFuture() + + if fut.finished(): + fut.completeFuture() else: if timeout.isZero(): retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) @@ -1490,7 +1519,7 @@ proc wait*[T](fut: Future[T], timeout = InfiniteDuration): Future[T] = timer = setTimer(moment, continuation, nil) fut.addCallback(continuation) - return retFuture + retFuture proc wait*[T](fut: Future[T], timeout = -1): Future[T] {. inline, deprecated: "Use wait(Future[T], Duration)".} = diff --git a/chronos/asyncmacro2.nim b/chronos/asyncmacro2.nim index 8e74073..a86147c 100644 --- a/chronos/asyncmacro2.nim +++ b/chronos/asyncmacro2.nim @@ -301,11 +301,6 @@ template await*[T](f: Future[T]): untyped = # transformation - `yield` gives control back to `futureContinue` which is # responsible for resuming execution once the yielded future is finished yield chronosInternalRetFuture.internalChild - - # `child` is guaranteed to have been `finished` after the yield - if chronosInternalRetFuture.internalMustCancel: - raise newCancelledError() - # `child` released by `futureContinue` chronosInternalRetFuture.internalChild.internalCheckComplete() when T isnot void: @@ -317,8 +312,6 @@ template awaitne*[T](f: Future[T]): Future[T] = when declared(chronosInternalRetFuture): chronosInternalRetFuture.internalChild = f yield chronosInternalRetFuture.internalChild - if chronosInternalRetFuture.internalMustCancel: - raise newCancelledError() cast[type(f)](chronosInternalRetFuture.internalChild) else: unsupported "awaitne is only available within {.async.}" diff --git a/chronos/asyncproc.nim b/chronos/asyncproc.nim index 8df8e33..3e2df88 100644 --- a/chronos/asyncproc.nim +++ b/chronos/asyncproc.nim @@ -1241,7 +1241,7 @@ proc closeWait*(p: AsyncProcessRef) {.async.} = # Here we ignore all possible errrors, because we do not want to raise # exceptions. discard closeProcessHandles(p.pipes, p.options, OSErrorCode(0)) - await p.pipes.closeProcessStreams(p.options) + await noCancel(p.pipes.closeProcessStreams(p.options)) discard p.closeThreadAndProcessHandle() untrackCounter(AsyncProcessTrackerName) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 5309846..0feb51e 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -736,13 +736,19 @@ proc close*(ab: AsyncEventQueue) {.raises: [].} = ab.queue.clear() proc closeWait*(ab: AsyncEventQueue): Future[void] {.raises: [].} = - var retFuture = newFuture[void]("AsyncEventQueue.closeWait()") + let retFuture = newFuture[void]("AsyncEventQueue.closeWait()", + {FutureFlag.OwnCancelSchedule}) proc continuation(udata: pointer) {.gcsafe.} = - if not(retFuture.finished()): - retFuture.complete() + retFuture.complete() + proc cancellation(udata: pointer) {.gcsafe.} = + # We are not going to change the state of `retFuture` to cancelled, so we + # will prevent the entire sequence of Futures from being cancelled. + discard + ab.close() # Schedule `continuation` to be called only after all the `reader` # notifications will be scheduled and processed. + retFuture.cancelCallback = cancellation callSoon(continuation) retFuture diff --git a/chronos/futures.nim b/chronos/futures.nim index 9b2667b..5f96867 100644 --- a/chronos/futures.nim +++ b/chronos/futures.nim @@ -37,6 +37,11 @@ type FutureState* {.pure.} = enum Pending, Completed, Cancelled, Failed + FutureFlag* {.pure.} = enum + OwnCancelSchedule + + FutureFlags* = set[FutureFlag] + InternalFutureBase* = object of RootObj # Internal untyped future representation - the fields are not part of the # public API and neither is `InternalFutureBase`, ie the inheritance @@ -47,8 +52,8 @@ type internalCancelcb*: CallbackFunc internalChild*: FutureBase internalState*: FutureState + internalFlags*: FutureFlags internalError*: ref CatchableError ## Stored exception - internalMustCancel*: bool internalClosure*: iterator(f: FutureBase): FutureBase {.closureIter.} when chronosFutureId: @@ -94,12 +99,11 @@ when chronosFutureTracking: var futureList* {.threadvar.}: FutureList # Internal utilities - these are not part of the stable API -proc internalInitFutureBase*( - fut: FutureBase, - loc: ptr SrcLoc, - state: FutureState) = +proc internalInitFutureBase*(fut: FutureBase, loc: ptr SrcLoc, + state: FutureState, flags: FutureFlags) = fut.internalState = state fut.internalLocation[LocationKind.Create] = loc + fut.internalFlags = flags if state != FutureState.Pending: fut.internalLocation[LocationKind.Finish] = loc @@ -128,21 +132,34 @@ template init*[T](F: type Future[T], fromProc: static[string] = ""): Future[T] = ## Specifying ``fromProc``, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. let res = Future[T]() - internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Pending) + internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Pending, {}) + res + +template init*[T](F: type Future[T], fromProc: static[string] = "", + flags: static[FutureFlags]): Future[T] = + ## Creates a new pending future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + let res = Future[T]() + internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Pending, + flags) res template completed*( F: type Future, fromProc: static[string] = ""): Future[void] = ## Create a new completed future - let res = Future[T]() - internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Completed) + let res = Future[void]() + internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Completed, + {}) res template completed*[T: not void]( F: type Future, valueParam: T, fromProc: static[string] = ""): Future[T] = ## Create a new completed future let res = Future[T](internalValue: valueParam) - internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Completed) + internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Completed, + {}) res template failed*[T]( @@ -150,19 +167,21 @@ template failed*[T]( fromProc: static[string] = ""): Future[T] = ## Create a new failed future let res = Future[T](internalError: errorParam) - internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Failed) + internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Failed, {}) when chronosStackTrace: res.internalErrorStackTrace = if getStackTrace(res.error) == "": getStackTrace() else: getStackTrace(res.error) - res func state*(future: FutureBase): FutureState = future.internalState +func flags*(future: FutureBase): FutureFlags = + future.internalFlags + func finished*(future: FutureBase): bool {.inline.} = ## Determines whether ``future`` has finished, i.e. ``future`` state changed ## from state ``Pending`` to one of the states (``Finished``, ``Cancelled``, diff --git a/chronos/ratelimit.nim b/chronos/ratelimit.nim index 4147db7..ad66c06 100644 --- a/chronos/ratelimit.nim +++ b/chronos/ratelimit.nim @@ -88,8 +88,8 @@ proc worker(bucket: TokenBucket) {.async.} = #buckets sleeper = sleepAsync(milliseconds(timeToTarget)) await sleeper or eventWaiter - sleeper.cancel() - eventWaiter.cancel() + sleeper.cancelSoon() + eventWaiter.cancelSoon() else: await eventWaiter diff --git a/chronos/streams/asyncstream.nim b/chronos/streams/asyncstream.nim index 191b36a..4698e83 100644 --- a/chronos/streams/asyncstream.nim +++ b/chronos/streams/asyncstream.nim @@ -913,7 +913,7 @@ proc close*(rw: AsyncStreamRW) = callSoon(continuation) else: rw.future.addCallback(continuation) - rw.future.cancel() + rw.future.cancelSoon() elif rw is AsyncStreamWriter: if isNil(rw.wsource) or isNil(rw.writerLoop) or isNil(rw.future): callSoon(continuation) @@ -922,12 +922,36 @@ proc close*(rw: AsyncStreamRW) = callSoon(continuation) else: rw.future.addCallback(continuation) - rw.future.cancel() + rw.future.cancelSoon() proc closeWait*(rw: AsyncStreamRW): Future[void] = ## Close and frees resources of stream ``rw``. + const FutureName = + when rw is AsyncStreamReader: + "async.stream.reader.closeWait" + else: + "async.stream.writer.closeWait" + + if rw.closed(): + return Future.completed(FutureName) + + let retFuture = newFuture[void](FutureName, {FutureFlag.OwnCancelSchedule}) + + proc continuation(udata: pointer) {.gcsafe, raises:[].} = + retFuture.complete() + + proc cancellation(udata: pointer) {.gcsafe, raises:[].} = + # We are not going to change the state of `retFuture` to cancelled, so we + # will prevent the entire sequence of Futures from being cancelled. + discard + rw.close() - rw.join() + if rw.future.finished(): + retFuture.complete() + else: + rw.future.addCallback(continuation, cast[pointer](retFuture)) + retFuture.cancelCallback = cancellation + retFuture proc startReader(rstream: AsyncStreamReader) = rstream.state = Running diff --git a/chronos/streams/tlsstream.nim b/chronos/streams/tlsstream.nim index 2999f7a..6432a10 100644 --- a/chronos/streams/tlsstream.nim +++ b/chronos/streams/tlsstream.nim @@ -267,19 +267,15 @@ template readAndReset(fut: untyped) = break proc cancelAndWait*(a, b, c, d: Future[TLSResult]): Future[void] = - var waiting: seq[Future[TLSResult]] + var waiting: seq[FutureBase] if not(isNil(a)) and not(a.finished()): - a.cancel() - waiting.add(a) + waiting.add(a.cancelAndWait()) if not(isNil(b)) and not(b.finished()): - b.cancel() - waiting.add(b) + waiting.add(b.cancelAndWait()) if not(isNil(c)) and not(c.finished()): - c.cancel() - waiting.add(c) + waiting.add(c.cancelAndWait()) if not(isNil(d)) and not(d.finished()): - d.cancel() - waiting.add(d) + waiting.add(d.cancelAndWait()) allFutures(waiting) proc dumpState*(state: cuint): string = @@ -432,7 +428,7 @@ proc tlsLoop*(stream: TLSAsyncStream) {.async.} = proc tlsWriteLoop(stream: AsyncStreamWriter) {.async.} = var wstream = TLSStreamWriter(stream) wstream.state = AsyncStreamState.Running - await stepsAsync(1) + await sleepAsync(0.milliseconds) if isNil(wstream.stream.mainLoop): wstream.stream.mainLoop = tlsLoop(wstream.stream) await wstream.stream.mainLoop @@ -440,7 +436,7 @@ proc tlsWriteLoop(stream: AsyncStreamWriter) {.async.} = proc tlsReadLoop(stream: AsyncStreamReader) {.async.} = var rstream = TLSStreamReader(stream) rstream.state = AsyncStreamState.Running - await stepsAsync(1) + await sleepAsync(0.milliseconds) if isNil(rstream.stream.mainLoop): rstream.stream.mainLoop = tlsLoop(rstream.stream) await rstream.stream.mainLoop diff --git a/chronos/transports/datagram.nim b/chronos/transports/datagram.nim index 665bc0e..af29c2a 100644 --- a/chronos/transports/datagram.nim +++ b/chronos/transports/datagram.nim @@ -690,8 +690,28 @@ proc join*(transp: DatagramTransport): Future[void] = proc closeWait*(transp: DatagramTransport): Future[void] = ## Close transport ``transp`` and release all resources. + const FutureName = "datagram.transport.closeWait" + + if {ReadClosed, WriteClosed} * transp.state != {}: + return Future.completed(FutureName) + + let retFuture = newFuture[void](FutureName, {FutureFlag.OwnCancelSchedule}) + + proc continuation(udata: pointer) {.gcsafe.} = + retFuture.complete() + + proc cancellation(udata: pointer) {.gcsafe.} = + # We are not going to change the state of `retFuture` to cancelled, so we + # will prevent the entire sequence of Futures from being cancelled. + discard + transp.close() - transp.join() + if transp.future.finished(): + retFuture.complete() + else: + transp.future.addCallback(continuation, cast[pointer](retFuture)) + retFuture.cancelCallback = cancellation + retFuture proc send*(transp: DatagramTransport, pbytes: pointer, nbytes: int): Future[void] = diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 44a39b2..f96650c 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -2588,15 +2588,34 @@ proc close*(transp: StreamTransport) = proc closeWait*(transp: StreamTransport): Future[void] = ## Close and frees resources of transport ``transp``. + const FutureName = "stream.transport.closeWait" + + if {ReadClosed, WriteClosed} * transp.state != {}: + return Future.completed(FutureName) + + let retFuture = newFuture[void](FutureName, {FutureFlag.OwnCancelSchedule}) + + proc continuation(udata: pointer) {.gcsafe.} = + retFuture.complete() + + proc cancellation(udata: pointer) {.gcsafe.} = + # We are not going to change the state of `retFuture` to cancelled, so we + # will prevent the entire sequence of Futures from being cancelled. + discard + transp.close() - transp.join() + if transp.future.finished(): + retFuture.complete() + else: + transp.future.addCallback(continuation, cast[pointer](retFuture)) + retFuture.cancelCallback = cancellation + retFuture proc shutdownWait*(transp: StreamTransport): Future[void] = ## Perform graceful shutdown of TCP connection backed by transport ``transp``. doAssert(transp.kind == TransportKind.Socket) let retFuture = newFuture[void]("stream.transport.shutdown") transp.checkClosed(retFuture) - transp.checkWriteEof(retFuture) when defined(windows): let loop = getThreadDispatcher() @@ -2636,7 +2655,14 @@ proc shutdownWait*(transp: StreamTransport): Future[void] = let res = osdefs.shutdown(SocketHandle(transp.fd), SHUT_WR) if res < 0: let err = osLastError() - retFuture.fail(getTransportOsError(err)) + case err + of ENOTCONN: + # The specified socket is not connected, it means that our initial + # goal is already happened. + transp.state.incl({WriteEof}) + callSoon(continuation, nil) + else: + retFuture.fail(getTransportOsError(err)) else: transp.state.incl({WriteEof}) callSoon(continuation, nil) diff --git a/tests/testbugs.nim b/tests/testbugs.nim index cf18a13..1f2a932 100644 --- a/tests/testbugs.nim +++ b/tests/testbugs.nim @@ -14,7 +14,7 @@ suite "Asynchronous issues test suite": const HELLO_PORT = 45679 const TEST_MSG = "testmsg" const MSG_LEN = TEST_MSG.len() - const TestsCount = 500 + const TestsCount = 100 type CustomData = ref object diff --git a/tests/testfut.nim b/tests/testfut.nim index a9fba05..bc61594 100644 --- a/tests/testfut.nim +++ b/tests/testfut.nim @@ -6,10 +6,15 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) import unittest2 +import stew/results import ../chronos, ../chronos/unittest2/asynctests {.used.} +type + TestFooConnection* = ref object + id*: int + suite "Future[T] behavior test suite": proc testFuture1(): Future[int] {.async.} = await sleepAsync(0.milliseconds) @@ -960,7 +965,7 @@ suite "Future[T] behavior test suite": let discarded {.used.} = await fut1 check res - asyncTest "cancel() async procedure test": + asyncTest "tryCancel() async procedure test": var completed = 0 proc client1() {.async.} = @@ -980,7 +985,7 @@ suite "Future[T] behavior test suite": inc(completed) var fut = client4() - fut.cancel() + discard fut.tryCancel() # Future must not be cancelled immediately, because it has many nested # futures. @@ -1031,7 +1036,7 @@ suite "Future[T] behavior test suite": var fut1 = client2() var fut2 = client2() - fut1.cancel() + discard fut1.tryCancel() await fut1 await cancelAndWait(fut2) check: @@ -1054,17 +1059,17 @@ suite "Future[T] behavior test suite": if not(retFuture.finished()): retFuture.complete() - proc cancel(udata: pointer) {.gcsafe.} = + proc cancellation(udata: pointer) {.gcsafe.} = inc(cancelled) if not(retFuture.finished()): removeTimer(moment, completion, cast[pointer](retFuture)) - retFuture.cancelCallback = cancel + retFuture.cancelCallback = cancellation discard setTimer(moment, completion, cast[pointer](retFuture)) return retFuture var fut = client1(100.milliseconds) - fut.cancel() + discard fut.tryCancel() await sleepAsync(500.milliseconds) check: fut.cancelled() @@ -1112,8 +1117,8 @@ suite "Future[T] behavior test suite": neverFlag3 = true res.addCallback(continuation) res.cancelCallback = cancellation - result = res neverFlag1 = true + res proc withTimeoutProc() {.async.} = try: @@ -1149,12 +1154,12 @@ suite "Future[T] behavior test suite": someFut = newFuture[void]() var raceFut3 = raceProc() - someFut.cancel() + discard someFut.tryCancel() await cancelAndWait(raceFut3) check: - raceFut1.state == FutureState.Cancelled - raceFut2.state == FutureState.Cancelled + raceFut1.state == FutureState.Completed + raceFut2.state == FutureState.Failed raceFut3.state == FutureState.Cancelled asyncTest "asyncSpawn() test": @@ -1255,12 +1260,12 @@ suite "Future[T] behavior test suite": (loc.procedure == procedure) check: - chk(loc10, "testfut.nim", 1221, "macroFuture") - chk(loc11, "testfut.nim", 1222, "") - chk(loc20, "testfut.nim", 1234, "template") - chk(loc21, "testfut.nim", 1237, "") - chk(loc30, "testfut.nim", 1231, "procedure") - chk(loc31, "testfut.nim", 1238, "") + chk(loc10, "testfut.nim", 1226, "macroFuture") + chk(loc11, "testfut.nim", 1227, "") + chk(loc20, "testfut.nim", 1239, "template") + chk(loc21, "testfut.nim", 1242, "") + chk(loc30, "testfut.nim", 1236, "procedure") + chk(loc31, "testfut.nim", 1243, "") asyncTest "withTimeout(fut) should wait cancellation test": proc futureNeverEnds(): Future[void] = @@ -1535,3 +1540,462 @@ suite "Future[T] behavior test suite": check: v1_u == 0'u v2_u + 1'u == 0'u + + asyncTest "wait() cancellation undefined behavior test #1": + proc testInnerFoo(fooFut: Future[void]): Future[TestFooConnection] {. + async.} = + await fooFut + return TestFooConnection() + + proc testFoo(fooFut: Future[void]) {.async.} = + let connection = + try: + let res = await testInnerFoo(fooFut).wait(10.seconds) + Result[TestFooConnection, int].ok(res) + except CancelledError: + Result[TestFooConnection, int].err(0) + except CatchableError: + Result[TestFooConnection, int].err(1) + check connection.isOk() + + var future = newFuture[void]("last.child.future") + var someFut = testFoo(future) + future.complete() + discard someFut.tryCancel() + await someFut + + asyncTest "wait() cancellation undefined behavior test #2": + proc testInnerFoo(fooFut: Future[void]): Future[TestFooConnection] {. + async.} = + await fooFut + return TestFooConnection() + + proc testMiddleFoo(fooFut: Future[void]): Future[TestFooConnection] {. + async.} = + await testInnerFoo(fooFut) + + proc testFoo(fooFut: Future[void]) {.async.} = + let connection = + try: + let res = await testMiddleFoo(fooFut).wait(10.seconds) + Result[TestFooConnection, int].ok(res) + except CancelledError: + Result[TestFooConnection, int].err(0) + except CatchableError: + Result[TestFooConnection, int].err(1) + check connection.isOk() + + var future = newFuture[void]("last.child.future") + var someFut = testFoo(future) + future.complete() + discard someFut.tryCancel() + await someFut + + asyncTest "withTimeout() cancellation undefined behavior test #1": + proc testInnerFoo(fooFut: Future[void]): Future[TestFooConnection] {. + async.} = + await fooFut + return TestFooConnection() + + proc testFoo(fooFut: Future[void]) {.async.} = + let connection = + try: + let + checkFut = testInnerFoo(fooFut) + res = await withTimeout(checkFut, 10.seconds) + if res: + Result[TestFooConnection, int].ok(checkFut.value) + else: + Result[TestFooConnection, int].err(0) + except CancelledError: + Result[TestFooConnection, int].err(1) + except CatchableError: + Result[TestFooConnection, int].err(2) + check connection.isOk() + + var future = newFuture[void]("last.child.future") + var someFut = testFoo(future) + future.complete() + discard someFut.tryCancel() + await someFut + + asyncTest "withTimeout() cancellation undefined behavior test #2": + proc testInnerFoo(fooFut: Future[void]): Future[TestFooConnection] {. + async.} = + await fooFut + return TestFooConnection() + + proc testMiddleFoo(fooFut: Future[void]): Future[TestFooConnection] {. + async.} = + await testInnerFoo(fooFut) + + proc testFoo(fooFut: Future[void]) {.async.} = + let connection = + try: + let + checkFut = testMiddleFoo(fooFut) + res = await withTimeout(checkFut, 10.seconds) + if res: + Result[TestFooConnection, int].ok(checkFut.value) + else: + Result[TestFooConnection, int].err(0) + except CancelledError: + Result[TestFooConnection, int].err(1) + except CatchableError: + Result[TestFooConnection, int].err(2) + check connection.isOk() + + var future = newFuture[void]("last.child.future") + var someFut = testFoo(future) + future.complete() + discard someFut.tryCancel() + await someFut + + asyncTest "Cancellation behavior test": + proc testInnerFoo(fooFut: Future[void]) {.async.} = + await fooFut + + proc testMiddleFoo(fooFut: Future[void]) {.async.} = + await testInnerFoo(fooFut) + + proc testOuterFoo(fooFut: Future[void]) {.async.} = + await testMiddleFoo(fooFut) + + block: + # Cancellation of pending Future + let future = newFuture[void]("last.child.pending.future") + await cancelAndWait(future) + check: + future.cancelled() == true + + block: + # Cancellation of completed Future + let future = newFuture[void]("last.child.completed.future") + future.complete() + await cancelAndWait(future) + check: + future.cancelled() == false + future.completed() == true + + block: + # Cancellation of failed Future + let future = newFuture[void]("last.child.failed.future") + future.fail(newException(ValueError, "ABCD")) + await cancelAndWait(future) + check: + future.cancelled() == false + future.failed() == true + + block: + # Cancellation of already cancelled Future + let future = newFuture[void]("last.child.cancelled.future") + future.cancelAndSchedule() + await cancelAndWait(future) + check: + future.cancelled() == true + + block: + # Cancellation of Pending->Pending->Pending->Pending sequence + let future = newFuture[void]("last.child.pending.future") + let testFut = testOuterFoo(future) + await cancelAndWait(testFut) + check: + testFut.cancelled() == true + + block: + # Cancellation of Pending->Pending->Pending->Completed sequence + let future = newFuture[void]("last.child.completed.future") + let testFut = testOuterFoo(future) + future.complete() + await cancelAndWait(testFut) + check: + testFut.cancelled() == false + testFut.completed() == true + + block: + # Cancellation of Pending->Pending->Pending->Failed sequence + let future = newFuture[void]("last.child.failed.future") + let testFut = testOuterFoo(future) + future.fail(newException(ValueError, "ABCD")) + await cancelAndWait(testFut) + check: + testFut.cancelled() == false + testFut.failed() == true + + block: + # Cancellation of Pending->Pending->Pending->Cancelled sequence + let future = newFuture[void]("last.child.cancelled.future") + let testFut = testOuterFoo(future) + future.cancelAndSchedule() + await cancelAndWait(testFut) + check: + testFut.cancelled() == true + + block: + # Cancellation of pending Future, when automatic scheduling disabled + let future = newFuture[void]("last.child.pending.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + discard + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + let cancelFut = cancelAndWait(future) + await sleepAsync(100.milliseconds) + check: + cancelFut.finished() == false + future.cancelled() == false + # Now we manually changing Future's state, so `cancelAndWait` could + # finish + future.complete() + await cancelFut + check: + cancelFut.finished() == true + future.cancelled() == false + future.finished() == true + + block: + # Cancellation of pending Future, which will fail Future on cancellation, + # when automatic scheduling disabled + let future = newFuture[void]("last.child.completed.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + future.complete() + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + await cancelAndWait(future) + check: + future.cancelled() == false + future.completed() == true + + block: + # Cancellation of pending Future, which will fail Future on cancellation, + # when automatic scheduling disabled + let future = newFuture[void]("last.child.failed.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + future.fail(newException(ValueError, "ABCD")) + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + await cancelAndWait(future) + check: + future.cancelled() == false + future.failed() == true + + block: + # Cancellation of pending Future, which will fail Future on cancellation, + # when automatic scheduling disabled + let future = newFuture[void]("last.child.cancelled.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + future.cancelAndSchedule() + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + await cancelAndWait(future) + check: + future.cancelled() == true + + block: + # Cancellation of pending Pending->Pending->Pending->Pending, when + # automatic scheduling disabled and Future do nothing in cancellation + # callback + let future = newFuture[void]("last.child.pending.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + discard + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + let testFut = testOuterFoo(future) + let cancelFut = cancelAndWait(testFut) + await sleepAsync(100.milliseconds) + check: + cancelFut.finished() == false + testFut.cancelled() == false + future.cancelled() == false + # Now we manually changing Future's state, so `cancelAndWait` could + # finish + future.complete() + await cancelFut + check: + cancelFut.finished() == true + future.cancelled() == false + future.finished() == true + testFut.cancelled() == false + testFut.finished() == true + + block: + # Cancellation of pending Pending->Pending->Pending->Pending, when + # automatic scheduling disabled and Future completes in cancellation + # callback + let future = newFuture[void]("last.child.pending.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + future.complete() + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + let testFut = testOuterFoo(future) + await cancelAndWait(testFut) + await sleepAsync(100.milliseconds) + check: + testFut.cancelled() == false + testFut.finished() == true + future.cancelled() == false + future.finished() == true + + block: + # Cancellation of pending Pending->Pending->Pending->Pending, when + # automatic scheduling disabled and Future fails in cancellation callback + let future = newFuture[void]("last.child.pending.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + future.fail(newException(ValueError, "ABCD")) + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + let testFut = testOuterFoo(future) + await cancelAndWait(testFut) + await sleepAsync(100.milliseconds) + check: + testFut.cancelled() == false + testFut.failed() == true + future.cancelled() == false + future.failed() == true + + block: + # Cancellation of pending Pending->Pending->Pending->Pending, when + # automatic scheduling disabled and Future fails in cancellation callback + let future = newFuture[void]("last.child.pending.future", + {FutureFlag.OwnCancelSchedule}) + proc cancellation(udata: pointer) {.gcsafe.} = + future.cancelAndSchedule() + future.cancelCallback = cancellation + # Note, future will never be finished in such case, until we manually not + # finish it + let testFut = testOuterFoo(future) + await cancelAndWait(testFut) + await sleepAsync(100.milliseconds) + check: + testFut.cancelled() == true + future.cancelled() == true + + test "Issue #334 test": + proc test(): bool = + var testres = "" + + proc a() {.async.} = + try: + await sleepAsync(seconds(1)) + except CatchableError as exc: + testres.add("A") + raise exc + + proc b() {.async.} = + try: + await a() + except CatchableError as exc: + testres.add("B") + raise exc + + proc c() {.async.} = + try: + echo $(await b().withTimeout(seconds(2))) + except CatchableError as exc: + testres.add("C") + raise exc + + let x = c() + x.cancelSoon() + + try: + waitFor x + except CatchableError: + testres.add("D") + + testres.add("E") + + waitFor sleepAsync(milliseconds(100)) + + testres == "ABCDE" + + check test() == true + + asyncTest "cancelAndWait() should be able to cancel test": + proc test1() {.async.} = + await noCancel sleepAsync(100.milliseconds) + await noCancel sleepAsync(100.milliseconds) + await sleepAsync(100.milliseconds) + + proc test2() {.async.} = + await noCancel sleepAsync(100.milliseconds) + await sleepAsync(100.milliseconds) + await noCancel sleepAsync(100.milliseconds) + + proc test3() {.async.} = + await sleepAsync(100.milliseconds) + await noCancel sleepAsync(100.milliseconds) + await noCancel sleepAsync(100.milliseconds) + + proc test4() {.async.} = + while true: + await noCancel sleepAsync(50.milliseconds) + await sleepAsync(0.milliseconds) + + proc test5() {.async.} = + while true: + await sleepAsync(0.milliseconds) + await noCancel sleepAsync(50.milliseconds) + + block: + let future1 = test1() + await cancelAndWait(future1) + let future2 = test1() + await sleepAsync(10.milliseconds) + await cancelAndWait(future2) + check: + future1.cancelled() == true + future2.cancelled() == true + + block: + let future1 = test2() + await cancelAndWait(future1) + let future2 = test2() + await sleepAsync(10.milliseconds) + await cancelAndWait(future2) + check: + future1.cancelled() == true + future2.cancelled() == true + + block: + let future1 = test3() + await cancelAndWait(future1) + let future2 = test3() + await sleepAsync(10.milliseconds) + await cancelAndWait(future2) + check: + future1.cancelled() == true + future2.cancelled() == true + + block: + let future1 = test4() + await cancelAndWait(future1) + let future2 = test4() + await sleepAsync(333.milliseconds) + await cancelAndWait(future2) + check: + future1.cancelled() == true + future2.cancelled() == true + + block: + let future1 = test5() + await cancelAndWait(future1) + let future2 = test5() + await sleepAsync(333.milliseconds) + await cancelAndWait(future2) + check: + future1.cancelled() == true + future2.cancelled() == true diff --git a/tests/testhttpclient.nim b/tests/testhttpclient.nim index 4daaf87..e10892e 100644 --- a/tests/testhttpclient.nim +++ b/tests/testhttpclient.nim @@ -704,6 +704,107 @@ suite "HTTP client testing suite": await server.closeWait() return "redirect-" & $res + proc testSendCancelLeaksTest(secure: bool): Future[bool] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async.} = + return defaultResponse() + + var server = createServer(initTAddress("127.0.0.1:0"), process, secure) + server.start() + let address = server.instance.localAddress() + + let ha = + if secure: + getAddress(address, HttpClientScheme.Secure, "/") + else: + getAddress(address, HttpClientScheme.NonSecure, "/") + + var counter = 0 + while true: + let + session = createSession(secure) + request = HttpClientRequestRef.new(session, ha, MethodGet) + requestFut = request.send() + + if counter > 0: + await stepsAsync(counter) + let exitLoop = + if not(requestFut.finished()): + await cancelAndWait(requestFut) + doAssert(cancelled(requestFut) or completed(requestFut), + "Future should be Cancelled or Completed at this point") + if requestFut.completed(): + let response = await requestFut + await response.closeWait() + + inc(counter) + false + else: + let response = await requestFut + await response.closeWait() + true + + await request.closeWait() + await session.closeWait() + + if exitLoop: + break + + await server.stop() + await server.closeWait() + return true + + proc testOpenCancelLeaksTest(secure: bool): Future[bool] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async.} = + return defaultResponse() + + var server = createServer(initTAddress("127.0.0.1:0"), process, secure) + server.start() + let address = server.instance.localAddress() + + let ha = + if secure: + getAddress(address, HttpClientScheme.Secure, "/") + else: + getAddress(address, HttpClientScheme.NonSecure, "/") + + var counter = 0 + while true: + let + session = createSession(secure) + request = HttpClientRequestRef.new(session, ha, MethodPost) + bodyFut = request.open() + + if counter > 0: + await stepsAsync(counter) + let exitLoop = + if not(bodyFut.finished()): + await cancelAndWait(bodyFut) + doAssert(cancelled(bodyFut) or completed(bodyFut), + "Future should be Cancelled or Completed at this point") + + if bodyFut.completed(): + let bodyWriter = await bodyFut + await bodyWriter.closeWait() + + inc(counter) + false + else: + let bodyWriter = await bodyFut + await bodyWriter.closeWait() + true + + await request.closeWait() + await session.closeWait() + + if exitLoop: + break + + await server.stop() + await server.closeWait() + return true + # proc testBasicAuthorization(): Future[bool] {.async.} = # let session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost}, # maxRedirections = 10) @@ -1243,6 +1344,18 @@ suite "HTTP client testing suite": test "HTTP(S) client maximum redirections test": check waitFor(testRequestRedirectTest(true, 4)) == "redirect-true" + test "HTTP send() cancellation leaks test": + check waitFor(testSendCancelLeaksTest(false)) == true + + test "HTTP(S) send() cancellation leaks test": + check waitFor(testSendCancelLeaksTest(true)) == true + + test "HTTP open() cancellation leaks test": + check waitFor(testOpenCancelLeaksTest(false)) == true + + test "HTTP(S) open() cancellation leaks test": + check waitFor(testOpenCancelLeaksTest(true)) == true + test "HTTPS basic authorization test": skip() # This test disabled because remote service is pretty flaky and fails pretty diff --git a/tests/testhttpserver.nim b/tests/testhttpserver.nim index 0ecc9aa..85aeee5 100644 --- a/tests/testhttpserver.nim +++ b/tests/testhttpserver.nim @@ -1326,32 +1326,31 @@ suite "HTTP server testing suite": server.start() var transp: StreamTransport - try: - transp = await connect(address) - block: - let response = await transp.httpClient2(test[0], 7) - check: - response.data == "TEST_OK" - response.headers.getString("connection") == test[3] - # We do this sleeping here just because we running both server and - # client in single process, so when we received response from server - # it does not mean that connection has been immediately closed - it - # takes some more calls, so we trying to get this calls happens. - await sleepAsync(50.milliseconds) - let connectionStillAvailable = - try: - let response {.used.} = await transp.httpClient2(test[0], 7) - true - except CatchableError: - false - check connectionStillAvailable == test[2] + transp = await connect(address) + block: + let response = await transp.httpClient2(test[0], 7) + check: + response.data == "TEST_OK" + response.headers.getString("connection") == test[3] + # We do this sleeping here just because we running both server and + # client in single process, so when we received response from server + # it does not mean that connection has been immediately closed - it + # takes some more calls, so we trying to get this calls happens. + await sleepAsync(50.milliseconds) + let connectionStillAvailable = + try: + let response {.used.} = await transp.httpClient2(test[0], 7) + true + except CatchableError: + false - finally: - if not(isNil(transp)): - await transp.closeWait() - await server.stop() - await server.closeWait() + check connectionStillAvailable == test[2] + + if not(isNil(transp)): + await transp.closeWait() + await server.stop() + await server.closeWait() asyncTest "HTTP debug tests": const @@ -1400,32 +1399,30 @@ suite "HTTP server testing suite": info.flags == {HttpServerFlags.Http11Pipeline} info.socketFlags == socketFlags - try: - var clientFutures: seq[Future[StreamTransport]] - for i in 0 ..< TestsCount: - clientFutures.add(client(address, TestRequest)) - await allFutures(clientFutures) + var clientFutures: seq[Future[StreamTransport]] + for i in 0 ..< TestsCount: + clientFutures.add(client(address, TestRequest)) + await allFutures(clientFutures) - let connections = server.getConnections() - check len(connections) == TestsCount - let currentTime = Moment.now() - for index, connection in connections.pairs(): - let transp = clientFutures[index].read() - check: - connection.remoteAddress.get() == transp.localAddress() - connection.localAddress.get() == transp.remoteAddress() - connection.connectionType == ConnectionType.NonSecure - connection.connectionState == ConnectionState.Alive - connection.query.get("") == "/httpdebug" - (currentTime - connection.createMoment.get()) != ZeroDuration - (currentTime - connection.acceptMoment) != ZeroDuration - var pending: seq[Future[void]] - for transpFut in clientFutures: - pending.add(closeWait(transpFut.read())) - await allFutures(pending) - finally: - await server.stop() - await server.closeWait() + let connections = server.getConnections() + check len(connections) == TestsCount + let currentTime = Moment.now() + for index, connection in connections.pairs(): + let transp = clientFutures[index].read() + check: + connection.remoteAddress.get() == transp.localAddress() + connection.localAddress.get() == transp.remoteAddress() + connection.connectionType == ConnectionType.NonSecure + connection.connectionState == ConnectionState.Alive + connection.query.get("") == "/httpdebug" + (currentTime - connection.createMoment.get()) != ZeroDuration + (currentTime - connection.acceptMoment) != ZeroDuration + var pending: seq[Future[void]] + for transpFut in clientFutures: + pending.add(closeWait(transpFut.read())) + await allFutures(pending) + await server.stop() + await server.closeWait() test "Leaks test": checkLeaks() diff --git a/tests/testratelimit.nim b/tests/testratelimit.nim index bf281ee..d284928 100644 --- a/tests/testratelimit.nim +++ b/tests/testratelimit.nim @@ -49,7 +49,7 @@ suite "Token Bucket": # Consume 10* the budget cap let beforeStart = Moment.now() waitFor(bucket.consume(1000).wait(5.seconds)) - check Moment.now() - beforeStart in 900.milliseconds .. 1500.milliseconds + check Moment.now() - beforeStart in 900.milliseconds .. 2200.milliseconds test "Sync manual replenish": var bucket = TokenBucket.new(1000, 0.seconds) @@ -96,7 +96,7 @@ suite "Token Bucket": futBlocker.finished == false fut2.finished == false - futBlocker.cancel() + futBlocker.cancelSoon() waitFor(fut2.wait(10.milliseconds)) test "Very long replenish": @@ -117,9 +117,14 @@ suite "Token Bucket": check bucket.tryConsume(1, fakeNow) == true test "Short replenish": - var bucket = TokenBucket.new(15000, 1.milliseconds) - let start = Moment.now() - check bucket.tryConsume(15000, start) - check bucket.tryConsume(1, start) == false + skip() + # TODO (cheatfate): This test was disabled, because it continuosly fails in + # Github Actions Windows x64 CI when using Nim 1.6.14 version. + # Unable to reproduce failure locally. - check bucket.tryConsume(15000, start + 1.milliseconds) == true + # var bucket = TokenBucket.new(15000, 1.milliseconds) + # let start = Moment.now() + # check bucket.tryConsume(15000, start) + # check bucket.tryConsume(1, start) == false + + # check bucket.tryConsume(15000, start + 1.milliseconds) == true diff --git a/tests/testsoon.nim b/tests/testsoon.nim index 88072c2..41a6e4e 100644 --- a/tests/testsoon.nim +++ b/tests/testsoon.nim @@ -11,75 +11,83 @@ import ../chronos {.used.} suite "callSoon() tests suite": - const CallSoonTests = 10 - var soonTest1 = 0'u - var timeoutsTest1 = 0 - var timeoutsTest2 = 0 - var soonTest2 = 0 - - proc callback1(udata: pointer) {.gcsafe.} = - soonTest1 = soonTest1 xor cast[uint](udata) - - proc test1(): uint = - callSoon(callback1, cast[pointer](0x12345678'u)) - callSoon(callback1, cast[pointer](0x23456789'u)) - callSoon(callback1, cast[pointer](0x3456789A'u)) - callSoon(callback1, cast[pointer](0x456789AB'u)) - callSoon(callback1, cast[pointer](0x56789ABC'u)) - callSoon(callback1, cast[pointer](0x6789ABCD'u)) - callSoon(callback1, cast[pointer](0x789ABCDE'u)) - callSoon(callback1, cast[pointer](0x89ABCDEF'u)) - callSoon(callback1, cast[pointer](0x9ABCDEF1'u)) - callSoon(callback1, cast[pointer](0xABCDEF12'u)) - callSoon(callback1, cast[pointer](0xBCDEF123'u)) - callSoon(callback1, cast[pointer](0xCDEF1234'u)) - callSoon(callback1, cast[pointer](0xDEF12345'u)) - callSoon(callback1, cast[pointer](0xEF123456'u)) - callSoon(callback1, cast[pointer](0xF1234567'u)) - callSoon(callback1, cast[pointer](0x12345678'u)) - ## All callbacks must be processed exactly with 1 poll() call. - poll() - result = soonTest1 - - proc testProc() {.async.} = - for i in 1..CallSoonTests: - await sleepAsync(100.milliseconds) - timeoutsTest1 += 1 - - var callbackproc: proc(udata: pointer) {.gcsafe, raises: [].} - callbackproc = proc (udata: pointer) {.gcsafe, raises: [].} = - timeoutsTest2 += 1 - {.gcsafe.}: - callSoon(callbackproc) - - proc test2(timers, callbacks: var int) = - callSoon(callbackproc) - waitFor(testProc()) - timers = timeoutsTest1 - callbacks = timeoutsTest2 - - proc testCallback(udata: pointer) = - soonTest2 = 987654321 - - proc test3(): bool = - callSoon(testCallback) - poll() - result = soonTest2 == 987654321 - test "User-defined callback argument test": - var values = [0x12345678'u, 0x23456789'u, 0x3456789A'u, 0x456789AB'u, - 0x56789ABC'u, 0x6789ABCD'u, 0x789ABCDE'u, 0x89ABCDEF'u, - 0x9ABCDEF1'u, 0xABCDEF12'u, 0xBCDEF123'u, 0xCDEF1234'u, - 0xDEF12345'u, 0xEF123456'u, 0xF1234567'u, 0x12345678'u] - var expect = 0'u - for item in values: - expect = expect xor item - check test1() == expect + proc test(): bool = + var soonTest = 0'u + + proc callback(udata: pointer) {.gcsafe.} = + soonTest = soonTest xor cast[uint](udata) + + callSoon(callback, cast[pointer](0x12345678'u)) + callSoon(callback, cast[pointer](0x23456789'u)) + callSoon(callback, cast[pointer](0x3456789A'u)) + callSoon(callback, cast[pointer](0x456789AB'u)) + callSoon(callback, cast[pointer](0x56789ABC'u)) + callSoon(callback, cast[pointer](0x6789ABCD'u)) + callSoon(callback, cast[pointer](0x789ABCDE'u)) + callSoon(callback, cast[pointer](0x89ABCDEF'u)) + callSoon(callback, cast[pointer](0x9ABCDEF1'u)) + callSoon(callback, cast[pointer](0xABCDEF12'u)) + callSoon(callback, cast[pointer](0xBCDEF123'u)) + callSoon(callback, cast[pointer](0xCDEF1234'u)) + callSoon(callback, cast[pointer](0xDEF12345'u)) + callSoon(callback, cast[pointer](0xEF123456'u)) + callSoon(callback, cast[pointer](0xF1234567'u)) + callSoon(callback, cast[pointer](0x12345678'u)) + ## All callbacks must be processed exactly with 1 poll() call. + poll() + + var values = [0x12345678'u, 0x23456789'u, 0x3456789A'u, 0x456789AB'u, + 0x56789ABC'u, 0x6789ABCD'u, 0x789ABCDE'u, 0x89ABCDEF'u, + 0x9ABCDEF1'u, 0xABCDEF12'u, 0xBCDEF123'u, 0xCDEF1234'u, + 0xDEF12345'u, 0xEF123456'u, 0xF1234567'u, 0x12345678'u] + var expect = 0'u + for item in values: + expect = expect xor item + + soonTest == expect + + check test() == true + test "`Asynchronous dead end` #7193 test": - var timers, callbacks: int - test2(timers, callbacks) - check: - timers == CallSoonTests - callbacks > CallSoonTests * 2 + const CallSoonTests = 5 + proc test() = + var + timeoutsTest1 = 0 + timeoutsTest2 = 0 + stopFlag = false + + var callbackproc: proc(udata: pointer) {.gcsafe, raises: [].} + callbackproc = proc (udata: pointer) {.gcsafe, raises: [].} = + timeoutsTest2 += 1 + if not(stopFlag): + callSoon(callbackproc) + + proc testProc() {.async.} = + for i in 1 .. CallSoonTests: + await sleepAsync(10.milliseconds) + timeoutsTest1 += 1 + + callSoon(callbackproc) + waitFor(testProc()) + stopFlag = true + poll() + + check: + timeoutsTest1 == CallSoonTests + timeoutsTest2 > CallSoonTests * 2 + + test() + test "`callSoon() is not working prior getGlobalDispatcher()` #7192 test": - check test3() == true + proc test(): bool = + var soonTest = 0 + + proc testCallback(udata: pointer) = + soonTest = 987654321 + + callSoon(testCallback) + poll() + soonTest == 987654321 + + check test() == true diff --git a/tests/teststream.nim b/tests/teststream.nim index 9e1ce55..762e996 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -1271,15 +1271,23 @@ suite "Stream Transport test suite": server2.start() server3.start() - # It works cause even though there's an active listening socket bound to dst3, we are using ReusePort - var transp1 = await connect(server1.local, localAddress = server3.local, flags={SocketFlags.ReusePort}) - var transp2 = await connect(server2.local, localAddress = server3.local, flags={SocketFlags.ReusePort}) + # It works cause even though there's an active listening socket bound to + # dst3, we are using ReusePort + var transp1 = await connect( + server1.localAddress(), localAddress = server3.localAddress(), + flags = {SocketFlags.ReusePort}) + var transp2 = await connect( + server2.localAddress(), localAddress = server3.localAddress(), + flags = {SocketFlags.ReusePort}) expect(TransportOsError): - var transp2 {.used.} = await connect(server2.local, localAddress = server3.local) + var transp2 {.used.} = await connect( + server2.localAddress(), localAddress = server3.localAddress()) expect(TransportOsError): - var transp3 {.used.} = await connect(server2.local, localAddress = initTAddress("::", server3.local.port)) + var transp3 {.used.} = await connect( + server2.localAddress(), + localAddress = initTAddress("::", server3.localAddress().port)) await transp1.closeWait() await transp2.closeWait() @@ -1293,6 +1301,77 @@ suite "Stream Transport test suite": server3.stop() await server3.closeWait() + proc testConnectCancelLeaksTest() {.async.} = + proc client(server: StreamServer, transp: StreamTransport) {.async.} = + await transp.closeWait() + + let + server = createStreamServer(initTAddress("127.0.0.1:0"), client) + address = server.localAddress() + + var counter = 0 + while true: + let transpFut = connect(address) + if counter > 0: + await stepsAsync(counter) + if not(transpFut.finished()): + await cancelAndWait(transpFut) + doAssert(cancelled(transpFut), + "Future should be Cancelled at this point") + inc(counter) + else: + let transp = await transpFut + await transp.closeWait() + break + server.stop() + await server.closeWait() + + proc testAcceptCancelLeaksTest() {.async.} = + var + counter = 0 + exitLoop = false + + # This timer will help to awake events poll in case its going to stuck + # usually happens on MacOS. + let sleepFut = sleepAsync(1.seconds) + + while not(exitLoop): + let + server = createStreamServer(initTAddress("127.0.0.1:0")) + address = server.localAddress() + + let + transpFut = connect(address) + acceptFut = server.accept() + + if counter > 0: + await stepsAsync(counter) + + exitLoop = + if not(acceptFut.finished()): + await cancelAndWait(acceptFut) + doAssert(cancelled(acceptFut), + "Future should be Cancelled at this point") + inc(counter) + false + else: + let transp = await acceptFut + await transp.closeWait() + true + + if not(transpFut.finished()): + await transpFut.cancelAndWait() + + if transpFut.completed(): + let transp = transpFut.value + await transp.closeWait() + + server.stop() + await server.closeWait() + + if not(sleepFut.finished()): + await cancelAndWait(sleepFut) + markFD = getCurrentFD() for i in 0.. Date: Mon, 16 Oct 2023 10:38:11 +0200 Subject: [PATCH 21/50] Complete futures in closure finally (fix #415) (#449) * Complete in closure finally * cleanup tests, add comment * handle defects * don't complete future on defect * complete future in test to avoid failure * fix with strict exceptions * fix regressions * fix nim 1.6 --- chronos/asyncfutures2.nim | 61 ++++--------- chronos/asyncmacro2.nim | 183 +++++++++++++++++++++++++------------- chronos/futures.nim | 7 +- tests/testmacro.nim | 130 +++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 110 deletions(-) diff --git a/chronos/asyncfutures2.nim b/chronos/asyncfutures2.nim index ee6e8e0..9674888 100644 --- a/chronos/asyncfutures2.nim +++ b/chronos/asyncfutures2.nim @@ -311,57 +311,30 @@ proc internalContinue(fut: pointer) {.raises: [], gcsafe.} = proc futureContinue*(fut: FutureBase) {.raises: [], gcsafe.} = # This function is responsible for calling the closure iterator generated by # the `{.async.}` transformation either until it has completed its iteration - # or raised and error / been cancelled. # # Every call to an `{.async.}` proc is redirected to call this function # instead with its original body captured in `fut.closure`. - var next: FutureBase - template iterate = - while true: - # Call closure to make progress on `fut` until it reaches `yield` (inside - # `await` typically) or completes / fails / is cancelled - next = fut.internalClosure(fut) - if fut.internalClosure.finished(): # Reached the end of the transformed proc - break + while true: + # Call closure to make progress on `fut` until it reaches `yield` (inside + # `await` typically) or completes / fails / is cancelled + let next: FutureBase = fut.internalClosure(fut) + if fut.internalClosure.finished(): # Reached the end of the transformed proc + break - if next == nil: - raiseAssert "Async procedure (" & ($fut.location[LocationKind.Create]) & - ") yielded `nil`, are you await'ing a `nil` Future?" + if next == nil: + raiseAssert "Async procedure (" & ($fut.location[LocationKind.Create]) & + ") yielded `nil`, are you await'ing a `nil` Future?" - if not next.finished(): - # We cannot make progress on `fut` until `next` has finished - schedule - # `fut` to continue running when that happens - GC_ref(fut) - next.addCallback(CallbackFunc(internalContinue), cast[pointer](fut)) + if not next.finished(): + # We cannot make progress on `fut` until `next` has finished - schedule + # `fut` to continue running when that happens + GC_ref(fut) + next.addCallback(CallbackFunc(internalContinue), cast[pointer](fut)) - # return here so that we don't remove the closure below - return + # return here so that we don't remove the closure below + return - # Continue while the yielded future is already finished. - - when chronosStrictException: - try: - iterate - except CancelledError: - fut.cancelAndSchedule() - except CatchableError as exc: - fut.fail(exc) - finally: - next = nil # GC hygiene - else: - try: - iterate - except CancelledError: - fut.cancelAndSchedule() - except CatchableError as exc: - fut.fail(exc) - except Exception as exc: - if exc of Defect: - raise (ref Defect)(exc) - - fut.fail((ref ValueError)(msg: exc.msg, parent: exc)) - finally: - next = nil # GC hygiene + # Continue while the yielded future is already finished. # `futureContinue` will not be called any more for this future so we can # clean it up diff --git a/chronos/asyncmacro2.nim b/chronos/asyncmacro2.nim index a86147c..d059404 100644 --- a/chronos/asyncmacro2.nim +++ b/chronos/asyncmacro2.nim @@ -9,60 +9,14 @@ import std/[macros] -# `quote do` will ruin line numbers so we avoid it using these helpers -proc completeWithResult(fut, baseType: NimNode): NimNode {.compileTime.} = - # when `baseType` is void: - # complete(`fut`) - # else: - # complete(`fut`, result) - if baseType.eqIdent("void"): - # Shortcut if we know baseType at macro expansion time - newCall(ident "complete", fut) - else: - # `baseType` might be generic and resolve to `void` - nnkWhenStmt.newTree( - nnkElifExpr.newTree( - nnkInfix.newTree(ident "is", baseType, ident "void"), - newCall(ident "complete", fut) - ), - nnkElseExpr.newTree( - newCall(ident "complete", fut, ident "result") - ) - ) - -proc completeWithNode(fut, baseType, node: NimNode): NimNode {.compileTime.} = - # when typeof(`node`) is void: - # `node` # statement / explicit return - # -> completeWithResult(fut, baseType) - # else: # expression / implicit return - # complete(`fut`, `node`) - if node.kind == nnkEmpty: # shortcut when known at macro expanstion time - completeWithResult(fut, baseType) - else: - # Handle both expressions and statements - since the type is not know at - # macro expansion time, we delegate this choice to a later compilation stage - # with `when`. - nnkWhenStmt.newTree( - nnkElifExpr.newTree( - nnkInfix.newTree( - ident "is", nnkTypeOfExpr.newTree(node), ident "void"), - newStmtList( - node, - completeWithResult(fut, baseType) - ) - ), - nnkElseExpr.newTree( - newCall(ident "complete", fut, node) - ) - ) - -proc processBody(node, fut, baseType: NimNode): NimNode {.compileTime.} = +proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} = #echo(node.treeRepr) case node.kind of nnkReturnStmt: let res = newNimNode(nnkStmtList, node) - res.add completeWithNode(fut, baseType, processBody(node[0], fut, baseType)) + if node[0].kind != nnkEmpty: + res.add newCall(setResultSym, processBody(node[0], setResultSym, baseType)) res.add newNimNode(nnkReturnStmt, node).add(newNilLit()) res @@ -71,12 +25,89 @@ proc processBody(node, fut, baseType: NimNode): NimNode {.compileTime.} = node else: for i in 0 ..< node.len: - # We must not transform nested procedures of any form, otherwise - # `fut` will be used for all nested procedures as their own - # `retFuture`. - node[i] = processBody(node[i], fut, baseType) + # We must not transform nested procedures of any form, since their + # returns are not meant for our futures + node[i] = processBody(node[i], setResultSym, baseType) node +proc wrapInTryFinally(fut, baseType, body: NimNode): NimNode {.compileTime.} = + # creates: + # var closureSucceeded = true + # try: `body` + # except CancelledError: closureSucceeded = false; `castFutureSym`.cancelAndSchedule() + # except CatchableError as exc: closureSucceeded = false; `castFutureSym`.fail(exc) + # except Defect as exc: + # closureSucceeded = false + # raise exc + # finally: + # if closureSucceeded: + # `castFutureSym`.complete(result) + + # we are completing inside finally to make sure the completion happens even + # after a `return` + let closureSucceeded = genSym(nskVar, "closureSucceeded") + var nTry = nnkTryStmt.newTree(body) + nTry.add nnkExceptBranch.newTree( + ident"CancelledError", + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + newCall(ident "cancelAndSchedule", fut) + ) + ) + + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", ident"CatchableError", ident"exc"), + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + newCall(ident "fail", fut, ident"exc") + ) + ) + + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", ident"Defect", ident"exc"), + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + nnkRaiseStmt.newTree(ident"exc") + ) + ) + + when not chronosStrictException: + # adds + # except Exception as exc: + # closureSucceeded = false + # fut.fail((ref ValueError)(msg: exc.msg, parent: exc)) + let excName = ident"exc" + + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", ident"Exception", ident"exc"), + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + newCall(ident "fail", fut, + quote do: (ref ValueError)(msg: `excName`.msg, parent: `excName`)), + ) + ) + + nTry.add nnkFinally.newTree( + nnkIfStmt.newTree( + nnkElifBranch.newTree( + closureSucceeded, + nnkWhenStmt.newTree( + nnkElifExpr.newTree( + nnkInfix.newTree(ident "is", baseType, ident "void"), + newCall(ident "complete", fut) + ), + nnkElseExpr.newTree( + newCall(ident "complete", fut, ident "result") + ) + ) + ) + ) + ) + return nnkStmtList.newTree( + newVarStmt(closureSucceeded, ident"true"), + nTry + ) + proc getName(node: NimNode): string {.compileTime.} = case node.kind of nnkSym: @@ -153,8 +184,9 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = if baseTypeIsVoid: futureVoidType else: returnType castFutureSym = nnkCast.newTree(internalFutureType, internalFutureSym) + setResultSym = ident"setResult" - procBody = prc.body.processBody(castFutureSym, baseType) + procBody = prc.body.processBody(setResultSym, baseType) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: @@ -199,9 +231,44 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = ) ) - completeDecl = completeWithNode(castFutureSym, baseType, procBodyBlck) + # generates: + # template `setResultSym`(code: untyped) {.used.} = + # when typeof(code) is void: code + # else: result = code + # + # this is useful to handle implicit returns, but also + # to bind the `result` to the one we declare here + setResultDecl = + nnkTemplateDef.newTree( + setResultSym, + newEmptyNode(), newEmptyNode(), + nnkFormalParams.newTree( + newEmptyNode(), + nnkIdentDefs.newTree( + ident"code", + ident"untyped", + newEmptyNode(), + ) + ), + nnkPragma.newTree(ident"used"), + newEmptyNode(), + nnkWhenStmt.newTree( + nnkElifBranch.newTree( + nnkInfix.newTree(ident"is", nnkTypeOfExpr.newTree(ident"code"), ident"void"), + ident"code" + ), + nnkElse.newTree( + newAssignment(ident"result", ident"code") + ) + ) + ) - closureBody = newStmtList(resultDecl, completeDecl) + completeDecl = wrapInTryFinally( + castFutureSym, baseType, + newCall(setResultSym, procBodyBlck) + ) + + closureBody = newStmtList(resultDecl, setResultDecl, completeDecl) internalFutureParameter = nnkIdentDefs.newTree( internalFutureSym, newIdentNode("FutureBase"), newEmptyNode()) @@ -225,10 +292,6 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # here the possibility of transporting more specific error types here # for example by casting exceptions coming out of `await`.. let raises = nnkBracket.newTree() - when chronosStrictException: - raises.add(newIdentNode("CatchableError")) - else: - raises.add(newIdentNode("Exception")) closureIterator.addPragma(nnkExprColonExpr.newTree( newIdentNode("raises"), diff --git a/chronos/futures.nim b/chronos/futures.nim index 5f96867..0af635f 100644 --- a/chronos/futures.nim +++ b/chronos/futures.nim @@ -17,11 +17,6 @@ export srcloc when chronosStackTrace: type StackTrace = string -when chronosStrictException: - {.pragma: closureIter, raises: [CatchableError], gcsafe.} -else: - {.pragma: closureIter, raises: [Exception], gcsafe.} - type LocationKind* {.pure.} = enum Create @@ -54,7 +49,7 @@ type internalState*: FutureState internalFlags*: FutureFlags internalError*: ref CatchableError ## Stored exception - internalClosure*: iterator(f: FutureBase): FutureBase {.closureIter.} + internalClosure*: iterator(f: FutureBase): FutureBase {.raises: [], gcsafe.} when chronosFutureId: internalId*: uint diff --git a/tests/testmacro.nim b/tests/testmacro.nim index ad4c22f..bd53078 100644 --- a/tests/testmacro.nim +++ b/tests/testmacro.nim @@ -94,6 +94,11 @@ proc testAwaitne(): Future[bool] {.async.} = return true +template returner = + # can't use `return 5` + result = 5 + return + suite "Macro transformations test suite": test "`await` command test": check waitFor(testAwait()) == true @@ -136,6 +141,131 @@ suite "Macro transformations test suite": check: waitFor(gen(int)) == default(int) + test "Nested return": + proc nr: Future[int] {.async.} = + return + if 1 == 1: + return 42 + else: + 33 + + check waitFor(nr()) == 42 + +suite "Macro transformations - completions": + test "Run closure to completion on return": # issue #415 + var x = 0 + proc test415 {.async.} = + try: + return + finally: + await sleepAsync(1.milliseconds) + x = 5 + waitFor(test415()) + check: x == 5 + + test "Run closure to completion on defer": + var x = 0 + proc testDefer {.async.} = + defer: + await sleepAsync(1.milliseconds) + x = 5 + return + waitFor(testDefer()) + check: x == 5 + + test "Run closure to completion with exceptions": + var x = 0 + proc testExceptionHandling {.async.} = + try: + return + finally: + try: + await sleepAsync(1.milliseconds) + raise newException(ValueError, "") + except ValueError: + await sleepAsync(1.milliseconds) + await sleepAsync(1.milliseconds) + x = 5 + waitFor(testExceptionHandling()) + check: x == 5 + + test "Correct return value when updating result after return": + proc testWeirdCase: int = + try: return 33 + finally: result = 55 + proc testWeirdCaseAsync: Future[int] {.async.} = + try: + await sleepAsync(1.milliseconds) + return 33 + finally: result = 55 + + check: + testWeirdCase() == waitFor(testWeirdCaseAsync()) + testWeirdCase() == 55 + + test "Generic & finally calling async": + proc testGeneric(T: type): Future[T] {.async.} = + try: + try: + await sleepAsync(1.milliseconds) + return + finally: + await sleepAsync(1.milliseconds) + await sleepAsync(1.milliseconds) + result = 11 + finally: + await sleepAsync(1.milliseconds) + await sleepAsync(1.milliseconds) + result = 12 + check waitFor(testGeneric(int)) == 12 + + proc testFinallyCallsAsync(T: type): Future[T] {.async.} = + try: + await sleepAsync(1.milliseconds) + return + finally: + result = await testGeneric(T) + check waitFor(testFinallyCallsAsync(int)) == 12 + + test "templates returning": + proc testReturner: Future[int] {.async.} = + returner + doAssert false + check waitFor(testReturner()) == 5 + + proc testReturner2: Future[int] {.async.} = + template returner2 = + return 6 + returner2 + doAssert false + check waitFor(testReturner2()) == 6 + + test "raising defects": + proc raiser {.async.} = + # sleeping to make sure our caller is the poll loop + await sleepAsync(0.milliseconds) + raise newException(Defect, "uh-oh") + + let fut = raiser() + expect(Defect): waitFor(fut) + check not fut.completed() + fut.complete() + + test "return result": + proc returnResult: Future[int] {.async.} = + var result: int + result = 12 + return result + check waitFor(returnResult()) == 12 + + test "async in async": + proc asyncInAsync: Future[int] {.async.} = + proc a2: Future[int] {.async.} = + result = 12 + result = await a2() + check waitFor(asyncInAsync()) == 12 + +suite "Macro transformations - implicit returns": test "Implicit return": proc implicit(): Future[int] {.async.} = 42 From a759c11ce4e9fd780f73d9264be1331a763e0fd8 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Tue, 17 Oct 2023 14:18:14 +0200 Subject: [PATCH 22/50] Raise tracking (#251) * Exception tracking v2 * some fixes * Nim 1.2 compat * simpler things * Fixes for libp2p * Fixes for strictException * better await exception check * Fix for template async proc * make async work with procTy * FuturEx is now a ref object type * add tests * update test * update readme * Switch to asyncraises pragma * Address tests review comments * Rename FuturEx to RaiseTrackingFuture * Fix typo * Split asyncraises into async, asyncraises * Add -d:chronosWarnMissingRaises * Add comment to RaiseTrackingFuture * Allow standalone asyncraises * CheckedFuture.fail type checking * First cleanup * Remove useless line * Review comments * nimble: Remove #head from unittest2 * Remove implict raises: CancelledError * Move checkFutureExceptions to asyncfutures2 * Small refacto * small cleanup * Complete in closure finally * cleanup tests, add comment * bump * chronos is not compatible with nim 1.2 anymore * re-add readme modifications * fix special exception handlers * also propagate excetion type in `read` * `RaiseTrackingFuture` -> `InternalRaisesFuture` Use internal naming scheme for RTF (this type should only be accessed via asyncraises) * use `internalError` for error reading * oops * 2.0 workarounds * again * remove try/finally for non-raising functions * Revert "remove try/finally for non-raising functions" This reverts commit 86bfeb5c972ef379a3bd34e4a16cd158a7455721. `finally` is needed if code returns early :/ * fixes * avoid exposing `newInternalRaisesFuture` in manual macro code * avoid unnecessary codegen for `Future[void]` * avoid reduntant block around async proc body * simplify body generation for forward declarations with comment but no body * avoid duplicate `gcsafe` annotiations * line info for return at end of async proc * expand tests * fix comments, add defer test --------- Co-authored-by: Jacek Sieka --- README.md | 35 +++ chronos.nimble | 2 +- chronos/asyncfutures2.nim | 146 +++++++++- chronos/asyncmacro2.nim | 554 +++++++++++++++++++++++--------------- tests/testfut.nim | 10 +- tests/testmacro.nim | 115 ++++++++ 6 files changed, 637 insertions(+), 225 deletions(-) diff --git a/README.md b/README.md index 3772c12..c06cfa9 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,41 @@ originating from tasks on the dispatcher queue. It is however possible that `Defect` that happen in tasks bubble up through `poll` as these are not caught by the transformation. +#### Checked exceptions + +By specifying a `asyncraises` list to an async procedure, you can check which +exceptions can be thrown by it. +```nim +proc p1(): Future[void] {.async, asyncraises: [IOError].} = + assert not (compiles do: raise newException(ValueError, "uh-uh")) + raise newException(IOError, "works") # Or any child of IOError +``` + +Under the hood, the return type of `p1` will be rewritten to another type, +which will convey raises informations to await. + +```nim +proc p2(): Future[void] {.async, asyncraises: [IOError].} = + await p1() # Works, because await knows that p1 + # can only raise IOError +``` + +The hidden type (`RaiseTrackingFuture`) is implicitely convertible into a Future. +However, it may causes issues when creating callback or methods +```nim +proc p3(): Future[void] {.async, asyncraises: [IOError].} = + let fut: Future[void] = p1() # works + assert not compiles(await fut) # await lost informations about raises, + # so it can raise anything + # Callbacks + assert not(compiles do: let cb1: proc(): Future[void] = p1) # doesn't work + let cb2: proc(): Future[void] {.async, asyncraises: [IOError].} = p1 # works + assert not(compiles do: + type c = proc(): Future[void] {.async, asyncraises: [IOError, ValueError].} + let cb3: c = p1 # doesn't work, the raises must match _exactly_ + ) +``` + ### Platform independence Several functions in `chronos` are backed by the operating system, such as diff --git a/chronos.nimble b/chronos.nimble index e9c1b11..f9e2617 100644 --- a/chronos.nimble +++ b/chronos.nimble @@ -7,7 +7,7 @@ description = "Networking framework with async/await support" license = "MIT or Apache License 2.0" skipDirs = @["tests"] -requires "nim >= 1.2.0", +requires "nim >= 1.6.0", "stew", "bearssl", "httputils", diff --git a/chronos/asyncfutures2.nim b/chronos/asyncfutures2.nim index 9674888..5a1383a 100644 --- a/chronos/asyncfutures2.nim +++ b/chronos/asyncfutures2.nim @@ -8,7 +8,7 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import std/sequtils +import std/[sequtils, macros] import stew/base10 when chronosStackTrace: @@ -35,6 +35,12 @@ func `[]`*(loc: array[LocationKind, ptr SrcLoc], v: int): ptr SrcLoc {. else: raiseAssert("Unknown source location " & $v) type + InternalRaisesFuture*[T, E] = ref object of Future[T] + ## Future with a tuple of possible exception types + ## eg InternalRaisesFuture[void, (ValueError, OSError)] + ## Will be injected by `asyncraises`, should generally + ## not be used manually + FutureStr*[T] = ref object of Future[T] ## Future to hold GC strings gcholder*: string @@ -59,6 +65,11 @@ proc newFutureImpl[T](loc: ptr SrcLoc, flags: FutureFlags): Future[T] = internalInitFutureBase(fut, loc, FutureState.Pending, flags) fut +proc newInternalRaisesFutureImpl[T, E](loc: ptr SrcLoc): InternalRaisesFuture[T, E] = + let fut = InternalRaisesFuture[T, E]() + internalInitFutureBase(fut, loc, FutureState.Pending, {}) + fut + proc newFutureSeqImpl[A, B](loc: ptr SrcLoc): FutureSeq[A, B] = let fut = FutureSeq[A, B]() internalInitFutureBase(fut, loc, FutureState.Pending, {}) @@ -70,12 +81,28 @@ proc newFutureStrImpl[T](loc: ptr SrcLoc): FutureStr[T] = fut template newFuture*[T](fromProc: static[string] = "", - flags: static[FutureFlags] = {}): Future[T] = + flags: static[FutureFlags] = {}): auto = ## Creates a new future. ## ## Specifying ``fromProc``, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. - newFutureImpl[T](getSrcLocation(fromProc), flags) + when declared(InternalRaisesFutureRaises): # injected by `asyncraises` + newInternalRaisesFutureImpl[T, InternalRaisesFutureRaises](getSrcLocation(fromProc)) + else: + newFutureImpl[T](getSrcLocation(fromProc), flags) + +macro getFutureExceptions(T: typedesc): untyped = + if getTypeInst(T)[1].len > 2: + getTypeInst(T)[1][2] + else: + ident"void" + +template newInternalRaisesFuture*[T](fromProc: static[string] = ""): auto = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + newInternalRaisesFutureImpl[T, getFutureExceptions(typeof(result))](getSrcLocation(fromProc)) template newFutureSeq*[A, B](fromProc: static[string] = ""): FutureSeq[A, B] = ## Create a new future which can hold/preserve GC sequence until future will @@ -188,6 +215,49 @@ template fail*(future: FutureBase, error: ref CatchableError) = ## Completes ``future`` with ``error``. fail(future, error, getSrcLocation()) +macro checkFailureType(future, error: typed): untyped = + let e = getTypeInst(future)[2] + let types = getType(e) + + if types.eqIdent("void"): + error("Can't raise exceptions on this Future") + + expectKind(types, nnkBracketExpr) + expectKind(types[0], nnkSym) + assert types[0].strVal == "tuple" + assert types.len > 1 + + expectKind(getTypeInst(error), nnkRefTy) + let toMatch = getTypeInst(error)[0] + + # Can't find a way to check `is` in the macro. (sameType doesn't + # work for inherited objects). Dirty hack here, for [IOError, OSError], + # this will generate: + # + # static: + # if not((`toMatch` is IOError) or (`toMatch` is OSError) + # or (`toMatch` is CancelledError) or false): + # raiseAssert("Can't fail with `toMatch`, only [IOError, OSError] is allowed") + var typeChecker = ident"false" + + for errorType in types[1..^1]: + typeChecker = newCall("or", typeChecker, newCall("is", toMatch, errorType)) + typeChecker = newCall( + "or", typeChecker, + newCall("is", toMatch, ident"CancelledError")) + + let errorMsg = "Can't fail with " & repr(toMatch) & ". Only " & repr(types[1..^1]) & " allowed" + + result = nnkStaticStmt.newNimNode(lineInfoFrom=error).add( + quote do: + if not(`typeChecker`): + raiseAssert(`errorMsg`) + ) + +template fail*[T, E](future: InternalRaisesFuture[T, E], error: ref CatchableError) = + checkFailureType(future, error) + fail(future, error, getSrcLocation()) + template newCancelledError(): ref CancelledError = (ref CancelledError)(msg: "Future operation cancelled!") @@ -429,6 +499,53 @@ proc internalCheckComplete*(fut: FutureBase) {.raises: [CatchableError].} = injectStacktrace(fut.internalError) raise fut.internalError +macro internalCheckComplete*(f: InternalRaisesFuture): untyped = + # For InternalRaisesFuture[void, (ValueError, OSError), will do: + # {.cast(raises: [ValueError, OSError]).}: + # if isNil(f.error): discard + # else: raise f.error + let e = getTypeInst(f)[2] + let types = getType(e) + + if types.eqIdent("void"): + return quote do: + if not(isNil(`f`.internalError)): + raiseAssert("Unhandled future exception: " & `f`.error.msg) + + expectKind(types, nnkBracketExpr) + expectKind(types[0], nnkSym) + assert types[0].strVal == "tuple" + assert types.len > 1 + + let ifRaise = nnkIfExpr.newTree( + nnkElifExpr.newTree( + quote do: isNil(`f`.internalError), + quote do: discard + ), + nnkElseExpr.newTree( + nnkRaiseStmt.newNimNode(lineInfoFrom=f).add( + quote do: (`f`.internalError) + ) + ) + ) + + nnkPragmaBlock.newTree( + nnkPragma.newTree( + nnkCast.newTree( + newEmptyNode(), + nnkExprColonExpr.newTree( + ident"raises", + block: + var res = nnkBracket.newTree() + for r in types[1..^1]: + res.add(r) + res + ) + ), + ), + ifRaise + ) + proc read*[T: not void](future: Future[T] ): lent T {.raises: [CatchableError].} = ## Retrieves the value of ``future``. Future must be finished otherwise ## this function will fail with a ``ValueError`` exception. @@ -452,6 +569,29 @@ proc read*(future: Future[void] ) {.raises: [CatchableError].} = # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") +proc read*[T: not void, E](future: InternalRaisesFuture[T, E] ): lent T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if not future.finished(): + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + + internalCheckComplete(future) + future.internalValue + +proc read*[E](future: InternalRaisesFuture[void, E]) = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + if future.finished(): + internalCheckComplete(future) + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + proc readError*(future: FutureBase): ref CatchableError {.raises: [ValueError].} = ## Retrieves the exception stored in ``future``. ## diff --git a/chronos/asyncmacro2.nim b/chronos/asyncmacro2.nim index d059404..499f847 100644 --- a/chronos/asyncmacro2.nim +++ b/chronos/asyncmacro2.nim @@ -2,108 +2,144 @@ # # Nim's Runtime Library # (c) Copyright 2015 Dominik Picheta +# (c) Copyright 2018-Present Status Research & Development GmbH # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -import std/[macros] +import std/algorithm proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} = - #echo(node.treeRepr) case node.kind of nnkReturnStmt: + # `return ...` -> `setResult(...); return` let res = newNimNode(nnkStmtList, node) if node[0].kind != nnkEmpty: res.add newCall(setResultSym, processBody(node[0], setResultSym, baseType)) - res.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + res.add newNimNode(nnkReturnStmt, node).add(newEmptyNode()) res of RoutineNodes-{nnkTemplateDef}: - # skip all the nested procedure definitions + # Skip nested routines since they have their own return value distinct from + # the Future we inject node else: for i in 0 ..< node.len: - # We must not transform nested procedures of any form, since their - # returns are not meant for our futures node[i] = processBody(node[i], setResultSym, baseType) node -proc wrapInTryFinally(fut, baseType, body: NimNode): NimNode {.compileTime.} = +proc wrapInTryFinally(fut, baseType, body, raisesTuple: NimNode): NimNode {.compileTime.} = # creates: - # var closureSucceeded = true # try: `body` - # except CancelledError: closureSucceeded = false; `castFutureSym`.cancelAndSchedule() - # except CatchableError as exc: closureSucceeded = false; `castFutureSym`.fail(exc) - # except Defect as exc: - # closureSucceeded = false - # raise exc + # [for raise in raisesTuple]: + # except `raise`: closureSucceeded = false; `castFutureSym`.fail(exc) # finally: # if closureSucceeded: # `castFutureSym`.complete(result) + # + # Calling `complete` inside `finally` ensures that all success paths + # (including early returns and code inside nested finally statements and + # defer) are completed with the final contents of `result` + let + closureSucceeded = genSym(nskVar, "closureSucceeded") + nTry = nnkTryStmt.newTree(body) + excName = ident"exc" - # we are completing inside finally to make sure the completion happens even - # after a `return` - let closureSucceeded = genSym(nskVar, "closureSucceeded") - var nTry = nnkTryStmt.newTree(body) - nTry.add nnkExceptBranch.newTree( - ident"CancelledError", + # Depending on the exception type, we must have at most one of each of these + # "special" exception handlers that are needed to implement cancellation and + # Defect propagation + var + hasDefect = false + hasCancelledError = false + hasCatchableError = false + + template addDefect = + if not hasDefect: + hasDefect = true + # When a Defect is raised, the program is in an undefined state and + # continuing running other tasks while the Future completion sits on the + # callback queue may lead to further damage so we re-raise them eagerly. + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", ident"Defect", excName), nnkStmtList.newTree( nnkAsgn.newTree(closureSucceeded, ident"false"), - newCall(ident "cancelAndSchedule", fut) + nnkRaiseStmt.newTree(excName) ) ) - - nTry.add nnkExceptBranch.newTree( - nnkInfix.newTree(ident"as", ident"CatchableError", ident"exc"), - nnkStmtList.newTree( - nnkAsgn.newTree(closureSucceeded, ident"false"), - newCall(ident "fail", fut, ident"exc") - ) - ) - - nTry.add nnkExceptBranch.newTree( - nnkInfix.newTree(ident"as", ident"Defect", ident"exc"), - nnkStmtList.newTree( - nnkAsgn.newTree(closureSucceeded, ident"false"), - nnkRaiseStmt.newTree(ident"exc") - ) - ) - - when not chronosStrictException: - # adds - # except Exception as exc: - # closureSucceeded = false - # fut.fail((ref ValueError)(msg: exc.msg, parent: exc)) - let excName = ident"exc" - - nTry.add nnkExceptBranch.newTree( - nnkInfix.newTree(ident"as", ident"Exception", ident"exc"), - nnkStmtList.newTree( - nnkAsgn.newTree(closureSucceeded, ident"false"), - newCall(ident "fail", fut, - quote do: (ref ValueError)(msg: `excName`.msg, parent: `excName`)), - ) - ) - - nTry.add nnkFinally.newTree( - nnkIfStmt.newTree( - nnkElifBranch.newTree( - closureSucceeded, - nnkWhenStmt.newTree( - nnkElifExpr.newTree( - nnkInfix.newTree(ident "is", baseType, ident "void"), - newCall(ident "complete", fut) - ), - nnkElseExpr.newTree( - newCall(ident "complete", fut, ident "result") - ) + template addCancelledError = + if not hasCancelledError: + hasCancelledError = true + nTry.add nnkExceptBranch.newTree( + ident"CancelledError", + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + newCall(ident "cancelAndSchedule", fut) ) ) + + template addCatchableError = + if not hasCatchableError: + hasCatchableError = true + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", ident"CatchableError", excName), + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + newCall(ident "fail", fut, excName) + )) + + for exc in raisesTuple: + if exc.eqIdent("Exception"): + addCancelledError + addCatchableError + addDefect + + # Because we store `CatchableError` in the Future, we cannot re-raise the + # original exception + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", ident"Exception", excName), + newCall(ident "fail", fut, + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + quote do: (ref ValueError)(msg: `excName`.msg, parent: `excName`))) + ) + elif exc.eqIdent("CancelledError"): + addCancelledError + elif exc.eqIdent("CatchableError"): + # Ensure cancellations are re-routed to the cancellation handler even if + # not explicitly specified in the raises list + addCancelledError + addCatchableError + else: + nTry.add nnkExceptBranch.newTree( + nnkInfix.newTree(ident"as", exc, excName), + nnkStmtList.newTree( + nnkAsgn.newTree(closureSucceeded, ident"false"), + newCall(ident "fail", fut, excName) + )) + + nTry.add nnkFinally.newTree( + nnkIfStmt.newTree( + nnkElifBranch.newTree( + closureSucceeded, + if baseType.eqIdent("void"): # shortcut for non-generic void + newCall(ident "complete", fut) + else: + nnkWhenStmt.newTree( + nnkElifExpr.newTree( + nnkInfix.newTree(ident "is", baseType, ident "void"), + newCall(ident "complete", fut) + ), + nnkElseExpr.newTree( + newCall(ident "complete", fut, ident "result") ) ) - return nnkStmtList.newTree( + ) + ) + ) + + nnkStmtList.newTree( newVarStmt(closureSucceeded, ident"true"), nTry ) @@ -144,6 +180,54 @@ proc cleanupOpenSymChoice(node: NimNode): NimNode {.compileTime.} = for child in node: result.add(cleanupOpenSymChoice(child)) +proc getAsyncCfg(prc: NimNode): tuple[raises: bool, async: bool, raisesTuple: NimNode] = + # reads the pragmas to extract the useful data + # and removes them + var + foundRaises = -1 + foundAsync = -1 + + for index, pragma in pragma(prc): + if pragma.kind == nnkExprColonExpr and pragma[0] == ident "asyncraises": + foundRaises = index + elif pragma.eqIdent("async"): + foundAsync = index + elif pragma.kind == nnkExprColonExpr and pragma[0] == ident "raises": + warning("The raises pragma doesn't work on async procedure. " & + "Please remove it or use asyncraises instead") + + result.raises = foundRaises >= 0 + result.async = foundAsync >= 0 + result.raisesTuple = nnkTupleConstr.newTree() + + if foundRaises >= 0: + for possibleRaise in pragma(prc)[foundRaises][1]: + result.raisesTuple.add(possibleRaise) + if result.raisesTuple.len == 0: + result.raisesTuple = ident("void") + else: + when defined(chronosWarnMissingRaises): + warning("Async proc miss asyncraises") + const defaultException = + when defined(chronosStrictException): "CatchableError" + else: "Exception" + result.raisesTuple.add(ident(defaultException)) + + let toRemoveList = @[foundRaises, foundAsync].filterIt(it >= 0).sorted().reversed() + for toRemove in toRemoveList: + pragma(prc).del(toRemove) + +proc isEmpty(n: NimNode): bool {.compileTime.} = + # true iff node recursively contains only comments or empties + case n.kind + of nnkEmpty, nnkCommentStmt: true + of nnkStmtList: + for child in n: + if not isEmpty(child): return false + true + else: + false + proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = ## This macro transforms a single procedure into a closure iterator. ## The ``async`` macro supports a stmtList holding multiple async procedures. @@ -158,7 +242,8 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = if returnType.kind == nnkEmpty: ident "void" elif not ( - returnType.kind == nnkBracketExpr and eqIdent(returnType[0], "Future")): + returnType.kind == nnkBracketExpr and + (eqIdent(returnType[0], "Future") or eqIdent(returnType[0], "InternalRaisesFuture"))): error( "Expected return type of 'Future' got '" & repr(returnType) & "'", prc) return @@ -168,77 +253,111 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = let baseTypeIsVoid = baseType.eqIdent("void") futureVoidType = nnkBracketExpr.newTree(ident "Future", ident "void") + (hasRaises, isAsync, raisesTuple) = getAsyncCfg(prc) - if prc.kind in {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: + if hasRaises: + # Store `asyncraises` types in InternalRaisesFuture + prc.params2[0] = nnkBracketExpr.newTree( + newIdentNode("InternalRaisesFuture"), + baseType, + raisesTuple + ) + elif baseTypeIsVoid: + # Adds the implicit Future[void] + prc.params2[0] = + newNimNode(nnkBracketExpr, prc). + add(newIdentNode("Future")). + add(newIdentNode("void")) + + if prc.kind notin {nnkProcTy, nnkLambda}: # TODO: Nim bug? + prc.addPragma(newColonExpr(ident "stackTrace", ident "off")) + + # The proc itself doesn't raise + prc.addPragma( + nnkExprColonExpr.newTree(newIdentNode("raises"), nnkBracket.newTree())) + + # `gcsafe` isn't deduced even though we require async code to be gcsafe + # https://github.com/nim-lang/RFCs/issues/435 + prc.addPragma(newIdentNode("gcsafe")) + + if isAsync == false: # `asyncraises` without `async` + # type InternalRaisesFutureRaises = `raisesTuple` + # `body` + prc.body = nnkStmtList.newTree( + nnkTypeSection.newTree( + nnkTypeDef.newTree( + ident"InternalRaisesFutureRaises", + newEmptyNode(), + raisesTuple + ) + ), + prc.body + ) + + return prc + + if prc.kind in {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo} and + not isEmpty(prc.body): + # don't do anything with forward bodies (empty) let prcName = prc.name.getName - outerProcBody = newNimNode(nnkStmtList, prc.body) - - # Copy comment for nimdoc - if prc.body.len > 0 and prc.body[0].kind == nnkCommentStmt: - outerProcBody.add(prc.body[0]) - - let + setResultSym = ident "setResult" + procBody = prc.body.processBody(setResultSym, baseType) internalFutureSym = ident "chronosInternalRetFuture" internalFutureType = if baseTypeIsVoid: futureVoidType else: returnType castFutureSym = nnkCast.newTree(internalFutureType, internalFutureSym) - setResultSym = ident"setResult" + resultIdent = ident "result" - procBody = prc.body.processBody(setResultSym, baseType) - - # don't do anything with forward bodies (empty) - if procBody.kind != nnkEmpty: - let - # fix #13899, `defer` should not escape its original scope - procBodyBlck = nnkBlockStmt.newTree(newEmptyNode(), procBody) - - resultDecl = nnkWhenStmt.newTree( - # when `baseType` is void: - nnkElifExpr.newTree( - nnkInfix.newTree(ident "is", baseType, ident "void"), - quote do: - template result: auto {.used.} = - {.fatal: "You should not reference the `result` variable inside" & - " a void async proc".} - ), - # else: - nnkElseExpr.newTree( - newStmtList( - quote do: {.push warning[resultshadowed]: off.}, - # var result {.used.}: `baseType` - # In the proc body, result may or may not end up being used - # depending on how the body is written - with implicit returns / - # expressions in particular, it is likely but not guaranteed that - # it is not used. Ideally, we would avoid emitting it in this - # case to avoid the default initializaiton. {.used.} typically - # works better than {.push.} which has a tendency to leak out of - # scope. - # TODO figure out if there's a way to detect `result` usage in - # the proc body _after_ template exapnsion, and therefore - # avoid creating this variable - one option is to create an - # addtional when branch witha fake `result` and check - # `compiles(procBody)` - this is not without cost though - nnkVarSection.newTree(nnkIdentDefs.newTree( - nnkPragmaExpr.newTree( - ident "result", - nnkPragma.newTree(ident "used")), - baseType, newEmptyNode()) - ), - quote do: {.pop.}, - ) + resultDecl = nnkWhenStmt.newTree( + # when `baseType` is void: + nnkElifExpr.newTree( + nnkInfix.newTree(ident "is", baseType, ident "void"), + quote do: + template result: auto {.used.} = + {.fatal: "You should not reference the `result` variable inside" & + " a void async proc".} + ), + # else: + nnkElseExpr.newTree( + newStmtList( + quote do: {.push warning[resultshadowed]: off.}, + # var result {.used.}: `baseType` + # In the proc body, result may or may not end up being used + # depending on how the body is written - with implicit returns / + # expressions in particular, it is likely but not guaranteed that + # it is not used. Ideally, we would avoid emitting it in this + # case to avoid the default initializaiton. {.used.} typically + # works better than {.push.} which has a tendency to leak out of + # scope. + # TODO figure out if there's a way to detect `result` usage in + # the proc body _after_ template exapnsion, and therefore + # avoid creating this variable - one option is to create an + # addtional when branch witha fake `result` and check + # `compiles(procBody)` - this is not without cost though + nnkVarSection.newTree(nnkIdentDefs.newTree( + nnkPragmaExpr.newTree( + resultIdent, + nnkPragma.newTree(ident "used")), + baseType, newEmptyNode()) + ), + quote do: {.pop.}, ) ) + ) - # generates: - # template `setResultSym`(code: untyped) {.used.} = - # when typeof(code) is void: code - # else: result = code - # - # this is useful to handle implicit returns, but also - # to bind the `result` to the one we declare here - setResultDecl = + # generates: + # template `setResultSym`(code: untyped) {.used.} = + # when typeof(code) is void: code + # else: `resultIdent` = code + # + # this is useful to handle implicit returns, but also + # to bind the `result` to the one we declare here + setResultDecl = + if baseTypeIsVoid: # shortcut for non-generic void + newEmptyNode() + else: nnkTemplateDef.newTree( setResultSym, newEmptyNode(), newEmptyNode(), @@ -254,107 +373,91 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = newEmptyNode(), nnkWhenStmt.newTree( nnkElifBranch.newTree( - nnkInfix.newTree(ident"is", nnkTypeOfExpr.newTree(ident"code"), ident"void"), + nnkInfix.newTree( + ident"is", nnkTypeOfExpr.newTree(ident"code"), ident"void"), ident"code" ), nnkElse.newTree( - newAssignment(ident"result", ident"code") + newAssignment(resultIdent, ident"code") ) ) ) - completeDecl = wrapInTryFinally( - castFutureSym, baseType, - newCall(setResultSym, procBodyBlck) - ) - - closureBody = newStmtList(resultDecl, setResultDecl, completeDecl) - - internalFutureParameter = nnkIdentDefs.newTree( - internalFutureSym, newIdentNode("FutureBase"), newEmptyNode()) - iteratorNameSym = genSym(nskIterator, $prcName) - closureIterator = newProc( - iteratorNameSym, - [newIdentNode("FutureBase"), internalFutureParameter], - closureBody, nnkIteratorDef) - - iteratorNameSym.copyLineInfo(prc) - - closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom=prc.body) - closureIterator.addPragma(newIdentNode("closure")) - - # `async` code must be gcsafe - closureIterator.addPragma(newIdentNode("gcsafe")) - - # TODO when push raises is active in a module, the iterator here inherits - # that annotation - here we explicitly disable it again which goes - # against the spirit of the raises annotation - one should investigate - # here the possibility of transporting more specific error types here - # for example by casting exceptions coming out of `await`.. - let raises = nnkBracket.newTree() - - closureIterator.addPragma(nnkExprColonExpr.newTree( - newIdentNode("raises"), - raises - )) - - # If proc has an explicit gcsafe pragma, we add it to iterator as well. - # TODO if these lines are not here, srcloc tests fail (!) - if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and - it.strVal == "gcsafe") != nil: - closureIterator.addPragma(newIdentNode("gcsafe")) - - outerProcBody.add(closureIterator) - - # -> let resultFuture = newFuture[T]() - # declared at the end to be sure that the closure - # doesn't reference it, avoid cyclic ref (#203) - let - retFutureSym = ident "resultFuture" - retFutureSym.copyLineInfo(prc) - # Do not change this code to `quote do` version because `instantiationInfo` - # will be broken for `newFuture()` call. - outerProcBody.add( - newLetStmt( - retFutureSym, - newCall(newTree(nnkBracketExpr, ident "newFuture", baseType), - newLit(prcName)) - ) - ) - # -> resultFuture.internalClosure = iterator - outerProcBody.add( - newAssignment( - newDotExpr(retFutureSym, newIdentNode("internalClosure")), - iteratorNameSym) + # Wrapping in try/finally ensures that early returns are handled properly + # and that `defer` is processed in the right scope + completeDecl = wrapInTryFinally( + castFutureSym, baseType, + if baseTypeIsVoid: procBody # shortcut for non-generic `void` + else: newCall(setResultSym, procBody), + raisesTuple ) - # -> futureContinue(resultFuture)) - outerProcBody.add( - newCall(newIdentNode("futureContinue"), retFutureSym) + closureBody = newStmtList(resultDecl, setResultDecl, completeDecl) + + internalFutureParameter = nnkIdentDefs.newTree( + internalFutureSym, newIdentNode("FutureBase"), newEmptyNode()) + iteratorNameSym = genSym(nskIterator, $prcName) + closureIterator = newProc( + iteratorNameSym, + [newIdentNode("FutureBase"), internalFutureParameter], + closureBody, nnkIteratorDef) + + outerProcBody = newNimNode(nnkStmtList, prc.body) + + # Copy comment for nimdoc + if prc.body.len > 0 and prc.body[0].kind == nnkCommentStmt: + outerProcBody.add(prc.body[0]) + + iteratorNameSym.copyLineInfo(prc) + + closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom=prc.body) + closureIterator.addPragma(newIdentNode("closure")) + + # `async` code must be gcsafe + closureIterator.addPragma(newIdentNode("gcsafe")) + + # Exceptions are caught inside the iterator and stored in the future + closureIterator.addPragma(nnkExprColonExpr.newTree( + newIdentNode("raises"), + nnkBracket.newTree() + )) + + outerProcBody.add(closureIterator) + + # -> let resultFuture = newInternalRaisesFuture[T]() + # declared at the end to be sure that the closure + # doesn't reference it, avoid cyclic ref (#203) + let + retFutureSym = ident "resultFuture" + retFutureSym.copyLineInfo(prc) + # Do not change this code to `quote do` version because `instantiationInfo` + # will be broken for `newFuture()` call. + outerProcBody.add( + newLetStmt( + retFutureSym, + newCall(newTree(nnkBracketExpr, ident "newInternalRaisesFuture", baseType), + newLit(prcName)) ) + ) + # -> resultFuture.internalClosure = iterator + outerProcBody.add( + newAssignment( + newDotExpr(retFutureSym, newIdentNode("internalClosure")), + iteratorNameSym) + ) - # -> return resultFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) + # -> futureContinue(resultFuture)) + outerProcBody.add( + newCall(newIdentNode("futureContinue"), retFutureSym) + ) - prc.body = outerProcBody + # -> return resultFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) - if prc.kind notin {nnkProcTy, nnkLambda}: # TODO: Nim bug? - prc.addPragma(newColonExpr(ident "stackTrace", ident "off")) - - # See **Remark 435** in this file. - # https://github.com/nim-lang/RFCs/issues/435 - prc.addPragma(newIdentNode("gcsafe")) - - prc.addPragma(nnkExprColonExpr.newTree( - newIdentNode("raises"), - nnkBracket.newTree() - )) - - if baseTypeIsVoid: - if returnType.kind == nnkEmpty: - # Add Future[void] - prc.params2[0] = futureVoidType + prc.body = outerProcBody + when chronosDumpAsync: + echo repr prc prc template await*[T](f: Future[T]): untyped = @@ -365,7 +468,8 @@ template await*[T](f: Future[T]): untyped = # responsible for resuming execution once the yielded future is finished yield chronosInternalRetFuture.internalChild # `child` released by `futureContinue` - chronosInternalRetFuture.internalChild.internalCheckComplete() + cast[type(f)](chronosInternalRetFuture.internalChild).internalCheckComplete() + when T isnot void: cast[type(f)](chronosInternalRetFuture.internalChild).value() else: @@ -385,8 +489,26 @@ macro async*(prc: untyped): untyped = if prc.kind == nnkStmtList: result = newStmtList() for oneProc in prc: + oneProc.addPragma(ident"async") result.add asyncSingleProc(oneProc) else: + prc.addPragma(ident"async") + result = asyncSingleProc(prc) + +macro asyncraises*(possibleExceptions, prc: untyped): untyped = + # Add back the pragma and let asyncSingleProc handle it + # Exerimental / subject to change and/or removal + if prc.kind == nnkStmtList: + result = newStmtList() + for oneProc in prc: + oneProc.addPragma(nnkExprColonExpr.newTree( + ident"asyncraises", + possibleExceptions + )) + result.add asyncSingleProc(oneProc) + else: + prc.addPragma(nnkExprColonExpr.newTree( + ident"asyncraises", + possibleExceptions + )) result = asyncSingleProc(prc) - when chronosDumpAsync: - echo repr result diff --git a/tests/testfut.nim b/tests/testfut.nim index bc61594..fc9d482 100644 --- a/tests/testfut.nim +++ b/tests/testfut.nim @@ -1223,11 +1223,11 @@ suite "Future[T] behavior test suite": test "location test": # WARNING: This test is very sensitive to line numbers and module name. - proc macroFuture() {.async.} = # LINE POSITION 1 - let someVar {.used.} = 5 # LINE POSITION 2 + proc macroFuture() {.async.} = + let someVar {.used.} = 5 # LINE POSITION 1 let someOtherVar {.used.} = 4 if true: - let otherVar {.used.} = 3 + let otherVar {.used.} = 3 # LINE POSITION 2 template templateFuture(): untyped = newFuture[void]("template") @@ -1260,8 +1260,8 @@ suite "Future[T] behavior test suite": (loc.procedure == procedure) check: - chk(loc10, "testfut.nim", 1226, "macroFuture") - chk(loc11, "testfut.nim", 1227, "") + chk(loc10, "testfut.nim", 1227, "macroFuture") + chk(loc11, "testfut.nim", 1230, "") chk(loc20, "testfut.nim", 1239, "template") chk(loc21, "testfut.nim", 1242, "") chk(loc30, "testfut.nim", 1236, "procedure") diff --git a/tests/testmacro.nim b/tests/testmacro.nim index bd53078..2d95a7f 100644 --- a/tests/testmacro.nim +++ b/tests/testmacro.nim @@ -151,6 +151,10 @@ suite "Macro transformations test suite": check waitFor(nr()) == 42 +# There are a few unreacheable statements to ensure that we don't regress in +# generated code +{.push warning[UnreachableCode]: off.} + suite "Macro transformations - completions": test "Run closure to completion on return": # issue #415 var x = 0 @@ -203,6 +207,21 @@ suite "Macro transformations - completions": testWeirdCase() == waitFor(testWeirdCaseAsync()) testWeirdCase() == 55 + test "Correct return value with result assignment in defer": + proc testWeirdCase: int = + defer: + result = 55 + result = 33 + proc testWeirdCaseAsync: Future[int] {.async.} = + defer: + result = 55 + await sleepAsync(1.milliseconds) + return 33 + + check: + testWeirdCase() == waitFor(testWeirdCaseAsync()) + testWeirdCase() == 55 + test "Generic & finally calling async": proc testGeneric(T: type): Future[T] {.async.} = try: @@ -264,6 +283,7 @@ suite "Macro transformations - completions": result = 12 result = await a2() check waitFor(asyncInAsync()) == 12 +{.pop.} suite "Macro transformations - implicit returns": test "Implicit return": @@ -362,3 +382,98 @@ suite "Closure iterator's exception transformation issues": waitFor(x()) +suite "Exceptions tracking": + template checkNotCompiles(body: untyped) = + check (not compiles(body)) + test "Can raise valid exception": + proc test1 {.async.} = raise newException(ValueError, "hey") + proc test2 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") + proc test3 {.async, asyncraises: [IOError, ValueError].} = + if 1 == 2: + raise newException(ValueError, "hey") + else: + raise newException(IOError, "hey") + + proc test4 {.async, asyncraises: [], used.} = raise newException(Defect, "hey") + proc test5 {.async, asyncraises: [].} = discard + proc test6 {.async, asyncraises: [].} = await test5() + + expect(ValueError): waitFor test1() + expect(ValueError): waitFor test2() + expect(IOError): waitFor test3() + waitFor test6() + + test "Cannot raise invalid exception": + checkNotCompiles: + proc test3 {.async, asyncraises: [IOError].} = raise newException(ValueError, "hey") + + test "Explicit return in non-raising proc": + proc test(): Future[int] {.async, asyncraises: [].} = return 12 + check: + waitFor(test()) == 12 + + test "Non-raising compatibility": + proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") + let testVar: Future[void] = test1() + + proc test2 {.async.} = raise newException(ValueError, "hey") + let testVar2: proc: Future[void] = test2 + + # Doesn't work unfortunately + #let testVar3: proc: Future[void] = test1 + + test "Cannot store invalid future types": + proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") + proc test2 {.async, asyncraises: [IOError].} = raise newException(IOError, "hey") + + var a = test1() + checkNotCompiles: + a = test2() + + test "Await raises the correct types": + proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") + proc test2 {.async, asyncraises: [ValueError, CancelledError].} = await test1() + checkNotCompiles: + proc test3 {.async, asyncraises: [CancelledError].} = await test1() + + test "Can create callbacks": + proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") + let callback: proc() {.async, asyncraises: [ValueError].} = test1 + + test "Can return values": + proc test1: Future[int] {.async, asyncraises: [ValueError].} = + if 1 == 0: raise newException(ValueError, "hey") + return 12 + proc test2: Future[int] {.async, asyncraises: [ValueError, IOError, CancelledError].} = + return await test1() + + checkNotCompiles: + proc test3: Future[int] {.async, asyncraises: [CancelledError].} = await test1() + + check waitFor(test2()) == 12 + + test "Manual tracking": + proc test1: Future[int] {.asyncraises: [ValueError].} = + result = newFuture[int]() + result.complete(12) + check waitFor(test1()) == 12 + + proc test2: Future[int] {.asyncraises: [IOError, OSError].} = + result = newFuture[int]() + result.fail(newException(IOError, "fail")) + result.fail(newException(OSError, "fail")) + checkNotCompiles: + result.fail(newException(ValueError, "fail")) + + proc test3: Future[void] {.asyncraises: [].} = + checkNotCompiles: + result.fail(newException(ValueError, "fail")) + + # Inheritance + proc test4: Future[void] {.asyncraises: [CatchableError].} = + result.fail(newException(IOError, "fail")) + + test "Reversed async, asyncraises": + proc test44 {.asyncraises: [ValueError], async.} = raise newException(ValueError, "hey") + checkNotCompiles: + proc test33 {.asyncraises: [IOError], async.} = raise newException(ValueError, "hey") From be9eef7a091da00720c8cf5c03f77d418c7c2b8a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 17 Oct 2023 14:19:20 +0200 Subject: [PATCH 23/50] move test data to c file (#448) * move test data to c file allows compiling with nlvm * more nlvm compat --- chronos/ioselects/ioselectors_epoll.nim | 2 +- tests/testasyncstream.c | 63 ++++++++++++++++++++++ tests/testasyncstream.nim | 69 ++----------------------- 3 files changed, 68 insertions(+), 66 deletions(-) create mode 100644 tests/testasyncstream.c diff --git a/chronos/ioselects/ioselectors_epoll.nim b/chronos/ioselects/ioselectors_epoll.nim index 161a5df..2156a39 100644 --- a/chronos/ioselects/ioselectors_epoll.nim +++ b/chronos/ioselects/ioselectors_epoll.nim @@ -411,7 +411,7 @@ proc registerProcess*[T](s: Selector, pid: int, data: T): SelectResult[cint] = s.freeKey(fdi32) s.freeProcess(int32(pid)) return err(res.error()) - s.pidFd = Opt.some(cast[cint](res.get())) + s.pidFd = Opt.some(res.get()) ok(cint(fdi32)) diff --git a/tests/testasyncstream.c b/tests/testasyncstream.c new file mode 100644 index 0000000..ecab9a9 --- /dev/null +++ b/tests/testasyncstream.c @@ -0,0 +1,63 @@ +#include + +// This is the X509TrustAnchor for the SelfSignedRsaCert above +// Generate by doing the following: +// 1. Compile `brssl` from BearSSL +// 2. Run `brssl ta filewithSelfSignedRsaCert.pem` +// 3. Paste the output in the emit block below +// 4. Rename `TAs` to `SelfSignedTAs` + +static const unsigned char TA0_DN[] = { + 0x30, 0x5F, 0x31, 0x0B, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, + 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, + 0x0C, 0x0A, 0x53, 0x6F, 0x6D, 0x65, 0x2D, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x31, 0x21, 0x30, 0x1F, 0x06, 0x03, 0x55, 0x04, 0x0A, 0x0C, 0x18, 0x49, + 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, + 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, 0x4C, 0x74, 0x64, 0x31, + 0x18, 0x30, 0x16, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0C, 0x0F, 0x31, 0x32, + 0x37, 0x2E, 0x30, 0x2E, 0x30, 0x2E, 0x31, 0x3A, 0x34, 0x33, 0x38, 0x30, + 0x38 +}; + +static const unsigned char TA0_RSA_N[] = { + 0xA7, 0xEE, 0xD5, 0xC6, 0x2C, 0xA3, 0x08, 0x33, 0x33, 0x86, 0xB5, 0x5C, + 0xD4, 0x8B, 0x16, 0xB1, 0xD7, 0xF7, 0xED, 0x95, 0x22, 0xDC, 0xA4, 0x40, + 0x24, 0x64, 0xC3, 0x91, 0xBA, 0x20, 0x82, 0x9D, 0x88, 0xED, 0x20, 0x98, + 0x46, 0x65, 0xDC, 0xD1, 0x15, 0x90, 0xBC, 0x7C, 0x19, 0x5F, 0x00, 0x96, + 0x69, 0x2C, 0x80, 0x0E, 0x7D, 0x7D, 0x8B, 0xD9, 0xFD, 0x49, 0x66, 0xEC, + 0x29, 0xC0, 0x39, 0x0E, 0x22, 0xF3, 0x6A, 0x28, 0xC0, 0x6B, 0x97, 0x93, + 0x2F, 0x92, 0x5E, 0x5A, 0xCC, 0xF4, 0xF4, 0xAE, 0xD9, 0xE3, 0xBB, 0x0A, + 0xDC, 0xA8, 0xDE, 0x4D, 0x16, 0xD6, 0xE6, 0x64, 0xF2, 0x85, 0x62, 0xF6, + 0xE3, 0x7B, 0x1D, 0x9A, 0x5C, 0x6A, 0xA3, 0x97, 0x93, 0x16, 0x9D, 0x02, + 0x2C, 0xFD, 0x90, 0x3E, 0xF8, 0x35, 0x44, 0x5E, 0x66, 0x8D, 0xF6, 0x80, + 0xF1, 0x71, 0x9B, 0x2F, 0x44, 0xC0, 0xCA, 0x7E, 0xB1, 0x90, 0x7F, 0xD8, + 0x8B, 0x7A, 0x85, 0x4B, 0xE3, 0xB1, 0xB1, 0xF4, 0xAA, 0x6A, 0x36, 0xA0, + 0xFF, 0x24, 0xB2, 0x27, 0xE0, 0xBA, 0x62, 0x7A, 0xE9, 0x95, 0xC9, 0x88, + 0x9D, 0x9B, 0xAB, 0xA4, 0x4C, 0xEA, 0x87, 0x46, 0xFA, 0xD6, 0x9B, 0x7E, + 0xB2, 0xE9, 0x5B, 0xCA, 0x5B, 0x84, 0xC4, 0xF7, 0xB4, 0xC7, 0x69, 0xC5, + 0x0B, 0x9A, 0x47, 0x9A, 0x86, 0xD4, 0xDF, 0xF3, 0x30, 0xC9, 0x6D, 0xB8, + 0x78, 0x10, 0xEF, 0xA0, 0x89, 0xF8, 0x30, 0x80, 0x9D, 0x96, 0x05, 0x44, + 0xB4, 0xFB, 0x98, 0x4C, 0x71, 0x6B, 0xBC, 0xD7, 0x5D, 0x66, 0x5E, 0x66, + 0xA7, 0x94, 0xE5, 0x65, 0x72, 0x85, 0xBC, 0x7C, 0x7F, 0x11, 0x98, 0xF8, + 0xCB, 0xD5, 0xE2, 0xB5, 0x67, 0x78, 0xF7, 0x49, 0x51, 0xC4, 0x7F, 0xBA, + 0x16, 0x66, 0xD2, 0x15, 0x5B, 0x98, 0x06, 0x03, 0x48, 0xD0, 0x9D, 0xF0, + 0x38, 0x2B, 0x9D, 0x51 +}; + +static const unsigned char TA0_RSA_E[] = { + 0x01, 0x00, 0x01 +}; + +const br_x509_trust_anchor SelfSignedTAs[1] = { + { + { (unsigned char *)TA0_DN, sizeof TA0_DN }, + BR_X509_TA_CA, + { + BR_KEYTYPE_RSA, + { .rsa = { + (unsigned char *)TA0_RSA_N, sizeof TA0_RSA_N, + (unsigned char *)TA0_RSA_E, sizeof TA0_RSA_E, + } } + } + } +}; diff --git a/tests/testasyncstream.nim b/tests/testasyncstream.nim index d90b688..c5701bb 100644 --- a/tests/testasyncstream.nim +++ b/tests/testasyncstream.nim @@ -73,69 +73,8 @@ N8r5CwGcIX/XPC3lKazzbZ8baA== -----END CERTIFICATE----- """ -# This is the X509TrustAnchor for the SelfSignedRsaCert above -# Generate by doing the following: -# 1. Compile `brssl` from BearSSL -# 2. Run `brssl ta filewithSelfSignedRsaCert.pem` -# 3. Paste the output in the emit block below -# 4. Rename `TAs` to `SelfSignedTAs` -{.emit: """ -static const unsigned char TA0_DN[] = { - 0x30, 0x5F, 0x31, 0x0B, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, - 0x02, 0x41, 0x55, 0x31, 0x13, 0x30, 0x11, 0x06, 0x03, 0x55, 0x04, 0x08, - 0x0C, 0x0A, 0x53, 0x6F, 0x6D, 0x65, 0x2D, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x31, 0x21, 0x30, 0x1F, 0x06, 0x03, 0x55, 0x04, 0x0A, 0x0C, 0x18, 0x49, - 0x6E, 0x74, 0x65, 0x72, 0x6E, 0x65, 0x74, 0x20, 0x57, 0x69, 0x64, 0x67, - 0x69, 0x74, 0x73, 0x20, 0x50, 0x74, 0x79, 0x20, 0x4C, 0x74, 0x64, 0x31, - 0x18, 0x30, 0x16, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0C, 0x0F, 0x31, 0x32, - 0x37, 0x2E, 0x30, 0x2E, 0x30, 0x2E, 0x31, 0x3A, 0x34, 0x33, 0x38, 0x30, - 0x38 -}; - -static const unsigned char TA0_RSA_N[] = { - 0xA7, 0xEE, 0xD5, 0xC6, 0x2C, 0xA3, 0x08, 0x33, 0x33, 0x86, 0xB5, 0x5C, - 0xD4, 0x8B, 0x16, 0xB1, 0xD7, 0xF7, 0xED, 0x95, 0x22, 0xDC, 0xA4, 0x40, - 0x24, 0x64, 0xC3, 0x91, 0xBA, 0x20, 0x82, 0x9D, 0x88, 0xED, 0x20, 0x98, - 0x46, 0x65, 0xDC, 0xD1, 0x15, 0x90, 0xBC, 0x7C, 0x19, 0x5F, 0x00, 0x96, - 0x69, 0x2C, 0x80, 0x0E, 0x7D, 0x7D, 0x8B, 0xD9, 0xFD, 0x49, 0x66, 0xEC, - 0x29, 0xC0, 0x39, 0x0E, 0x22, 0xF3, 0x6A, 0x28, 0xC0, 0x6B, 0x97, 0x93, - 0x2F, 0x92, 0x5E, 0x5A, 0xCC, 0xF4, 0xF4, 0xAE, 0xD9, 0xE3, 0xBB, 0x0A, - 0xDC, 0xA8, 0xDE, 0x4D, 0x16, 0xD6, 0xE6, 0x64, 0xF2, 0x85, 0x62, 0xF6, - 0xE3, 0x7B, 0x1D, 0x9A, 0x5C, 0x6A, 0xA3, 0x97, 0x93, 0x16, 0x9D, 0x02, - 0x2C, 0xFD, 0x90, 0x3E, 0xF8, 0x35, 0x44, 0x5E, 0x66, 0x8D, 0xF6, 0x80, - 0xF1, 0x71, 0x9B, 0x2F, 0x44, 0xC0, 0xCA, 0x7E, 0xB1, 0x90, 0x7F, 0xD8, - 0x8B, 0x7A, 0x85, 0x4B, 0xE3, 0xB1, 0xB1, 0xF4, 0xAA, 0x6A, 0x36, 0xA0, - 0xFF, 0x24, 0xB2, 0x27, 0xE0, 0xBA, 0x62, 0x7A, 0xE9, 0x95, 0xC9, 0x88, - 0x9D, 0x9B, 0xAB, 0xA4, 0x4C, 0xEA, 0x87, 0x46, 0xFA, 0xD6, 0x9B, 0x7E, - 0xB2, 0xE9, 0x5B, 0xCA, 0x5B, 0x84, 0xC4, 0xF7, 0xB4, 0xC7, 0x69, 0xC5, - 0x0B, 0x9A, 0x47, 0x9A, 0x86, 0xD4, 0xDF, 0xF3, 0x30, 0xC9, 0x6D, 0xB8, - 0x78, 0x10, 0xEF, 0xA0, 0x89, 0xF8, 0x30, 0x80, 0x9D, 0x96, 0x05, 0x44, - 0xB4, 0xFB, 0x98, 0x4C, 0x71, 0x6B, 0xBC, 0xD7, 0x5D, 0x66, 0x5E, 0x66, - 0xA7, 0x94, 0xE5, 0x65, 0x72, 0x85, 0xBC, 0x7C, 0x7F, 0x11, 0x98, 0xF8, - 0xCB, 0xD5, 0xE2, 0xB5, 0x67, 0x78, 0xF7, 0x49, 0x51, 0xC4, 0x7F, 0xBA, - 0x16, 0x66, 0xD2, 0x15, 0x5B, 0x98, 0x06, 0x03, 0x48, 0xD0, 0x9D, 0xF0, - 0x38, 0x2B, 0x9D, 0x51 -}; - -static const unsigned char TA0_RSA_E[] = { - 0x01, 0x00, 0x01 -}; - -static const br_x509_trust_anchor SelfSignedTAs[1] = { - { - { (unsigned char *)TA0_DN, sizeof TA0_DN }, - BR_X509_TA_CA, - { - BR_KEYTYPE_RSA, - { .rsa = { - (unsigned char *)TA0_RSA_N, sizeof TA0_RSA_N, - (unsigned char *)TA0_RSA_E, sizeof TA0_RSA_E, - } } - } - } -}; -""".} -var SelfSignedTrustAnchors {.importc: "SelfSignedTAs", nodecl.}: array[1, X509TrustAnchor] +let SelfSignedTrustAnchors {.importc: "SelfSignedTAs".}: array[1, X509TrustAnchor] +{.compile: "testasyncstream.c".} proc createBigMessage(message: string, size: int): seq[byte] = var res = newSeq[byte](size) @@ -983,7 +922,7 @@ suite "TLSStream test suite": test "Simple server with RSA self-signed certificate": let res = waitFor(checkSSLServer(SelfSignedRsaKey, SelfSignedRsaCert)) check res == true - + test "Custom TrustAnchors test": proc checkTrustAnchors(testMessage: string): Future[string] {.async.} = var key = TLSPrivateKey.init(SelfSignedRsaKey) @@ -1025,7 +964,7 @@ suite "TLSStream test suite": return cast[string](res) let res = waitFor checkTrustAnchors("Some message") check res == "Some message\r\n" - + test "TLSStream leaks test": checkLeaks() From e3c5a86a14ac39f4092ea399b491d15f844bc8dd Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 17 Oct 2023 20:25:25 +0200 Subject: [PATCH 24/50] Introduce chronos/internals, move some code (#453) * Introduce chronos/internals, move some code This PR breaks the include dependencies between `asyncfutures2` and `asyncmacros2` by moving the dispatcher and some other code to a new module. This step makes it easier to implement `asyncraises` support for future utilities like `allFutures` etc avoiding the need to play tricks with include order etc. Future PR:s may further articulate the difference between "internal" stuff subject to API breakage and regular public API intended for end users (rather than advanced integrators). * names * windows fix --- chronos/asyncloop.nim | 1543 +---------------- chronos/internal/asyncengine.nim | 1232 +++++++++++++ .../asyncfutures.nim} | 322 +++- .../asyncmacro.nim} | 4 +- chronos/internal/errors.nim | 5 + 5 files changed, 1564 insertions(+), 1542 deletions(-) create mode 100644 chronos/internal/asyncengine.nim rename chronos/{asyncfutures2.nim => internal/asyncfutures.nim} (80%) rename chronos/{asyncmacro2.nim => internal/asyncmacro.nim} (99%) create mode 100644 chronos/internal/errors.nim diff --git a/chronos/asyncloop.nim b/chronos/asyncloop.nim index fecec39..b4d48af 100644 --- a/chronos/asyncloop.nim +++ b/chronos/asyncloop.nim @@ -10,16 +10,6 @@ {.push raises: [].} -from nativesockets import Port -import std/[tables, heapqueue, deques] -import stew/results -import "."/[config, futures, osdefs, oserrno, osutils, timer] - -export Port -export futures, timer, results - -#{.injectStmt: newGcInvariant().} - ## Chronos ## ************* ## @@ -138,1534 +128,7 @@ export futures, timer, results ## ## * The effect system (``raises: []``) does not work with async procedures. -# TODO: Check if yielded future is nil and throw a more meaningful exception +import ./internal/[asyncengine, asyncfutures, asyncmacro, errors] -const - MaxEventsCount* = 64 - -when defined(windows): - import std/[sets, hashes] -elif defined(macosx) or defined(freebsd) or defined(netbsd) or - defined(openbsd) or defined(dragonfly) or defined(macos) or - defined(linux) or defined(android) or defined(solaris): - import "."/selectors2 - export SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, - SIGBUS, SIGFPE, SIGKILL, SIGUSR1, SIGSEGV, SIGUSR2, - SIGPIPE, SIGALRM, SIGTERM, SIGPIPE - export oserrno - -type - AsyncCallback = InternalAsyncCallback - - AsyncError* = object of CatchableError - ## Generic async exception - AsyncTimeoutError* = object of AsyncError - ## Timeout exception - - TimerCallback* = ref object - finishAt*: Moment - function*: AsyncCallback - - TrackerBase* = ref object of RootRef - id*: string - dump*: proc(): string {.gcsafe, raises: [].} - isLeaked*: proc(): bool {.gcsafe, raises: [].} - - TrackerCounter* = object - opened*: uint64 - closed*: uint64 - - PDispatcherBase = ref object of RootRef - timers*: HeapQueue[TimerCallback] - callbacks*: Deque[AsyncCallback] - idlers*: Deque[AsyncCallback] - ticks*: Deque[AsyncCallback] - trackers*: Table[string, TrackerBase] - counters*: Table[string, TrackerCounter] - -proc sentinelCallbackImpl(arg: pointer) {.gcsafe, noreturn.} = - raiseAssert "Sentinel callback MUST not be scheduled" - -const - SentinelCallback = AsyncCallback(function: sentinelCallbackImpl, - udata: nil) - -proc isSentinel(acb: AsyncCallback): bool = - acb == SentinelCallback - -proc `<`(a, b: TimerCallback): bool = - result = a.finishAt < b.finishAt - -func getAsyncTimestamp*(a: Duration): auto {.inline.} = - ## Return rounded up value of duration with milliseconds resolution. - ## - ## This function also take care on int32 overflow, because Linux and Windows - ## accepts signed 32bit integer as timeout. - let milsec = Millisecond.nanoseconds() - let nansec = a.nanoseconds() - var res = nansec div milsec - let mid = nansec mod milsec - when defined(windows): - res = min(int64(high(int32) - 1), res) - result = cast[DWORD](res) - result += DWORD(min(1'i32, cast[int32](mid))) - else: - res = min(int64(high(int32) - 1), res) - result = cast[int32](res) - result += min(1, cast[int32](mid)) - -template processTimersGetTimeout(loop, timeout: untyped) = - var lastFinish = curTime - while loop.timers.len > 0: - if loop.timers[0].function.function.isNil: - discard loop.timers.pop() - continue - - lastFinish = loop.timers[0].finishAt - if curTime < lastFinish: - break - - loop.callbacks.addLast(loop.timers.pop().function) - - if loop.timers.len > 0: - timeout = (lastFinish - curTime).getAsyncTimestamp() - - if timeout == 0: - if (len(loop.callbacks) == 0) and (len(loop.idlers) == 0): - when defined(windows): - timeout = INFINITE - else: - timeout = -1 - else: - if (len(loop.callbacks) != 0) or (len(loop.idlers) != 0): - timeout = 0 - -template processTimers(loop: untyped) = - var curTime = Moment.now() - while loop.timers.len > 0: - if loop.timers[0].function.function.isNil: - discard loop.timers.pop() - continue - - if curTime < loop.timers[0].finishAt: - break - loop.callbacks.addLast(loop.timers.pop().function) - -template processIdlers(loop: untyped) = - if len(loop.idlers) > 0: - loop.callbacks.addLast(loop.idlers.popFirst()) - -template processTicks(loop: untyped) = - while len(loop.ticks) > 0: - loop.callbacks.addLast(loop.ticks.popFirst()) - -template processCallbacks(loop: untyped) = - while true: - let callable = loop.callbacks.popFirst() # len must be > 0 due to sentinel - if isSentinel(callable): - break - if not(isNil(callable.function)): - callable.function(callable.udata) - -proc raiseAsDefect*(exc: ref Exception, msg: string) {.noreturn, noinline.} = - # Reraise an exception as a Defect, where it's unexpected and can't be handled - # We include the stack trace in the message because otherwise, it's easily - # lost - Nim doesn't print it for `parent` exceptions for example (!) - raise (ref Defect)( - msg: msg & "\n" & exc.msg & "\n" & exc.getStackTrace(), parent: exc) - -proc raiseOsDefect*(error: OSErrorCode, msg = "") {.noreturn, noinline.} = - # Reraise OS error code as a Defect, where it's unexpected and can't be - # handled. We include the stack trace in the message because otherwise, - # it's easily lost. - raise (ref Defect)(msg: msg & "\n[" & $int(error) & "] " & osErrorMsg(error) & - "\n" & getStackTrace()) - -func toPointer(error: OSErrorCode): pointer = - when sizeof(int) == 8: - cast[pointer](uint64(uint32(error))) - else: - cast[pointer](uint32(error)) - -func toException*(v: OSErrorCode): ref OSError = newOSError(v) - # This helper will allow to use `tryGet()` and raise OSError for - # Result[T, OSErrorCode] values. - -when defined(windows): - {.pragma: stdcallbackFunc, stdcall, gcsafe, raises: [].} - - export SIGINT, SIGQUIT, SIGTERM - type - CompletionKey = ULONG_PTR - - CompletionData* = object - cb*: CallbackFunc - errCode*: OSErrorCode - bytesCount*: uint32 - udata*: pointer - - CustomOverlapped* = object of OVERLAPPED - data*: CompletionData - - DispatcherFlag* = enum - SignalHandlerInstalled - - PDispatcher* = ref object of PDispatcherBase - ioPort: HANDLE - handles: HashSet[AsyncFD] - connectEx*: WSAPROC_CONNECTEX - acceptEx*: WSAPROC_ACCEPTEX - getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS - transmitFile*: WSAPROC_TRANSMITFILE - getQueuedCompletionStatusEx*: LPFN_GETQUEUEDCOMPLETIONSTATUSEX - disconnectEx*: WSAPROC_DISCONNECTEX - flags: set[DispatcherFlag] - - PtrCustomOverlapped* = ptr CustomOverlapped - - RefCustomOverlapped* = ref CustomOverlapped - - PostCallbackData = object - ioPort: HANDLE - handleFd: AsyncFD - waitFd: HANDLE - udata: pointer - ovlref: RefCustomOverlapped - ovl: pointer - - WaitableHandle* = ref PostCallbackData - ProcessHandle* = distinct WaitableHandle - SignalHandle* = distinct WaitableHandle - - WaitableResult* {.pure.} = enum - Ok, Timeout - - AsyncFD* = distinct int - - proc hash(x: AsyncFD): Hash {.borrow.} - proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow, gcsafe.} - - proc getFunc(s: SocketHandle, fun: var pointer, guid: GUID): bool = - var bytesRet: DWORD - fun = nil - wsaIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, unsafeAddr(guid), - DWORD(sizeof(GUID)), addr fun, DWORD(sizeof(pointer)), - addr(bytesRet), nil, nil) == 0 - - proc globalInit() = - var wsa = WSAData() - let res = wsaStartup(0x0202'u16, addr wsa) - if res != 0: - raiseOsDefect(osLastError(), - "globalInit(): Unable to initialize Windows Sockets API") - - proc initAPI(loop: PDispatcher) = - var funcPointer: pointer = nil - - let kernel32 = getModuleHandle(newWideCString("kernel32.dll")) - loop.getQueuedCompletionStatusEx = cast[LPFN_GETQUEUEDCOMPLETIONSTATUSEX]( - getProcAddress(kernel32, "GetQueuedCompletionStatusEx")) - - let sock = osdefs.socket(osdefs.AF_INET, 1, 6) - if sock == osdefs.INVALID_SOCKET: - raiseOsDefect(osLastError(), "initAPI(): Unable to create control socket") - - block: - let res = getFunc(sock, funcPointer, WSAID_CONNECTEX) - if not(res): - raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & - "dispatcher's ConnectEx()") - loop.connectEx = cast[WSAPROC_CONNECTEX](funcPointer) - - block: - let res = getFunc(sock, funcPointer, WSAID_ACCEPTEX) - if not(res): - raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & - "dispatcher's AcceptEx()") - loop.acceptEx = cast[WSAPROC_ACCEPTEX](funcPointer) - - block: - let res = getFunc(sock, funcPointer, WSAID_GETACCEPTEXSOCKADDRS) - if not(res): - raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & - "dispatcher's GetAcceptExSockAddrs()") - loop.getAcceptExSockAddrs = - cast[WSAPROC_GETACCEPTEXSOCKADDRS](funcPointer) - - block: - let res = getFunc(sock, funcPointer, WSAID_TRANSMITFILE) - if not(res): - raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & - "dispatcher's TransmitFile()") - loop.transmitFile = cast[WSAPROC_TRANSMITFILE](funcPointer) - - block: - let res = getFunc(sock, funcPointer, WSAID_DISCONNECTEX) - if not(res): - raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & - "dispatcher's DisconnectEx()") - loop.disconnectEx = cast[WSAPROC_DISCONNECTEX](funcPointer) - - if closeFd(sock) != 0: - raiseOsDefect(osLastError(), "initAPI(): Unable to close control socket") - - proc newDispatcher*(): PDispatcher = - ## Creates a new Dispatcher instance. - let port = createIoCompletionPort(osdefs.INVALID_HANDLE_VALUE, - HANDLE(0), 0, 1) - if port == osdefs.INVALID_HANDLE_VALUE: - raiseOsDefect(osLastError(), "newDispatcher(): Unable to create " & - "IOCP port") - var res = PDispatcher( - ioPort: port, - handles: initHashSet[AsyncFD](), - timers: initHeapQueue[TimerCallback](), - callbacks: initDeque[AsyncCallback](64), - idlers: initDeque[AsyncCallback](), - ticks: initDeque[AsyncCallback](), - trackers: initTable[string, TrackerBase](), - counters: initTable[string, TrackerCounter]() - ) - res.callbacks.addLast(SentinelCallback) - initAPI(res) - res - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setThreadDispatcher*(disp: PDispatcher) {.gcsafe, raises: [].} - proc getThreadDispatcher*(): PDispatcher {.gcsafe, raises: [].} - - proc getIoHandler*(disp: PDispatcher): HANDLE = - ## Returns the underlying IO Completion Port handle (Windows) or selector - ## (Unix) for the specified dispatcher. - disp.ioPort - - proc register2*(fd: AsyncFD): Result[void, OSErrorCode] = - ## Register file descriptor ``fd`` in thread's dispatcher. - let loop = getThreadDispatcher() - if createIoCompletionPort(HANDLE(fd), loop.ioPort, cast[CompletionKey](fd), - 1) == osdefs.INVALID_HANDLE_VALUE: - return err(osLastError()) - loop.handles.incl(fd) - ok() - - proc register*(fd: AsyncFD) {.raises: [OSError].} = - ## Register file descriptor ``fd`` in thread's dispatcher. - register2(fd).tryGet() - - proc unregister*(fd: AsyncFD) = - ## Unregisters ``fd``. - getThreadDispatcher().handles.excl(fd) - - {.push stackTrace: off.} - proc waitableCallback(param: pointer, timerOrWaitFired: WINBOOL) {. - stdcallbackFunc.} = - # This procedure will be executed in `wait thread`, so it must not use - # GC related objects. - # We going to ignore callbacks which was spawned when `isNil(param) == true` - # because we unable to indicate this error. - if isNil(param): return - var wh = cast[ptr PostCallbackData](param) - # We ignore result of postQueueCompletionStatus() call because we unable to - # indicate error. - discard postQueuedCompletionStatus(wh[].ioPort, DWORD(timerOrWaitFired), - ULONG_PTR(wh[].handleFd), - wh[].ovl) - {.pop.} - - proc registerWaitable( - handle: HANDLE, - flags: ULONG, - timeout: Duration, - cb: CallbackFunc, - udata: pointer - ): Result[WaitableHandle, OSErrorCode] = - ## Register handle of (Change notification, Console input, Event, - ## Memory resource notification, Mutex, Process, Semaphore, Thread, - ## Waitable timer) for waiting, using specific Windows' ``flags`` and - ## ``timeout`` value. - ## - ## Callback ``cb`` will be scheduled with ``udata`` parameter when - ## ``handle`` become signaled. - ## - ## Result of this procedure call ``WaitableHandle`` should be closed using - ## closeWaitable() call. - ## - ## NOTE: This is private procedure, not supposed to be publicly available, - ## please use ``waitForSingleObject()``. - let loop = getThreadDispatcher() - var ovl = RefCustomOverlapped(data: CompletionData(cb: cb)) - - var whandle = (ref PostCallbackData)( - ioPort: loop.getIoHandler(), - handleFd: AsyncFD(handle), - udata: udata, - ovlref: ovl, - ovl: cast[pointer](ovl) - ) - - ovl.data.udata = cast[pointer](whandle) - - let dwordTimeout = - if timeout == InfiniteDuration: - DWORD(INFINITE) - else: - DWORD(timeout.milliseconds) - - if registerWaitForSingleObject(addr(whandle[].waitFd), handle, - cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](whandle), - dwordTimeout, - flags) == WINBOOL(0): - ovl.data.udata = nil - whandle.ovlref = nil - whandle.ovl = nil - return err(osLastError()) - - ok(WaitableHandle(whandle)) - - proc closeWaitable(wh: WaitableHandle): Result[void, OSErrorCode] = - ## Close waitable handle ``wh`` and clear all the resources. It is safe - ## to close this handle, even if wait operation is pending. - ## - ## NOTE: This is private procedure, not supposed to be publicly available, - ## please use ``waitForSingleObject()``. - doAssert(not(isNil(wh))) - - let pdata = (ref PostCallbackData)(wh) - # We are not going to clear `ref` fields in PostCallbackData object because - # it possible that callback is already scheduled. - if unregisterWait(pdata.waitFd) == 0: - let res = osLastError() - if res != ERROR_IO_PENDING: - return err(res) - ok() - - proc addProcess2*(pid: int, cb: CallbackFunc, - udata: pointer = nil): Result[ProcessHandle, OSErrorCode] = - ## Registers callback ``cb`` to be called when process with process - ## identifier ``pid`` exited. Returns process identifier, which can be - ## used to clear process callback via ``removeProcess``. - doAssert(pid > 0, "Process identifier must be positive integer") - let - hProcess = openProcess(SYNCHRONIZE, WINBOOL(0), DWORD(pid)) - flags = WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE - - var wh: WaitableHandle = nil - - if hProcess == HANDLE(0): - return err(osLastError()) - - proc continuation(udata: pointer) {.gcsafe.} = - doAssert(not(isNil(udata))) - doAssert(not(isNil(wh))) - discard closeFd(hProcess) - cb(wh[].udata) - - wh = - block: - let res = registerWaitable(hProcess, flags, InfiniteDuration, - continuation, udata) - if res.isErr(): - discard closeFd(hProcess) - return err(res.error()) - res.get() - ok(ProcessHandle(wh)) - - proc removeProcess2*(procHandle: ProcessHandle): Result[void, OSErrorCode] = - ## Remove process' watching using process' descriptor ``procHandle``. - let waitableHandle = WaitableHandle(procHandle) - doAssert(not(isNil(waitableHandle))) - ? closeWaitable(waitableHandle) - ok() - - proc addProcess*(pid: int, cb: CallbackFunc, - udata: pointer = nil): ProcessHandle {. - raises: [OSError].} = - ## Registers callback ``cb`` to be called when process with process - ## identifier ``pid`` exited. Returns process identifier, which can be - ## used to clear process callback via ``removeProcess``. - addProcess2(pid, cb, udata).tryGet() - - proc removeProcess*(procHandle: ProcessHandle) {. - raises: [ OSError].} = - ## Remove process' watching using process' descriptor ``procHandle``. - removeProcess2(procHandle).tryGet() - - {.push stackTrace: off.} - proc consoleCtrlEventHandler(dwCtrlType: DWORD): uint32 {.stdcallbackFunc.} = - ## This procedure will be executed in different thread, so it MUST not use - ## any GC related features (strings, seqs, echo etc.). - case dwCtrlType - of CTRL_C_EVENT: - return - (if raiseSignal(SIGINT).valueOr(false): TRUE else: FALSE) - of CTRL_BREAK_EVENT: - return - (if raiseSignal(SIGINT).valueOr(false): TRUE else: FALSE) - of CTRL_CLOSE_EVENT: - return - (if raiseSignal(SIGTERM).valueOr(false): TRUE else: FALSE) - of CTRL_LOGOFF_EVENT: - return - (if raiseSignal(SIGQUIT).valueOr(false): TRUE else: FALSE) - else: - FALSE - {.pop.} - - proc addSignal2*(signal: int, cb: CallbackFunc, - udata: pointer = nil): Result[SignalHandle, OSErrorCode] = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb`` with specified argument ``udata``. Returns signal - ## identifier code, which can be used to remove signal callback - ## via ``removeSignal``. - ## - ## NOTE: On Windows only subset of signals are supported: SIGINT, SIGTERM, - ## SIGQUIT - const supportedSignals = [SIGINT, SIGTERM, SIGQUIT] - doAssert(cint(signal) in supportedSignals, "Signal is not supported") - let loop = getThreadDispatcher() - var hWait: WaitableHandle = nil - - proc continuation(ucdata: pointer) {.gcsafe.} = - doAssert(not(isNil(ucdata))) - doAssert(not(isNil(hWait))) - cb(hWait[].udata) - - if SignalHandlerInstalled notin loop.flags: - if getConsoleCP() != 0'u32: - # Console application, we going to cleanup Nim default signal handlers. - if setConsoleCtrlHandler(consoleCtrlEventHandler, TRUE) == FALSE: - return err(osLastError()) - loop.flags.incl(SignalHandlerInstalled) - else: - return err(ERROR_NOT_SUPPORTED) - - let - flags = WT_EXECUTEINWAITTHREAD - hEvent = ? openEvent($getSignalName(signal)) - - hWait = registerWaitable(hEvent, flags, InfiniteDuration, - continuation, udata).valueOr: - discard closeFd(hEvent) - return err(error) - ok(SignalHandle(hWait)) - - proc removeSignal2*(signalHandle: SignalHandle): Result[void, OSErrorCode] = - ## Remove watching signal ``signal``. - ? closeWaitable(WaitableHandle(signalHandle)) - ok() - - proc addSignal*(signal: int, cb: CallbackFunc, - udata: pointer = nil): SignalHandle {. - raises: [ValueError].} = - ## Registers callback ``cb`` to be called when signal ``signal`` will be - ## raised. Returns signal identifier, which can be used to clear signal - ## callback via ``removeSignal``. - addSignal2(signal, cb, udata).valueOr: - raise newException(ValueError, osErrorMsg(error)) - - proc removeSignal*(signalHandle: SignalHandle) {. - raises: [ValueError].} = - ## Remove signal's watching using signal descriptor ``signalfd``. - let res = removeSignal2(signalHandle) - if res.isErr(): - raise newException(ValueError, osErrorMsg(res.error())) - - proc poll*() = - ## Perform single asynchronous step, processing timers and completing - ## tasks. Blocks until at least one event has completed. - ## - ## Exceptions raised here indicate that waiting for tasks to be unblocked - ## failed - exceptions from within tasks are instead propagated through - ## their respective futures and not allowed to interrrupt the poll call. - let loop = getThreadDispatcher() - var - curTime = Moment.now() - curTimeout = DWORD(0) - events: array[MaxEventsCount, osdefs.OVERLAPPED_ENTRY] - - # On reentrant `poll` calls from `processCallbacks`, e.g., `waitFor`, - # complete pending work of the outer `processCallbacks` call. - # On non-reentrant `poll` calls, this only removes sentinel element. - processCallbacks(loop) - - # Moving expired timers to `loop.callbacks` and calculate timeout - loop.processTimersGetTimeout(curTimeout) - - let networkEventsCount = - if isNil(loop.getQueuedCompletionStatusEx): - let res = getQueuedCompletionStatus( - loop.ioPort, - addr events[0].dwNumberOfBytesTransferred, - addr events[0].lpCompletionKey, - cast[ptr POVERLAPPED](addr events[0].lpOverlapped), - curTimeout - ) - if res == FALSE: - let errCode = osLastError() - if not(isNil(events[0].lpOverlapped)): - 1 - else: - if uint32(errCode) != WAIT_TIMEOUT: - raiseOsDefect(errCode, "poll(): Unable to get OS events") - 0 - else: - 1 - else: - var eventsReceived = ULONG(0) - let res = loop.getQueuedCompletionStatusEx( - loop.ioPort, - addr events[0], - ULONG(len(events)), - eventsReceived, - curTimeout, - WINBOOL(0) - ) - if res == FALSE: - let errCode = osLastError() - if uint32(errCode) != WAIT_TIMEOUT: - raiseOsDefect(errCode, "poll(): Unable to get OS events") - 0 - else: - int(eventsReceived) - - for i in 0 ..< networkEventsCount: - var customOverlapped = PtrCustomOverlapped(events[i].lpOverlapped) - customOverlapped.data.errCode = - block: - let res = cast[uint64](customOverlapped.internal) - if res == 0'u64: - OSErrorCode(-1) - else: - OSErrorCode(rtlNtStatusToDosError(res)) - customOverlapped.data.bytesCount = events[i].dwNumberOfBytesTransferred - let acb = AsyncCallback(function: customOverlapped.data.cb, - udata: cast[pointer](customOverlapped)) - loop.callbacks.addLast(acb) - - # Moving expired timers to `loop.callbacks`. - loop.processTimers() - - # We move idle callbacks to `loop.callbacks` only if there no pending - # network events. - if networkEventsCount == 0: - loop.processIdlers() - - # We move tick callbacks to `loop.callbacks` always. - processTicks(loop) - - # All callbacks which will be added during `processCallbacks` will be - # scheduled after the sentinel and are processed on next `poll()` call. - loop.callbacks.addLast(SentinelCallback) - processCallbacks(loop) - - # All callbacks done, skip `processCallbacks` at start. - loop.callbacks.addFirst(SentinelCallback) - - proc closeSocket*(fd: AsyncFD, aftercb: CallbackFunc = nil) = - ## Closes a socket and ensures that it is unregistered. - let loop = getThreadDispatcher() - loop.handles.excl(fd) - let - param = toPointer( - if closeFd(SocketHandle(fd)) == 0: - OSErrorCode(0) - else: - osLastError() - ) - if not(isNil(aftercb)): - loop.callbacks.addLast(AsyncCallback(function: aftercb, udata: param)) - - proc closeHandle*(fd: AsyncFD, aftercb: CallbackFunc = nil) = - ## Closes a (pipe/file) handle and ensures that it is unregistered. - let loop = getThreadDispatcher() - loop.handles.excl(fd) - let - param = toPointer( - if closeFd(HANDLE(fd)) == 0: - OSErrorCode(0) - else: - osLastError() - ) - - if not(isNil(aftercb)): - loop.callbacks.addLast(AsyncCallback(function: aftercb, udata: param)) - - proc contains*(disp: PDispatcher, fd: AsyncFD): bool = - ## Returns ``true`` if ``fd`` is registered in thread's dispatcher. - fd in disp.handles - -elif defined(macosx) or defined(freebsd) or defined(netbsd) or - defined(openbsd) or defined(dragonfly) or defined(macos) or - defined(linux) or defined(android) or defined(solaris): - const - SIG_IGN = cast[proc(x: cint) {.raises: [], noconv, gcsafe.}](1) - - type - AsyncFD* = distinct cint - - SelectorData* = object - reader*: AsyncCallback - writer*: AsyncCallback - - PDispatcher* = ref object of PDispatcherBase - selector: Selector[SelectorData] - keys: seq[ReadyKey] - - proc `==`*(x, y: AsyncFD): bool {.borrow, gcsafe.} - - proc globalInit() = - # We are ignoring SIGPIPE signal, because we are working with EPIPE. - signal(cint(SIGPIPE), SIG_IGN) - - proc initAPI(disp: PDispatcher) = - discard - - proc newDispatcher*(): PDispatcher = - ## Create new dispatcher. - let selector = - block: - let res = Selector.new(SelectorData) - if res.isErr(): raiseOsDefect(res.error(), - "Could not initialize selector") - res.get() - - var res = PDispatcher( - selector: selector, - timers: initHeapQueue[TimerCallback](), - callbacks: initDeque[AsyncCallback](chronosEventsCount), - idlers: initDeque[AsyncCallback](), - keys: newSeq[ReadyKey](chronosEventsCount), - trackers: initTable[string, TrackerBase](), - counters: initTable[string, TrackerCounter]() - ) - res.callbacks.addLast(SentinelCallback) - initAPI(res) - res - - var gDisp{.threadvar.}: PDispatcher ## Global dispatcher - - proc setThreadDispatcher*(disp: PDispatcher) {.gcsafe, raises: [].} - proc getThreadDispatcher*(): PDispatcher {.gcsafe, raises: [].} - - proc getIoHandler*(disp: PDispatcher): Selector[SelectorData] = - ## Returns system specific OS queue. - disp.selector - - proc contains*(disp: PDispatcher, fd: AsyncFD): bool {.inline.} = - ## Returns ``true`` if ``fd`` is registered in thread's dispatcher. - cint(fd) in disp.selector - - proc register2*(fd: AsyncFD): Result[void, OSErrorCode] = - ## Register file descriptor ``fd`` in thread's dispatcher. - var data: SelectorData - getThreadDispatcher().selector.registerHandle2(cint(fd), {}, data) - - proc unregister2*(fd: AsyncFD): Result[void, OSErrorCode] = - ## Unregister file descriptor ``fd`` from thread's dispatcher. - getThreadDispatcher().selector.unregister2(cint(fd)) - - proc addReader2*(fd: AsyncFD, cb: CallbackFunc, - udata: pointer = nil): Result[void, OSErrorCode] = - ## Start watching the file descriptor ``fd`` for read availability and then - ## call the callback ``cb`` with specified argument ``udata``. - let loop = getThreadDispatcher() - var newEvents = {Event.Read} - withData(loop.selector, cint(fd), adata) do: - let acb = AsyncCallback(function: cb, udata: udata) - adata.reader = acb - if not(isNil(adata.writer.function)): - newEvents.incl(Event.Write) - do: - return err(osdefs.EBADF) - loop.selector.updateHandle2(cint(fd), newEvents) - - proc removeReader2*(fd: AsyncFD): Result[void, OSErrorCode] = - ## Stop watching the file descriptor ``fd`` for read availability. - let loop = getThreadDispatcher() - var newEvents: set[Event] - withData(loop.selector, cint(fd), adata) do: - # We need to clear `reader` data, because `selectors` don't do it - adata.reader = default(AsyncCallback) - if not(isNil(adata.writer.function)): - newEvents.incl(Event.Write) - do: - return err(osdefs.EBADF) - loop.selector.updateHandle2(cint(fd), newEvents) - - proc addWriter2*(fd: AsyncFD, cb: CallbackFunc, - udata: pointer = nil): Result[void, OSErrorCode] = - ## Start watching the file descriptor ``fd`` for write availability and then - ## call the callback ``cb`` with specified argument ``udata``. - let loop = getThreadDispatcher() - var newEvents = {Event.Write} - withData(loop.selector, cint(fd), adata) do: - let acb = AsyncCallback(function: cb, udata: udata) - adata.writer = acb - if not(isNil(adata.reader.function)): - newEvents.incl(Event.Read) - do: - return err(osdefs.EBADF) - loop.selector.updateHandle2(cint(fd), newEvents) - - proc removeWriter2*(fd: AsyncFD): Result[void, OSErrorCode] = - ## Stop watching the file descriptor ``fd`` for write availability. - let loop = getThreadDispatcher() - var newEvents: set[Event] - withData(loop.selector, cint(fd), adata) do: - # We need to clear `writer` data, because `selectors` don't do it - adata.writer = default(AsyncCallback) - if not(isNil(adata.reader.function)): - newEvents.incl(Event.Read) - do: - return err(osdefs.EBADF) - loop.selector.updateHandle2(cint(fd), newEvents) - - proc register*(fd: AsyncFD) {.raises: [OSError].} = - ## Register file descriptor ``fd`` in thread's dispatcher. - register2(fd).tryGet() - - proc unregister*(fd: AsyncFD) {.raises: [OSError].} = - ## Unregister file descriptor ``fd`` from thread's dispatcher. - unregister2(fd).tryGet() - - proc addReader*(fd: AsyncFD, cb: CallbackFunc, udata: pointer = nil) {. - raises: [OSError].} = - ## Start watching the file descriptor ``fd`` for read availability and then - ## call the callback ``cb`` with specified argument ``udata``. - addReader2(fd, cb, udata).tryGet() - - proc removeReader*(fd: AsyncFD) {.raises: [OSError].} = - ## Stop watching the file descriptor ``fd`` for read availability. - removeReader2(fd).tryGet() - - proc addWriter*(fd: AsyncFD, cb: CallbackFunc, udata: pointer = nil) {. - raises: [OSError].} = - ## Start watching the file descriptor ``fd`` for write availability and then - ## call the callback ``cb`` with specified argument ``udata``. - addWriter2(fd, cb, udata).tryGet() - - proc removeWriter*(fd: AsyncFD) {.raises: [OSError].} = - ## Stop watching the file descriptor ``fd`` for write availability. - removeWriter2(fd).tryGet() - - proc unregisterAndCloseFd*(fd: AsyncFD): Result[void, OSErrorCode] = - ## Unregister from system queue and close asynchronous socket. - ## - ## NOTE: Use this function to close temporary sockets/pipes only (which - ## are not exposed to the public and not supposed to be used/reused). - ## Please use closeSocket(AsyncFD) and closeHandle(AsyncFD) instead. - doAssert(fd != AsyncFD(osdefs.INVALID_SOCKET)) - ? unregister2(fd) - if closeFd(cint(fd)) != 0: - err(osLastError()) - else: - ok() - - proc closeSocket*(fd: AsyncFD, aftercb: CallbackFunc = nil) = - ## Close asynchronous socket. - ## - ## Please note, that socket is not closed immediately. To avoid bugs with - ## closing socket, while operation pending, socket will be closed as - ## soon as all pending operations will be notified. - let loop = getThreadDispatcher() - - proc continuation(udata: pointer) = - let - param = toPointer( - if SocketHandle(fd) in loop.selector: - let ures = unregister2(fd) - if ures.isErr(): - discard closeFd(cint(fd)) - ures.error() - else: - if closeFd(cint(fd)) != 0: - osLastError() - else: - OSErrorCode(0) - else: - osdefs.EBADF - ) - if not(isNil(aftercb)): aftercb(param) - - withData(loop.selector, cint(fd), adata) do: - # We are scheduling reader and writer callbacks to be called - # explicitly, so they can get an error and continue work. - # Callbacks marked as deleted so we don't need to get REAL notifications - # from system queue for this reader and writer. - - if not(isNil(adata.reader.function)): - loop.callbacks.addLast(adata.reader) - adata.reader = default(AsyncCallback) - - if not(isNil(adata.writer.function)): - loop.callbacks.addLast(adata.writer) - adata.writer = default(AsyncCallback) - - # We can't unregister file descriptor from system queue here, because - # in such case processing queue will stuck on poll() call, because there - # can be no file descriptors registered in system queue. - var acb = AsyncCallback(function: continuation) - loop.callbacks.addLast(acb) - - proc closeHandle*(fd: AsyncFD, aftercb: CallbackFunc = nil) = - ## Close asynchronous file/pipe handle. - ## - ## Please note, that socket is not closed immediately. To avoid bugs with - ## closing socket, while operation pending, socket will be closed as - ## soon as all pending operations will be notified. - ## You can execute ``aftercb`` before actual socket close operation. - closeSocket(fd, aftercb) - - when chronosEventEngine in ["epoll", "kqueue"]: - type - ProcessHandle* = distinct int - SignalHandle* = distinct int - - proc addSignal2*( - signal: int, - cb: CallbackFunc, - udata: pointer = nil - ): Result[SignalHandle, OSErrorCode] = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb`` with specified argument ``udata``. Returns signal - ## identifier code, which can be used to remove signal callback - ## via ``removeSignal``. - let loop = getThreadDispatcher() - var data: SelectorData - let sigfd = ? loop.selector.registerSignal(signal, data) - withData(loop.selector, sigfd, adata) do: - adata.reader = AsyncCallback(function: cb, udata: udata) - do: - return err(osdefs.EBADF) - ok(SignalHandle(sigfd)) - - proc addProcess2*( - pid: int, - cb: CallbackFunc, - udata: pointer = nil - ): Result[ProcessHandle, OSErrorCode] = - ## Registers callback ``cb`` to be called when process with process - ## identifier ``pid`` exited. Returns process' descriptor, which can be - ## used to clear process callback via ``removeProcess``. - let loop = getThreadDispatcher() - var data: SelectorData - let procfd = ? loop.selector.registerProcess(pid, data) - withData(loop.selector, procfd, adata) do: - adata.reader = AsyncCallback(function: cb, udata: udata) - do: - return err(osdefs.EBADF) - ok(ProcessHandle(procfd)) - - proc removeSignal2*(signalHandle: SignalHandle): Result[void, OSErrorCode] = - ## Remove watching signal ``signal``. - getThreadDispatcher().selector.unregister2(cint(signalHandle)) - - proc removeProcess2*(procHandle: ProcessHandle): Result[void, OSErrorCode] = - ## Remove process' watching using process' descriptor ``procfd``. - getThreadDispatcher().selector.unregister2(cint(procHandle)) - - proc addSignal*(signal: int, cb: CallbackFunc, - udata: pointer = nil): SignalHandle {. - raises: [OSError].} = - ## Start watching signal ``signal``, and when signal appears, call the - ## callback ``cb`` with specified argument ``udata``. Returns signal - ## identifier code, which can be used to remove signal callback - ## via ``removeSignal``. - addSignal2(signal, cb, udata).tryGet() - - proc removeSignal*(signalHandle: SignalHandle) {. - raises: [OSError].} = - ## Remove watching signal ``signal``. - removeSignal2(signalHandle).tryGet() - - proc addProcess*(pid: int, cb: CallbackFunc, - udata: pointer = nil): ProcessHandle {. - raises: [OSError].} = - ## Registers callback ``cb`` to be called when process with process - ## identifier ``pid`` exited. Returns process identifier, which can be - ## used to clear process callback via ``removeProcess``. - addProcess2(pid, cb, udata).tryGet() - - proc removeProcess*(procHandle: ProcessHandle) {. - raises: [OSError].} = - ## Remove process' watching using process' descriptor ``procHandle``. - removeProcess2(procHandle).tryGet() - - proc poll*() {.gcsafe.} = - ## Perform single asynchronous step. - let loop = getThreadDispatcher() - var curTime = Moment.now() - var curTimeout = 0 - - # On reentrant `poll` calls from `processCallbacks`, e.g., `waitFor`, - # complete pending work of the outer `processCallbacks` call. - # On non-reentrant `poll` calls, this only removes sentinel element. - processCallbacks(loop) - - # Moving expired timers to `loop.callbacks` and calculate timeout. - loop.processTimersGetTimeout(curTimeout) - - # Processing IO descriptors and all hardware events. - let count = - block: - let res = loop.selector.selectInto2(curTimeout, loop.keys) - if res.isErr(): - raiseOsDefect(res.error(), "poll(): Unable to get OS events") - res.get() - - for i in 0 ..< count: - let fd = loop.keys[i].fd - let events = loop.keys[i].events - - withData(loop.selector, cint(fd), adata) do: - if (Event.Read in events) or (events == {Event.Error}): - if not isNil(adata.reader.function): - loop.callbacks.addLast(adata.reader) - - if (Event.Write in events) or (events == {Event.Error}): - if not isNil(adata.writer.function): - loop.callbacks.addLast(adata.writer) - - if Event.User in events: - if not isNil(adata.reader.function): - loop.callbacks.addLast(adata.reader) - - when chronosEventEngine in ["epoll", "kqueue"]: - let customSet = {Event.Timer, Event.Signal, Event.Process, - Event.Vnode} - if customSet * events != {}: - if not isNil(adata.reader.function): - loop.callbacks.addLast(adata.reader) - - # Moving expired timers to `loop.callbacks`. - loop.processTimers() - - # We move idle callbacks to `loop.callbacks` only if there no pending - # network events. - if count == 0: - loop.processIdlers() - - # We move tick callbacks to `loop.callbacks` always. - processTicks(loop) - - # All callbacks which will be added during `processCallbacks` will be - # scheduled after the sentinel and are processed on next `poll()` call. - loop.callbacks.addLast(SentinelCallback) - processCallbacks(loop) - - # All callbacks done, skip `processCallbacks` at start. - loop.callbacks.addFirst(SentinelCallback) - -else: - proc initAPI() = discard - proc globalInit() = discard - -proc setThreadDispatcher*(disp: PDispatcher) = - ## Set current thread's dispatcher instance to ``disp``. - if not(gDisp.isNil()): - doAssert gDisp.callbacks.len == 0 - gDisp = disp - -proc getThreadDispatcher*(): PDispatcher = - ## Returns current thread's dispatcher instance. - if gDisp.isNil(): - setThreadDispatcher(newDispatcher()) - gDisp - -proc setGlobalDispatcher*(disp: PDispatcher) {. - gcsafe, deprecated: "Use setThreadDispatcher() instead".} = - setThreadDispatcher(disp) - -proc getGlobalDispatcher*(): PDispatcher {. - gcsafe, deprecated: "Use getThreadDispatcher() instead".} = - getThreadDispatcher() - -proc setTimer*(at: Moment, cb: CallbackFunc, - udata: pointer = nil): TimerCallback = - ## Arrange for the callback ``cb`` to be called at the given absolute - ## timestamp ``at``. You can also pass ``udata`` to callback. - let loop = getThreadDispatcher() - result = TimerCallback(finishAt: at, - function: AsyncCallback(function: cb, udata: udata)) - loop.timers.push(result) - -proc clearTimer*(timer: TimerCallback) {.inline.} = - timer.function = default(AsyncCallback) - -proc addTimer*(at: Moment, cb: CallbackFunc, udata: pointer = nil) {. - inline, deprecated: "Use setTimer/clearTimer instead".} = - ## Arrange for the callback ``cb`` to be called at the given absolute - ## timestamp ``at``. You can also pass ``udata`` to callback. - discard setTimer(at, cb, udata) - -proc addTimer*(at: int64, cb: CallbackFunc, udata: pointer = nil) {. - inline, deprecated: "Use addTimer(Duration, cb, udata)".} = - discard setTimer(Moment.init(at, Millisecond), cb, udata) - -proc addTimer*(at: uint64, cb: CallbackFunc, udata: pointer = nil) {. - inline, deprecated: "Use addTimer(Duration, cb, udata)".} = - discard setTimer(Moment.init(int64(at), Millisecond), cb, udata) - -proc removeTimer*(at: Moment, cb: CallbackFunc, udata: pointer = nil) = - ## Remove timer callback ``cb`` with absolute timestamp ``at`` from waiting - ## queue. - let loop = getThreadDispatcher() - var list = cast[seq[TimerCallback]](loop.timers) - var index = -1 - for i in 0.. 0, "Number should be positive integer") - var - retFuture = newFuture[void]("chronos.stepsAsync(int)") - counter = 0 - continuation: proc(data: pointer) {.gcsafe, raises: [].} - - continuation = proc(data: pointer) {.gcsafe, raises: [].} = - if not(retFuture.finished()): - inc(counter) - if counter < number: - internalCallTick(continuation) - else: - retFuture.complete() - - if number <= 0: - retFuture.complete() - else: - internalCallTick(continuation) - - retFuture - -proc idleAsync*(): Future[void] = - ## Suspends the execution of the current asynchronous task until "idle" time. - ## - ## "idle" time its moment of time, when no network events were processed by - ## ``poll()`` call. - var retFuture = newFuture[void]("chronos.idleAsync()") - - proc continuation(data: pointer) {.gcsafe.} = - if not(retFuture.finished()): - retFuture.complete() - - proc cancellation(udata: pointer) {.gcsafe.} = - discard - - retFuture.cancelCallback = cancellation - callIdle(continuation, nil) - retFuture - -proc withTimeout*[T](fut: Future[T], timeout: Duration): Future[bool] = - ## Returns a future which will complete once ``fut`` completes or after - ## ``timeout`` milliseconds has elapsed. - ## - ## If ``fut`` completes first the returned future will hold true, - ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned - ## future will hold false. - var - retFuture = newFuture[bool]("chronos.withTimeout", - {FutureFlag.OwnCancelSchedule}) - moment: Moment - timer: TimerCallback - timeouted = false - - template completeFuture(fut: untyped): untyped = - if fut.failed() or fut.completed(): - retFuture.complete(true) - else: - retFuture.cancelAndSchedule() - - # TODO: raises annotation shouldn't be needed, but likely similar issue as - # https://github.com/nim-lang/Nim/issues/17369 - proc continuation(udata: pointer) {.gcsafe, raises: [].} = - if not(retFuture.finished()): - if timeouted: - retFuture.complete(false) - return - if not(fut.finished()): - # Timer exceeded first, we going to cancel `fut` and wait until it - # not completes. - timeouted = true - fut.cancelSoon() - else: - # Future `fut` completed/failed/cancelled first. - if not(isNil(timer)): - clearTimer(timer) - fut.completeFuture() - - # TODO: raises annotation shouldn't be needed, but likely similar issue as - # https://github.com/nim-lang/Nim/issues/17369 - proc cancellation(udata: pointer) {.gcsafe, raises: [].} = - if not(fut.finished()): - if not isNil(timer): - clearTimer(timer) - fut.cancelSoon() - else: - fut.completeFuture() - - if fut.finished(): - retFuture.complete(true) - else: - if timeout.isZero(): - retFuture.complete(false) - elif timeout.isInfinite(): - retFuture.cancelCallback = cancellation - fut.addCallback(continuation) - else: - moment = Moment.fromNow(timeout) - retFuture.cancelCallback = cancellation - timer = setTimer(moment, continuation, nil) - fut.addCallback(continuation) - - retFuture - -proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] {. - inline, deprecated: "Use withTimeout(Future[T], Duration)".} = - withTimeout(fut, timeout.milliseconds()) - -proc wait*[T](fut: Future[T], timeout = InfiniteDuration): Future[T] = - ## Returns a future which will complete once future ``fut`` completes - ## or if timeout of ``timeout`` milliseconds has been expired. - ## - ## If ``timeout`` is ``-1``, then statement ``await wait(fut)`` is - ## equal to ``await fut``. - ## - ## TODO: In case when ``fut`` got cancelled, what result Future[T] - ## should return, because it can't be cancelled too. - var - retFuture = newFuture[T]("chronos.wait()", {FutureFlag.OwnCancelSchedule}) - moment: Moment - timer: TimerCallback - timeouted = false - - template completeFuture(fut: untyped): untyped = - if fut.failed(): - retFuture.fail(fut.error) - elif fut.cancelled(): - retFuture.cancelAndSchedule() - else: - when T is void: - retFuture.complete() - else: - retFuture.complete(fut.value) - - proc continuation(udata: pointer) {.raises: [].} = - if not(retFuture.finished()): - if timeouted: - retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) - return - if not(fut.finished()): - # Timer exceeded first. - timeouted = true - fut.cancelSoon() - else: - # Future `fut` completed/failed/cancelled first. - if not(isNil(timer)): - clearTimer(timer) - fut.completeFuture() - - var cancellation: proc(udata: pointer) {.gcsafe, raises: [].} - cancellation = proc(udata: pointer) {.gcsafe, raises: [].} = - if not(fut.finished()): - if not(isNil(timer)): - clearTimer(timer) - fut.cancelSoon() - else: - fut.completeFuture() - - if fut.finished(): - fut.completeFuture() - else: - if timeout.isZero(): - retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) - elif timeout.isInfinite(): - retFuture.cancelCallback = cancellation - fut.addCallback(continuation) - else: - moment = Moment.fromNow(timeout) - retFuture.cancelCallback = cancellation - timer = setTimer(moment, continuation, nil) - fut.addCallback(continuation) - - retFuture - -proc wait*[T](fut: Future[T], timeout = -1): Future[T] {. - inline, deprecated: "Use wait(Future[T], Duration)".} = - if timeout == -1: - wait(fut, InfiniteDuration) - elif timeout == 0: - wait(fut, ZeroDuration) - else: - wait(fut, timeout.milliseconds()) - -include asyncmacro2 - -proc runForever*() = - ## Begins a never ending global dispatcher poll loop. - ## Raises different exceptions depending on the platform. - while true: - poll() - -proc waitFor*[T](fut: Future[T]): T {.raises: [CatchableError].} = - ## **Blocks** the current thread until the specified future completes. - ## There's no way to tell if poll or read raised the exception - while not(fut.finished()): - poll() - - fut.read() - -proc addTracker*[T](id: string, tracker: T) {. - deprecated: "Please use trackCounter facility instead".} = - ## Add new ``tracker`` object to current thread dispatcher with identifier - ## ``id``. - getThreadDispatcher().trackers[id] = tracker - -proc getTracker*(id: string): TrackerBase {. - deprecated: "Please use getTrackerCounter() instead".} = - ## Get ``tracker`` from current thread dispatcher using identifier ``id``. - getThreadDispatcher().trackers.getOrDefault(id, nil) - -proc trackCounter*(name: string) {.noinit.} = - ## Increase tracker counter with name ``name`` by 1. - let tracker = TrackerCounter(opened: 0'u64, closed: 0'u64) - inc(getThreadDispatcher().counters.mgetOrPut(name, tracker).opened) - -proc untrackCounter*(name: string) {.noinit.} = - ## Decrease tracker counter with name ``name`` by 1. - let tracker = TrackerCounter(opened: 0'u64, closed: 0'u64) - inc(getThreadDispatcher().counters.mgetOrPut(name, tracker).closed) - -proc getTrackerCounter*(name: string): TrackerCounter {.noinit.} = - ## Return value of counter with name ``name``. - let tracker = TrackerCounter(opened: 0'u64, closed: 0'u64) - getThreadDispatcher().counters.getOrDefault(name, tracker) - -proc isCounterLeaked*(name: string): bool {.noinit.} = - ## Returns ``true`` if leak is detected, number of `opened` not equal to - ## number of `closed` requests. - let tracker = TrackerCounter(opened: 0'u64, closed: 0'u64) - let res = getThreadDispatcher().counters.getOrDefault(name, tracker) - res.opened != res.closed - -iterator trackerCounters*( - loop: PDispatcher - ): tuple[name: string, value: TrackerCounter] = - ## Iterates over `loop` thread dispatcher tracker counter table, returns all - ## the tracker counter's names and values. - doAssert(not(isNil(loop))) - for key, value in loop.counters.pairs(): - yield (key, value) - -iterator trackerCounterKeys*(loop: PDispatcher): string = - doAssert(not(isNil(loop))) - ## Iterates over `loop` thread dispatcher tracker counter table, returns all - ## tracker names. - for key in loop.counters.keys(): - yield key - -when chronosFutureTracking: - iterator pendingFutures*(): FutureBase = - ## Iterates over the list of pending Futures (Future[T] objects which not - ## yet completed, cancelled or failed). - var slider = futureList.head - while not(isNil(slider)): - yield slider - slider = slider.next - - proc pendingFuturesCount*(): uint = - ## Returns number of pending Futures (Future[T] objects which not yet - ## completed, cancelled or failed). - futureList.count - -when defined(windows): - proc waitForSingleObject*(handle: HANDLE, - timeout: Duration): Future[WaitableResult] {. - raises: [].} = - ## Waits until the specified object is in the signaled state or the - ## time-out interval elapses. WaitForSingleObject() for asynchronous world. - let flags = WT_EXECUTEONLYONCE - - var - retFuture = newFuture[WaitableResult]("chronos.waitForSingleObject()") - waitHandle: WaitableHandle = nil - - proc continuation(udata: pointer) {.gcsafe.} = - doAssert(not(isNil(waitHandle))) - if not(retFuture.finished()): - let - ovl = cast[PtrCustomOverlapped](udata) - returnFlag = WINBOOL(ovl.data.bytesCount) - res = closeWaitable(waitHandle) - if res.isErr(): - retFuture.fail(newException(AsyncError, osErrorMsg(res.error()))) - else: - if returnFlag == TRUE: - retFuture.complete(WaitableResult.Timeout) - else: - retFuture.complete(WaitableResult.Ok) - - proc cancellation(udata: pointer) {.gcsafe.} = - doAssert(not(isNil(waitHandle))) - if not(retFuture.finished()): - discard closeWaitable(waitHandle) - - let wres = uint32(waitForSingleObject(handle, DWORD(0))) - if wres == WAIT_OBJECT_0: - retFuture.complete(WaitableResult.Ok) - return retFuture - elif wres == WAIT_ABANDONED: - retFuture.fail(newException(AsyncError, "Handle was abandoned")) - return retFuture - elif wres == WAIT_FAILED: - retFuture.fail(newException(AsyncError, osErrorMsg(osLastError()))) - return retFuture - - if timeout == ZeroDuration: - retFuture.complete(WaitableResult.Timeout) - return retFuture - - waitHandle = - block: - let res = registerWaitable(handle, flags, timeout, continuation, nil) - if res.isErr(): - retFuture.fail(newException(AsyncError, osErrorMsg(res.error()))) - return retFuture - res.get() - - retFuture.cancelCallback = cancellation - return retFuture - -# Perform global per-module initialization. -globalInit() +export asyncfutures, asyncengine, errors +export asyncmacro.async, asyncmacro.await, asyncmacro.awaitne, asyncraises diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim new file mode 100644 index 0000000..5a46f04 --- /dev/null +++ b/chronos/internal/asyncengine.nim @@ -0,0 +1,1232 @@ +# +# Chronos +# +# (c) Copyright 2015 Dominik Picheta +# (c) Copyright 2018-Present Status Research & Development GmbH +# +# Licensed under either of +# Apache License, version 2.0, (LICENSE-APACHEv2) +# MIT license (LICENSE-MIT) + +{.push raises: [].} + +from nativesockets import Port +import std/[tables, heapqueue, deques] +import stew/results +import ".."/[config, futures, osdefs, oserrno, osutils, timer] + +import ./[asyncmacro, errors] + +export Port +export deques, errors, futures, timer, results + +export + asyncmacro.async, asyncmacro.await, asyncmacro.awaitne, asyncmacro.asyncraises + +const + MaxEventsCount* = 64 + +when defined(windows): + import std/[sets, hashes] +elif defined(macosx) or defined(freebsd) or defined(netbsd) or + defined(openbsd) or defined(dragonfly) or defined(macos) or + defined(linux) or defined(android) or defined(solaris): + import ../selectors2 + export SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, + SIGBUS, SIGFPE, SIGKILL, SIGUSR1, SIGSEGV, SIGUSR2, + SIGPIPE, SIGALRM, SIGTERM, SIGPIPE + export oserrno + +type + AsyncCallback* = InternalAsyncCallback + + TimerCallback* = ref object + finishAt*: Moment + function*: AsyncCallback + + TrackerBase* = ref object of RootRef + id*: string + dump*: proc(): string {.gcsafe, raises: [].} + isLeaked*: proc(): bool {.gcsafe, raises: [].} + + TrackerCounter* = object + opened*: uint64 + closed*: uint64 + + PDispatcherBase = ref object of RootRef + timers*: HeapQueue[TimerCallback] + callbacks*: Deque[AsyncCallback] + idlers*: Deque[AsyncCallback] + ticks*: Deque[AsyncCallback] + trackers*: Table[string, TrackerBase] + counters*: Table[string, TrackerCounter] + +proc sentinelCallbackImpl(arg: pointer) {.gcsafe, noreturn.} = + raiseAssert "Sentinel callback MUST not be scheduled" + +const + SentinelCallback = AsyncCallback(function: sentinelCallbackImpl, + udata: nil) + +proc isSentinel(acb: AsyncCallback): bool = + acb == SentinelCallback + +proc `<`(a, b: TimerCallback): bool = + result = a.finishAt < b.finishAt + +func getAsyncTimestamp*(a: Duration): auto {.inline.} = + ## Return rounded up value of duration with milliseconds resolution. + ## + ## This function also take care on int32 overflow, because Linux and Windows + ## accepts signed 32bit integer as timeout. + let milsec = Millisecond.nanoseconds() + let nansec = a.nanoseconds() + var res = nansec div milsec + let mid = nansec mod milsec + when defined(windows): + res = min(int64(high(int32) - 1), res) + result = cast[DWORD](res) + result += DWORD(min(1'i32, cast[int32](mid))) + else: + res = min(int64(high(int32) - 1), res) + result = cast[int32](res) + result += min(1, cast[int32](mid)) + +template processTimersGetTimeout(loop, timeout: untyped) = + var lastFinish = curTime + while loop.timers.len > 0: + if loop.timers[0].function.function.isNil: + discard loop.timers.pop() + continue + + lastFinish = loop.timers[0].finishAt + if curTime < lastFinish: + break + + loop.callbacks.addLast(loop.timers.pop().function) + + if loop.timers.len > 0: + timeout = (lastFinish - curTime).getAsyncTimestamp() + + if timeout == 0: + if (len(loop.callbacks) == 0) and (len(loop.idlers) == 0): + when defined(windows): + timeout = INFINITE + else: + timeout = -1 + else: + if (len(loop.callbacks) != 0) or (len(loop.idlers) != 0): + timeout = 0 + +template processTimers(loop: untyped) = + var curTime = Moment.now() + while loop.timers.len > 0: + if loop.timers[0].function.function.isNil: + discard loop.timers.pop() + continue + + if curTime < loop.timers[0].finishAt: + break + loop.callbacks.addLast(loop.timers.pop().function) + +template processIdlers(loop: untyped) = + if len(loop.idlers) > 0: + loop.callbacks.addLast(loop.idlers.popFirst()) + +template processTicks(loop: untyped) = + while len(loop.ticks) > 0: + loop.callbacks.addLast(loop.ticks.popFirst()) + +template processCallbacks(loop: untyped) = + while true: + let callable = loop.callbacks.popFirst() # len must be > 0 due to sentinel + if isSentinel(callable): + break + if not(isNil(callable.function)): + callable.function(callable.udata) + +proc raiseAsDefect*(exc: ref Exception, msg: string) {.noreturn, noinline.} = + # Reraise an exception as a Defect, where it's unexpected and can't be handled + # We include the stack trace in the message because otherwise, it's easily + # lost - Nim doesn't print it for `parent` exceptions for example (!) + raise (ref Defect)( + msg: msg & "\n" & exc.msg & "\n" & exc.getStackTrace(), parent: exc) + +proc raiseOsDefect*(error: OSErrorCode, msg = "") {.noreturn, noinline.} = + # Reraise OS error code as a Defect, where it's unexpected and can't be + # handled. We include the stack trace in the message because otherwise, + # it's easily lost. + raise (ref Defect)(msg: msg & "\n[" & $int(error) & "] " & osErrorMsg(error) & + "\n" & getStackTrace()) + +func toPointer(error: OSErrorCode): pointer = + when sizeof(int) == 8: + cast[pointer](uint64(uint32(error))) + else: + cast[pointer](uint32(error)) + +func toException*(v: OSErrorCode): ref OSError = newOSError(v) + # This helper will allow to use `tryGet()` and raise OSError for + # Result[T, OSErrorCode] values. + +when defined(windows): + {.pragma: stdcallbackFunc, stdcall, gcsafe, raises: [].} + + export SIGINT, SIGQUIT, SIGTERM + type + CompletionKey = ULONG_PTR + + CompletionData* = object + cb*: CallbackFunc + errCode*: OSErrorCode + bytesCount*: uint32 + udata*: pointer + + CustomOverlapped* = object of OVERLAPPED + data*: CompletionData + + DispatcherFlag* = enum + SignalHandlerInstalled + + PDispatcher* = ref object of PDispatcherBase + ioPort: HANDLE + handles: HashSet[AsyncFD] + connectEx*: WSAPROC_CONNECTEX + acceptEx*: WSAPROC_ACCEPTEX + getAcceptExSockAddrs*: WSAPROC_GETACCEPTEXSOCKADDRS + transmitFile*: WSAPROC_TRANSMITFILE + getQueuedCompletionStatusEx*: LPFN_GETQUEUEDCOMPLETIONSTATUSEX + disconnectEx*: WSAPROC_DISCONNECTEX + flags: set[DispatcherFlag] + + PtrCustomOverlapped* = ptr CustomOverlapped + + RefCustomOverlapped* = ref CustomOverlapped + + PostCallbackData = object + ioPort: HANDLE + handleFd: AsyncFD + waitFd: HANDLE + udata: pointer + ovlref: RefCustomOverlapped + ovl: pointer + + WaitableHandle* = ref PostCallbackData + ProcessHandle* = distinct WaitableHandle + SignalHandle* = distinct WaitableHandle + + WaitableResult* {.pure.} = enum + Ok, Timeout + + AsyncFD* = distinct int + + proc hash(x: AsyncFD): Hash {.borrow.} + proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow, gcsafe.} + + proc getFunc(s: SocketHandle, fun: var pointer, guid: GUID): bool = + var bytesRet: DWORD + fun = nil + wsaIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, unsafeAddr(guid), + DWORD(sizeof(GUID)), addr fun, DWORD(sizeof(pointer)), + addr(bytesRet), nil, nil) == 0 + + proc globalInit() = + var wsa = WSAData() + let res = wsaStartup(0x0202'u16, addr wsa) + if res != 0: + raiseOsDefect(osLastError(), + "globalInit(): Unable to initialize Windows Sockets API") + + proc initAPI(loop: PDispatcher) = + var funcPointer: pointer = nil + + let kernel32 = getModuleHandle(newWideCString("kernel32.dll")) + loop.getQueuedCompletionStatusEx = cast[LPFN_GETQUEUEDCOMPLETIONSTATUSEX]( + getProcAddress(kernel32, "GetQueuedCompletionStatusEx")) + + let sock = osdefs.socket(osdefs.AF_INET, 1, 6) + if sock == osdefs.INVALID_SOCKET: + raiseOsDefect(osLastError(), "initAPI(): Unable to create control socket") + + block: + let res = getFunc(sock, funcPointer, WSAID_CONNECTEX) + if not(res): + raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & + "dispatcher's ConnectEx()") + loop.connectEx = cast[WSAPROC_CONNECTEX](funcPointer) + + block: + let res = getFunc(sock, funcPointer, WSAID_ACCEPTEX) + if not(res): + raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & + "dispatcher's AcceptEx()") + loop.acceptEx = cast[WSAPROC_ACCEPTEX](funcPointer) + + block: + let res = getFunc(sock, funcPointer, WSAID_GETACCEPTEXSOCKADDRS) + if not(res): + raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & + "dispatcher's GetAcceptExSockAddrs()") + loop.getAcceptExSockAddrs = + cast[WSAPROC_GETACCEPTEXSOCKADDRS](funcPointer) + + block: + let res = getFunc(sock, funcPointer, WSAID_TRANSMITFILE) + if not(res): + raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & + "dispatcher's TransmitFile()") + loop.transmitFile = cast[WSAPROC_TRANSMITFILE](funcPointer) + + block: + let res = getFunc(sock, funcPointer, WSAID_DISCONNECTEX) + if not(res): + raiseOsDefect(osLastError(), "initAPI(): Unable to initialize " & + "dispatcher's DisconnectEx()") + loop.disconnectEx = cast[WSAPROC_DISCONNECTEX](funcPointer) + + if closeFd(sock) != 0: + raiseOsDefect(osLastError(), "initAPI(): Unable to close control socket") + + proc newDispatcher*(): PDispatcher = + ## Creates a new Dispatcher instance. + let port = createIoCompletionPort(osdefs.INVALID_HANDLE_VALUE, + HANDLE(0), 0, 1) + if port == osdefs.INVALID_HANDLE_VALUE: + raiseOsDefect(osLastError(), "newDispatcher(): Unable to create " & + "IOCP port") + var res = PDispatcher( + ioPort: port, + handles: initHashSet[AsyncFD](), + timers: initHeapQueue[TimerCallback](), + callbacks: initDeque[AsyncCallback](64), + idlers: initDeque[AsyncCallback](), + ticks: initDeque[AsyncCallback](), + trackers: initTable[string, TrackerBase](), + counters: initTable[string, TrackerCounter]() + ) + res.callbacks.addLast(SentinelCallback) + initAPI(res) + res + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + + proc setThreadDispatcher*(disp: PDispatcher) {.gcsafe, raises: [].} + proc getThreadDispatcher*(): PDispatcher {.gcsafe, raises: [].} + + proc getIoHandler*(disp: PDispatcher): HANDLE = + ## Returns the underlying IO Completion Port handle (Windows) or selector + ## (Unix) for the specified dispatcher. + disp.ioPort + + proc register2*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Register file descriptor ``fd`` in thread's dispatcher. + let loop = getThreadDispatcher() + if createIoCompletionPort(HANDLE(fd), loop.ioPort, cast[CompletionKey](fd), + 1) == osdefs.INVALID_HANDLE_VALUE: + return err(osLastError()) + loop.handles.incl(fd) + ok() + + proc register*(fd: AsyncFD) {.raises: [OSError].} = + ## Register file descriptor ``fd`` in thread's dispatcher. + register2(fd).tryGet() + + proc unregister*(fd: AsyncFD) = + ## Unregisters ``fd``. + getThreadDispatcher().handles.excl(fd) + + {.push stackTrace: off.} + proc waitableCallback(param: pointer, timerOrWaitFired: WINBOOL) {. + stdcallbackFunc.} = + # This procedure will be executed in `wait thread`, so it must not use + # GC related objects. + # We going to ignore callbacks which was spawned when `isNil(param) == true` + # because we unable to indicate this error. + if isNil(param): return + var wh = cast[ptr PostCallbackData](param) + # We ignore result of postQueueCompletionStatus() call because we unable to + # indicate error. + discard postQueuedCompletionStatus(wh[].ioPort, DWORD(timerOrWaitFired), + ULONG_PTR(wh[].handleFd), + wh[].ovl) + {.pop.} + + proc registerWaitable*( + handle: HANDLE, + flags: ULONG, + timeout: Duration, + cb: CallbackFunc, + udata: pointer + ): Result[WaitableHandle, OSErrorCode] = + ## Register handle of (Change notification, Console input, Event, + ## Memory resource notification, Mutex, Process, Semaphore, Thread, + ## Waitable timer) for waiting, using specific Windows' ``flags`` and + ## ``timeout`` value. + ## + ## Callback ``cb`` will be scheduled with ``udata`` parameter when + ## ``handle`` become signaled. + ## + ## Result of this procedure call ``WaitableHandle`` should be closed using + ## closeWaitable() call. + ## + ## NOTE: This is private procedure, not supposed to be publicly available, + ## please use ``waitForSingleObject()``. + let loop = getThreadDispatcher() + var ovl = RefCustomOverlapped(data: CompletionData(cb: cb)) + + var whandle = (ref PostCallbackData)( + ioPort: loop.getIoHandler(), + handleFd: AsyncFD(handle), + udata: udata, + ovlref: ovl, + ovl: cast[pointer](ovl) + ) + + ovl.data.udata = cast[pointer](whandle) + + let dwordTimeout = + if timeout == InfiniteDuration: + DWORD(INFINITE) + else: + DWORD(timeout.milliseconds) + + if registerWaitForSingleObject(addr(whandle[].waitFd), handle, + cast[WAITORTIMERCALLBACK](waitableCallback), + cast[pointer](whandle), + dwordTimeout, + flags) == WINBOOL(0): + ovl.data.udata = nil + whandle.ovlref = nil + whandle.ovl = nil + return err(osLastError()) + + ok(WaitableHandle(whandle)) + + proc closeWaitable*(wh: WaitableHandle): Result[void, OSErrorCode] = + ## Close waitable handle ``wh`` and clear all the resources. It is safe + ## to close this handle, even if wait operation is pending. + ## + ## NOTE: This is private procedure, not supposed to be publicly available, + ## please use ``waitForSingleObject()``. + doAssert(not(isNil(wh))) + + let pdata = (ref PostCallbackData)(wh) + # We are not going to clear `ref` fields in PostCallbackData object because + # it possible that callback is already scheduled. + if unregisterWait(pdata.waitFd) == 0: + let res = osLastError() + if res != ERROR_IO_PENDING: + return err(res) + ok() + + proc addProcess2*(pid: int, cb: CallbackFunc, + udata: pointer = nil): Result[ProcessHandle, OSErrorCode] = + ## Registers callback ``cb`` to be called when process with process + ## identifier ``pid`` exited. Returns process identifier, which can be + ## used to clear process callback via ``removeProcess``. + doAssert(pid > 0, "Process identifier must be positive integer") + let + hProcess = openProcess(SYNCHRONIZE, WINBOOL(0), DWORD(pid)) + flags = WT_EXECUTEINWAITTHREAD or WT_EXECUTEONLYONCE + + var wh: WaitableHandle = nil + + if hProcess == HANDLE(0): + return err(osLastError()) + + proc continuation(udata: pointer) {.gcsafe.} = + doAssert(not(isNil(udata))) + doAssert(not(isNil(wh))) + discard closeFd(hProcess) + cb(wh[].udata) + + wh = + block: + let res = registerWaitable(hProcess, flags, InfiniteDuration, + continuation, udata) + if res.isErr(): + discard closeFd(hProcess) + return err(res.error()) + res.get() + ok(ProcessHandle(wh)) + + proc removeProcess2*(procHandle: ProcessHandle): Result[void, OSErrorCode] = + ## Remove process' watching using process' descriptor ``procHandle``. + let waitableHandle = WaitableHandle(procHandle) + doAssert(not(isNil(waitableHandle))) + ? closeWaitable(waitableHandle) + ok() + + proc addProcess*(pid: int, cb: CallbackFunc, + udata: pointer = nil): ProcessHandle {. + raises: [OSError].} = + ## Registers callback ``cb`` to be called when process with process + ## identifier ``pid`` exited. Returns process identifier, which can be + ## used to clear process callback via ``removeProcess``. + addProcess2(pid, cb, udata).tryGet() + + proc removeProcess*(procHandle: ProcessHandle) {. + raises: [ OSError].} = + ## Remove process' watching using process' descriptor ``procHandle``. + removeProcess2(procHandle).tryGet() + + {.push stackTrace: off.} + proc consoleCtrlEventHandler(dwCtrlType: DWORD): uint32 {.stdcallbackFunc.} = + ## This procedure will be executed in different thread, so it MUST not use + ## any GC related features (strings, seqs, echo etc.). + case dwCtrlType + of CTRL_C_EVENT: + return + (if raiseSignal(SIGINT).valueOr(false): TRUE else: FALSE) + of CTRL_BREAK_EVENT: + return + (if raiseSignal(SIGINT).valueOr(false): TRUE else: FALSE) + of CTRL_CLOSE_EVENT: + return + (if raiseSignal(SIGTERM).valueOr(false): TRUE else: FALSE) + of CTRL_LOGOFF_EVENT: + return + (if raiseSignal(SIGQUIT).valueOr(false): TRUE else: FALSE) + else: + FALSE + {.pop.} + + proc addSignal2*(signal: int, cb: CallbackFunc, + udata: pointer = nil): Result[SignalHandle, OSErrorCode] = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb`` with specified argument ``udata``. Returns signal + ## identifier code, which can be used to remove signal callback + ## via ``removeSignal``. + ## + ## NOTE: On Windows only subset of signals are supported: SIGINT, SIGTERM, + ## SIGQUIT + const supportedSignals = [SIGINT, SIGTERM, SIGQUIT] + doAssert(cint(signal) in supportedSignals, "Signal is not supported") + let loop = getThreadDispatcher() + var hWait: WaitableHandle = nil + + proc continuation(ucdata: pointer) {.gcsafe.} = + doAssert(not(isNil(ucdata))) + doAssert(not(isNil(hWait))) + cb(hWait[].udata) + + if SignalHandlerInstalled notin loop.flags: + if getConsoleCP() != 0'u32: + # Console application, we going to cleanup Nim default signal handlers. + if setConsoleCtrlHandler(consoleCtrlEventHandler, TRUE) == FALSE: + return err(osLastError()) + loop.flags.incl(SignalHandlerInstalled) + else: + return err(ERROR_NOT_SUPPORTED) + + let + flags = WT_EXECUTEINWAITTHREAD + hEvent = ? openEvent($getSignalName(signal)) + + hWait = registerWaitable(hEvent, flags, InfiniteDuration, + continuation, udata).valueOr: + discard closeFd(hEvent) + return err(error) + ok(SignalHandle(hWait)) + + proc removeSignal2*(signalHandle: SignalHandle): Result[void, OSErrorCode] = + ## Remove watching signal ``signal``. + ? closeWaitable(WaitableHandle(signalHandle)) + ok() + + proc addSignal*(signal: int, cb: CallbackFunc, + udata: pointer = nil): SignalHandle {. + raises: [ValueError].} = + ## Registers callback ``cb`` to be called when signal ``signal`` will be + ## raised. Returns signal identifier, which can be used to clear signal + ## callback via ``removeSignal``. + addSignal2(signal, cb, udata).valueOr: + raise newException(ValueError, osErrorMsg(error)) + + proc removeSignal*(signalHandle: SignalHandle) {. + raises: [ValueError].} = + ## Remove signal's watching using signal descriptor ``signalfd``. + let res = removeSignal2(signalHandle) + if res.isErr(): + raise newException(ValueError, osErrorMsg(res.error())) + + proc poll*() = + ## Perform single asynchronous step, processing timers and completing + ## tasks. Blocks until at least one event has completed. + ## + ## Exceptions raised here indicate that waiting for tasks to be unblocked + ## failed - exceptions from within tasks are instead propagated through + ## their respective futures and not allowed to interrrupt the poll call. + let loop = getThreadDispatcher() + var + curTime = Moment.now() + curTimeout = DWORD(0) + events: array[MaxEventsCount, osdefs.OVERLAPPED_ENTRY] + + # On reentrant `poll` calls from `processCallbacks`, e.g., `waitFor`, + # complete pending work of the outer `processCallbacks` call. + # On non-reentrant `poll` calls, this only removes sentinel element. + processCallbacks(loop) + + # Moving expired timers to `loop.callbacks` and calculate timeout + loop.processTimersGetTimeout(curTimeout) + + let networkEventsCount = + if isNil(loop.getQueuedCompletionStatusEx): + let res = getQueuedCompletionStatus( + loop.ioPort, + addr events[0].dwNumberOfBytesTransferred, + addr events[0].lpCompletionKey, + cast[ptr POVERLAPPED](addr events[0].lpOverlapped), + curTimeout + ) + if res == FALSE: + let errCode = osLastError() + if not(isNil(events[0].lpOverlapped)): + 1 + else: + if uint32(errCode) != WAIT_TIMEOUT: + raiseOsDefect(errCode, "poll(): Unable to get OS events") + 0 + else: + 1 + else: + var eventsReceived = ULONG(0) + let res = loop.getQueuedCompletionStatusEx( + loop.ioPort, + addr events[0], + ULONG(len(events)), + eventsReceived, + curTimeout, + WINBOOL(0) + ) + if res == FALSE: + let errCode = osLastError() + if uint32(errCode) != WAIT_TIMEOUT: + raiseOsDefect(errCode, "poll(): Unable to get OS events") + 0 + else: + int(eventsReceived) + + for i in 0 ..< networkEventsCount: + var customOverlapped = PtrCustomOverlapped(events[i].lpOverlapped) + customOverlapped.data.errCode = + block: + let res = cast[uint64](customOverlapped.internal) + if res == 0'u64: + OSErrorCode(-1) + else: + OSErrorCode(rtlNtStatusToDosError(res)) + customOverlapped.data.bytesCount = events[i].dwNumberOfBytesTransferred + let acb = AsyncCallback(function: customOverlapped.data.cb, + udata: cast[pointer](customOverlapped)) + loop.callbacks.addLast(acb) + + # Moving expired timers to `loop.callbacks`. + loop.processTimers() + + # We move idle callbacks to `loop.callbacks` only if there no pending + # network events. + if networkEventsCount == 0: + loop.processIdlers() + + # We move tick callbacks to `loop.callbacks` always. + processTicks(loop) + + # All callbacks which will be added during `processCallbacks` will be + # scheduled after the sentinel and are processed on next `poll()` call. + loop.callbacks.addLast(SentinelCallback) + processCallbacks(loop) + + # All callbacks done, skip `processCallbacks` at start. + loop.callbacks.addFirst(SentinelCallback) + + proc closeSocket*(fd: AsyncFD, aftercb: CallbackFunc = nil) = + ## Closes a socket and ensures that it is unregistered. + let loop = getThreadDispatcher() + loop.handles.excl(fd) + let + param = toPointer( + if closeFd(SocketHandle(fd)) == 0: + OSErrorCode(0) + else: + osLastError() + ) + if not(isNil(aftercb)): + loop.callbacks.addLast(AsyncCallback(function: aftercb, udata: param)) + + proc closeHandle*(fd: AsyncFD, aftercb: CallbackFunc = nil) = + ## Closes a (pipe/file) handle and ensures that it is unregistered. + let loop = getThreadDispatcher() + loop.handles.excl(fd) + let + param = toPointer( + if closeFd(HANDLE(fd)) == 0: + OSErrorCode(0) + else: + osLastError() + ) + + if not(isNil(aftercb)): + loop.callbacks.addLast(AsyncCallback(function: aftercb, udata: param)) + + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = + ## Returns ``true`` if ``fd`` is registered in thread's dispatcher. + fd in disp.handles + +elif defined(macosx) or defined(freebsd) or defined(netbsd) or + defined(openbsd) or defined(dragonfly) or defined(macos) or + defined(linux) or defined(android) or defined(solaris): + const + SIG_IGN = cast[proc(x: cint) {.raises: [], noconv, gcsafe.}](1) + + type + AsyncFD* = distinct cint + + SelectorData* = object + reader*: AsyncCallback + writer*: AsyncCallback + + PDispatcher* = ref object of PDispatcherBase + selector: Selector[SelectorData] + keys: seq[ReadyKey] + + proc `==`*(x, y: AsyncFD): bool {.borrow, gcsafe.} + + proc globalInit() = + # We are ignoring SIGPIPE signal, because we are working with EPIPE. + signal(cint(SIGPIPE), SIG_IGN) + + proc initAPI(disp: PDispatcher) = + discard + + proc newDispatcher*(): PDispatcher = + ## Create new dispatcher. + let selector = + block: + let res = Selector.new(SelectorData) + if res.isErr(): raiseOsDefect(res.error(), + "Could not initialize selector") + res.get() + + var res = PDispatcher( + selector: selector, + timers: initHeapQueue[TimerCallback](), + callbacks: initDeque[AsyncCallback](chronosEventsCount), + idlers: initDeque[AsyncCallback](), + keys: newSeq[ReadyKey](chronosEventsCount), + trackers: initTable[string, TrackerBase](), + counters: initTable[string, TrackerCounter]() + ) + res.callbacks.addLast(SentinelCallback) + initAPI(res) + res + + var gDisp{.threadvar.}: PDispatcher ## Global dispatcher + + proc setThreadDispatcher*(disp: PDispatcher) {.gcsafe, raises: [].} + proc getThreadDispatcher*(): PDispatcher {.gcsafe, raises: [].} + + proc getIoHandler*(disp: PDispatcher): Selector[SelectorData] = + ## Returns system specific OS queue. + disp.selector + + proc contains*(disp: PDispatcher, fd: AsyncFD): bool {.inline.} = + ## Returns ``true`` if ``fd`` is registered in thread's dispatcher. + cint(fd) in disp.selector + + proc register2*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Register file descriptor ``fd`` in thread's dispatcher. + var data: SelectorData + getThreadDispatcher().selector.registerHandle2(cint(fd), {}, data) + + proc unregister2*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Unregister file descriptor ``fd`` from thread's dispatcher. + getThreadDispatcher().selector.unregister2(cint(fd)) + + proc addReader2*(fd: AsyncFD, cb: CallbackFunc, + udata: pointer = nil): Result[void, OSErrorCode] = + ## Start watching the file descriptor ``fd`` for read availability and then + ## call the callback ``cb`` with specified argument ``udata``. + let loop = getThreadDispatcher() + var newEvents = {Event.Read} + withData(loop.selector, cint(fd), adata) do: + let acb = AsyncCallback(function: cb, udata: udata) + adata.reader = acb + if not(isNil(adata.writer.function)): + newEvents.incl(Event.Write) + do: + return err(osdefs.EBADF) + loop.selector.updateHandle2(cint(fd), newEvents) + + proc removeReader2*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Stop watching the file descriptor ``fd`` for read availability. + let loop = getThreadDispatcher() + var newEvents: set[Event] + withData(loop.selector, cint(fd), adata) do: + # We need to clear `reader` data, because `selectors` don't do it + adata.reader = default(AsyncCallback) + if not(isNil(adata.writer.function)): + newEvents.incl(Event.Write) + do: + return err(osdefs.EBADF) + loop.selector.updateHandle2(cint(fd), newEvents) + + proc addWriter2*(fd: AsyncFD, cb: CallbackFunc, + udata: pointer = nil): Result[void, OSErrorCode] = + ## Start watching the file descriptor ``fd`` for write availability and then + ## call the callback ``cb`` with specified argument ``udata``. + let loop = getThreadDispatcher() + var newEvents = {Event.Write} + withData(loop.selector, cint(fd), adata) do: + let acb = AsyncCallback(function: cb, udata: udata) + adata.writer = acb + if not(isNil(adata.reader.function)): + newEvents.incl(Event.Read) + do: + return err(osdefs.EBADF) + loop.selector.updateHandle2(cint(fd), newEvents) + + proc removeWriter2*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Stop watching the file descriptor ``fd`` for write availability. + let loop = getThreadDispatcher() + var newEvents: set[Event] + withData(loop.selector, cint(fd), adata) do: + # We need to clear `writer` data, because `selectors` don't do it + adata.writer = default(AsyncCallback) + if not(isNil(adata.reader.function)): + newEvents.incl(Event.Read) + do: + return err(osdefs.EBADF) + loop.selector.updateHandle2(cint(fd), newEvents) + + proc register*(fd: AsyncFD) {.raises: [OSError].} = + ## Register file descriptor ``fd`` in thread's dispatcher. + register2(fd).tryGet() + + proc unregister*(fd: AsyncFD) {.raises: [OSError].} = + ## Unregister file descriptor ``fd`` from thread's dispatcher. + unregister2(fd).tryGet() + + proc addReader*(fd: AsyncFD, cb: CallbackFunc, udata: pointer = nil) {. + raises: [OSError].} = + ## Start watching the file descriptor ``fd`` for read availability and then + ## call the callback ``cb`` with specified argument ``udata``. + addReader2(fd, cb, udata).tryGet() + + proc removeReader*(fd: AsyncFD) {.raises: [OSError].} = + ## Stop watching the file descriptor ``fd`` for read availability. + removeReader2(fd).tryGet() + + proc addWriter*(fd: AsyncFD, cb: CallbackFunc, udata: pointer = nil) {. + raises: [OSError].} = + ## Start watching the file descriptor ``fd`` for write availability and then + ## call the callback ``cb`` with specified argument ``udata``. + addWriter2(fd, cb, udata).tryGet() + + proc removeWriter*(fd: AsyncFD) {.raises: [OSError].} = + ## Stop watching the file descriptor ``fd`` for write availability. + removeWriter2(fd).tryGet() + + proc unregisterAndCloseFd*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Unregister from system queue and close asynchronous socket. + ## + ## NOTE: Use this function to close temporary sockets/pipes only (which + ## are not exposed to the public and not supposed to be used/reused). + ## Please use closeSocket(AsyncFD) and closeHandle(AsyncFD) instead. + doAssert(fd != AsyncFD(osdefs.INVALID_SOCKET)) + ? unregister2(fd) + if closeFd(cint(fd)) != 0: + err(osLastError()) + else: + ok() + + proc closeSocket*(fd: AsyncFD, aftercb: CallbackFunc = nil) = + ## Close asynchronous socket. + ## + ## Please note, that socket is not closed immediately. To avoid bugs with + ## closing socket, while operation pending, socket will be closed as + ## soon as all pending operations will be notified. + let loop = getThreadDispatcher() + + proc continuation(udata: pointer) = + let + param = toPointer( + if SocketHandle(fd) in loop.selector: + let ures = unregister2(fd) + if ures.isErr(): + discard closeFd(cint(fd)) + ures.error() + else: + if closeFd(cint(fd)) != 0: + osLastError() + else: + OSErrorCode(0) + else: + osdefs.EBADF + ) + if not(isNil(aftercb)): aftercb(param) + + withData(loop.selector, cint(fd), adata) do: + # We are scheduling reader and writer callbacks to be called + # explicitly, so they can get an error and continue work. + # Callbacks marked as deleted so we don't need to get REAL notifications + # from system queue for this reader and writer. + + if not(isNil(adata.reader.function)): + loop.callbacks.addLast(adata.reader) + adata.reader = default(AsyncCallback) + + if not(isNil(adata.writer.function)): + loop.callbacks.addLast(adata.writer) + adata.writer = default(AsyncCallback) + + # We can't unregister file descriptor from system queue here, because + # in such case processing queue will stuck on poll() call, because there + # can be no file descriptors registered in system queue. + var acb = AsyncCallback(function: continuation) + loop.callbacks.addLast(acb) + + proc closeHandle*(fd: AsyncFD, aftercb: CallbackFunc = nil) = + ## Close asynchronous file/pipe handle. + ## + ## Please note, that socket is not closed immediately. To avoid bugs with + ## closing socket, while operation pending, socket will be closed as + ## soon as all pending operations will be notified. + ## You can execute ``aftercb`` before actual socket close operation. + closeSocket(fd, aftercb) + + when chronosEventEngine in ["epoll", "kqueue"]: + type + ProcessHandle* = distinct int + SignalHandle* = distinct int + + proc addSignal2*( + signal: int, + cb: CallbackFunc, + udata: pointer = nil + ): Result[SignalHandle, OSErrorCode] = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb`` with specified argument ``udata``. Returns signal + ## identifier code, which can be used to remove signal callback + ## via ``removeSignal``. + let loop = getThreadDispatcher() + var data: SelectorData + let sigfd = ? loop.selector.registerSignal(signal, data) + withData(loop.selector, sigfd, adata) do: + adata.reader = AsyncCallback(function: cb, udata: udata) + do: + return err(osdefs.EBADF) + ok(SignalHandle(sigfd)) + + proc addProcess2*( + pid: int, + cb: CallbackFunc, + udata: pointer = nil + ): Result[ProcessHandle, OSErrorCode] = + ## Registers callback ``cb`` to be called when process with process + ## identifier ``pid`` exited. Returns process' descriptor, which can be + ## used to clear process callback via ``removeProcess``. + let loop = getThreadDispatcher() + var data: SelectorData + let procfd = ? loop.selector.registerProcess(pid, data) + withData(loop.selector, procfd, adata) do: + adata.reader = AsyncCallback(function: cb, udata: udata) + do: + return err(osdefs.EBADF) + ok(ProcessHandle(procfd)) + + proc removeSignal2*(signalHandle: SignalHandle): Result[void, OSErrorCode] = + ## Remove watching signal ``signal``. + getThreadDispatcher().selector.unregister2(cint(signalHandle)) + + proc removeProcess2*(procHandle: ProcessHandle): Result[void, OSErrorCode] = + ## Remove process' watching using process' descriptor ``procfd``. + getThreadDispatcher().selector.unregister2(cint(procHandle)) + + proc addSignal*(signal: int, cb: CallbackFunc, + udata: pointer = nil): SignalHandle {. + raises: [OSError].} = + ## Start watching signal ``signal``, and when signal appears, call the + ## callback ``cb`` with specified argument ``udata``. Returns signal + ## identifier code, which can be used to remove signal callback + ## via ``removeSignal``. + addSignal2(signal, cb, udata).tryGet() + + proc removeSignal*(signalHandle: SignalHandle) {. + raises: [OSError].} = + ## Remove watching signal ``signal``. + removeSignal2(signalHandle).tryGet() + + proc addProcess*(pid: int, cb: CallbackFunc, + udata: pointer = nil): ProcessHandle {. + raises: [OSError].} = + ## Registers callback ``cb`` to be called when process with process + ## identifier ``pid`` exited. Returns process identifier, which can be + ## used to clear process callback via ``removeProcess``. + addProcess2(pid, cb, udata).tryGet() + + proc removeProcess*(procHandle: ProcessHandle) {. + raises: [OSError].} = + ## Remove process' watching using process' descriptor ``procHandle``. + removeProcess2(procHandle).tryGet() + + proc poll*() {.gcsafe.} = + ## Perform single asynchronous step. + let loop = getThreadDispatcher() + var curTime = Moment.now() + var curTimeout = 0 + + # On reentrant `poll` calls from `processCallbacks`, e.g., `waitFor`, + # complete pending work of the outer `processCallbacks` call. + # On non-reentrant `poll` calls, this only removes sentinel element. + processCallbacks(loop) + + # Moving expired timers to `loop.callbacks` and calculate timeout. + loop.processTimersGetTimeout(curTimeout) + + # Processing IO descriptors and all hardware events. + let count = + block: + let res = loop.selector.selectInto2(curTimeout, loop.keys) + if res.isErr(): + raiseOsDefect(res.error(), "poll(): Unable to get OS events") + res.get() + + for i in 0 ..< count: + let fd = loop.keys[i].fd + let events = loop.keys[i].events + + withData(loop.selector, cint(fd), adata) do: + if (Event.Read in events) or (events == {Event.Error}): + if not isNil(adata.reader.function): + loop.callbacks.addLast(adata.reader) + + if (Event.Write in events) or (events == {Event.Error}): + if not isNil(adata.writer.function): + loop.callbacks.addLast(adata.writer) + + if Event.User in events: + if not isNil(adata.reader.function): + loop.callbacks.addLast(adata.reader) + + when chronosEventEngine in ["epoll", "kqueue"]: + let customSet = {Event.Timer, Event.Signal, Event.Process, + Event.Vnode} + if customSet * events != {}: + if not isNil(adata.reader.function): + loop.callbacks.addLast(adata.reader) + + # Moving expired timers to `loop.callbacks`. + loop.processTimers() + + # We move idle callbacks to `loop.callbacks` only if there no pending + # network events. + if count == 0: + loop.processIdlers() + + # We move tick callbacks to `loop.callbacks` always. + processTicks(loop) + + # All callbacks which will be added during `processCallbacks` will be + # scheduled after the sentinel and are processed on next `poll()` call. + loop.callbacks.addLast(SentinelCallback) + processCallbacks(loop) + + # All callbacks done, skip `processCallbacks` at start. + loop.callbacks.addFirst(SentinelCallback) + +else: + proc initAPI() = discard + proc globalInit() = discard + +proc setThreadDispatcher*(disp: PDispatcher) = + ## Set current thread's dispatcher instance to ``disp``. + if not(gDisp.isNil()): + doAssert gDisp.callbacks.len == 0 + gDisp = disp + +proc getThreadDispatcher*(): PDispatcher = + ## Returns current thread's dispatcher instance. + if gDisp.isNil(): + setThreadDispatcher(newDispatcher()) + gDisp + +proc setGlobalDispatcher*(disp: PDispatcher) {. + gcsafe, deprecated: "Use setThreadDispatcher() instead".} = + setThreadDispatcher(disp) + +proc getGlobalDispatcher*(): PDispatcher {. + gcsafe, deprecated: "Use getThreadDispatcher() instead".} = + getThreadDispatcher() + +proc setTimer*(at: Moment, cb: CallbackFunc, + udata: pointer = nil): TimerCallback = + ## Arrange for the callback ``cb`` to be called at the given absolute + ## timestamp ``at``. You can also pass ``udata`` to callback. + let loop = getThreadDispatcher() + result = TimerCallback(finishAt: at, + function: AsyncCallback(function: cb, udata: udata)) + loop.timers.push(result) + +proc clearTimer*(timer: TimerCallback) {.inline.} = + timer.function = default(AsyncCallback) + +proc addTimer*(at: Moment, cb: CallbackFunc, udata: pointer = nil) {. + inline, deprecated: "Use setTimer/clearTimer instead".} = + ## Arrange for the callback ``cb`` to be called at the given absolute + ## timestamp ``at``. You can also pass ``udata`` to callback. + discard setTimer(at, cb, udata) + +proc addTimer*(at: int64, cb: CallbackFunc, udata: pointer = nil) {. + inline, deprecated: "Use addTimer(Duration, cb, udata)".} = + discard setTimer(Moment.init(at, Millisecond), cb, udata) + +proc addTimer*(at: uint64, cb: CallbackFunc, udata: pointer = nil) {. + inline, deprecated: "Use addTimer(Duration, cb, udata)".} = + discard setTimer(Moment.init(int64(at), Millisecond), cb, udata) + +proc removeTimer*(at: Moment, cb: CallbackFunc, udata: pointer = nil) = + ## Remove timer callback ``cb`` with absolute timestamp ``at`` from waiting + ## queue. + let loop = getThreadDispatcher() + var list = cast[seq[TimerCallback]](loop.timers) + var index = -1 + for i in 0.. 0, "Number should be positive integer") + var + retFuture = newFuture[void]("chronos.stepsAsync(int)") + counter = 0 + continuation: proc(data: pointer) {.gcsafe, raises: [].} + + continuation = proc(data: pointer) {.gcsafe, raises: [].} = + if not(retFuture.finished()): + inc(counter) + if counter < number: + internalCallTick(continuation) + else: + retFuture.complete() + + if number <= 0: + retFuture.complete() + else: + internalCallTick(continuation) + + retFuture + +proc idleAsync*(): Future[void] = + ## Suspends the execution of the current asynchronous task until "idle" time. + ## + ## "idle" time its moment of time, when no network events were processed by + ## ``poll()`` call. + var retFuture = newFuture[void]("chronos.idleAsync()") + + proc continuation(data: pointer) {.gcsafe.} = + if not(retFuture.finished()): + retFuture.complete() + + proc cancellation(udata: pointer) {.gcsafe.} = + discard + + retFuture.cancelCallback = cancellation + callIdle(continuation, nil) + retFuture + +proc withTimeout*[T](fut: Future[T], timeout: Duration): Future[bool] = + ## Returns a future which will complete once ``fut`` completes or after + ## ``timeout`` milliseconds has elapsed. + ## + ## If ``fut`` completes first the returned future will hold true, + ## otherwise, if ``timeout`` milliseconds has elapsed first, the returned + ## future will hold false. + var + retFuture = newFuture[bool]("chronos.withTimeout", + {FutureFlag.OwnCancelSchedule}) + moment: Moment + timer: TimerCallback + timeouted = false + + template completeFuture(fut: untyped): untyped = + if fut.failed() or fut.completed(): + retFuture.complete(true) + else: + retFuture.cancelAndSchedule() + + # TODO: raises annotation shouldn't be needed, but likely similar issue as + # https://github.com/nim-lang/Nim/issues/17369 + proc continuation(udata: pointer) {.gcsafe, raises: [].} = + if not(retFuture.finished()): + if timeouted: + retFuture.complete(false) + return + if not(fut.finished()): + # Timer exceeded first, we going to cancel `fut` and wait until it + # not completes. + timeouted = true + fut.cancelSoon() + else: + # Future `fut` completed/failed/cancelled first. + if not(isNil(timer)): + clearTimer(timer) + fut.completeFuture() + + # TODO: raises annotation shouldn't be needed, but likely similar issue as + # https://github.com/nim-lang/Nim/issues/17369 + proc cancellation(udata: pointer) {.gcsafe, raises: [].} = + if not(fut.finished()): + if not isNil(timer): + clearTimer(timer) + fut.cancelSoon() + else: + fut.completeFuture() + + if fut.finished(): + retFuture.complete(true) + else: + if timeout.isZero(): + retFuture.complete(false) + elif timeout.isInfinite(): + retFuture.cancelCallback = cancellation + fut.addCallback(continuation) + else: + moment = Moment.fromNow(timeout) + retFuture.cancelCallback = cancellation + timer = setTimer(moment, continuation, nil) + fut.addCallback(continuation) + + retFuture + +proc withTimeout*[T](fut: Future[T], timeout: int): Future[bool] {. + inline, deprecated: "Use withTimeout(Future[T], Duration)".} = + withTimeout(fut, timeout.milliseconds()) + +proc wait*[T](fut: Future[T], timeout = InfiniteDuration): Future[T] = + ## Returns a future which will complete once future ``fut`` completes + ## or if timeout of ``timeout`` milliseconds has been expired. + ## + ## If ``timeout`` is ``-1``, then statement ``await wait(fut)`` is + ## equal to ``await fut``. + ## + ## TODO: In case when ``fut`` got cancelled, what result Future[T] + ## should return, because it can't be cancelled too. + var + retFuture = newFuture[T]("chronos.wait()", {FutureFlag.OwnCancelSchedule}) + moment: Moment + timer: TimerCallback + timeouted = false + + template completeFuture(fut: untyped): untyped = + if fut.failed(): + retFuture.fail(fut.error) + elif fut.cancelled(): + retFuture.cancelAndSchedule() + else: + when T is void: + retFuture.complete() + else: + retFuture.complete(fut.value) + + proc continuation(udata: pointer) {.raises: [].} = + if not(retFuture.finished()): + if timeouted: + retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) + return + if not(fut.finished()): + # Timer exceeded first. + timeouted = true + fut.cancelSoon() + else: + # Future `fut` completed/failed/cancelled first. + if not(isNil(timer)): + clearTimer(timer) + fut.completeFuture() + + var cancellation: proc(udata: pointer) {.gcsafe, raises: [].} + cancellation = proc(udata: pointer) {.gcsafe, raises: [].} = + if not(fut.finished()): + if not(isNil(timer)): + clearTimer(timer) + fut.cancelSoon() + else: + fut.completeFuture() + + if fut.finished(): + fut.completeFuture() + else: + if timeout.isZero(): + retFuture.fail(newException(AsyncTimeoutError, "Timeout exceeded!")) + elif timeout.isInfinite(): + retFuture.cancelCallback = cancellation + fut.addCallback(continuation) + else: + moment = Moment.fromNow(timeout) + retFuture.cancelCallback = cancellation + timer = setTimer(moment, continuation, nil) + fut.addCallback(continuation) + + retFuture + +proc wait*[T](fut: Future[T], timeout = -1): Future[T] {. + inline, deprecated: "Use wait(Future[T], Duration)".} = + if timeout == -1: + wait(fut, InfiniteDuration) + elif timeout == 0: + wait(fut, ZeroDuration) + else: + wait(fut, timeout.milliseconds()) + + +when defined(windows): + import ../osdefs + + proc waitForSingleObject*(handle: HANDLE, + timeout: Duration): Future[WaitableResult] {. + raises: [].} = + ## Waits until the specified object is in the signaled state or the + ## time-out interval elapses. WaitForSingleObject() for asynchronous world. + let flags = WT_EXECUTEONLYONCE + + var + retFuture = newFuture[WaitableResult]("chronos.waitForSingleObject()") + waitHandle: WaitableHandle = nil + + proc continuation(udata: pointer) {.gcsafe.} = + doAssert(not(isNil(waitHandle))) + if not(retFuture.finished()): + let + ovl = cast[PtrCustomOverlapped](udata) + returnFlag = WINBOOL(ovl.data.bytesCount) + res = closeWaitable(waitHandle) + if res.isErr(): + retFuture.fail(newException(AsyncError, osErrorMsg(res.error()))) + else: + if returnFlag == TRUE: + retFuture.complete(WaitableResult.Timeout) + else: + retFuture.complete(WaitableResult.Ok) + + proc cancellation(udata: pointer) {.gcsafe.} = + doAssert(not(isNil(waitHandle))) + if not(retFuture.finished()): + discard closeWaitable(waitHandle) + + let wres = uint32(waitForSingleObject(handle, DWORD(0))) + if wres == WAIT_OBJECT_0: + retFuture.complete(WaitableResult.Ok) + return retFuture + elif wres == WAIT_ABANDONED: + retFuture.fail(newException(AsyncError, "Handle was abandoned")) + return retFuture + elif wres == WAIT_FAILED: + retFuture.fail(newException(AsyncError, osErrorMsg(osLastError()))) + return retFuture + + if timeout == ZeroDuration: + retFuture.complete(WaitableResult.Timeout) + return retFuture + + waitHandle = + block: + let res = registerWaitable(handle, flags, timeout, continuation, nil) + if res.isErr(): + retFuture.fail(newException(AsyncError, osErrorMsg(res.error()))) + return retFuture + res.get() + + retFuture.cancelCallback = cancellation + return retFuture diff --git a/chronos/asyncmacro2.nim b/chronos/internal/asyncmacro.nim similarity index 99% rename from chronos/asyncmacro2.nim rename to chronos/internal/asyncmacro.nim index 499f847..8d99155 100644 --- a/chronos/asyncmacro2.nim +++ b/chronos/internal/asyncmacro.nim @@ -8,7 +8,9 @@ # distribution, for details about the copyright. # -import std/algorithm +import + std/[algorithm, macros, sequtils], + ../[futures, config] proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} = case node.kind diff --git a/chronos/internal/errors.nim b/chronos/internal/errors.nim new file mode 100644 index 0000000..083f7a2 --- /dev/null +++ b/chronos/internal/errors.nim @@ -0,0 +1,5 @@ +type + AsyncError* = object of CatchableError + ## Generic async exception + AsyncTimeoutError* = object of AsyncError + ## Timeout exception From f56d2866877314ded1179a06fda63c651a7e3d72 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 24 Oct 2023 16:21:07 +0200 Subject: [PATCH 25/50] introduce `asyncraises` to core future utilities (#454) * introduce `asyncraises` to core future utilities Similar to the introduction of `raises` into a codebase, `asyncraises` needs to be introduced gradually across all functionality before deriving benefit. This is a first introduction along with utilities to manage raises lists and transform them at compile time. Several scenarios ensue: * for trivial cases, adding `asyncraises` is enough and the framework deduces the rest * some functions "add" new asyncraises (similar to what `raise` does in "normal" code) - for example `wait` may raise all exceptions of the future passed to it and additionally a few of its own - this requires extending the raises list * som functions "remove" raises (similar to what `try/except` does) such as `nocancel` with blocks cancellations and therefore reduce the raising set Both of the above cases are currently handled by a macro, but depending on the situation lead to code organisation issues around return types and pragma limitations - in particular, to keep `asyncraises` backwards-compatibility, some code needs to exist in two versions which somewhat complicates the implementation. * add `asyncraises` versions for several `asyncfutures` utilities * when assigning exceptions to a `Future` via `fail`, check at compile time if possible and at runtime if not that the exception matches constraints * fix `waitFor` comments * move async raises to separate module, implement `or` --- README.md | 30 ++- chronos/internal/asyncfutures.nim | 330 ++++++++++++++++------------- chronos/internal/asyncmacro.nim | 9 +- chronos/internal/raisesfutures.nim | 124 +++++++++++ tests/testmacro.nim | 60 ++++++ 5 files changed, 396 insertions(+), 157 deletions(-) create mode 100644 chronos/internal/raisesfutures.nim diff --git a/README.md b/README.md index c06cfa9..0a23ea1 100644 --- a/README.md +++ b/README.md @@ -222,8 +222,8 @@ proc p1(): Future[void] {.async, asyncraises: [IOError].} = raise newException(IOError, "works") # Or any child of IOError ``` -Under the hood, the return type of `p1` will be rewritten to another type, -which will convey raises informations to await. +Under the hood, the return type of `p1` will be rewritten to an internal type, +which will convey raises informations to `await`. ```nim proc p2(): Future[void] {.async, asyncraises: [IOError].} = @@ -231,8 +231,10 @@ proc p2(): Future[void] {.async, asyncraises: [IOError].} = # can only raise IOError ``` -The hidden type (`RaiseTrackingFuture`) is implicitely convertible into a Future. -However, it may causes issues when creating callback or methods +Raw functions and callbacks that don't go through the `async` transformation but +still return a `Future` and interact with the rest of the framework also need to +be annotated with `asyncraises` to participate in the checked exception scheme: + ```nim proc p3(): Future[void] {.async, asyncraises: [IOError].} = let fut: Future[void] = p1() # works @@ -247,6 +249,24 @@ proc p3(): Future[void] {.async, asyncraises: [IOError].} = ) ``` +When `chronos` performs the `async` transformation, all code is placed in a +a special `try/except` clause that re-routes exception handling to the `Future`. + +Beacuse of this re-routing, functions that return a `Future` instance manually +never directly raise exceptions themselves - instead, exceptions are handled +indirectly via `await` or `Future.read`. When writing raw async functions, they +too must not raise exceptions - instead, they must store exceptions in the +future they return: + +```nim +proc p4(): Future[void] {.asyncraises: [ValueError].} = + let fut = newFuture[void] + + # Equivalent of `raise (ref ValueError)()` in raw async functions: + fut.fail((ref ValueError)(msg: "raising in raw async function")) + fut +``` + ### Platform independence Several functions in `chronos` are backed by the operating system, such as @@ -268,7 +288,7 @@ Because of this, the effect system thinks no exceptions are "leaking" because in fact, exception _handling_ is deferred to when the future is being read. Effectively, this means that while code can be compiled with -`{.push raises: [Defect]}`, the intended effect propagation and checking is +`{.push raises: []}`, the intended effect propagation and checking is **disabled** for `async` functions. To enable checking exception effects in `async` code, enable strict mode with diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index 860b8b6..abf28c7 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -8,12 +8,16 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) +{.push raises: [].} + import std/[sequtils, macros] import stew/base10 -import ./asyncengine +import ./[asyncengine, raisesfutures] import ../[config, futures] +export raisesfutures.InternalRaisesFuture + when chronosStackTrace: import std/strutils when defined(nimHasStacktracesModule): @@ -38,12 +42,6 @@ func `[]`*(loc: array[LocationKind, ptr SrcLoc], v: int): ptr SrcLoc {. else: raiseAssert("Unknown source location " & $v) type - InternalRaisesFuture*[T, E] = ref object of Future[T] - ## Future with a tuple of possible exception types - ## eg InternalRaisesFuture[void, (ValueError, OSError)] - ## Will be injected by `asyncraises`, should generally - ## not be used manually - FutureStr*[T] = ref object of Future[T] ## Future to hold GC strings gcholder*: string @@ -52,6 +50,8 @@ type ## Future to hold GC seqs gcholder*: seq[B] + SomeFuture = Future|InternalRaisesFuture + # Backwards compatibility for old FutureState name template Finished* {.deprecated: "Use Completed instead".} = Completed template Finished*(T: type FutureState): FutureState {. @@ -68,11 +68,18 @@ proc newFutureImpl[T](loc: ptr SrcLoc, flags: FutureFlags): Future[T] = internalInitFutureBase(fut, loc, FutureState.Pending, flags) fut -proc newInternalRaisesFutureImpl[T, E](loc: ptr SrcLoc): InternalRaisesFuture[T, E] = +proc newInternalRaisesFutureImpl[T, E]( + loc: ptr SrcLoc): InternalRaisesFuture[T, E] = let fut = InternalRaisesFuture[T, E]() internalInitFutureBase(fut, loc, FutureState.Pending, {}) fut +proc newInternalRaisesFutureImpl[T, E]( + loc: ptr SrcLoc, flags: FutureFlags): InternalRaisesFuture[T, E] = + let fut = InternalRaisesFuture[T, E]() + internalInitFutureBase(fut, loc, FutureState.Pending, flags) + fut + proc newFutureSeqImpl[A, B](loc: ptr SrcLoc): FutureSeq[A, B] = let fut = FutureSeq[A, B]() internalInitFutureBase(fut, loc, FutureState.Pending, {}) @@ -90,7 +97,8 @@ template newFuture*[T](fromProc: static[string] = "", ## Specifying ``fromProc``, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. when declared(InternalRaisesFutureRaises): # injected by `asyncraises` - newInternalRaisesFutureImpl[T, InternalRaisesFutureRaises](getSrcLocation(fromProc)) + newInternalRaisesFutureImpl[T, InternalRaisesFutureRaises]( + getSrcLocation(fromProc), flags) else: newFutureImpl[T](getSrcLocation(fromProc), flags) @@ -214,53 +222,11 @@ proc fail(future: FutureBase, error: ref CatchableError, loc: ptr SrcLoc) = getStackTrace(error) future.finish(FutureState.Failed) -template fail*(future: FutureBase, error: ref CatchableError) = +template fail*( + future: FutureBase, error: ref CatchableError, warn: static bool = false) = ## Completes ``future`` with ``error``. fail(future, error, getSrcLocation()) -macro checkFailureType(future, error: typed): untyped = - let e = getTypeInst(future)[2] - let types = getType(e) - - if types.eqIdent("void"): - error("Can't raise exceptions on this Future") - - expectKind(types, nnkBracketExpr) - expectKind(types[0], nnkSym) - assert types[0].strVal == "tuple" - assert types.len > 1 - - expectKind(getTypeInst(error), nnkRefTy) - let toMatch = getTypeInst(error)[0] - - # Can't find a way to check `is` in the macro. (sameType doesn't - # work for inherited objects). Dirty hack here, for [IOError, OSError], - # this will generate: - # - # static: - # if not((`toMatch` is IOError) or (`toMatch` is OSError) - # or (`toMatch` is CancelledError) or false): - # raiseAssert("Can't fail with `toMatch`, only [IOError, OSError] is allowed") - var typeChecker = ident"false" - - for errorType in types[1..^1]: - typeChecker = newCall("or", typeChecker, newCall("is", toMatch, errorType)) - typeChecker = newCall( - "or", typeChecker, - newCall("is", toMatch, ident"CancelledError")) - - let errorMsg = "Can't fail with " & repr(toMatch) & ". Only " & repr(types[1..^1]) & " allowed" - - result = nnkStaticStmt.newNimNode(lineInfoFrom=error).add( - quote do: - if not(`typeChecker`): - raiseAssert(`errorMsg`) - ) - -template fail*[T, E](future: InternalRaisesFuture[T, E], error: ref CatchableError) = - checkFailureType(future, error) - fail(future, error, getSrcLocation()) - template newCancelledError(): ref CancelledError = (ref CancelledError)(msg: "Future operation cancelled!") @@ -572,29 +538,6 @@ proc read*(future: Future[void] ) {.raises: [CatchableError].} = # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") -proc read*[T: not void, E](future: InternalRaisesFuture[T, E] ): lent T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if not future.finished(): - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - - internalCheckComplete(future) - future.internalValue - -proc read*[E](future: InternalRaisesFuture[void, E]) = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished(): - internalCheckComplete(future) - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - proc readError*(future: FutureBase): ref CatchableError {.raises: [ValueError].} = ## Retrieves the exception stored in ``future``. ## @@ -621,8 +564,9 @@ template taskCancelMessage(future: FutureBase): string = "Asynchronous task " & taskFutureLocation(future) & " was cancelled!" proc waitFor*[T](fut: Future[T]): T {.raises: [CatchableError].} = - ## **Blocks** the current thread until the specified future completes. - ## There's no way to tell if poll or read raised the exception + ## **Blocks** the current thread until the specified future finishes and + ## reads it, potentially raising an exception if the future failed or was + ## cancelled. while not(fut.finished()): poll() @@ -716,6 +660,47 @@ proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] {. retFuture.cancelCallback = cancellation return retFuture +template orImpl*[T, Y](fut1: Future[T], fut2: Future[Y]): untyped = + var cb: proc(udata: pointer) {.gcsafe, raises: [].} + cb = proc(udata: pointer) {.gcsafe, raises: [].} = + if not(retFuture.finished()): + var fut = cast[FutureBase](udata) + if cast[pointer](fut1) == udata: + fut2.removeCallback(cb) + else: + fut1.removeCallback(cb) + if fut.failed(): + retFuture.fail(fut.error, warn = false) + else: + retFuture.complete() + + proc cancellation(udata: pointer) = + # On cancel we remove all our callbacks only. + if not(fut1.finished()): + fut1.removeCallback(cb) + if not(fut2.finished()): + fut2.removeCallback(cb) + + if fut1.finished(): + if fut1.failed(): + retFuture.fail(fut1.error, warn = false) + else: + retFuture.complete() + return retFuture + + if fut2.finished(): + if fut2.failed(): + retFuture.fail(fut2.error, warn = false) + else: + retFuture.complete() + return retFuture + + fut1.addCallback(cb) + fut2.addCallback(cb) + + retFuture.cancelCallback = cancellation + return retFuture + proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = ## Returns a future which will complete once either ``fut1`` or ``fut2`` ## finish. @@ -730,45 +715,8 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = ## ## If cancelled, ``fut1`` and ``fut2`` futures WILL NOT BE cancelled. var retFuture = newFuture[void]("chronos.or") - var cb: proc(udata: pointer) {.gcsafe, raises: [].} - cb = proc(udata: pointer) {.gcsafe, raises: [].} = - if not(retFuture.finished()): - var fut = cast[FutureBase](udata) - if cast[pointer](fut1) == udata: - fut2.removeCallback(cb) - else: - fut1.removeCallback(cb) - if fut.failed(): - retFuture.fail(fut.error) - else: - retFuture.complete() + orImpl(fut1, fut2) - proc cancellation(udata: pointer) = - # On cancel we remove all our callbacks only. - if not(fut1.finished()): - fut1.removeCallback(cb) - if not(fut2.finished()): - fut2.removeCallback(cb) - - if fut1.finished(): - if fut1.failed(): - retFuture.fail(fut1.error) - else: - retFuture.complete() - return retFuture - - if fut2.finished(): - if fut2.failed(): - retFuture.fail(fut2.error) - else: - retFuture.complete() - return retFuture - - fut1.addCallback(cb) - fut2.addCallback(cb) - - retFuture.cancelCallback = cancellation - return retFuture proc all*[T](futs: varargs[Future[T]]): auto {. deprecated: "Use allFutures(varargs[Future[T]])".} = @@ -908,7 +856,7 @@ proc oneValue*[T](futs: varargs[Future[T]]): Future[T] {. return retFuture proc cancelSoon(future: FutureBase, aftercb: CallbackFunc, udata: pointer, - loc: ptr SrcLoc) = + loc: ptr SrcLoc) {.raises: [].} = ## Perform cancellation ``future`` and call ``aftercb`` callback when ## ``future`` become finished (completed with value, failed or cancelled). ## @@ -965,7 +913,8 @@ template cancel*(future: FutureBase) {. ## Cancel ``future``. cancelSoon(future, nil, nil, getSrcLocation()) -proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] = +proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] {. + asyncraises: [CancelledError].} = ## Perform cancellation ``future`` return Future which will be completed when ## ``future`` become finished (completed with value, failed or cancelled). ## @@ -989,7 +938,7 @@ template cancelAndWait*(future: FutureBase): Future[void] = ## Cancel ``future``. cancelAndWait(future, getSrcLocation()) -proc noCancel*[T](future: Future[T]): Future[T] = +proc noCancel*[F: SomeFuture](future: F): auto = # asyncraises: asyncraiseOf(future) - CancelledError ## Prevent cancellation requests from propagating to ``future`` while ## forwarding its value or error when it finishes. ## @@ -997,16 +946,25 @@ proc noCancel*[T](future: Future[T]): Future[T] = ## should not be cancelled at all cost, for example closing sockets, pipes, ## connections or servers. Usually it become useful in exception or finally ## blocks. - let retFuture = newFuture[T]("chronos.noCancel(T)", - {FutureFlag.OwnCancelSchedule}) + when F is InternalRaisesFuture: + type + E = F.E + InternalRaisesFutureRaises = E.remove(CancelledError) + + let retFuture = newFuture[F.T]("chronos.noCancel(T)", + {FutureFlag.OwnCancelSchedule}) template completeFuture() = if future.completed(): - when T is void: + when F.T is void: retFuture.complete() else: retFuture.complete(future.value) elif future.failed(): - retFuture.fail(future.error) + when F is Future: + retFuture.fail(future.error, warn = false) + when declared(InternalRaisesFutureRaises): + when InternalRaisesFutureRaises isnot void: + retFuture.fail(future.error, warn = false) else: raiseAssert("Unexpected future state [" & $future.state & "]") @@ -1019,7 +977,8 @@ proc noCancel*[T](future: Future[T]): Future[T] = future.addCallback(continuation) retFuture -proc allFutures*(futs: varargs[FutureBase]): Future[void] = +proc allFutures*(futs: varargs[FutureBase]): Future[void] {. + asyncraises: [CancelledError].} = ## Returns a future which will complete only when all futures in ``futs`` ## will be completed, failed or canceled. ## @@ -1057,7 +1016,8 @@ proc allFutures*(futs: varargs[FutureBase]): Future[void] = retFuture -proc allFutures*[T](futs: varargs[Future[T]]): Future[void] = +proc allFutures*[T](futs: varargs[Future[T]]): Future[void] {. + asyncraises: [CancelledError].} = ## Returns a future which will complete only when all futures in ``futs`` ## will be completed, failed or canceled. ## @@ -1070,7 +1030,8 @@ proc allFutures*[T](futs: varargs[Future[T]]): Future[void] = nfuts.add(future) allFutures(nfuts) -proc allFinished*[T](futs: varargs[Future[T]]): Future[seq[Future[T]]] = +proc allFinished*[F: SomeFuture](futs: varargs[F]): Future[seq[F]] {. + asyncraises: [CancelledError].} = ## Returns a future which will complete only when all futures in ``futs`` ## will be completed, failed or canceled. ## @@ -1080,7 +1041,7 @@ proc allFinished*[T](futs: varargs[Future[T]]): Future[seq[Future[T]]] = ## If the argument is empty, the returned future COMPLETES immediately. ## ## On cancel all the awaited futures ``futs`` WILL NOT BE cancelled. - var retFuture = newFuture[seq[Future[T]]]("chronos.allFinished()") + var retFuture = newFuture[seq[F]]("chronos.allFinished()") let totalFutures = len(futs) var finishedFutures = 0 @@ -1110,7 +1071,8 @@ proc allFinished*[T](futs: varargs[Future[T]]): Future[seq[Future[T]]] = return retFuture -proc one*[T](futs: varargs[Future[T]]): Future[Future[T]] = +proc one*[F: SomeFuture](futs: varargs[F]): Future[F] {. + asyncraises: [ValueError, CancelledError].} = ## Returns a future which will complete and return completed Future[T] inside, ## when one of the futures in ``futs`` will be completed, failed or canceled. ## @@ -1119,7 +1081,7 @@ proc one*[T](futs: varargs[Future[T]]): Future[Future[T]] = ## On success returned Future will hold finished Future[T]. ## ## On cancel futures in ``futs`` WILL NOT BE cancelled. - var retFuture = newFuture[Future[T]]("chronos.one()") + var retFuture = newFuture[F]("chronos.one()") if len(futs) == 0: retFuture.fail(newException(ValueError, "Empty Future[T] list")) @@ -1137,7 +1099,7 @@ proc one*[T](futs: varargs[Future[T]]): Future[Future[T]] = var cb: proc(udata: pointer) {.gcsafe, raises: [].} cb = proc(udata: pointer) {.gcsafe, raises: [].} = if not(retFuture.finished()): - var res: Future[T] + var res: F var rfut = cast[FutureBase](udata) for i in 0..= 1 + + types + +macro checkRaises*[T: CatchableError]( + future: InternalRaisesFuture, error: ref T, warn: static bool = true): untyped = + ## Generate code that checks that the given error is compatible with the + ## raises restrictions of `future`. + ## + ## This check is done either at compile time or runtime depending on the + ## information available at compile time - in particular, if the raises + ## inherit from `error`, we end up with the equivalent of a downcast which + ## raises a Defect if it fails. + let raises = getRaises(future) + + expectKind(getTypeInst(error), nnkRefTy) + let toMatch = getTypeInst(error)[0] + + var + typeChecker = ident"false" + maybeChecker = ident"false" + runtimeChecker = ident"false" + + for errorType in raises[1..^1]: + typeChecker = infix(typeChecker, "or", infix(toMatch, "is", errorType)) + maybeChecker = infix(maybeChecker, "or", infix(errorType, "is", toMatch)) + runtimeChecker = infix( + runtimeChecker, "or", + infix(error, "of", nnkBracketExpr.newTree(ident"typedesc", errorType))) + + let + errorMsg = "`fail`: `" & repr(toMatch) & "` incompatible with `asyncraises: " & repr(raises[1..^1]) & "`" + warningMsg = "Can't verify `fail` exception type at compile time - expected one of " & repr(raises[1..^1]) & ", got `" & repr(toMatch) & "`" + # A warning from this line means exception type will be verified at runtime + warning = if warn: + quote do: {.warning: `warningMsg`.} + else: newEmptyNode() + + # Cannot check inhertance in macro so we let `static` do the heavy lifting + quote do: + when not(`typeChecker`): + when not(`maybeChecker`): + static: + {.error: `errorMsg`.} + else: + `warning` + assert(`runtimeChecker`, `errorMsg`) diff --git a/tests/testmacro.nim b/tests/testmacro.nim index 2d95a7f..2fc24be 100644 --- a/tests/testmacro.nim +++ b/tests/testmacro.nim @@ -477,3 +477,63 @@ suite "Exceptions tracking": proc test44 {.asyncraises: [ValueError], async.} = raise newException(ValueError, "hey") checkNotCompiles: proc test33 {.asyncraises: [IOError], async.} = raise newException(ValueError, "hey") + + test "or errors": + proc testit {.asyncraises: [ValueError], async.} = + raise (ref ValueError)() + + proc testit2 {.asyncraises: [IOError], async.} = + raise (ref IOError)() + + proc test {.async, asyncraises: [ValueError, IOError].} = + await testit() or testit2() + + proc noraises() {.raises: [].} = + expect(ValueError): + try: + let f = test() + waitFor(f) + except IOError: + doAssert false + + noraises() + + test "Wait errors": + proc testit {.asyncraises: [ValueError], async.} = raise newException(ValueError, "hey") + + proc test {.async, asyncraises: [ValueError, AsyncTimeoutError, CancelledError].} = + await wait(testit(), 1000.milliseconds) + + proc noraises() {.raises: [].} = + try: + expect(ValueError): waitFor(test()) + except CancelledError: doAssert false + except AsyncTimeoutError: doAssert false + + noraises() + + test "Nocancel errors": + proc testit {.asyncraises: [ValueError, CancelledError], async.} = + await sleepAsync(5.milliseconds) + raise (ref ValueError)() + + proc test {.async, asyncraises: [ValueError].} = + await noCancel testit() + + proc noraises() {.raises: [].} = + expect(ValueError): + let f = test() + waitFor(f.cancelAndWait()) + waitFor(f) + + noraises() + + test "Defect on wrong exception type at runtime": + {.push warning[User]: off} + let f = InternalRaisesFuture[void, (ValueError,)]() + expect(Defect): f.fail((ref CatchableError)()) + {.pop.} + check: not f.finished() + + expect(Defect): f.fail((ref CatchableError)(), warn = false) + check: not f.finished() From 12dc36cfeee3ac487aef1278c9b324cc082dcfeb Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 25 Oct 2023 15:16:10 +0200 Subject: [PATCH 26/50] Update README regarding cancellation (#450) * Update README regarding cancellation * Apply suggestions from code review Co-authored-by: Eugene Kabanov --------- Co-authored-by: Jacek Sieka Co-authored-by: Eugene Kabanov --- README.md | 81 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 0a23ea1..495f9f8 100644 --- a/README.md +++ b/README.md @@ -301,45 +301,64 @@ effects on forward declarations, callbacks and methods using ### Cancellation support -Any running `Future` can be cancelled. This can be used to launch multiple -futures, and wait for one of them to finish, and cancel the rest of them, -to add timeout, or to let the user cancel a running task. +Any running `Future` can be cancelled. This can be used for timeouts, +to let a user cancel a running task, to start multiple futures in parallel +and cancel them as soon as one finishes, etc. ```nim -# Simple cancellation -let future = sleepAsync(10.minutes) -future.cancel() +import chronos/apps/http/httpclient -# Wait for cancellation -let future2 = sleepAsync(10.minutes) -await future2.cancelAndWait() +proc cancellationExample() {.async.} = + # Simple cancellation + let future = sleepAsync(10.minutes) + future.cancelSoon() + # `cancelSoon` will not wait for the cancellation + # to be finished, so the Future could still be + # pending at this point. -# Race between futures -proc retrievePage(uri: string): Future[string] {.async.} = - # requires to import uri, chronos/apps/http/httpclient, stew/byteutils - let httpSession = HttpSessionRef.new() - try: - resp = await httpSession.fetch(parseUri(uri)) - result = string.fromBytes(resp.data) - finally: - # be sure to always close the session - await httpSession.closeWait() + # Wait for cancellation + let future2 = sleepAsync(10.minutes) + await future2.cancelAndWait() + # Using `cancelAndWait`, we know that future2 isn't + # pending anymore. However, it could have completed + # before cancellation happened (in which case, it + # will hold a value) -let - futs = - @[ - retrievePage("https://duckduckgo.com/?q=chronos"), - retrievePage("https://www.google.fr/search?q=chronos") - ] + # Race between futures + proc retrievePage(uri: string): Future[string] {.async.} = + let httpSession = HttpSessionRef.new() + try: + let resp = await httpSession.fetch(parseUri(uri)) + return bytesToString(resp.data) + finally: + # be sure to always close the session + # `finally` will run also during cancellation - + # `noCancel` ensures that `closeWait` doesn't get cancelled + await noCancel(httpSession.closeWait()) -let finishedFut = await one(futs) -for fut in futs: - if not fut.finished: - fut.cancel() -echo "Result: ", await finishedFut + let + futs = + @[ + retrievePage("https://duckduckgo.com/?q=chronos"), + retrievePage("https://www.google.fr/search?q=chronos") + ] + + let finishedFut = await one(futs) + for fut in futs: + if not fut.finished: + fut.cancelSoon() + echo "Result: ", await finishedFut + +waitFor(cancellationExample()) ``` -When an `await` is cancelled, it will raise a `CancelledError`: +Even if cancellation is initiated, it is not guaranteed that +the operation gets cancelled - the future might still be completed +or fail depending on the ordering of events and the specifics of +the operation. + +If the future indeed gets cancelled, `await` will raise a +`CancelledError` as is likely to happen in the following example: ```nim proc c1 {.async.} = echo "Before sleep" From 8375770fe578fb880570838cc558753bd6af6809 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 30 Oct 2023 15:27:25 +0200 Subject: [PATCH 27/50] Fix unreachable code places. (#459) * Fix unreachable code. * Use implicit returns instead. --- chronos/osdefs.nim | 2 ++ chronos/streams/tlsstream.nim | 34 ++++++++++++---------------------- chronos/transports/osnet.nim | 4 ++-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/chronos/osdefs.nim b/chronos/osdefs.nim index 75ceb67..78de4b7 100644 --- a/chronos/osdefs.nim +++ b/chronos/osdefs.nim @@ -1617,6 +1617,8 @@ elif defined(linux): # RTA_PRIORITY* = 6'u16 RTA_PREFSRC* = 7'u16 # RTA_METRICS* = 8'u16 + RTM_NEWLINK* = 16'u16 + RTM_NEWROUTE* = 24'u16 RTM_F_LOOKUP_TABLE* = 0x1000 diff --git a/chronos/streams/tlsstream.nim b/chronos/streams/tlsstream.nim index 6432a10..0c8efb9 100644 --- a/chronos/streams/tlsstream.nim +++ b/chronos/streams/tlsstream.nim @@ -157,17 +157,15 @@ proc tlsWriteRec(engine: ptr SslEngineContext, doAssert(length != 0 and not isNil(buf)) await writer.wsource.write(buf, int(length)) sslEngineSendrecAck(engine[], length) - return TLSResult.Success + TLSResult.Success except AsyncStreamError as exc: writer.state = AsyncStreamState.Error writer.error = exc - return TLSResult.Error + TLSResult.Error except CancelledError: if writer.state == AsyncStreamState.Running: writer.state = AsyncStreamState.Stopped - return TLSResult.Stopped - - return TLSResult.Error + TLSResult.Stopped proc tlsWriteApp(engine: ptr SslEngineContext, writer: TLSStreamWriter): Future[TLSResult] {.async.} = @@ -182,7 +180,6 @@ proc tlsWriteApp(engine: ptr SslEngineContext, # (and discarded). writer.state = AsyncStreamState.Finished return TLSResult.WriteEof - let toWrite = min(int(length), item.size) copyOut(buf, item, toWrite) if int(length) >= item.size: @@ -190,7 +187,6 @@ proc tlsWriteApp(engine: ptr SslEngineContext, sslEngineSendappAck(engine[], uint(item.size)) sslEngineFlush(engine[], 0) item.future.complete() - return TLSResult.Success else: # BearSSL is not ready to accept whole item, so we will send # only part of item and adjust offset. @@ -198,17 +194,15 @@ proc tlsWriteApp(engine: ptr SslEngineContext, item.size = item.size - int(length) writer.queue.addFirstNoWait(item) sslEngineSendappAck(engine[], length) - return TLSResult.Success + TLSResult.Success else: sslEngineClose(engine[]) item.future.complete() - return TLSResult.Success + TLSResult.Success except CancelledError: if writer.state == AsyncStreamState.Running: writer.state = AsyncStreamState.Stopped - return TLSResult.Stopped - - return TLSResult.Error + TLSResult.Stopped proc tlsReadRec(engine: ptr SslEngineContext, reader: TLSStreamReader): Future[TLSResult] {.async.} = @@ -219,19 +213,17 @@ proc tlsReadRec(engine: ptr SslEngineContext, sslEngineRecvrecAck(engine[], uint(res)) if res == 0: sslEngineClose(engine[]) - return TLSResult.ReadEof + TLSResult.ReadEof else: - return TLSResult.Success + TLSResult.Success except AsyncStreamError as exc: reader.state = AsyncStreamState.Error reader.error = exc - return TLSResult.Error + TLSResult.Error except CancelledError: if reader.state == AsyncStreamState.Running: reader.state = AsyncStreamState.Stopped - return TLSResult.Stopped - - return TLSResult.Error + TLSResult.Stopped proc tlsReadApp(engine: ptr SslEngineContext, reader: TLSStreamReader): Future[TLSResult] {.async.} = @@ -240,13 +232,11 @@ proc tlsReadApp(engine: ptr SslEngineContext, var buf = sslEngineRecvappBuf(engine[], length) await upload(addr reader.buffer, buf, int(length)) sslEngineRecvappAck(engine[], length) - return TLSResult.Success + TLSResult.Success except CancelledError: if reader.state == AsyncStreamState.Running: reader.state = AsyncStreamState.Stopped - return TLSResult.Stopped - - return TLSResult.Error + TLSResult.Stopped template readAndReset(fut: untyped) = if fut.finished(): diff --git a/chronos/transports/osnet.nim b/chronos/transports/osnet.nim index 21adb65..99dabd7 100644 --- a/chronos/transports/osnet.nim +++ b/chronos/transports/osnet.nim @@ -677,10 +677,10 @@ when defined(linux): var msg = cast[ptr NlMsgHeader](addr data[0]) var endflag = false while NLMSG_OK(msg, length): - if msg.nlmsg_type == NLMSG_ERROR: + if msg.nlmsg_type in [uint16(NLMSG_DONE), uint16(NLMSG_ERROR)]: endflag = true break - else: + elif msg.nlmsg_type == RTM_NEWROUTE: res = processRoute(msg) endflag = true break From a70b145964dddd64d1d0af567da30d0572f2a10e Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 30 Oct 2023 15:27:50 +0200 Subject: [PATCH 28/50] IPv4/IPv6 dualstack (#456) * Initial commit. * Fix tests. * Fix linux compilation issue. * Add getDomain() implementation. Add getDomain() tests. Add datagram tests. * Fix style errors. * Deprecate NetFlag. Deprecate new flags in ServerFlags. Add isAvailable(). Fix setDualstack() to ignore errors on `Auto`. Updatetests. * Deprecate some old procedures. Improve datagram transport a bit. * Address review comments, and fix tests. * Fix setDescriptorBlocking() issue. Recover connect() dualstack behavior. Add test for connect() IPv6-[IPv4 mapped] addresses. * Fix alignment code issue. Fix TcpNoDelay was not available on Windows. * Add dualstack support to HTTP/HTTPS client/server. --- chronos/apps/http/httpclient.nim | 10 +- chronos/apps/http/httpserver.nim | 5 +- chronos/apps/http/shttpserver.nim | 5 +- chronos/handles.nim | 180 ++++++++++++++------- chronos/internal/asyncengine.nim | 13 ++ chronos/osdefs.nim | 96 +++++++---- chronos/oserrno.nim | 1 + chronos/osutils.nim | 4 + chronos/transports/common.nim | 79 ++++++++- chronos/transports/datagram.nim | 193 ++++++++++++---------- chronos/transports/stream.nim | 258 ++++++++++++++++-------------- tests/testdatagram.nim | 126 +++++++++++++++ tests/teststream.nim | 127 +++++++++++++++ 13 files changed, 795 insertions(+), 302 deletions(-) diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 01e2bab..34089c7 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -126,6 +126,7 @@ type connectionsCount*: int socketFlags*: set[SocketFlags] flags*: HttpClientFlags + dualstack*: DualStackType HttpAddress* = object id*: string @@ -263,7 +264,8 @@ proc new*(t: typedesc[HttpSessionRef], maxConnections = -1, idleTimeout = HttpConnectionIdleTimeout, idlePeriod = HttpConnectionCheckPeriod, - socketFlags: set[SocketFlags] = {}): HttpSessionRef {. + socketFlags: set[SocketFlags] = {}, + dualstack = DualStackType.Auto): HttpSessionRef {. raises: [] .} = ## Create new HTTP session object. ## @@ -283,7 +285,8 @@ proc new*(t: typedesc[HttpSessionRef], idleTimeout: idleTimeout, idlePeriod: idlePeriod, connections: initTable[string, seq[HttpClientConnectionRef]](), - socketFlags: socketFlags + socketFlags: socketFlags, + dualstack: dualstack ) res.watcherFut = if HttpClientFlag.Http11Pipeline in flags: @@ -620,7 +623,8 @@ proc connect(session: HttpSessionRef, let transp = try: await connect(address, bufferSize = session.connectionBufferSize, - flags = session.socketFlags) + flags = session.socketFlags, + dualstack = session.dualstack) except CancelledError as exc: raise exc except CatchableError: diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index f0788e2..2ab5317 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -191,7 +191,8 @@ proc new*(htype: typedesc[HttpServerRef], backlogSize: int = DefaultBacklogSize, httpHeadersTimeout = 10.seconds, maxHeadersSize: int = 8192, - maxRequestBodySize: int = 1_048_576): HttpResult[HttpServerRef] {. + maxRequestBodySize: int = 1_048_576, + dualstack = DualStackType.Auto): HttpResult[HttpServerRef] {. raises: [].} = let serverUri = @@ -206,7 +207,7 @@ proc new*(htype: typedesc[HttpServerRef], let serverInstance = try: createStreamServer(address, flags = socketFlags, bufferSize = bufferSize, - backlog = backlogSize) + backlog = backlogSize, dualstack = dualstack) except TransportOsError as exc: return err(exc.msg) except CatchableError as exc: diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index 6d321a0..0300597 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -92,7 +92,8 @@ proc new*(htype: typedesc[SecureHttpServerRef], backlogSize: int = DefaultBacklogSize, httpHeadersTimeout = 10.seconds, maxHeadersSize: int = 8192, - maxRequestBodySize: int = 1_048_576 + maxRequestBodySize: int = 1_048_576, + dualstack = DualStackType.Auto ): HttpResult[SecureHttpServerRef] {.raises: [].} = doAssert(not(isNil(tlsPrivateKey)), "TLS private key must not be nil!") @@ -110,7 +111,7 @@ proc new*(htype: typedesc[SecureHttpServerRef], let serverInstance = try: createStreamServer(address, flags = socketFlags, bufferSize = bufferSize, - backlog = backlogSize) + backlog = backlogSize, dualstack = dualstack) except TransportOsError as exc: return err(exc.msg) except CatchableError as exc: diff --git a/chronos/handles.nim b/chronos/handles.nim index 2348b33..afa57fb 100644 --- a/chronos/handles.nim +++ b/chronos/handles.nim @@ -21,66 +21,113 @@ const asyncInvalidSocket* = AsyncFD(osdefs.INVALID_SOCKET) asyncInvalidPipe* = asyncInvalidSocket -proc setSocketBlocking*(s: SocketHandle, blocking: bool): bool = +proc setSocketBlocking*(s: SocketHandle, blocking: bool): bool {. + deprecated: "Please use setDescriptorBlocking() instead".} = ## Sets blocking mode on socket. - when defined(windows) or defined(nimdoc): - var mode = clong(ord(not blocking)) - if osdefs.ioctlsocket(s, osdefs.FIONBIO, addr(mode)) == -1: - false - else: - true - else: - let x: int = osdefs.fcntl(s, osdefs.F_GETFL, 0) - if x == -1: - false - else: - let mode = - if blocking: x and not osdefs.O_NONBLOCK else: x or osdefs.O_NONBLOCK - if osdefs.fcntl(s, osdefs.F_SETFL, mode) == -1: - false - else: - true + setDescriptorBlocking(s, blocking).isOkOr: + return false + true -proc setSockOpt*(socket: AsyncFD, level, optname, optval: int): bool = - ## `setsockopt()` for integer options. - ## Returns ``true`` on success, ``false`` on error. +proc setSockOpt2*(socket: AsyncFD, + level, optname, optval: int): Result[void, OSErrorCode] = var value = cint(optval) - osdefs.setsockopt(SocketHandle(socket), cint(level), cint(optname), - addr(value), SockLen(sizeof(value))) >= cint(0) + let res = osdefs.setsockopt(SocketHandle(socket), cint(level), cint(optname), + addr(value), SockLen(sizeof(value))) + if res == -1: + return err(osLastError()) + ok() -proc setSockOpt*(socket: AsyncFD, level, optname: int, value: pointer, - valuelen: int): bool = +proc setSockOpt2*(socket: AsyncFD, level, optname: int, value: pointer, + valuelen: int): Result[void, OSErrorCode] = ## `setsockopt()` for custom options (pointer and length). ## Returns ``true`` on success, ``false`` on error. - osdefs.setsockopt(SocketHandle(socket), cint(level), cint(optname), value, - SockLen(valuelen)) >= cint(0) + let res = osdefs.setsockopt(SocketHandle(socket), cint(level), cint(optname), + value, SockLen(valuelen)) + if res == -1: + return err(osLastError()) + ok() -proc getSockOpt*(socket: AsyncFD, level, optname: int, value: var int): bool = +proc setSockOpt*(socket: AsyncFD, level, optname, optval: int): bool {. + deprecated: "Please use setSockOpt2() instead".} = + ## `setsockopt()` for integer options. + ## Returns ``true`` on success, ``false`` on error. + setSockOpt2(socket, level, optname, optval).isOk + +proc setSockOpt*(socket: AsyncFD, level, optname: int, value: pointer, + valuelen: int): bool {. + deprecated: "Please use setSockOpt2() instead".} = + ## `setsockopt()` for custom options (pointer and length). + ## Returns ``true`` on success, ``false`` on error. + setSockOpt2(socket, level, optname, value, valuelen).isOk + +proc getSockOpt2*(socket: AsyncFD, + level, optname: int): Result[cint, OSErrorCode] = + var + value: cint + size = SockLen(sizeof(value)) + let res = osdefs.getsockopt(SocketHandle(socket), cint(level), cint(optname), + addr(value), addr(size)) + if res == -1: + return err(osLastError()) + ok(value) + +proc getSockOpt2*(socket: AsyncFD, level, optname: int, + T: type): Result[T, OSErrorCode] = + var + value = default(T) + size = SockLen(sizeof(value)) + let res = osdefs.getsockopt(SocketHandle(socket), cint(level), cint(optname), + cast[ptr byte](addr(value)), addr(size)) + if res == -1: + return err(osLastError()) + ok(value) + +proc getSockOpt*(socket: AsyncFD, level, optname: int, value: var int): bool {. + deprecated: "Please use getSockOpt2() instead".} = ## `getsockopt()` for integer options. ## Returns ``true`` on success, ``false`` on error. - var res: cint - var size = SockLen(sizeof(res)) - if osdefs.getsockopt(SocketHandle(socket), cint(level), cint(optname), - addr(res), addr(size)) >= cint(0): - value = int(res) - true - else: - false + value = getSockOpt2(socket, level, optname).valueOr: + return false + true -proc getSockOpt*(socket: AsyncFD, level, optname: int, value: pointer, - valuelen: var int): bool = +proc getSockOpt*(socket: AsyncFD, level, optname: int, value: var pointer, + valuelen: var int): bool {. + deprecated: "Please use getSockOpt2() instead".} = ## `getsockopt()` for custom options (pointer and length). ## Returns ``true`` on success, ``false`` on error. osdefs.getsockopt(SocketHandle(socket), cint(level), cint(optname), value, cast[ptr SockLen](addr valuelen)) >= cint(0) -proc getSocketError*(socket: AsyncFD, err: var int): bool = +proc getSocketError*(socket: AsyncFD, err: var int): bool {. + deprecated: "Please use getSocketError() instead".} = ## Recover error code associated with socket handle ``socket``. - getSockOpt(socket, cint(osdefs.SOL_SOCKET), cint(osdefs.SO_ERROR), err) + err = getSockOpt2(socket, cint(osdefs.SOL_SOCKET), + cint(osdefs.SO_ERROR)).valueOr: + return false + true + +proc getSocketError2*(socket: AsyncFD): Result[cint, OSErrorCode] = + getSockOpt2(socket, cint(osdefs.SOL_SOCKET), cint(osdefs.SO_ERROR)) + +proc isAvailable*(domain: Domain): bool = + when defined(windows): + let fd = wsaSocket(toInt(domain), toInt(SockType.SOCK_STREAM), + toInt(Protocol.IPPROTO_TCP), nil, GROUP(0), 0'u32) + if fd == osdefs.INVALID_SOCKET: + return if osLastError() == osdefs.WSAEAFNOSUPPORT: false else: true + discard closeFd(fd) + true + else: + let fd = osdefs.socket(toInt(domain), toInt(SockType.SOCK_STREAM), + toInt(Protocol.IPPROTO_TCP)) + if fd == -1: + return if osLastError() == osdefs.EAFNOSUPPORT: false else: true + discard closeFd(fd) + true proc createAsyncSocket2*(domain: Domain, sockType: SockType, - protocol: Protocol, - inherit = true): Result[AsyncFD, OSErrorCode] = + protocol: Protocol, + inherit = true): Result[AsyncFD, OSErrorCode] = ## Creates new asynchronous socket. when defined(windows): let flags = @@ -93,15 +140,12 @@ proc createAsyncSocket2*(domain: Domain, sockType: SockType, if fd == osdefs.INVALID_SOCKET: return err(osLastError()) - let bres = setDescriptorBlocking(fd, false) - if bres.isErr(): + setDescriptorBlocking(fd, false).isOkOr: discard closeFd(fd) - return err(bres.error()) - - let res = register2(AsyncFD(fd)) - if res.isErr(): + return err(error) + register2(AsyncFD(fd)).isOkOr: discard closeFd(fd) - return err(res.error()) + return err(error) ok(AsyncFD(fd)) else: @@ -114,23 +158,20 @@ proc createAsyncSocket2*(domain: Domain, sockType: SockType, let fd = osdefs.socket(toInt(domain), socketType, toInt(protocol)) if fd == -1: return err(osLastError()) - let res = register2(AsyncFD(fd)) - if res.isErr(): + register2(AsyncFD(fd)).isOkOr: discard closeFd(fd) - return err(res.error()) + return err(error) ok(AsyncFD(fd)) else: let fd = osdefs.socket(toInt(domain), toInt(sockType), toInt(protocol)) if fd == -1: return err(osLastError()) - let bres = setDescriptorFlags(cint(fd), true, true) - if bres.isErr(): + setDescriptorFlags(cint(fd), true, true).isOkOr: discard closeFd(fd) - return err(bres.error()) - let res = register2(AsyncFD(fd)) - if res.isErr(): + return err(error) + register2(AsyncFD(fd)).isOkOr: discard closeFd(fd) - return err(bres.error()) + return err(error) ok(AsyncFD(fd)) proc wrapAsyncSocket2*(sock: cint|SocketHandle): Result[AsyncFD, OSErrorCode] = @@ -230,3 +271,26 @@ proc createAsyncPipe*(): tuple[read: AsyncFD, write: AsyncFD] = else: let pipes = res.get() (read: AsyncFD(pipes.read), write: AsyncFD(pipes.write)) + +proc getDualstack*(fd: AsyncFD): Result[bool, OSErrorCode] = + ## Returns `true` if `IPV6_V6ONLY` socket option set to `false`. + var + flag = cint(0) + size = SockLen(sizeof(flag)) + let res = osdefs.getsockopt(SocketHandle(fd), cint(osdefs.IPPROTO_IPV6), + cint(osdefs.IPV6_V6ONLY), addr(flag), addr(size)) + if res == -1: + return err(osLastError()) + ok(flag == cint(0)) + +proc setDualstack*(fd: AsyncFD, value: bool): Result[void, OSErrorCode] = + ## Sets `IPV6_V6ONLY` socket option value to `false` if `value == true` and + ## to `true` if `value == false`. + var + flag = cint(if value: 0 else: 1) + size = SockLen(sizeof(flag)) + let res = osdefs.setsockopt(SocketHandle(fd), cint(osdefs.IPPROTO_IPV6), + cint(osdefs.IPV6_V6ONLY), addr(flag), size) + if res == -1: + return err(osLastError()) + ok() diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim index 5a46f04..ebcc278 100644 --- a/chronos/internal/asyncengine.nim +++ b/chronos/internal/asyncengine.nim @@ -670,6 +670,19 @@ when defined(windows): if not(isNil(aftercb)): loop.callbacks.addLast(AsyncCallback(function: aftercb, udata: param)) + proc unregisterAndCloseFd*(fd: AsyncFD): Result[void, OSErrorCode] = + ## Unregister from system queue and close asynchronous socket. + ## + ## NOTE: Use this function to close temporary sockets/pipes only (which + ## are not exposed to the public and not supposed to be used/reused). + ## Please use closeSocket(AsyncFD) and closeHandle(AsyncFD) instead. + doAssert(fd != AsyncFD(osdefs.INVALID_SOCKET)) + unregister(fd) + if closeFd(SocketHandle(fd)) != 0: + err(osLastError()) + else: + ok() + proc contains*(disp: PDispatcher, fd: AsyncFD): bool = ## Returns ``true`` if ``fd`` is registered in thread's dispatcher. fd in disp.handles diff --git a/chronos/osdefs.nim b/chronos/osdefs.nim index 78de4b7..ab07721 100644 --- a/chronos/osdefs.nim +++ b/chronos/osdefs.nim @@ -122,6 +122,7 @@ when defined(windows): SO_UPDATE_ACCEPT_CONTEXT* = 0x700B SO_CONNECT_TIME* = 0x700C SO_UPDATE_CONNECT_CONTEXT* = 0x7010 + SO_PROTOCOL_INFOW* = 0x2005 FILE_FLAG_FIRST_PIPE_INSTANCE* = 0x00080000'u32 FILE_FLAG_OPEN_NO_RECALL* = 0x00100000'u32 @@ -258,6 +259,9 @@ when defined(windows): FIONBIO* = WSAIOW(102, 126) HANDLE_FLAG_INHERIT* = 1'u32 + IPV6_V6ONLY* = 27 + MAX_PROTOCOL_CHAIN* = 7 + WSAPROTOCOL_LEN* = 255 type LONG* = int32 @@ -441,6 +445,32 @@ when defined(windows): prefix*: SOCKADDR_INET prefixLength*: uint8 + WSAPROTOCOLCHAIN* {.final, pure.} = object + chainLen*: int32 + chainEntries*: array[MAX_PROTOCOL_CHAIN, DWORD] + + WSAPROTOCOL_INFO* {.final, pure.} = object + dwServiceFlags1*: uint32 + dwServiceFlags2*: uint32 + dwServiceFlags3*: uint32 + dwServiceFlags4*: uint32 + dwProviderFlags*: uint32 + providerId*: GUID + dwCatalogEntryId*: DWORD + protocolChain*: WSAPROTOCOLCHAIN + iVersion*: int32 + iAddressFamily*: int32 + iMaxSockAddr*: int32 + iMinSockAddr*: int32 + iSocketType*: int32 + iProtocol*: int32 + iProtocolMaxOffset*: int32 + iNetworkByteOrder*: int32 + iSecurityScheme*: int32 + dwMessageSize*: uint32 + dwProviderReserved*: uint32 + szProtocol*: array[WSAPROTOCOL_LEN + 1, WCHAR] + MibIpForwardRow2* {.final, pure.} = object interfaceLuid*: uint64 interfaceIndex*: uint32 @@ -890,7 +920,7 @@ elif defined(macos) or defined(macosx): O_NONBLOCK, SOL_SOCKET, SOCK_RAW, SOCK_DGRAM, SOCK_STREAM, MSG_NOSIGNAL, MSG_PEEK, AF_INET, AF_INET6, AF_UNIX, SO_ERROR, SO_REUSEADDR, - SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, + SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, SOCK_DGRAM, RLIMIT_NOFILE, SIG_BLOCK, SIG_UNBLOCK, SHUT_RD, SHUT_WR, SHUT_RDWR, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, @@ -915,7 +945,7 @@ elif defined(macos) or defined(macosx): O_NONBLOCK, SOL_SOCKET, SOCK_RAW, SOCK_DGRAM, SOCK_STREAM, MSG_NOSIGNAL, MSG_PEEK, AF_INET, AF_INET6, AF_UNIX, SO_ERROR, SO_REUSEADDR, - SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, + SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, SOCK_DGRAM, RLIMIT_NOFILE, SIG_BLOCK, SIG_UNBLOCK, SHUT_RD, SHUT_WR, SHUT_RDWR, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, @@ -977,7 +1007,8 @@ elif defined(linux): SOL_SOCKET, SO_ERROR, RLIMIT_NOFILE, MSG_NOSIGNAL, MSG_PEEK, AF_INET, AF_INET6, AF_UNIX, SO_REUSEADDR, SO_REUSEPORT, - SO_BROADCAST, IPPROTO_IP, IPV6_MULTICAST_HOPS, + SO_BROADCAST, IPPROTO_IP, IPPROTO_IPV6, + IPV6_MULTICAST_HOPS, SOCK_DGRAM, SOCK_STREAM, SHUT_RD, SHUT_WR, SHUT_RDWR, POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, @@ -1005,7 +1036,7 @@ elif defined(linux): SOL_SOCKET, SO_ERROR, RLIMIT_NOFILE, MSG_NOSIGNAL, MSG_PEEK, AF_INET, AF_INET6, AF_UNIX, SO_REUSEADDR, SO_REUSEPORT, - SO_BROADCAST, IPPROTO_IP, IPV6_MULTICAST_HOPS, + SO_BROADCAST, IPPROTO_IP, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, SOCK_DGRAM, SOCK_STREAM, SHUT_RD, SHUT_WR, SHUT_RDWR, POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL, SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGTRAP, SIGABRT, @@ -1127,7 +1158,7 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or O_NONBLOCK, SOL_SOCKET, SOCK_RAW, SOCK_DGRAM, SOCK_STREAM, MSG_NOSIGNAL, MSG_PEEK, AF_INET, AF_INET6, AF_UNIX, SO_ERROR, SO_REUSEADDR, - SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, + SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, SOCK_DGRAM, RLIMIT_NOFILE, SIG_BLOCK, SIG_UNBLOCK, CLOCK_MONOTONIC, SHUT_RD, SHUT_WR, SHUT_RDWR, @@ -1154,7 +1185,7 @@ elif defined(freebsd) or defined(openbsd) or defined(netbsd) or O_NONBLOCK, SOL_SOCKET, SOCK_RAW, SOCK_DGRAM, SOCK_STREAM, MSG_NOSIGNAL, MSG_PEEK, AF_INET, AF_INET6, AF_UNIX, SO_ERROR, SO_REUSEADDR, - SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, + SO_REUSEPORT, SO_BROADCAST, IPPROTO_IP, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, SOCK_DGRAM, RLIMIT_NOFILE, SIG_BLOCK, SIG_UNBLOCK, CLOCK_MONOTONIC, SHUT_RD, SHUT_WR, SHUT_RDWR, @@ -1182,47 +1213,52 @@ when defined(linux): SOCK_CLOEXEC* = 0x80000 TCP_NODELAY* = cint(1) IPPROTO_TCP* = 6 -elif defined(freebsd) or defined(netbsd) or defined(dragonfly): + O_CLOEXEC* = 0x80000 + POSIX_SPAWN_USEVFORK* = 0x40 + IPV6_V6ONLY* = 26 +elif defined(freebsd): const SOCK_NONBLOCK* = 0x20000000 SOCK_CLOEXEC* = 0x10000000 TCP_NODELAY* = cint(1) IPPROTO_TCP* = 6 + O_CLOEXEC* = 0x00100000 + POSIX_SPAWN_USEVFORK* = 0x00 + IPV6_V6ONLY* = 27 +elif defined(netbsd): + const + SOCK_NONBLOCK* = 0x20000000 + SOCK_CLOEXEC* = 0x10000000 + TCP_NODELAY* = cint(1) + IPPROTO_TCP* = 6 + O_CLOEXEC* = 0x00400000 + POSIX_SPAWN_USEVFORK* = 0x00 + IPV6_V6ONLY* = 27 +elif defined(dragonfly): + const + SOCK_NONBLOCK* = 0x20000000 + SOCK_CLOEXEC* = 0x10000000 + TCP_NODELAY* = cint(1) + IPPROTO_TCP* = 6 + O_CLOEXEC* = 0x00020000 + POSIX_SPAWN_USEVFORK* = 0x00 + IPV6_V6ONLY* = 27 elif defined(openbsd): const SOCK_CLOEXEC* = 0x8000 SOCK_NONBLOCK* = 0x4000 TCP_NODELAY* = cint(1) IPPROTO_TCP* = 6 + O_CLOEXEC* = 0x10000 + POSIX_SPAWN_USEVFORK* = 0x00 + IPV6_V6ONLY* = 27 elif defined(macos) or defined(macosx): const TCP_NODELAY* = cint(1) IP_MULTICAST_TTL* = cint(10) IPPROTO_TCP* = 6 - -when defined(linux): - const - O_CLOEXEC* = 0x80000 - POSIX_SPAWN_USEVFORK* = 0x40 -elif defined(freebsd): - const - O_CLOEXEC* = 0x00100000 - POSIX_SPAWN_USEVFORK* = 0x00 -elif defined(openbsd): - const - O_CLOEXEC* = 0x10000 - POSIX_SPAWN_USEVFORK* = 0x00 -elif defined(netbsd): - const - O_CLOEXEC* = 0x00400000 - POSIX_SPAWN_USEVFORK* = 0x00 -elif defined(dragonfly): - const - O_CLOEXEC* = 0x00020000 - POSIX_SPAWN_USEVFORK* = 0x00 -elif defined(macos) or defined(macosx): - const POSIX_SPAWN_USEVFORK* = 0x00 + IPV6_V6ONLY* = 27 when defined(linux) or defined(macos) or defined(macosx) or defined(freebsd) or defined(openbsd) or defined(netbsd) or defined(dragonfly): diff --git a/chronos/oserrno.nim b/chronos/oserrno.nim index 4f1c765..2a9f82c 100644 --- a/chronos/oserrno.nim +++ b/chronos/oserrno.nim @@ -1328,6 +1328,7 @@ elif defined(windows): ERROR_CONNECTION_REFUSED* = OSErrorCode(1225) ERROR_CONNECTION_ABORTED* = OSErrorCode(1236) WSAEMFILE* = OSErrorCode(10024) + WSAEAFNOSUPPORT* = OSErrorCode(10047) WSAENETDOWN* = OSErrorCode(10050) WSAENETRESET* = OSErrorCode(10052) WSAECONNABORTED* = OSErrorCode(10053) diff --git a/chronos/osutils.nim b/chronos/osutils.nim index 86505c2..f9c09f2 100644 --- a/chronos/osutils.nim +++ b/chronos/osutils.nim @@ -346,6 +346,10 @@ else: return err(osLastError()) ok() + proc setDescriptorBlocking*(s: SocketHandle, + value: bool): Result[void, OSErrorCode] = + setDescriptorBlocking(cint(s), value) + proc setDescriptorInheritance*(s: cint, value: bool): Result[void, OSErrorCode] = let flags = handleEintr(osdefs.fcntl(s, osdefs.F_GETFD)) diff --git a/chronos/transports/common.nim b/chronos/transports/common.nim index 4b4be7d..b7776e5 100644 --- a/chronos/transports/common.nim +++ b/chronos/transports/common.nim @@ -11,7 +11,7 @@ import std/[strutils] import stew/[base10, byteutils] -import ".."/[asyncloop, osdefs, oserrno] +import ".."/[asyncloop, osdefs, oserrno, handles] from std/net import Domain, `==`, IpAddress, IpAddressFamily, parseIpAddress, SockType, Protocol, Port, `$` @@ -31,6 +31,9 @@ type ReuseAddr, ReusePort, TcpNoDelay, NoAutoRead, GCUserData, FirstPipe, NoPipeFlash, Broadcast + DualStackType* {.pure.} = enum + Auto, Enabled, Disabled, Default + AddressFamily* {.pure.} = enum None, IPv4, IPv6, Unix @@ -76,6 +79,7 @@ when defined(windows) or defined(nimdoc): asock*: AsyncFD # Current AcceptEx() socket errorCode*: OSErrorCode # Current error code abuffer*: array[128, byte] # Windows AcceptEx() buffer + dualstack*: DualStackType # IPv4/IPv6 dualstack parameters when defined(windows): aovl*: CustomOverlapped # AcceptEx OVERLAPPED structure else: @@ -90,6 +94,7 @@ else: bufferSize*: int # Size of internal transports' buffer loopFuture*: Future[void] # Server's main Future errorCode*: OSErrorCode # Current error code + dualstack*: DualStackType # IPv4/IPv6 dualstack parameters type TransportError* = object of AsyncError @@ -720,3 +725,75 @@ proc raiseTransportError*(ecode: OSErrorCode) {. raise getTransportTooManyError(ecode) else: raise getTransportOsError(ecode) + +proc isAvailable*(family: AddressFamily): bool = + case family + of AddressFamily.None: + raiseAssert "Invalid address family" + of AddressFamily.IPv4: + isAvailable(Domain.AF_INET) + of AddressFamily.IPv6: + isAvailable(Domain.AF_INET6) + of AddressFamily.Unix: + isAvailable(Domain.AF_UNIX) + +proc getDomain*(socket: AsyncFD): Result[AddressFamily, OSErrorCode] = + ## Returns address family which is used to create socket ``socket``. + ## + ## Note: `chronos` supports only `AF_INET`, `AF_INET6` and `AF_UNIX` sockets. + ## For all other types of sockets this procedure returns + ## `EAFNOSUPPORT/WSAEAFNOSUPPORT` error. + when defined(windows): + let protocolInfo = ? getSockOpt2(socket, cint(osdefs.SOL_SOCKET), + cint(osdefs.SO_PROTOCOL_INFOW), + WSAPROTOCOL_INFO) + if protocolInfo.iAddressFamily == toInt(Domain.AF_INET): + ok(AddressFamily.IPv4) + elif protocolInfo.iAddressFamily == toInt(Domain.AF_INET6): + ok(AddressFamily.IPv6) + else: + err(WSAEAFNOSUPPORT) + else: + var + saddr = Sockaddr_storage() + slen = SockLen(sizeof(saddr)) + if getsockname(SocketHandle(socket), cast[ptr SockAddr](addr saddr), + addr slen) != 0: + return err(osLastError()) + if int(saddr.ss_family) == toInt(Domain.AF_INET): + ok(AddressFamily.IPv4) + elif int(saddr.ss_family) == toInt(Domain.AF_INET6): + ok(AddressFamily.IPv6) + elif int(saddr.ss_family) == toInt(Domain.AF_UNIX): + ok(AddressFamily.Unix) + else: + err(EAFNOSUPPORT) + +proc setDualstack*(socket: AsyncFD, family: AddressFamily, + flag: DualStackType): Result[void, OSErrorCode] = + if family == AddressFamily.IPv6: + case flag + of DualStackType.Auto: + # In case of `Auto` we going to ignore all the errors. + discard setDualstack(socket, true) + ok() + of DualStackType.Enabled: + ? setDualstack(socket, true) + ok() + of DualStackType.Disabled: + ? setDualstack(socket, false) + ok() + of DualStackType.Default: + ok() + else: + ok() + +proc setDualstack*(socket: AsyncFD, + flag: DualStackType): Result[void, OSErrorCode] = + let family = + case flag + of DualStackType.Auto: + getDomain(socket).get(AddressFamily.IPv6) + else: + ? getDomain(socket) + setDualstack(socket, family, flag) diff --git a/chronos/transports/datagram.nim b/chronos/transports/datagram.nim index af29c2a..aec18ae 100644 --- a/chronos/transports/datagram.nim +++ b/chronos/transports/datagram.nim @@ -11,7 +11,7 @@ import std/deques when not(defined(windows)): import ".."/selectors2 -import ".."/[asyncloop, osdefs, oserrno, handles] +import ".."/[asyncloop, osdefs, oserrno, osutils, handles] import "."/common type @@ -247,57 +247,65 @@ when defined(windows): udata: pointer, child: DatagramTransport, bufferSize: int, - ttl: int): DatagramTransport {. + ttl: int, + dualstack = DualStackType.Auto + ): DatagramTransport {. raises: [TransportOsError].} = - var localSock: AsyncFD - doAssert(remote.family == local.family) doAssert(not isNil(cbproc)) - doAssert(remote.family in {AddressFamily.IPv4, AddressFamily.IPv6}) - var res = if isNil(child): DatagramTransport() else: child - if sock == asyncInvalidSocket: - localSock = createAsyncSocket(local.getDomain(), SockType.SOCK_DGRAM, - Protocol.IPPROTO_UDP) - - if localSock == asyncInvalidSocket: - raiseTransportOsError(osLastError()) - else: - if not setSocketBlocking(SocketHandle(sock), false): - raiseTransportOsError(osLastError()) - localSock = sock - let bres = register2(localSock) - if bres.isErr(): - raiseTransportOsError(bres.error()) + let localSock = + if sock == asyncInvalidSocket: + let proto = + if local.family == AddressFamily.Unix: + Protocol.IPPROTO_IP + else: + Protocol.IPPROTO_UDP + let res = createAsyncSocket2(local.getDomain(), SockType.SOCK_DGRAM, + proto) + if res.isErr(): + raiseTransportOsError(res.error) + res.get() + else: + setDescriptorBlocking(SocketHandle(sock), false).isOkOr: + raiseTransportOsError(error) + register2(sock).isOkOr: + raiseTransportOsError(error) + sock ## Apply ServerFlags here if ServerFlags.ReuseAddr in flags: - if not setSockOpt(localSock, osdefs.SOL_SOCKET, osdefs.SO_REUSEADDR, 1): - let err = osLastError() + setSockOpt2(localSock, SOL_SOCKET, SO_REUSEADDR, 1).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) if ServerFlags.ReusePort in flags: - if not setSockOpt(localSock, osdefs.SOL_SOCKET, osdefs.SO_REUSEPORT, 1): - let err = osLastError() + setSockOpt2(localSock, SOL_SOCKET, SO_REUSEPORT, 1).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) if ServerFlags.Broadcast in flags: - if not setSockOpt(localSock, osdefs.SOL_SOCKET, osdefs.SO_BROADCAST, 1): - let err = osLastError() + setSockOpt2(localSock, SOL_SOCKET, SO_BROADCAST, 1).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) if ttl > 0: - if not setSockOpt(localSock, osdefs.IPPROTO_IP, osdefs.IP_TTL, ttl): - let err = osLastError() + setSockOpt2(localSock, osdefs.IPPROTO_IP, osdefs.IP_TTL, ttl).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) + + ## IPV6_V6ONLY + if sock == asyncInvalidSocket: + setDualstack(localSock, local.family, dualstack).isOkOr: + closeSocket(localSock) + raiseTransportOsError(error) + else: + setDualstack(localSock, dualstack).isOkOr: + raiseTransportOsError(error) ## Fix for Q263823. var bytesRet: DWORD @@ -457,70 +465,75 @@ else: udata: pointer, child: DatagramTransport, bufferSize: int, - ttl: int): DatagramTransport {. + ttl: int, + dualstack = DualStackType.Auto + ): DatagramTransport {. raises: [TransportOsError].} = - var localSock: AsyncFD - doAssert(remote.family == local.family) doAssert(not isNil(cbproc)) - var res = if isNil(child): DatagramTransport() else: child - if sock == asyncInvalidSocket: - let proto = - if local.family == AddressFamily.Unix: - Protocol.IPPROTO_IP - else: - Protocol.IPPROTO_UDP - localSock = createAsyncSocket(local.getDomain(), SockType.SOCK_DGRAM, - proto) - if localSock == asyncInvalidSocket: - raiseTransportOsError(osLastError()) - else: - if not setSocketBlocking(SocketHandle(sock), false): - raiseTransportOsError(osLastError()) - localSock = sock - let bres = register2(localSock) - if bres.isErr(): - raiseTransportOsError(bres.error()) + let localSock = + if sock == asyncInvalidSocket: + let proto = + if local.family == AddressFamily.Unix: + Protocol.IPPROTO_IP + else: + Protocol.IPPROTO_UDP + let res = createAsyncSocket2(local.getDomain(), SockType.SOCK_DGRAM, + proto) + if res.isErr(): + raiseTransportOsError(res.error) + res.get() + else: + setDescriptorBlocking(SocketHandle(sock), false).isOkOr: + raiseTransportOsError(error) + register2(sock).isOkOr: + raiseTransportOsError(error) + sock ## Apply ServerFlags here if ServerFlags.ReuseAddr in flags: - if not setSockOpt(localSock, osdefs.SOL_SOCKET, osdefs.SO_REUSEADDR, 1): - let err = osLastError() + setSockOpt2(localSock, SOL_SOCKET, SO_REUSEADDR, 1).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) if ServerFlags.ReusePort in flags: - if not setSockOpt(localSock, osdefs.SOL_SOCKET, osdefs.SO_REUSEPORT, 1): - let err = osLastError() + setSockOpt2(localSock, SOL_SOCKET, SO_REUSEPORT, 1).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) if ServerFlags.Broadcast in flags: - if not setSockOpt(localSock, osdefs.SOL_SOCKET, osdefs.SO_BROADCAST, 1): - let err = osLastError() + setSockOpt2(localSock, SOL_SOCKET, SO_BROADCAST, 1).isOkOr: if sock == asyncInvalidSocket: closeSocket(localSock) - raiseTransportOsError(err) + raiseTransportOsError(error) if ttl > 0: - let tres = - if local.family == AddressFamily.IPv4: - setSockOpt(localSock, osdefs.IPPROTO_IP, osdefs.IP_MULTICAST_TTL, - cint(ttl)) - elif local.family == AddressFamily.IPv6: - setSockOpt(localSock, osdefs.IPPROTO_IP, osdefs.IPV6_MULTICAST_HOPS, - cint(ttl)) - else: - raiseAssert "Unsupported address bound to local socket" + if local.family == AddressFamily.IPv4: + setSockOpt2(localSock, osdefs.IPPROTO_IP, osdefs.IP_MULTICAST_TTL, + cint(ttl)).isOkOr: + if sock == asyncInvalidSocket: + closeSocket(localSock) + raiseTransportOsError(error) + elif local.family == AddressFamily.IPv6: + setSockOpt2(localSock, osdefs.IPPROTO_IP, osdefs.IPV6_MULTICAST_HOPS, + cint(ttl)).isOkOr: + if sock == asyncInvalidSocket: + closeSocket(localSock) + raiseTransportOsError(error) + else: + raiseAssert "Unsupported address bound to local socket" - if not tres: - let err = osLastError() - if sock == asyncInvalidSocket: - closeSocket(localSock) - raiseTransportOsError(err) + ## IPV6_V6ONLY + if sock == asyncInvalidSocket: + setDualstack(localSock, local.family, dualstack).isOkOr: + closeSocket(localSock) + raiseTransportOsError(error) + else: + setDualstack(localSock, dualstack).isOkOr: + raiseTransportOsError(error) if local.family != AddressFamily.None: var saddr: Sockaddr_storage @@ -594,8 +607,9 @@ proc newDatagramTransport*(cbproc: DatagramCallback, udata: pointer = nil, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, - ttl: int = 0 - ): DatagramTransport {. + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. raises: [TransportOsError].} = ## Create new UDP datagram transport (IPv4). ## @@ -610,7 +624,7 @@ proc newDatagramTransport*(cbproc: DatagramCallback, ## ``ttl`` - TTL for UDP datagram packet (only usable when flags has ## ``Broadcast`` option). newDatagramTransportCommon(cbproc, remote, local, sock, flags, udata, child, - bufSize, ttl) + bufSize, ttl, dualstack) proc newDatagramTransport*[T](cbproc: DatagramCallback, udata: ref T, @@ -620,13 +634,15 @@ proc newDatagramTransport*[T](cbproc: DatagramCallback, flags: set[ServerFlags] = {}, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, - ttl: int = 0 - ): DatagramTransport {. + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. raises: [TransportOsError].} = var fflags = flags + {GCUserData} GC_ref(udata) newDatagramTransportCommon(cbproc, remote, local, sock, fflags, - cast[pointer](udata), child, bufSize, ttl) + cast[pointer](udata), child, bufSize, ttl, + dualstack) proc newDatagramTransport6*(cbproc: DatagramCallback, remote: TransportAddress = AnyAddress6, @@ -636,8 +652,9 @@ proc newDatagramTransport6*(cbproc: DatagramCallback, udata: pointer = nil, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, - ttl: int = 0 - ): DatagramTransport {. + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. raises: [TransportOsError].} = ## Create new UDP datagram transport (IPv6). ## @@ -652,7 +669,7 @@ proc newDatagramTransport6*(cbproc: DatagramCallback, ## ``ttl`` - TTL for UDP datagram packet (only usable when flags has ## ``Broadcast`` option). newDatagramTransportCommon(cbproc, remote, local, sock, flags, udata, child, - bufSize, ttl) + bufSize, ttl, dualstack) proc newDatagramTransport6*[T](cbproc: DatagramCallback, udata: ref T, @@ -662,13 +679,15 @@ proc newDatagramTransport6*[T](cbproc: DatagramCallback, flags: set[ServerFlags] = {}, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, - ttl: int = 0 - ): DatagramTransport {. + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. raises: [TransportOsError].} = var fflags = flags + {GCUserData} GC_ref(udata) newDatagramTransportCommon(cbproc, remote, local, sock, fflags, - cast[pointer](udata), child, bufSize, ttl) + cast[pointer](udata), child, bufSize, ttl, + dualstack) proc join*(transp: DatagramTransport): Future[void] = ## Wait until the transport ``transp`` will be closed. diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index f96650c..8982b99 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -639,7 +639,8 @@ when defined(windows): child: StreamTransport = nil, localAddress = TransportAddress(), flags: set[SocketFlags] = {}, - ): Future[StreamTransport] = + dualstack = DualStackType.Auto + ): Future[StreamTransport] = ## Open new connection to remote peer with address ``address`` and create ## new transport object ``StreamTransport`` for established connection. ## ``bufferSize`` is size of internal buffer for transport. @@ -658,24 +659,33 @@ when defined(windows): toSAddr(raddress, saddr, slen) proto = Protocol.IPPROTO_TCP - sock = createAsyncSocket(raddress.getDomain(), SockType.SOCK_STREAM, - proto) - if sock == asyncInvalidSocket: - retFuture.fail(getTransportOsError(osLastError())) + sock = createAsyncSocket2(raddress.getDomain(), SockType.SOCK_STREAM, + proto).valueOr: + retFuture.fail(getTransportOsError(error)) return retFuture + if address.family in {AddressFamily.IPv4, AddressFamily.IPv6}: + if SocketFlags.TcpNoDelay in flags: + setSockOpt2(sock, osdefs.IPPROTO_TCP, osdefs.TCP_NODELAY, 1).isOkOr: + sock.closeSocket() + retFuture.fail(getTransportOsError(error)) + return retFuture + if SocketFlags.ReuseAddr in flags: - if not(setSockOpt(sock, SOL_SOCKET, SO_REUSEADDR, 1)): - let err = osLastError() + setSockOpt2(sock, SOL_SOCKET, SO_REUSEADDR, 1).isOkOr: sock.closeSocket() - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture if SocketFlags.ReusePort in flags: - if not(setSockOpt(sock, SOL_SOCKET, SO_REUSEPORT, 1)): - let err = osLastError() + setSockOpt2(sock, SOL_SOCKET, SO_REUSEPORT, 1).isOkOr: sock.closeSocket() - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture + # IPV6_V6ONLY. + setDualstack(sock, address.family, dualstack).isOkOr: + sock.closeSocket() + retFuture.fail(getTransportOsError(error)) + return retFuture if localAddress != TransportAddress(): if localAddress.family != address.family: @@ -966,14 +976,9 @@ when defined(windows): if server.status notin {ServerStatus.Stopped, ServerStatus.Closed}: server.apending = true # TODO No way to report back errors! - server.asock = - block: - let sock = createAsyncSocket(server.domain, SockType.SOCK_STREAM, - Protocol.IPPROTO_TCP) - if sock == asyncInvalidSocket: - raiseOsDefect(osLastError(), - "acceptLoop(): Unablet to create new socket") - sock + server.asock = createAsyncSocket2(server.domain, SockType.SOCK_STREAM, + Protocol.IPPROTO_TCP).valueOr: + raiseOsDefect(error, "acceptLoop(): Unablet to create new socket") var dwBytesReceived = DWORD(0) let dwReceiveDataLength = DWORD(0) @@ -1167,15 +1172,13 @@ when defined(windows): if server.local.family in {AddressFamily.IPv4, AddressFamily.IPv6}: # TCP Sockets part var loop = getThreadDispatcher() - server.asock = createAsyncSocket(server.domain, SockType.SOCK_STREAM, - Protocol.IPPROTO_TCP) - if server.asock == asyncInvalidSocket: - let err = osLastError() - case err + server.asock = createAsyncSocket2(server.domain, SockType.SOCK_STREAM, + Protocol.IPPROTO_TCP).valueOr: + case error of ERROR_TOO_MANY_OPEN_FILES, WSAENOBUFS, WSAEMFILE: - retFuture.fail(getTransportTooManyError(err)) + retFuture.fail(getTransportTooManyError(error)) else: - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture var dwBytesReceived = DWORD(0) @@ -1468,7 +1471,8 @@ else: child: StreamTransport = nil, localAddress = TransportAddress(), flags: set[SocketFlags] = {}, - ): Future[StreamTransport] = + dualstack = DualStackType.Auto, + ): Future[StreamTransport] = ## Open new connection to remote peer with address ``address`` and create ## new transport object ``StreamTransport`` for established connection. ## ``bufferSize`` - size of internal buffer for transport. @@ -1483,36 +1487,37 @@ else: else: Protocol.IPPROTO_TCP - let sock = createAsyncSocket(address.getDomain(), SockType.SOCK_STREAM, - proto) - if sock == asyncInvalidSocket: - let err = osLastError() - case err + let sock = createAsyncSocket2(address.getDomain(), SockType.SOCK_STREAM, + proto).valueOr: + case error of oserrno.EMFILE: retFuture.fail(getTransportTooManyError()) else: - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture if address.family in {AddressFamily.IPv4, AddressFamily.IPv6}: if SocketFlags.TcpNoDelay in flags: - if not(setSockOpt(sock, osdefs.IPPROTO_TCP, osdefs.TCP_NODELAY, 1)): - let err = osLastError() + setSockOpt2(sock, osdefs.IPPROTO_TCP, osdefs.TCP_NODELAY, 1).isOkOr: sock.closeSocket() - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture + if SocketFlags.ReuseAddr in flags: - if not(setSockOpt(sock, SOL_SOCKET, SO_REUSEADDR, 1)): - let err = osLastError() + setSockOpt2(sock, SOL_SOCKET, SO_REUSEADDR, 1).isOkOr: sock.closeSocket() - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture if SocketFlags.ReusePort in flags: - if not(setSockOpt(sock, SOL_SOCKET, SO_REUSEPORT, 1)): - let err = osLastError() + setSockOpt2(sock, SOL_SOCKET, SO_REUSEPORT, 1).isOkOr: sock.closeSocket() - retFuture.fail(getTransportOsError(err)) + retFuture.fail(getTransportOsError(error)) return retFuture + # IPV6_V6ONLY. + setDualstack(sock, address.family, dualstack).isOkOr: + sock.closeSocket() + retFuture.fail(getTransportOsError(error)) + return retFuture if localAddress != TransportAddress(): if localAddress.family != address.family: @@ -1532,17 +1537,14 @@ else: proc continuation(udata: pointer) = if not(retFuture.finished()): - var err = 0 - - let res = removeWriter2(sock) - if res.isErr(): + removeWriter2(sock).isOkOr: discard unregisterAndCloseFd(sock) - retFuture.fail(getTransportOsError(res.error())) + retFuture.fail(getTransportOsError(error)) return - if not(sock.getSocketError(err)): + let err = sock.getSocketError2().valueOr: discard unregisterAndCloseFd(sock) - retFuture.fail(getTransportOsError(res.error())) + retFuture.fail(getTransportOsError(error)) return if err != 0: @@ -1578,10 +1580,9 @@ else: # http://www.madore.org/~david/computers/connect-intr.html case errorCode of oserrno.EINPROGRESS, oserrno.EINTR: - let res = addWriter2(sock, continuation) - if res.isErr(): + addWriter2(sock, continuation).isOkOr: discard unregisterAndCloseFd(sock) - retFuture.fail(getTransportOsError(res.error())) + retFuture.fail(getTransportOsError(error)) return retFuture retFuture.cancelCallback = cancel break @@ -1782,11 +1783,13 @@ proc connect*(address: TransportAddress, bufferSize = DefaultStreamBufferSize, child: StreamTransport = nil, flags: set[TransportFlags], - localAddress = TransportAddress()): Future[StreamTransport] = + localAddress = TransportAddress(), + dualstack = DualStackType.Auto + ): Future[StreamTransport] = # Retro compatibility with TransportFlags var mappedFlags: set[SocketFlags] if TcpNoDelay in flags: mappedFlags.incl(SocketFlags.TcpNoDelay) - address.connect(bufferSize, child, localAddress, mappedFlags) + connect(address, bufferSize, child, localAddress, mappedFlags, dualstack) proc close*(server: StreamServer) = ## Release ``server`` resources. @@ -1848,7 +1851,8 @@ proc createStreamServer*(host: TransportAddress, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil, - udata: pointer = nil): StreamServer {. + udata: pointer = nil, + dualstack = DualStackType.Auto): StreamServer {. raises: [TransportOsError].} = ## Create new TCP stream server. ## @@ -1874,42 +1878,48 @@ proc createStreamServer*(host: TransportAddress, elif defined(windows): # Windows if host.family in {AddressFamily.IPv4, AddressFamily.IPv6}: - if sock == asyncInvalidSocket: - serverSocket = createAsyncSocket(host.getDomain(), - SockType.SOCK_STREAM, - Protocol.IPPROTO_TCP) - - if serverSocket == asyncInvalidSocket: - raiseTransportOsError(osLastError()) - else: - let bres = setDescriptorBlocking(SocketHandle(sock), false) - if bres.isErr(): - raiseTransportOsError(bres.error()) - let wres = register2(sock) - if wres.isErr(): - raiseTransportOsError(wres.error()) - serverSocket = sock - # SO_REUSEADDR is not useful for Unix domain sockets. + serverSocket = + if sock == asyncInvalidSocket: + # TODO (cheatfate): `valueOr` generates weird compile error. + let res = createAsyncSocket2(host.getDomain(), SockType.SOCK_STREAM, + Protocol.IPPROTO_TCP) + if res.isErr(): + raiseTransportOsError(res.error()) + res.get() + else: + setDescriptorBlocking(SocketHandle(sock), false).isOkOr: + raiseTransportOsError(error) + register2(sock).isOkOr: + raiseTransportOsError(error) + sock + # SO_REUSEADDR if ServerFlags.ReuseAddr in flags: - if not(setSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, 1)): - let err = osLastError() + setSockOpt2(serverSocket, SOL_SOCKET, SO_REUSEADDR, 1).isOkOr: if sock == asyncInvalidSocket: discard closeFd(SocketHandle(serverSocket)) - raiseTransportOsError(err) + raiseTransportOsError(error) + # SO_REUSEPORT if ServerFlags.ReusePort in flags: - if not(setSockOpt(serverSocket, SOL_SOCKET, SO_REUSEPORT, 1)): - let err = osLastError() + setSockOpt2(serverSocket, SOL_SOCKET, SO_REUSEPORT, 1).isOkOr: if sock == asyncInvalidSocket: discard closeFd(SocketHandle(serverSocket)) - raiseTransportOsError(err) - # TCP flags are not useful for Unix domain sockets. + raiseTransportOsError(error) + # TCP_NODELAY if ServerFlags.TcpNoDelay in flags: - if not(setSockOpt(serverSocket, osdefs.IPPROTO_TCP, - osdefs.TCP_NODELAY, 1)): - let err = osLastError() + setSockOpt2(serverSocket, osdefs.IPPROTO_TCP, + osdefs.TCP_NODELAY, 1).isOkOr: if sock == asyncInvalidSocket: discard closeFd(SocketHandle(serverSocket)) - raiseTransportOsError(err) + raiseTransportOsError(error) + # IPV6_V6ONLY. + if sock == asyncInvalidSocket: + setDualstack(serverSocket, host.family, dualstack).isOkOr: + discard closeFd(SocketHandle(serverSocket)) + raiseTransportOsError(error) + else: + setDualstack(serverSocket, dualstack).isOkOr: + raiseTransportOsError(error) + host.toSAddr(saddr, slen) if bindSocket(SocketHandle(serverSocket), cast[ptr SockAddr](addr saddr), slen) != 0: @@ -1936,47 +1946,54 @@ proc createStreamServer*(host: TransportAddress, serverSocket = AsyncFD(0) else: # Posix - if sock == asyncInvalidSocket: - let proto = if host.family == AddressFamily.Unix: - Protocol.IPPROTO_IP + serverSocket = + if sock == asyncInvalidSocket: + let proto = if host.family == AddressFamily.Unix: + Protocol.IPPROTO_IP + else: + Protocol.IPPROTO_TCP + # TODO (cheatfate): `valueOr` generates weird compile error. + let res = createAsyncSocket2(host.getDomain(), SockType.SOCK_STREAM, + proto) + if res.isErr(): + raiseTransportOsError(res.error()) + res.get() else: - Protocol.IPPROTO_TCP - serverSocket = createAsyncSocket(host.getDomain(), - SockType.SOCK_STREAM, - proto) - if serverSocket == asyncInvalidSocket: - raiseTransportOsError(osLastError()) - else: - let bres = setDescriptorFlags(cint(sock), true, true) - if bres.isErr(): - raiseTransportOsError(osLastError()) - let rres = register2(sock) - if rres.isErr(): - raiseTransportOsError(osLastError()) - serverSocket = sock + setDescriptorFlags(cint(sock), true, true).isOkOr: + raiseTransportOsError(error) + register2(sock).isOkOr: + raiseTransportOsError(error) + sock if host.family in {AddressFamily.IPv4, AddressFamily.IPv6}: - # SO_REUSEADDR and SO_REUSEPORT are not useful for Unix domain sockets. + # SO_REUSEADDR if ServerFlags.ReuseAddr in flags: - if not(setSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, 1)): - let err = osLastError() + setSockOpt2(serverSocket, SOL_SOCKET, SO_REUSEADDR, 1).isOkOr: if sock == asyncInvalidSocket: discard unregisterAndCloseFd(serverSocket) - raiseTransportOsError(err) + raiseTransportOsError(error) + # SO_REUSEPORT if ServerFlags.ReusePort in flags: - if not(setSockOpt(serverSocket, SOL_SOCKET, SO_REUSEPORT, 1)): - let err = osLastError() + setSockOpt2(serverSocket, SOL_SOCKET, SO_REUSEPORT, 1).isOkOr: if sock == asyncInvalidSocket: discard unregisterAndCloseFd(serverSocket) - raiseTransportOsError(err) - # TCP flags are not useful for Unix domain sockets. + raiseTransportOsError(error) + # TCP_NODELAY if ServerFlags.TcpNoDelay in flags: - if not(setSockOpt(serverSocket, osdefs.IPPROTO_TCP, - osdefs.TCP_NODELAY, 1)): - let err = osLastError() + setSockOpt2(serverSocket, osdefs.IPPROTO_TCP, + osdefs.TCP_NODELAY, 1).isOkOr: if sock == asyncInvalidSocket: discard unregisterAndCloseFd(serverSocket) - raiseTransportOsError(err) + raiseTransportOsError(error) + # IPV6_V6ONLY + if sock == asyncInvalidSocket: + setDualstack(serverSocket, host.family, dualstack).isOkOr: + discard closeFd(SocketHandle(serverSocket)) + raiseTransportOsError(error) + else: + setDualstack(serverSocket, dualstack).isOkOr: + raiseTransportOsError(error) + elif host.family in {AddressFamily.Unix}: # We do not care about result here, because if file cannot be removed, # `bindSocket` will return EADDRINUSE. @@ -2016,6 +2033,7 @@ proc createStreamServer*(host: TransportAddress, sres.status = Starting sres.loopFuture = newFuture[void]("stream.transport.server") sres.udata = udata + sres.dualstack = dualstack if localAddress.family == AddressFamily.None: sres.local = host else: @@ -2029,8 +2047,7 @@ proc createStreamServer*(host: TransportAddress, cb = acceptPipeLoop if not(isNil(cbproc)): - sres.aovl.data = CompletionData(cb: cb, - udata: cast[pointer](sres)) + sres.aovl.data = CompletionData(cb: cb, udata: cast[pointer](sres)) else: if host.family == AddressFamily.Unix: sres.sock = @@ -2055,10 +2072,11 @@ proc createStreamServer*(host: TransportAddress, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil, - udata: pointer = nil): StreamServer {. + udata: pointer = nil, + dualstack = DualStackType.Auto): StreamServer {. raises: [CatchableError].} = createStreamServer(host, nil, flags, sock, backlog, bufferSize, - child, init, cast[pointer](udata)) + child, init, cast[pointer](udata), dualstack) proc createStreamServer*[T](host: TransportAddress, cbproc: StreamCallback, @@ -2068,12 +2086,13 @@ proc createStreamServer*[T](host: TransportAddress, backlog: int = DefaultBacklogSize, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, - init: TransportInitCallback = nil): StreamServer {. + init: TransportInitCallback = nil, + dualstack = DualStackType.Auto): StreamServer {. raises: [CatchableError].} = var fflags = flags + {GCUserData} GC_ref(udata) createStreamServer(host, cbproc, fflags, sock, backlog, bufferSize, - child, init, cast[pointer](udata)) + child, init, cast[pointer](udata), dualstack) proc createStreamServer*[T](host: TransportAddress, flags: set[ServerFlags] = {}, @@ -2082,12 +2101,13 @@ proc createStreamServer*[T](host: TransportAddress, backlog: int = DefaultBacklogSize, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, - init: TransportInitCallback = nil): StreamServer {. + init: TransportInitCallback = nil, + dualstack = DualStackType.Auto): StreamServer {. raises: [CatchableError].} = var fflags = flags + {GCUserData} GC_ref(udata) createStreamServer(host, nil, fflags, sock, backlog, bufferSize, - child, init, cast[pointer](udata)) + child, init, cast[pointer](udata), dualstack) proc getUserData*[T](server: StreamServer): T {.inline.} = ## Obtain user data stored in ``server`` object. diff --git a/tests/testdatagram.nim b/tests/testdatagram.nim index 7db04f9..ae7ab23 100644 --- a/tests/testdatagram.nim +++ b/tests/testdatagram.nim @@ -533,6 +533,54 @@ suite "Datagram Transport test suite": result = res + proc performDualstackTest( + sstack: DualStackType, saddr: TransportAddress, + cstack: DualStackType, caddr: TransportAddress + ): Future[bool] {.async.} = + var + expectStr = "ANYADDRESS MESSAGE" + event = newAsyncEvent() + res = 0 + + proc process1(transp: DatagramTransport, + raddr: TransportAddress): Future[void] {.async.} = + var bmsg = transp.getMessage() + var smsg = cast[string](bmsg) + if smsg == expectStr: + inc(res) + event.fire() + + proc process2(transp: DatagramTransport, + raddr: TransportAddress): Future[void] {.async.} = + discard + + let + sdgram = newDatagramTransport(process1, local = saddr, + dualstack = sstack) + localcaddr = + if caddr.family == AddressFamily.IPv4: + AnyAddress + else: + AnyAddress6 + + cdgram = newDatagramTransport(process2, local = localcaddr, + dualstack = cstack) + + var address = caddr + address.port = sdgram.localAddress().port + + try: + await cdgram.sendTo(address, addr expectStr[0], len(expectStr)) + except CatchableError: + discard + try: + await event.wait().wait(500.milliseconds) + except CatchableError: + discard + + await allFutures(sdgram.closeWait(), cdgram.closeWait()) + res == 1 + test "close(transport) test": check waitFor(testTransportClose()) == true test m1: @@ -557,5 +605,83 @@ suite "Datagram Transport test suite": check waitFor(testBroadcast()) == 1 test "0.0.0.0/::0 (INADDR_ANY) test": check waitFor(testAnyAddress()) == 6 + asyncTest "[IP] getDomain(socket) [SOCK_DGRAM] test": + if isAvailable(AddressFamily.IPv4) and isAvailable(AddressFamily.IPv6): + block: + let res = createAsyncSocket2(Domain.AF_INET, SockType.SOCK_DGRAM, + Protocol.IPPROTO_UDP) + check res.isOk() + let fres = getDomain(res.get()) + check fres.isOk() + discard unregisterAndCloseFd(res.get()) + check fres.get() == AddressFamily.IPv4 + + block: + let res = createAsyncSocket2(Domain.AF_INET6, SockType.SOCK_DGRAM, + Protocol.IPPROTO_UDP) + check res.isOk() + let fres = getDomain(res.get()) + check fres.isOk() + discard unregisterAndCloseFd(res.get()) + check fres.get() == AddressFamily.IPv6 + + when not(defined(windows)): + block: + let res = createAsyncSocket2(Domain.AF_UNIX, SockType.SOCK_DGRAM, + Protocol.IPPROTO_IP) + check res.isOk() + let fres = getDomain(res.get()) + check fres.isOk() + discard unregisterAndCloseFd(res.get()) + check fres.get() == AddressFamily.Unix + else: + skip() + asyncTest "[IP] DualStack [UDP] server [DualStackType.Auto] test": + if isAvailable(AddressFamily.IPv4) and isAvailable(AddressFamily.IPv6): + let serverAddress = initTAddress("[::]:0") + check: + (await performDualstackTest( + DualStackType.Auto, serverAddress, + DualStackType.Auto, initTAddress("127.0.0.1:0"))) == true + check: + (await performDualstackTest( + DualStackType.Auto, serverAddress, + DualStackType.Auto, initTAddress("127.0.0.1:0").toIPv6())) == true + check: + (await performDualstackTest( + DualStackType.Auto, serverAddress, + DualStackType.Auto, initTAddress("[::1]:0"))) == true + else: + skip() + asyncTest "[IP] DualStack [UDP] server [DualStackType.Enabled] test": + if isAvailable(AddressFamily.IPv4) and isAvailable(AddressFamily.IPv6): + let serverAddress = initTAddress("[::]:0") + check: + (await performDualstackTest( + DualStackType.Enabled, serverAddress, + DualStackType.Auto, initTAddress("127.0.0.1:0"))) == true + (await performDualstackTest( + DualStackType.Enabled, serverAddress, + DualStackType.Auto, initTAddress("127.0.0.1:0").toIPv6())) == true + (await performDualstackTest( + DualStackType.Enabled, serverAddress, + DualStackType.Auto, initTAddress("[::1]:0"))) == true + else: + skip() + asyncTest "[IP] DualStack [UDP] server [DualStackType.Disabled] test": + if isAvailable(AddressFamily.IPv4) and isAvailable(AddressFamily.IPv6): + let serverAddress = initTAddress("[::]:0") + check: + (await performDualstackTest( + DualStackType.Disabled, serverAddress, + DualStackType.Auto, initTAddress("127.0.0.1:0"))) == false + (await performDualstackTest( + DualStackType.Disabled, serverAddress, + DualStackType.Auto, initTAddress("127.0.0.1:0").toIPv6())) == false + (await performDualstackTest( + DualStackType.Disabled, serverAddress, + DualStackType.Auto, initTAddress("[::1]:0"))) == true + else: + skip() test "Transports leak test": checkLeaks() diff --git a/tests/teststream.nim b/tests/teststream.nim index 762e996..b042792 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -1372,6 +1372,42 @@ suite "Stream Transport test suite": if not(sleepFut.finished()): await cancelAndWait(sleepFut) + proc performDualstackTest( + sstack: DualStackType, saddr: TransportAddress, + cstack: DualStackType, caddr: TransportAddress + ): Future[bool] {.async.} = + let server = createStreamServer(saddr, dualstack = sstack) + var address = caddr + address.port = server.localAddress().port + var acceptFut = server.accept() + let + clientTransp = + try: + let res = await connect(address, + dualstack = cstack).wait(500.milliseconds) + Opt.some(res) + except CatchableError: + Opt.none(StreamTransport) + serverTransp = + if clientTransp.isSome(): + let res = await acceptFut + Opt.some(res) + else: + Opt.none(StreamTransport) + + let testResult = clientTransp.isSome() and serverTransp.isSome() + var pending: seq[FutureBase] + if clientTransp.isSome(): + pending.add(closeWait(clientTransp.get())) + if serverTransp.isSome(): + pending.add(closeWait(serverTransp.get())) + else: + pending.add(cancelAndWait(acceptFut)) + await allFutures(pending) + server.stop() + await server.closeWait() + testResult + markFD = getCurrentFD() for i in 0.. Date: Tue, 31 Oct 2023 03:43:58 +0200 Subject: [PATCH 29/50] Consider ERROR_NETNAME_DELETED as ConnectionAbortedError. (#460) --- chronos/transports/stream.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 8982b99..7471a44 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -1096,7 +1096,7 @@ when defined(windows): retFuture.fail(getServerUseClosedError()) server.clean() of WSAENETDOWN, WSAENETRESET, WSAECONNABORTED, WSAECONNRESET, - WSAETIMEDOUT: + WSAETIMEDOUT, ERROR_NETNAME_DELETED: server.asock.closeSocket() retFuture.fail(getConnectionAbortedError(ovl.data.errCode)) server.clean() From cd6369c0488e1bc1dd6b6ce2fbc3b372a8adb74f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 7 Nov 2023 11:12:59 +0100 Subject: [PATCH 30/50] `asyncraises` -> `async: (raises: ..., raw: ...)` (#455) Per discussion in https://github.com/status-im/nim-chronos/pull/251#issuecomment-1559233139, `async: (parameters..)` is introduced as a way to customize the async transformation instead of relying on separate keywords (like asyncraises). Two parameters are available as of now: `raises`: controls the exception effect tracking `raw`: disables body transformation Parameters are added to `async` as a tuple allowing more params to be added easily in the future: ```nim: proc f() {.async: (name: value, ...).}` ``` --- README.md | 110 +++++++++------- chronos/asyncloop.nim | 2 +- chronos/internal/asyncengine.nim | 2 +- chronos/internal/asyncfutures.nim | 43 +++---- chronos/internal/asyncmacro.nim | 195 +++++++++++++++-------------- chronos/internal/raisesfutures.nim | 28 +++-- tests/testmacro.nim | 64 +++++----- 7 files changed, 239 insertions(+), 205 deletions(-) diff --git a/README.md b/README.md index 495f9f8..f59a6c8 100644 --- a/README.md +++ b/README.md @@ -46,18 +46,18 @@ Submit a PR to add yours! ### Concepts -Chronos implements the async/await paradigm in a self-contained library, using -macros, with no specific helpers from the compiler. +Chronos implements the async/await paradigm in a self-contained library using +the macro and closure iterator transformation features provided by Nim. -Our event loop is called a "dispatcher" and a single instance per thread is +The event loop is called a "dispatcher" and a single instance per thread is created, as soon as one is needed. To trigger a dispatcher's processing step, we need to call `poll()` - either -directly or through a wrapper like `runForever()` or `waitFor()`. This step +directly or through a wrapper like `runForever()` or `waitFor()`. Each step handles any file descriptors, timers and callbacks that are ready to be processed. -`Future` objects encapsulate the result of an async procedure, upon successful +`Future` objects encapsulate the result of an `async` procedure upon successful completion, and a list of callbacks to be scheduled after any type of completion - be that success, failure or cancellation. @@ -156,7 +156,7 @@ Exceptions inheriting from `CatchableError` are caught by hidden `try` blocks and placed in the `Future.error` field, changing the future's status to `Failed`. -When a future is awaited, that exception is re-raised, only to be caught again +When a future is awaited, that exception is re-raised only to be caught again by a hidden `try` block in the calling async procedure. That's how these exceptions move up the async chain. @@ -214,57 +214,81 @@ by the transformation. #### Checked exceptions -By specifying a `asyncraises` list to an async procedure, you can check which -exceptions can be thrown by it. +By specifying a `raises` list to an async procedure, you can check which +exceptions can be raised by it: + ```nim -proc p1(): Future[void] {.async, asyncraises: [IOError].} = +proc p1(): Future[void] {.async: (raises: [IOError]).} = assert not (compiles do: raise newException(ValueError, "uh-uh")) raise newException(IOError, "works") # Or any child of IOError -``` -Under the hood, the return type of `p1` will be rewritten to an internal type, -which will convey raises informations to `await`. - -```nim -proc p2(): Future[void] {.async, asyncraises: [IOError].} = +proc p2(): Future[void] {.async, (raises: [IOError]).} = await p1() # Works, because await knows that p1 # can only raise IOError ``` -Raw functions and callbacks that don't go through the `async` transformation but -still return a `Future` and interact with the rest of the framework also need to -be annotated with `asyncraises` to participate in the checked exception scheme: +Under the hood, the return type of `p1` will be rewritten to an internal type +which will convey raises informations to `await`. + +### Raw functions + +Raw functions are those that interact with `chronos` via the `Future` type but +whose body does not go through the async transformation. + +Such functions are created by adding `raw: true` to the `async` parameters: ```nim -proc p3(): Future[void] {.async, asyncraises: [IOError].} = - let fut: Future[void] = p1() # works - assert not compiles(await fut) # await lost informations about raises, - # so it can raise anything - # Callbacks - assert not(compiles do: let cb1: proc(): Future[void] = p1) # doesn't work - let cb2: proc(): Future[void] {.async, asyncraises: [IOError].} = p1 # works - assert not(compiles do: - type c = proc(): Future[void] {.async, asyncraises: [IOError, ValueError].} - let cb3: c = p1 # doesn't work, the raises must match _exactly_ - ) +proc rawAsync(): Future[void] {.async: (raw: true).} = + let future = newFuture[void]("rawAsync") + future.complete() + return future ``` -When `chronos` performs the `async` transformation, all code is placed in a -a special `try/except` clause that re-routes exception handling to the `Future`. - -Beacuse of this re-routing, functions that return a `Future` instance manually -never directly raise exceptions themselves - instead, exceptions are handled -indirectly via `await` or `Future.read`. When writing raw async functions, they -too must not raise exceptions - instead, they must store exceptions in the -future they return: +Raw functions must not raise exceptions directly - they are implicitly declared +as `raises: []` - instead they should store exceptions in the returned `Future`: ```nim -proc p4(): Future[void] {.asyncraises: [ValueError].} = - let fut = newFuture[void] +proc rawFailure(): Future[void] {.async: (raw: true).} = + let future = newFuture[void]("rawAsync") + future.fail((ref ValueError)(msg: "Oh no!")) + return future +``` - # Equivalent of `raise (ref ValueError)()` in raw async functions: - fut.fail((ref ValueError)(msg: "raising in raw async function")) - fut +Raw functions can also use checked exceptions: + +```nim +proc rawAsyncRaises(): Future[void] {.async: (raw: true, raises: [IOError]).} = + let fut = newFuture[void]() + assert not (compiles do: fut.fail((ref ValueError)(msg: "uh-uh"))) + fut.fail((ref IOError)(msg: "IO")) + return fut +``` + +### Callbacks and closures + +Callback/closure types are declared using the `async` annotation as usual: + +```nim +type MyCallback = proc(): Future[void] {.async.} + +proc runCallback(cb: MyCallback) {.async: (raises: []).} = + try: + await cb() + except CatchableError: + discard # handle errors as usual +``` + +When calling a callback, it is important to remember that the given function +may raise and exceptions need to be handled. + +Checked exceptions can be used to limit the exceptions that a callback can +raise: + +```nim +type MyEasyCallback = proc: Future[void] {.async: (raises: []).} + +proc runCallback(cb: MyEasyCallback) {.async: (raises: [])} = + await cb() ``` ### Platform independence @@ -278,7 +302,7 @@ annotated as raising `CatchableError` only raise on _some_ platforms - in order to work on all platforms, calling code must assume that they will raise even when they don't seem to do so on one platform. -### Exception effects +### Strict exception mode `chronos` currently offers minimal support for exception effects and `raises` annotations. In general, during the `async` transformation, a generic diff --git a/chronos/asyncloop.nim b/chronos/asyncloop.nim index b4d48af..428252c 100644 --- a/chronos/asyncloop.nim +++ b/chronos/asyncloop.nim @@ -131,4 +131,4 @@ import ./internal/[asyncengine, asyncfutures, asyncmacro, errors] export asyncfutures, asyncengine, errors -export asyncmacro.async, asyncmacro.await, asyncmacro.awaitne, asyncraises +export asyncmacro.async, asyncmacro.await, asyncmacro.awaitne diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim index ebcc278..23d7c6a 100644 --- a/chronos/internal/asyncengine.nim +++ b/chronos/internal/asyncengine.nim @@ -21,7 +21,7 @@ export Port export deques, errors, futures, timer, results export - asyncmacro.async, asyncmacro.await, asyncmacro.awaitne, asyncmacro.asyncraises + asyncmacro.async, asyncmacro.await, asyncmacro.awaitne const MaxEventsCount* = 64 diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index abf28c7..b144cea 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -102,18 +102,12 @@ template newFuture*[T](fromProc: static[string] = "", else: newFutureImpl[T](getSrcLocation(fromProc), flags) -macro getFutureExceptions(T: typedesc): untyped = - if getTypeInst(T)[1].len > 2: - getTypeInst(T)[1][2] - else: - ident"void" - -template newInternalRaisesFuture*[T](fromProc: static[string] = ""): auto = +template newInternalRaisesFuture*[T, E](fromProc: static[string] = ""): auto = ## Creates a new future. ## ## Specifying ``fromProc``, which is a string specifying the name of the proc ## that this future belongs to, is a good habit as it helps with debugging. - newInternalRaisesFutureImpl[T, getFutureExceptions(typeof(result))](getSrcLocation(fromProc)) + newInternalRaisesFutureImpl[T, E](getSrcLocation(fromProc)) template newFutureSeq*[A, B](fromProc: static[string] = ""): FutureSeq[A, B] = ## Create a new future which can hold/preserve GC sequence until future will @@ -476,7 +470,7 @@ macro internalCheckComplete*(f: InternalRaisesFuture): untyped = let e = getTypeInst(f)[2] let types = getType(e) - if types.eqIdent("void"): + if isNoRaises(types): return quote do: if not(isNil(`f`.internalError)): raiseAssert("Unhandled future exception: " & `f`.error.msg) @@ -484,7 +478,6 @@ macro internalCheckComplete*(f: InternalRaisesFuture): untyped = expectKind(types, nnkBracketExpr) expectKind(types[0], nnkSym) assert types[0].strVal == "tuple" - assert types.len > 1 let ifRaise = nnkIfExpr.newTree( nnkElifExpr.newTree( @@ -914,7 +907,7 @@ template cancel*(future: FutureBase) {. cancelSoon(future, nil, nil, getSrcLocation()) proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Perform cancellation ``future`` return Future which will be completed when ## ``future`` become finished (completed with value, failed or cancelled). ## @@ -938,7 +931,7 @@ template cancelAndWait*(future: FutureBase): Future[void] = ## Cancel ``future``. cancelAndWait(future, getSrcLocation()) -proc noCancel*[F: SomeFuture](future: F): auto = # asyncraises: asyncraiseOf(future) - CancelledError +proc noCancel*[F: SomeFuture](future: F): auto = # async: (raw: true, raises: asyncraiseOf(future) - CancelledError ## Prevent cancellation requests from propagating to ``future`` while ## forwarding its value or error when it finishes. ## @@ -978,7 +971,7 @@ proc noCancel*[F: SomeFuture](future: F): auto = # asyncraises: asyncraiseOf(fut retFuture proc allFutures*(futs: varargs[FutureBase]): Future[void] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Returns a future which will complete only when all futures in ``futs`` ## will be completed, failed or canceled. ## @@ -1017,7 +1010,7 @@ proc allFutures*(futs: varargs[FutureBase]): Future[void] {. retFuture proc allFutures*[T](futs: varargs[Future[T]]): Future[void] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Returns a future which will complete only when all futures in ``futs`` ## will be completed, failed or canceled. ## @@ -1031,7 +1024,7 @@ proc allFutures*[T](futs: varargs[Future[T]]): Future[void] {. allFutures(nfuts) proc allFinished*[F: SomeFuture](futs: varargs[F]): Future[seq[F]] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Returns a future which will complete only when all futures in ``futs`` ## will be completed, failed or canceled. ## @@ -1072,7 +1065,7 @@ proc allFinished*[F: SomeFuture](futs: varargs[F]): Future[seq[F]] {. return retFuture proc one*[F: SomeFuture](futs: varargs[F]): Future[F] {. - asyncraises: [ValueError, CancelledError].} = + async: (raw: true, raises: [ValueError, CancelledError]).} = ## Returns a future which will complete and return completed Future[T] inside, ## when one of the futures in ``futs`` will be completed, failed or canceled. ## @@ -1121,7 +1114,7 @@ proc one*[F: SomeFuture](futs: varargs[F]): Future[F] {. return retFuture proc race*(futs: varargs[FutureBase]): Future[FutureBase] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Returns a future which will complete and return completed FutureBase, ## when one of the futures in ``futs`` will be completed, failed or canceled. ## @@ -1173,7 +1166,8 @@ proc race*(futs: varargs[FutureBase]): Future[FutureBase] {. when (chronosEventEngine in ["epoll", "kqueue"]) or defined(windows): import std/os - proc waitSignal*(signal: int): Future[void] {.asyncraises: [AsyncError, CancelledError].} = + proc waitSignal*(signal: int): Future[void] {. + async: (raw: true, raises: [AsyncError, CancelledError]).} = var retFuture = newFuture[void]("chronos.waitSignal()") var signalHandle: Opt[SignalHandle] @@ -1208,7 +1202,7 @@ when (chronosEventEngine in ["epoll", "kqueue"]) or defined(windows): retFuture proc sleepAsync*(duration: Duration): Future[void] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Suspends the execution of the current async procedure for the next ## ``duration`` time. var retFuture = newFuture[void]("chronos.sleepAsync(Duration)") @@ -1228,10 +1222,12 @@ proc sleepAsync*(duration: Duration): Future[void] {. return retFuture proc sleepAsync*(ms: int): Future[void] {. - inline, deprecated: "Use sleepAsync(Duration)", asyncraises: [CancelledError].} = + inline, deprecated: "Use sleepAsync(Duration)", + async: (raw: true, raises: [CancelledError]).} = result = sleepAsync(ms.milliseconds()) -proc stepsAsync*(number: int): Future[void] {.asyncraises: [CancelledError].} = +proc stepsAsync*(number: int): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Suspends the execution of the current async procedure for the next ## ``number`` of asynchronous steps (``poll()`` calls). ## @@ -1258,7 +1254,8 @@ proc stepsAsync*(number: int): Future[void] {.asyncraises: [CancelledError].} = retFuture -proc idleAsync*(): Future[void] {.asyncraises: [CancelledError].} = +proc idleAsync*(): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Suspends the execution of the current asynchronous task until "idle" time. ## ## "idle" time its moment of time, when no network events were processed by @@ -1277,7 +1274,7 @@ proc idleAsync*(): Future[void] {.asyncraises: [CancelledError].} = retFuture proc withTimeout*[T](fut: Future[T], timeout: Duration): Future[bool] {. - asyncraises: [CancelledError].} = + async: (raw: true, raises: [CancelledError]).} = ## Returns a future which will complete once ``fut`` completes or after ## ``timeout`` milliseconds has elapsed. ## diff --git a/chronos/internal/asyncmacro.nim b/chronos/internal/asyncmacro.nim index f313f6f..c110847 100644 --- a/chronos/internal/asyncmacro.nim +++ b/chronos/internal/asyncmacro.nim @@ -9,8 +9,9 @@ # import - std/[algorithm, macros, sequtils], - ../[futures, config] + std/[macros], + ../[futures, config], + ./raisesfutures proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} = case node.kind @@ -32,10 +33,10 @@ proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} node[i] = processBody(node[i], setResultSym, baseType) node -proc wrapInTryFinally(fut, baseType, body, raisesTuple: NimNode): NimNode {.compileTime.} = +proc wrapInTryFinally(fut, baseType, body, raises: NimNode): NimNode {.compileTime.} = # creates: # try: `body` - # [for raise in raisesTuple]: + # [for raise in raises]: # except `raise`: closureSucceeded = false; `castFutureSym`.fail(exc) # finally: # if closureSucceeded: @@ -91,7 +92,17 @@ proc wrapInTryFinally(fut, baseType, body, raisesTuple: NimNode): NimNode {.comp newCall(ident "fail", fut, excName) )) - for exc in raisesTuple: + let raises = if raises == nil: + const defaultException = + when defined(chronosStrictException): "CatchableError" + else: "Exception" + nnkTupleConstr.newTree(ident(defaultException)) + elif isNoRaises(raises): + nnkTupleConstr.newTree() + else: + raises + + for exc in raises: if exc.eqIdent("Exception"): addCancelledError addCatchableError @@ -182,42 +193,33 @@ proc cleanupOpenSymChoice(node: NimNode): NimNode {.compileTime.} = for child in node: result.add(cleanupOpenSymChoice(child)) -proc getAsyncCfg(prc: NimNode): tuple[raises: bool, async: bool, raisesTuple: NimNode] = - # reads the pragmas to extract the useful data - # and removes them +proc decodeParams(params: NimNode): tuple[raw: bool, raises: NimNode] = + # decodes the parameter tuple given in `async: (name: value, ...)` to its + # recognised parts + params.expectKind(nnkTupleConstr) + var - foundRaises = -1 - foundAsync = -1 + raw = false + raises: NimNode = nil - for index, pragma in pragma(prc): - if pragma.kind == nnkExprColonExpr and pragma[0] == ident "asyncraises": - foundRaises = index - elif pragma.eqIdent("async"): - foundAsync = index - elif pragma.kind == nnkExprColonExpr and pragma[0] == ident "raises": - warning("The raises pragma doesn't work on async procedure. " & - "Please remove it or use asyncraises instead") + for param in params: + param.expectKind(nnkExprColonExpr) - result.raises = foundRaises >= 0 - result.async = foundAsync >= 0 - result.raisesTuple = nnkTupleConstr.newTree() + if param[0].eqIdent("raises"): + param[1].expectKind(nnkBracket) + if param[1].len == 0: + raises = makeNoRaises() + else: + raises = nnkTupleConstr.newTree() + for possibleRaise in param[1]: + raises.add(possibleRaise) + elif param[0].eqIdent("raw"): + # boolVal doesn't work in untyped macros it seems.. + raw = param[1].eqIdent("true") + else: + warning("Unrecognised async parameter: " & repr(param[0]), param) - if foundRaises >= 0: - for possibleRaise in pragma(prc)[foundRaises][1]: - result.raisesTuple.add(possibleRaise) - if result.raisesTuple.len == 0: - result.raisesTuple = ident("void") - else: - when defined(chronosWarnMissingRaises): - warning("Async proc miss asyncraises") - const defaultException = - when defined(chronosStrictException): "CatchableError" - else: "Exception" - result.raisesTuple.add(ident(defaultException)) - - let toRemoveList = @[foundRaises, foundAsync].filterIt(it >= 0).sorted().reversed() - for toRemove in toRemoveList: - pragma(prc).del(toRemove) + (raw, raises) proc isEmpty(n: NimNode): bool {.compileTime.} = # true iff node recursively contains only comments or empties @@ -230,13 +232,18 @@ proc isEmpty(n: NimNode): bool {.compileTime.} = else: false -proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = +proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = ## This macro transforms a single procedure into a closure iterator. ## The ``async`` macro supports a stmtList holding multiple async procedures. if prc.kind notin {nnkProcTy, nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform " & $prc.kind & " into an async proc." & " proc/method definition or lambda node expected.", prc) + for pragma in prc.pragma(): + if pragma.kind == nnkExprColonExpr and pragma[0].eqIdent("raises"): + warning("The raises pragma doesn't work on async procedures - use " & + "`async: (raises: [...]) instead.", prc) + let returnType = cleanupOpenSymChoice(prc.params2[0]) # Verify that the return type is a Future[T] @@ -254,22 +261,24 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = let baseTypeIsVoid = baseType.eqIdent("void") - futureVoidType = nnkBracketExpr.newTree(ident "Future", ident "void") - (hasRaises, isAsync, raisesTuple) = getAsyncCfg(prc) - - if hasRaises: - # Store `asyncraises` types in InternalRaisesFuture - prc.params2[0] = nnkBracketExpr.newTree( - newIdentNode("InternalRaisesFuture"), - baseType, - raisesTuple - ) - elif baseTypeIsVoid: - # Adds the implicit Future[void] - prc.params2[0] = + (raw, raises) = decodeParams(params) + internalFutureType = + if baseTypeIsVoid: newNimNode(nnkBracketExpr, prc). add(newIdentNode("Future")). - add(newIdentNode("void")) + add(baseType) + else: + returnType + internalReturnType = if raises == nil: + internalFutureType + else: + nnkBracketExpr.newTree( + newIdentNode("InternalRaisesFuture"), + baseType, + raises + ) + + prc.params2[0] = internalReturnType if prc.kind notin {nnkProcTy, nnkLambda}: # TODO: Nim bug? prc.addPragma(newColonExpr(ident "stackTrace", ident "off")) @@ -282,24 +291,28 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = # https://github.com/nim-lang/RFCs/issues/435 prc.addPragma(newIdentNode("gcsafe")) - if isAsync == false: # `asyncraises` without `async` - # type InternalRaisesFutureRaises {.used.} = `raisesTuple` - # `body` - prc.body = nnkStmtList.newTree( - nnkTypeSection.newTree( - nnkTypeDef.newTree( - nnkPragmaExpr.newTree( - ident"InternalRaisesFutureRaises", - nnkPragma.newTree( - newIdentNode("used") - ) - ), - newEmptyNode(), - raisesTuple - ) - ), - prc.body - ) + if raw: # raw async = body is left as-is + if raises != nil and prc.kind notin {nnkProcTy, nnkLambda} and not isEmpty(prc.body): + # Inject `raises` type marker that causes `newFuture` to return a raise- + # tracking future instead of an ordinary future: + # + # type InternalRaisesFutureRaises = `raisesTuple` + # `body` + prc.body = nnkStmtList.newTree( + nnkTypeSection.newTree( + nnkTypeDef.newTree( + nnkPragmaExpr.newTree( + ident"InternalRaisesFutureRaises", + nnkPragma.newTree(ident "used")), + newEmptyNode(), + raises, + ) + ), + prc.body + ) + + when chronosDumpAsync: + echo repr prc return prc @@ -311,9 +324,6 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = setResultSym = ident "setResult" procBody = prc.body.processBody(setResultSym, baseType) internalFutureSym = ident "chronosInternalRetFuture" - internalFutureType = - if baseTypeIsVoid: futureVoidType - else: returnType castFutureSym = nnkCast.newTree(internalFutureType, internalFutureSym) resultIdent = ident "result" @@ -396,7 +406,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = castFutureSym, baseType, if baseTypeIsVoid: procBody # shortcut for non-generic `void` else: newCall(setResultSym, procBody), - raisesTuple + raises ) closureBody = newStmtList(resultDecl, setResultDecl, completeDecl) @@ -431,19 +441,22 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = outerProcBody.add(closureIterator) - # -> let resultFuture = newInternalRaisesFuture[T]() + # -> let resultFuture = newInternalRaisesFuture[T, E]() # declared at the end to be sure that the closure # doesn't reference it, avoid cyclic ref (#203) let retFutureSym = ident "resultFuture" + newFutProc = if raises == nil: + newTree(nnkBracketExpr, ident "newFuture", baseType) + else: + newTree(nnkBracketExpr, ident "newInternalRaisesFuture", baseType, raises) retFutureSym.copyLineInfo(prc) # Do not change this code to `quote do` version because `instantiationInfo` # will be broken for `newFuture()` call. outerProcBody.add( newLetStmt( retFutureSym, - newCall(newTree(nnkBracketExpr, ident "newInternalRaisesFuture", baseType), - newLit(prcName)) + newCall(newFutProc, newLit(prcName)) ) ) # -> resultFuture.internalClosure = iterator @@ -465,6 +478,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = when chronosDumpAsync: echo repr prc + prc template await*[T](f: Future[T]): untyped = @@ -490,32 +504,23 @@ template awaitne*[T](f: Future[T]): Future[T] = else: unsupported "awaitne is only available within {.async.}" -macro async*(prc: untyped): untyped = +macro async*(params, prc: untyped): untyped = ## Macro which processes async procedures into the appropriate ## iterators and yield statements. if prc.kind == nnkStmtList: result = newStmtList() for oneProc in prc: - oneProc.addPragma(ident"async") - result.add asyncSingleProc(oneProc) + result.add asyncSingleProc(oneProc, params) else: - prc.addPragma(ident"async") - result = asyncSingleProc(prc) + result = asyncSingleProc(prc, params) + +macro async*(prc: untyped): untyped = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. -macro asyncraises*(possibleExceptions, prc: untyped): untyped = - # Add back the pragma and let asyncSingleProc handle it - # Exerimental / subject to change and/or removal if prc.kind == nnkStmtList: result = newStmtList() for oneProc in prc: - oneProc.addPragma(nnkExprColonExpr.newTree( - ident"asyncraises", - possibleExceptions - )) - result.add asyncSingleProc(oneProc) + result.add asyncSingleProc(oneProc, nnkTupleConstr.newTree()) else: - prc.addPragma(nnkExprColonExpr.newTree( - ident"asyncraises", - possibleExceptions - )) - result = asyncSingleProc(prc) + result = asyncSingleProc(prc, nnkTupleConstr.newTree()) diff --git a/chronos/internal/raisesfutures.nim b/chronos/internal/raisesfutures.nim index 6a85581..ad811f7 100644 --- a/chronos/internal/raisesfutures.nim +++ b/chronos/internal/raisesfutures.nim @@ -7,13 +7,23 @@ type ## Future with a tuple of possible exception types ## eg InternalRaisesFuture[void, (ValueError, OSError)] ## - ## This type gets injected by `asyncraises` and similar utilities and - ## should not be used manually as the internal exception representation is - ## subject to change in future chronos versions. + ## This type gets injected by `async: (raises: ...)` and similar utilities + ## and should not be used manually as the internal exception representation + ## is subject to change in future chronos versions. + +proc makeNoRaises*(): NimNode {.compileTime.} = + # An empty tuple would have been easier but... + # https://github.com/nim-lang/Nim/issues/22863 + # https://github.com/nim-lang/Nim/issues/22865 + + ident"void" + +proc isNoRaises*(n: NimNode): bool {.compileTime.} = + n.eqIdent("void") iterator members(tup: NimNode): NimNode = # Given a typedesc[tuple] = (A, B, C), yields the tuple members (A, B C) - if not tup.eqIdent("void"): + if not isNoRaises(tup): for n in getType(getTypeInst(tup)[1])[1..^1]: yield n @@ -40,7 +50,7 @@ macro prepend*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = result.add err if result.len == 0: - result = ident"void" + result = makeNoRaises() macro remove*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = result = nnkTupleConstr.newTree() @@ -49,7 +59,7 @@ macro remove*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = result.add err if result.len == 0: - result = ident"void" + result = makeNoRaises() macro union*(tup0: typedesc[tuple], tup1: typedesc[tuple]): typedesc = ## Join the types of the two tuples deduplicating the entries @@ -65,11 +75,13 @@ macro union*(tup0: typedesc[tuple], tup1: typedesc[tuple]): typedesc = for err2 in getType(getTypeInst(tup1)[1])[1..^1]: result.add err2 + if result.len == 0: + result = makeNoRaises() proc getRaises*(future: NimNode): NimNode {.compileTime.} = # Given InternalRaisesFuture[T, (A, B, C)], returns (A, B, C) let types = getType(getTypeInst(future)[2]) - if types.eqIdent("void"): + if isNoRaises(types): nnkBracketExpr.newTree(newEmptyNode()) else: expectKind(types, nnkBracketExpr) @@ -106,7 +118,7 @@ macro checkRaises*[T: CatchableError]( infix(error, "of", nnkBracketExpr.newTree(ident"typedesc", errorType))) let - errorMsg = "`fail`: `" & repr(toMatch) & "` incompatible with `asyncraises: " & repr(raises[1..^1]) & "`" + errorMsg = "`fail`: `" & repr(toMatch) & "` incompatible with `raises: " & repr(raises[1..^1]) & "`" warningMsg = "Can't verify `fail` exception type at compile time - expected one of " & repr(raises[1..^1]) & ", got `" & repr(toMatch) & "`" # A warning from this line means exception type will be verified at runtime warning = if warn: diff --git a/tests/testmacro.nim b/tests/testmacro.nim index 2fc24be..c9b45dd 100644 --- a/tests/testmacro.nim +++ b/tests/testmacro.nim @@ -387,16 +387,16 @@ suite "Exceptions tracking": check (not compiles(body)) test "Can raise valid exception": proc test1 {.async.} = raise newException(ValueError, "hey") - proc test2 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") - proc test3 {.async, asyncraises: [IOError, ValueError].} = + proc test2 {.async: (raises: [ValueError]).} = raise newException(ValueError, "hey") + proc test3 {.async: (raises: [IOError, ValueError]).} = if 1 == 2: raise newException(ValueError, "hey") else: raise newException(IOError, "hey") - proc test4 {.async, asyncraises: [], used.} = raise newException(Defect, "hey") - proc test5 {.async, asyncraises: [].} = discard - proc test6 {.async, asyncraises: [].} = await test5() + proc test4 {.async: (raises: []), used.} = raise newException(Defect, "hey") + proc test5 {.async: (raises: []).} = discard + proc test6 {.async: (raises: []).} = await test5() expect(ValueError): waitFor test1() expect(ValueError): waitFor test2() @@ -405,15 +405,15 @@ suite "Exceptions tracking": test "Cannot raise invalid exception": checkNotCompiles: - proc test3 {.async, asyncraises: [IOError].} = raise newException(ValueError, "hey") + proc test3 {.async: (raises: [IOError]).} = raise newException(ValueError, "hey") test "Explicit return in non-raising proc": - proc test(): Future[int] {.async, asyncraises: [].} = return 12 + proc test(): Future[int] {.async: (raises: []).} = return 12 check: waitFor(test()) == 12 test "Non-raising compatibility": - proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") + proc test1 {.async: (raises: [ValueError]).} = raise newException(ValueError, "hey") let testVar: Future[void] = test1() proc test2 {.async.} = raise newException(ValueError, "hey") @@ -423,69 +423,64 @@ suite "Exceptions tracking": #let testVar3: proc: Future[void] = test1 test "Cannot store invalid future types": - proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") - proc test2 {.async, asyncraises: [IOError].} = raise newException(IOError, "hey") + proc test1 {.async: (raises: [ValueError]).} = raise newException(ValueError, "hey") + proc test2 {.async: (raises: [IOError]).} = raise newException(IOError, "hey") var a = test1() checkNotCompiles: a = test2() test "Await raises the correct types": - proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") - proc test2 {.async, asyncraises: [ValueError, CancelledError].} = await test1() + proc test1 {.async: (raises: [ValueError]).} = raise newException(ValueError, "hey") + proc test2 {.async: (raises: [ValueError, CancelledError]).} = await test1() checkNotCompiles: - proc test3 {.async, asyncraises: [CancelledError].} = await test1() + proc test3 {.async: (raises: [CancelledError]).} = await test1() test "Can create callbacks": - proc test1 {.async, asyncraises: [ValueError].} = raise newException(ValueError, "hey") - let callback: proc() {.async, asyncraises: [ValueError].} = test1 + proc test1 {.async: (raises: [ValueError]).} = raise newException(ValueError, "hey") + let callback: proc() {.async: (raises: [ValueError]).} = test1 test "Can return values": - proc test1: Future[int] {.async, asyncraises: [ValueError].} = + proc test1: Future[int] {.async: (raises: [ValueError]).} = if 1 == 0: raise newException(ValueError, "hey") return 12 - proc test2: Future[int] {.async, asyncraises: [ValueError, IOError, CancelledError].} = + proc test2: Future[int] {.async: (raises: [ValueError, IOError, CancelledError]).} = return await test1() checkNotCompiles: - proc test3: Future[int] {.async, asyncraises: [CancelledError].} = await test1() + proc test3: Future[int] {.async: (raises: [CancelledError]).} = await test1() check waitFor(test2()) == 12 test "Manual tracking": - proc test1: Future[int] {.asyncraises: [ValueError].} = + proc test1: Future[int] {.async: (raw: true, raises: [ValueError]).} = result = newFuture[int]() result.complete(12) check waitFor(test1()) == 12 - proc test2: Future[int] {.asyncraises: [IOError, OSError].} = + proc test2: Future[int] {.async: (raw: true, raises: [IOError, OSError]).} = result = newFuture[int]() result.fail(newException(IOError, "fail")) result.fail(newException(OSError, "fail")) checkNotCompiles: result.fail(newException(ValueError, "fail")) - proc test3: Future[void] {.asyncraises: [].} = + proc test3: Future[void] {.async: (raw: true, raises: []).} = checkNotCompiles: result.fail(newException(ValueError, "fail")) # Inheritance - proc test4: Future[void] {.asyncraises: [CatchableError].} = + proc test4: Future[void] {.async: (raw: true, raises: [CatchableError]).} = result.fail(newException(IOError, "fail")) - test "Reversed async, asyncraises": - proc test44 {.asyncraises: [ValueError], async.} = raise newException(ValueError, "hey") - checkNotCompiles: - proc test33 {.asyncraises: [IOError], async.} = raise newException(ValueError, "hey") - test "or errors": - proc testit {.asyncraises: [ValueError], async.} = + proc testit {.async: (raises: [ValueError]).} = raise (ref ValueError)() - proc testit2 {.asyncraises: [IOError], async.} = + proc testit2 {.async: (raises: [IOError]).} = raise (ref IOError)() - proc test {.async, asyncraises: [ValueError, IOError].} = + proc test {.async: (raises: [ValueError, IOError]).} = await testit() or testit2() proc noraises() {.raises: [].} = @@ -499,9 +494,10 @@ suite "Exceptions tracking": noraises() test "Wait errors": - proc testit {.asyncraises: [ValueError], async.} = raise newException(ValueError, "hey") + proc testit {.async: (raises: [ValueError]).} = + raise newException(ValueError, "hey") - proc test {.async, asyncraises: [ValueError, AsyncTimeoutError, CancelledError].} = + proc test {.async: (raises: [ValueError, AsyncTimeoutError, CancelledError]).} = await wait(testit(), 1000.milliseconds) proc noraises() {.raises: [].} = @@ -513,11 +509,11 @@ suite "Exceptions tracking": noraises() test "Nocancel errors": - proc testit {.asyncraises: [ValueError, CancelledError], async.} = + proc testit {.async: (raises: [ValueError, CancelledError]).} = await sleepAsync(5.milliseconds) raise (ref ValueError)() - proc test {.async, asyncraises: [ValueError].} = + proc test {.async: (raises: [ValueError]).} = await noCancel testit() proc noraises() {.raises: [].} = From 5ebd771d35464832eb9edf603b616fd34ad158cd Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 8 Nov 2023 15:12:32 +0100 Subject: [PATCH 31/50] per-function `Exception` handling (#457) This PR replaces the global strict exception mode with an option to handle `Exception` per function while at the same time enabling strict exception checking globally by default as has been planned for v4. `handleException` mode raises `AsyncExceptionError` to distinguish it from `ValueError` which may originate from user code. * remove obsolete 1.2 config options --- README.md | 105 ++++++++++++++--------- chronos/config.nim | 145 ++++++++++++++------------------ chronos/internal/asyncmacro.nim | 40 ++++++--- chronos/internal/errors.nim | 4 + tests/testmacro.nim | 11 +++ 5 files changed, 168 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index f59a6c8..c80f826 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,12 @@ Chronos is an efficient [async/await](https://en.wikipedia.org/wiki/Async/await) framework for Nim. Features include: -* Efficient dispatch pipeline for asynchronous execution +* Asynchronous socket and process I/O * HTTP server with SSL/TLS support out of the box (no OpenSSL needed) -* Cancellation support * Synchronization primitivies like queues, events and locks -* FIFO processing order of dispatch queue -* Minimal exception effect support (see [exception effects](#exception-effects)) +* Cancellation +* Efficient dispatch pipeline with excellent multi-platform support +* Exception effect support (see [exception effects](#exception-effects)) ## Installation @@ -152,16 +152,13 @@ feet, in a certain section, is to not use `await` in it. ### Error handling -Exceptions inheriting from `CatchableError` are caught by hidden `try` blocks -and placed in the `Future.error` field, changing the future's status to -`Failed`. +Exceptions inheriting from [`CatchableError`](https://nim-lang.org/docs/system.html#CatchableError) +interrupt execution of the `async` procedure. The exception is placed in the +`Future.error` field while changing the status of the `Future` to `Failed` +and callbacks are scheduled. -When a future is awaited, that exception is re-raised only to be caught again -by a hidden `try` block in the calling async procedure. That's how these -exceptions move up the async chain. - -A failed future's callbacks will still be scheduled, but it's not possible to -resume execution from the point an exception was raised. +When a future is awaited, the exception is re-raised, traversing the `async` +execution chain until handled. ```nim proc p1() {.async.} = @@ -206,11 +203,11 @@ proc p3() {.async.} = await fut2 ``` -Chronos does not allow that future continuations and other callbacks raise -`CatchableError` - as such, calls to `poll` will never raise exceptions caused -originating from tasks on the dispatcher queue. It is however possible that -`Defect` that happen in tasks bubble up through `poll` as these are not caught -by the transformation. +Because `chronos` ensures that all exceptions are re-routed to the `Future`, +`poll` will not itself raise exceptions. + +`poll` may still panic / raise `Defect` if such are raised in user code due to +undefined behavior. #### Checked exceptions @@ -230,6 +227,53 @@ proc p2(): Future[void] {.async, (raises: [IOError]).} = Under the hood, the return type of `p1` will be rewritten to an internal type which will convey raises informations to `await`. +#### The `Exception` type + +Exceptions deriving from `Exception` are not caught by default as these may +include `Defect` and other forms undefined or uncatchable behavior. + +Because exception effect tracking is turned on for `async` functions, this may +sometimes lead to compile errors around forward declarations, methods and +closures as Nim conservatively asssumes that any `Exception` might be raised +from those. + +Make sure to excplicitly annotate these with `{.raises.}`: + +```nim +# Forward declarations need to explicitly include a raises list: +proc myfunction() {.raises: [ValueError].} + +# ... as do `proc` types +type MyClosure = proc() {.raises: [ValueError].} + +proc myfunction() = + raise (ref ValueError)(msg: "Implementation here") + +let closure: MyClosure = myfunction +``` + +For compatibility, `async` functions can be instructed to handle `Exception` as +well, specifying `handleException: true`. `Exception` that is not a `Defect` and +not a `CatchableError` will then be caught and remapped to +`AsyncExceptionError`: + +```nim +proc raiseException() {.async: (handleException: true, raises: [AsyncExceptionError]).} = + raise (ref Exception)(msg: "Raising Exception is UB") + +proc callRaiseException() {.async: (raises: []).} = + try: + raiseException() + except AsyncExceptionError as exc: + # The original Exception is available from the `parent` field + echo exc.parent.msg +``` + +This mode can be enabled globally with `-d:chronosHandleException` as a help +when porting code to `chronos` but should generally be avoided as global +configuration settings may interfere with libraries that use `chronos` leading +to unexpected behavior. + ### Raw functions Raw functions are those that interact with `chronos` via the `Future` type but @@ -302,27 +346,6 @@ annotated as raising `CatchableError` only raise on _some_ platforms - in order to work on all platforms, calling code must assume that they will raise even when they don't seem to do so on one platform. -### Strict exception mode - -`chronos` currently offers minimal support for exception effects and `raises` -annotations. In general, during the `async` transformation, a generic -`except CatchableError` handler is added around the entire function being -transformed, in order to catch any exceptions and transfer them to the `Future`. -Because of this, the effect system thinks no exceptions are "leaking" because in -fact, exception _handling_ is deferred to when the future is being read. - -Effectively, this means that while code can be compiled with -`{.push raises: []}`, the intended effect propagation and checking is -**disabled** for `async` functions. - -To enable checking exception effects in `async` code, enable strict mode with -`-d:chronosStrictException`. - -In the strict mode, `async` functions are checked such that they only raise -`CatchableError` and thus must make sure to explicitly specify exception -effects on forward declarations, callbacks and methods using -`{.raises: [CatchableError].}` (or more strict) annotations. - ### Cancellation support Any running `Future` can be cancelled. This can be used for timeouts, @@ -379,9 +402,9 @@ waitFor(cancellationExample()) Even if cancellation is initiated, it is not guaranteed that the operation gets cancelled - the future might still be completed or fail depending on the ordering of events and the specifics of -the operation. +the operation. -If the future indeed gets cancelled, `await` will raise a +If the future indeed gets cancelled, `await` will raise a `CancelledError` as is likely to happen in the following example: ```nim proc c1 {.async.} = diff --git a/chronos/config.nim b/chronos/config.nim index bd6c2b9..6af3e31 100644 --- a/chronos/config.nim +++ b/chronos/config.nim @@ -11,100 +11,77 @@ ## `chronosDebug` can be defined to enable several debugging helpers that come ## with a runtime cost - it is recommeneded to not enable these in production ## code. -when (NimMajor, NimMinor) >= (1, 4): - const - chronosStrictException* {.booldefine.}: bool = defined(chronosPreviewV4) - ## Require that `async` code raises only derivatives of `CatchableError` - ## and not `Exception` - forward declarations, methods and `proc` types - ## used from within `async` code may need to be be explicitly annotated - ## with `raises: [CatchableError]` when this mode is enabled. +const + chronosHandleException* {.booldefine.}: bool = false + ## Remap `Exception` to `AsyncExceptionError` for all `async` functions. + ## + ## This modes provides backwards compatibility when using functions with + ## inaccurate `{.raises.}` effects such as unannotated forward declarations, + ## methods and `proc` types - it is recommened to annotate such code + ## explicitly as the `Exception` handling mode may introduce surprising + ## behavior in exception handlers, should `Exception` actually be raised. + ## + ## The setting provides the default for the per-function-based + ## `handleException` parameter which has precedence over this global setting. + ## + ## `Exception` handling may be removed in future chronos versions. - chronosStrictFutureAccess* {.booldefine.}: bool = defined(chronosPreviewV4) + chronosStrictFutureAccess* {.booldefine.}: bool = defined(chronosPreviewV4) - chronosStackTrace* {.booldefine.}: bool = defined(chronosDebug) - ## Include stack traces in futures for creation and completion points + chronosStackTrace* {.booldefine.}: bool = defined(chronosDebug) + ## Include stack traces in futures for creation and completion points - chronosFutureId* {.booldefine.}: bool = defined(chronosDebug) - ## Generate a unique `id` for every future - when disabled, the address of - ## the future will be used instead + chronosFutureId* {.booldefine.}: bool = defined(chronosDebug) + ## Generate a unique `id` for every future - when disabled, the address of + ## the future will be used instead - chronosFutureTracking* {.booldefine.}: bool = defined(chronosDebug) - ## Keep track of all pending futures and allow iterating over them - - ## useful for detecting hung tasks + chronosFutureTracking* {.booldefine.}: bool = defined(chronosDebug) + ## Keep track of all pending futures and allow iterating over them - + ## useful for detecting hung tasks - chronosDumpAsync* {.booldefine.}: bool = defined(nimDumpAsync) - ## Print code generated by {.async.} transformation + chronosDumpAsync* {.booldefine.}: bool = defined(nimDumpAsync) + ## Print code generated by {.async.} transformation - chronosProcShell* {.strdefine.}: string = - when defined(windows): - "cmd.exe" + chronosProcShell* {.strdefine.}: string = + when defined(windows): + "cmd.exe" + else: + when defined(android): + "/system/bin/sh" else: - when defined(android): - "/system/bin/sh" - else: - "/bin/sh" - ## Default shell binary path. - ## - ## The shell is used as command for command line when process started - ## using `AsyncProcessOption.EvalCommand` and API calls such as - ## ``execCommand(command)`` and ``execCommandEx(command)``. + "/bin/sh" + ## Default shell binary path. + ## + ## The shell is used as command for command line when process started + ## using `AsyncProcessOption.EvalCommand` and API calls such as + ## ``execCommand(command)`` and ``execCommandEx(command)``. - chronosEventsCount* {.intdefine.} = 64 - ## Number of OS poll events retrieved by syscall (epoll, kqueue, poll). + chronosEventsCount* {.intdefine.} = 64 + ## Number of OS poll events retrieved by syscall (epoll, kqueue, poll). - chronosInitialSize* {.intdefine.} = 64 - ## Initial size of Selector[T]'s array of file descriptors. + chronosInitialSize* {.intdefine.} = 64 + ## Initial size of Selector[T]'s array of file descriptors. - chronosEventEngine* {.strdefine.}: string = - when defined(linux) and not(defined(android) or defined(emscripten)): - "epoll" - elif defined(macosx) or defined(macos) or defined(ios) or - defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly): - "kqueue" - elif defined(android) or defined(emscripten): - "poll" - elif defined(posix): - "poll" - else: - "" - ## OS polling engine type which is going to be used by chronos. + chronosEventEngine* {.strdefine.}: string = + when defined(linux) and not(defined(android) or defined(emscripten)): + "epoll" + elif defined(macosx) or defined(macos) or defined(ios) or + defined(freebsd) or defined(netbsd) or defined(openbsd) or + defined(dragonfly): + "kqueue" + elif defined(android) or defined(emscripten): + "poll" + elif defined(posix): + "poll" + else: + "" + ## OS polling engine type which is going to be used by chronos. -else: - # 1.2 doesn't support `booldefine` in `when` properly - const - chronosStrictException*: bool = - defined(chronosPreviewV4) or defined(chronosStrictException) - chronosStrictFutureAccess*: bool = - defined(chronosPreviewV4) or defined(chronosStrictFutureAccess) - chronosStackTrace*: bool = defined(chronosDebug) or defined(chronosStackTrace) - chronosFutureId*: bool = defined(chronosDebug) or defined(chronosFutureId) - chronosFutureTracking*: bool = - defined(chronosDebug) or defined(chronosFutureTracking) - chronosDumpAsync*: bool = defined(nimDumpAsync) - chronosProcShell* {.strdefine.}: string = - when defined(windows): - "cmd.exe" - else: - when defined(android): - "/system/bin/sh" - else: - "/bin/sh" - chronosEventsCount*: int = 64 - chronosInitialSize*: int = 64 - chronosEventEngine* {.strdefine.}: string = - when defined(linux) and not(defined(android) or defined(emscripten)): - "epoll" - elif defined(macosx) or defined(macos) or defined(ios) or - defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly): - "kqueue" - elif defined(android) or defined(emscripten): - "poll" - elif defined(posix): - "poll" - else: - "" +when defined(chronosStrictException): + {.warning: "-d:chronosStrictException has been deprecated in favor of handleException".} + # In chronos v3, this setting was used as the opposite of + # `chronosHandleException` - the setting is deprecated to encourage + # migration to the new mode. when defined(debug) or defined(chronosConfig): import std/macros @@ -113,7 +90,7 @@ when defined(debug) or defined(chronosConfig): hint("Chronos configuration:") template printOption(name: string, value: untyped) = hint(name & ": " & $value) - printOption("chronosStrictException", chronosStrictException) + printOption("chronosHandleException", chronosHandleException) printOption("chronosStackTrace", chronosStackTrace) printOption("chronosFutureId", chronosFutureId) printOption("chronosFutureTracking", chronosFutureTracking) diff --git a/chronos/internal/asyncmacro.nim b/chronos/internal/asyncmacro.nim index c110847..11daf33 100644 --- a/chronos/internal/asyncmacro.nim +++ b/chronos/internal/asyncmacro.nim @@ -33,7 +33,9 @@ proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} node[i] = processBody(node[i], setResultSym, baseType) node -proc wrapInTryFinally(fut, baseType, body, raises: NimNode): NimNode {.compileTime.} = +proc wrapInTryFinally( + fut, baseType, body, raises: NimNode, + handleException: bool): NimNode {.compileTime.} = # creates: # try: `body` # [for raise in raises]: @@ -92,15 +94,15 @@ proc wrapInTryFinally(fut, baseType, body, raises: NimNode): NimNode {.compileTi newCall(ident "fail", fut, excName) )) - let raises = if raises == nil: - const defaultException = - when defined(chronosStrictException): "CatchableError" - else: "Exception" - nnkTupleConstr.newTree(ident(defaultException)) + var raises = if raises == nil: + nnkTupleConstr.newTree(ident"CatchableError") elif isNoRaises(raises): nnkTupleConstr.newTree() else: - raises + raises.copyNimTree() + + if handleException: + raises.add(ident"Exception") for exc in raises: if exc.eqIdent("Exception"): @@ -115,7 +117,9 @@ proc wrapInTryFinally(fut, baseType, body, raises: NimNode): NimNode {.compileTi newCall(ident "fail", fut, nnkStmtList.newTree( nnkAsgn.newTree(closureSucceeded, ident"false"), - quote do: (ref ValueError)(msg: `excName`.msg, parent: `excName`))) + quote do: + (ref AsyncExceptionError)( + msg: `excName`.msg, parent: `excName`))) ) elif exc.eqIdent("CancelledError"): addCancelledError @@ -132,6 +136,8 @@ proc wrapInTryFinally(fut, baseType, body, raises: NimNode): NimNode {.compileTi newCall(ident "fail", fut, excName) )) + addDefect # Must not complete future on defect + nTry.add nnkFinally.newTree( nnkIfStmt.newTree( nnkElifBranch.newTree( @@ -193,7 +199,13 @@ proc cleanupOpenSymChoice(node: NimNode): NimNode {.compileTime.} = for child in node: result.add(cleanupOpenSymChoice(child)) -proc decodeParams(params: NimNode): tuple[raw: bool, raises: NimNode] = +type + AsyncParams = tuple + raw: bool + raises: NimNode + handleException: bool + +proc decodeParams(params: NimNode): AsyncParams = # decodes the parameter tuple given in `async: (name: value, ...)` to its # recognised parts params.expectKind(nnkTupleConstr) @@ -201,6 +213,7 @@ proc decodeParams(params: NimNode): tuple[raw: bool, raises: NimNode] = var raw = false raises: NimNode = nil + handleException = chronosHandleException for param in params: param.expectKind(nnkExprColonExpr) @@ -216,10 +229,12 @@ proc decodeParams(params: NimNode): tuple[raw: bool, raises: NimNode] = elif param[0].eqIdent("raw"): # boolVal doesn't work in untyped macros it seems.. raw = param[1].eqIdent("true") + elif param[0].eqIdent("handleException"): + handleException = param[1].eqIdent("true") else: warning("Unrecognised async parameter: " & repr(param[0]), param) - (raw, raises) + (raw, raises, handleException) proc isEmpty(n: NimNode): bool {.compileTime.} = # true iff node recursively contains only comments or empties @@ -261,7 +276,7 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = let baseTypeIsVoid = baseType.eqIdent("void") - (raw, raises) = decodeParams(params) + (raw, raises, handleException) = decodeParams(params) internalFutureType = if baseTypeIsVoid: newNimNode(nnkBracketExpr, prc). @@ -406,7 +421,8 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = castFutureSym, baseType, if baseTypeIsVoid: procBody # shortcut for non-generic `void` else: newCall(setResultSym, procBody), - raises + raises, + handleException ) closureBody = newStmtList(resultDecl, setResultDecl, completeDecl) diff --git a/chronos/internal/errors.nim b/chronos/internal/errors.nim index 083f7a2..8e6443e 100644 --- a/chronos/internal/errors.nim +++ b/chronos/internal/errors.nim @@ -3,3 +3,7 @@ type ## Generic async exception AsyncTimeoutError* = object of AsyncError ## Timeout exception + + AsyncExceptionError* = object of AsyncError + ## Error raised in `handleException` mode - the original exception is + ## available from the `parent` field. diff --git a/tests/testmacro.nim b/tests/testmacro.nim index c9b45dd..1361193 100644 --- a/tests/testmacro.nim +++ b/tests/testmacro.nim @@ -533,3 +533,14 @@ suite "Exceptions tracking": expect(Defect): f.fail((ref CatchableError)(), warn = false) check: not f.finished() + + test "handleException behavior": + proc raiseException() {. + async: (handleException: true, raises: [AsyncExceptionError]).} = + raise (ref Exception)(msg: "Raising Exception is UB and support for it may change in the future") + + proc callCatchAll() {.async: (raises: []).} = + expect(AsyncExceptionError): + await raiseException() + + waitFor(callCatchAll()) From 53690f4717369d6b5bf2aa3eea97e35b5fe52e0a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 8 Nov 2023 16:14:33 +0100 Subject: [PATCH 32/50] run tests outside of nim compilation (#463) else we need memory for both compiler and test --- chronos.nimble | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chronos.nimble b/chronos.nimble index f9e2617..e2fa998 100644 --- a/chronos.nimble +++ b/chronos.nimble @@ -13,6 +13,8 @@ requires "nim >= 1.6.0", "httputils", "unittest2" +import os + let nimc = getEnv("NIMC", "nim") # Which nim compiler to use let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js) let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler @@ -44,7 +46,8 @@ proc build(args, path: string) = exec nimc & " " & lang & " " & cfg & " " & flags & " " & args & " " & path proc run(args, path: string) = - build args & " -r", path + build args, path + exec "build/" & path.splitPath[1] task test, "Run all tests": for args in testArguments: From c252ce68d8e36a705f1f72b21c195dbf4ebcb176 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 8 Nov 2023 16:15:11 +0100 Subject: [PATCH 33/50] verbose test output on actions rerun (#462) --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7aa0fa..2d2cace 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,12 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Enable debug verbosity + if: runner.debug == '1' + run: | + echo "V=1" >> $GITHUB_ENV + echo "UNITTEST2_OUTPUT_LVL=VERBOSE" >> $GITHUB_ENV + - name: Install build dependencies (Linux i386) if: runner.os == 'Linux' && matrix.target.cpu == 'i386' run: | From 9759f01016c5d6b511c6eae3bf8376cc456fc0de Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 8 Nov 2023 21:20:24 +0100 Subject: [PATCH 34/50] doc generation fixes (#464) * doc generation fixes * fix --- .github/workflows/doc.yml | 6 ++--- .gitignore | 1 + chronos/config.nim | 4 ++- chronos/internal/asyncengine.nim | 42 +++++++++++++++++++++++++------- chronos/osdefs.nim | 2 ++ chronos/selectors2.nim | 17 ++++++------- chronos/transports/common.nim | 2 +- 7 files changed, 51 insertions(+), 23 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 1668eb0..6e1510a 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -15,13 +15,13 @@ jobs: continue-on-error: true steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true - uses: jiro4989/setup-nim-action@v1 with: - nim-version: '1.6.6' + nim-version: '1.6.16' - name: Generate doc run: | @@ -35,7 +35,7 @@ jobs: ls docs - name: Clone the gh-pages branch - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: status-im/nim-chronos ref: gh-pages diff --git a/.gitignore b/.gitignore index c631551..b599536 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ nimble.develop nimble.paths /build/ nimbledeps +/docs diff --git a/chronos/config.nim b/chronos/config.nim index 6af3e31..4055361 100644 --- a/chronos/config.nim +++ b/chronos/config.nim @@ -63,7 +63,9 @@ const ## Initial size of Selector[T]'s array of file descriptors. chronosEventEngine* {.strdefine.}: string = - when defined(linux) and not(defined(android) or defined(emscripten)): + when defined(nimdoc): + "" + elif defined(linux) and not(defined(android) or defined(emscripten)): "epoll" elif defined(macosx) or defined(macos) or defined(ios) or defined(freebsd) or defined(netbsd) or defined(openbsd) or diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim index 23d7c6a..0a15799 100644 --- a/chronos/internal/asyncengine.nim +++ b/chronos/internal/asyncengine.nim @@ -169,7 +169,36 @@ func toException*(v: OSErrorCode): ref OSError = newOSError(v) # This helper will allow to use `tryGet()` and raise OSError for # Result[T, OSErrorCode] values. -when defined(windows): +when defined(nimdoc): + type + PDispatcher* = ref object of PDispatcherBase + AsyncFD* = distinct cint + + var gDisp {.threadvar.}: PDispatcher + + proc newDispatcher*(): PDispatcher = discard + proc poll*() = discard + ## Perform single asynchronous step, processing timers and completing + ## tasks. Blocks until at least one event has completed. + ## + ## Exceptions raised during `async` task exection are stored as outcome + ## in the corresponding `Future` - `poll` itself does not raise. + + proc register2*(fd: AsyncFD): Result[void, OSErrorCode] = discard + proc unregister2*(fd: AsyncFD): Result[void, OSErrorCode] = discard + proc addReader2*(fd: AsyncFD, cb: CallbackFunc, + udata: pointer = nil): Result[void, OSErrorCode] = discard + proc removeReader2*(fd: AsyncFD): Result[void, OSErrorCode] = discard + proc addWriter2*(fd: AsyncFD, cb: CallbackFunc, + udata: pointer = nil): Result[void, OSErrorCode] = discard + proc removeWriter2*(fd: AsyncFD): Result[void, OSErrorCode] = discard + proc closeHandle*(fd: AsyncFD, aftercb: CallbackFunc = nil) = discard + proc closeSocket*(fd: AsyncFD, aftercb: CallbackFunc = nil) = discard + proc unregisterAndCloseFd*(fd: AsyncFD): Result[void, OSErrorCode] = discard + + proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow, gcsafe.} + +elif defined(windows): {.pragma: stdcallbackFunc, stdcall, gcsafe, raises: [].} export SIGINT, SIGQUIT, SIGTERM @@ -551,12 +580,6 @@ when defined(windows): raise newException(ValueError, osErrorMsg(res.error())) proc poll*() = - ## Perform single asynchronous step, processing timers and completing - ## tasks. Blocks until at least one event has completed. - ## - ## Exceptions raised here indicate that waiting for tasks to be unblocked - ## failed - exceptions from within tasks are instead propagated through - ## their respective futures and not allowed to interrrupt the poll call. let loop = getThreadDispatcher() var curTime = Moment.now() @@ -1241,5 +1264,6 @@ when chronosFutureTracking: ## completed, cancelled or failed). futureList.count -# Perform global per-module initialization. -globalInit() +when not defined(nimdoc): + # Perform global per-module initialization. + globalInit() diff --git a/chronos/osdefs.nim b/chronos/osdefs.nim index ab07721..40a6365 100644 --- a/chronos/osdefs.nim +++ b/chronos/osdefs.nim @@ -1526,6 +1526,8 @@ when defined(posix): INVALID_HANDLE_VALUE* = cint(-1) proc `==`*(x: SocketHandle, y: int): bool = int(x) == y +when defined(nimdoc): + proc `==`*(x: SocketHandle, y: SocketHandle): bool {.borrow.} when defined(macosx) or defined(macos) or defined(bsd): const diff --git a/chronos/selectors2.nim b/chronos/selectors2.nim index c5918fd..5cb8a57 100644 --- a/chronos/selectors2.nim +++ b/chronos/selectors2.nim @@ -36,7 +36,6 @@ import config, osdefs, osutils, oserrno export results, oserrno when defined(nimdoc): - type Selector*[T] = ref object ## An object which holds descriptors to be checked for read/write status @@ -306,11 +305,11 @@ else: doAssert((timeout >= min) and (timeout <= max), "Cannot select with incorrect timeout value, got " & $timeout) -when chronosEventEngine == "epoll": - include ./ioselects/ioselectors_epoll -elif chronosEventEngine == "kqueue": - include ./ioselects/ioselectors_kqueue -elif chronosEventEngine == "poll": - include ./ioselects/ioselectors_poll -else: - {.fatal: "Event engine `" & chronosEventEngine & "` is not supported!".} + when chronosEventEngine == "epoll": + include ./ioselects/ioselectors_epoll + elif chronosEventEngine == "kqueue": + include ./ioselects/ioselectors_kqueue + elif chronosEventEngine == "poll": + include ./ioselects/ioselectors_poll + else: + {.fatal: "Event engine `" & chronosEventEngine & "` is not supported!".} diff --git a/chronos/transports/common.nim b/chronos/transports/common.nim index b7776e5..d8263af 100644 --- a/chronos/transports/common.nim +++ b/chronos/transports/common.nim @@ -199,7 +199,7 @@ proc `$`*(address: TransportAddress): string = "None" proc toHex*(address: TransportAddress): string = - ## Returns hexadecimal representation of ``address`. + ## Returns hexadecimal representation of ``address``. case address.family of AddressFamily.IPv4: "0x" & address.address_v4.toHex() From 9896316599290a820289329f254c914303f7251b Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Thu, 9 Nov 2023 18:01:43 +0200 Subject: [PATCH 35/50] Remove deprecated AsyncEventBus. (#461) * Remove deprecated AsyncEventBus. Change number of tests for ThreadSignal. * Recover 1000 tests count. --- chronos/asyncsync.nim | 228 ------------------------------------------ 1 file changed, 228 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 0feb51e..fa23471 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -62,50 +62,6 @@ type AsyncLockError* = object of AsyncError ## ``AsyncLock`` is either locked or unlocked. - EventBusSubscription*[T] = proc(bus: AsyncEventBus, - payload: EventPayload[T]): Future[void] {. - gcsafe, raises: [].} - ## EventBus subscription callback type. - - EventBusAllSubscription* = proc(bus: AsyncEventBus, - event: AwaitableEvent): Future[void] {. - gcsafe, raises: [].} - ## EventBus subscription callback type. - - EventBusCallback = proc(bus: AsyncEventBus, event: string, key: EventBusKey, - data: EventPayloadBase) {. - gcsafe, raises: [].} - - EventBusKey* = object - ## Unique subscription key. - eventName: string - typeName: string - unique: uint64 - cb: EventBusCallback - - EventItem = object - waiters: seq[FutureBase] - subscribers: seq[EventBusKey] - - AsyncEventBus* = ref object of RootObj - ## An eventbus object. - counter: uint64 - events: Table[string, EventItem] - subscribers: seq[EventBusKey] - waiters: seq[Future[AwaitableEvent]] - - EventPayloadBase* = ref object of RootObj - loc: ptr SrcLoc - - EventPayload*[T] = ref object of EventPayloadBase - ## Eventbus' event payload object - value: T - - AwaitableEvent* = object - ## Eventbus' event payload object - eventName: string - payload: EventPayloadBase - AsyncEventQueueFullError* = object of AsyncError EventQueueKey* = distinct uint64 @@ -471,190 +427,6 @@ proc `$`*[T](aq: AsyncQueue[T]): string = res.add("]") res -template generateKey(typeName, eventName: string): string = - "type[" & typeName & "]-key[" & eventName & "]" - -proc newAsyncEventBus*(): AsyncEventBus {. - deprecated: "Implementation has unfixable flaws, please use" & - "AsyncEventQueue[T] instead".} = - ## Creates new ``AsyncEventBus``. - AsyncEventBus(counter: 0'u64, events: initTable[string, EventItem]()) - -template get*[T](payload: EventPayload[T]): T = - ## Returns event payload data. - payload.value - -template location*(payload: EventPayloadBase): SrcLoc = - ## Returns source location address of event emitter. - payload.loc[] - -proc get*(event: AwaitableEvent, T: typedesc): T {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue[T] instead".} = - ## Returns event's payload of type ``T`` from event ``event``. - cast[EventPayload[T]](event.payload).value - -template event*(event: AwaitableEvent): string = - ## Returns event's name from event ``event``. - event.eventName - -template location*(event: AwaitableEvent): SrcLoc = - ## Returns source location address of event emitter. - event.payload.loc[] - -proc waitEvent*(bus: AsyncEventBus, T: typedesc, event: string): Future[T] {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue[T] instead".} = - ## Wait for the event from AsyncEventBus ``bus`` with name ``event``. - ## - ## Returned ``Future[T]`` will hold event's payload of type ``T``. - var default: EventItem - var retFuture = newFuture[T]("AsyncEventBus.waitEvent") - let eventKey = generateKey(T.name, event) - proc cancellation(udata: pointer) {.gcsafe, raises: [].} = - if not(retFuture.finished()): - bus.events.withValue(eventKey, item): - item.waiters.keepItIf(it != cast[FutureBase](retFuture)) - retFuture.cancelCallback = cancellation - let baseFuture = cast[FutureBase](retFuture) - bus.events.mgetOrPut(eventKey, default).waiters.add(baseFuture) - retFuture - -proc waitAllEvents*(bus: AsyncEventBus): Future[AwaitableEvent] {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue[T] instead".} = - ## Wait for any event from AsyncEventBus ``bus``. - ## - ## Returns ``Future`` which holds helper object. Using this object you can - ## retrieve event's name and payload. - var retFuture = newFuture[AwaitableEvent]("AsyncEventBus.waitAllEvents") - proc cancellation(udata: pointer) {.gcsafe, raises: [].} = - if not(retFuture.finished()): - bus.waiters.keepItIf(it != retFuture) - retFuture.cancelCallback = cancellation - bus.waiters.add(retFuture) - retFuture - -proc subscribe*[T](bus: AsyncEventBus, event: string, - callback: EventBusSubscription[T]): EventBusKey {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue[T] instead".} = - ## Subscribe to the event ``event`` passed through eventbus ``bus`` with - ## callback ``callback``. - ## - ## Returns key that can be used to unsubscribe. - proc trampoline(tbus: AsyncEventBus, event: string, key: EventBusKey, - data: EventPayloadBase) {.gcsafe, raises: [].} = - let payload = cast[EventPayload[T]](data) - asyncSpawn callback(bus, payload) - - let subkey = - block: - inc(bus.counter) - EventBusKey(eventName: event, typeName: T.name, unique: bus.counter, - cb: trampoline) - - var default: EventItem - let eventKey = generateKey(T.name, event) - bus.events.mgetOrPut(eventKey, default).subscribers.add(subkey) - subkey - -proc subscribeAll*(bus: AsyncEventBus, - callback: EventBusAllSubscription): EventBusKey {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue instead".} = - ## Subscribe to all events passed through eventbus ``bus`` with callback - ## ``callback``. - ## - ## Returns key that can be used to unsubscribe. - proc trampoline(tbus: AsyncEventBus, event: string, key: EventBusKey, - data: EventPayloadBase) {.gcsafe, raises: [].} = - let event = AwaitableEvent(eventName: event, payload: data) - asyncSpawn callback(bus, event) - - let subkey = - block: - inc(bus.counter) - EventBusKey(eventName: "", typeName: "", unique: bus.counter, - cb: trampoline) - bus.subscribers.add(subkey) - subkey - -proc unsubscribe*(bus: AsyncEventBus, key: EventBusKey) {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue instead".} = - ## Cancel subscription of subscriber with key ``key`` from eventbus ``bus``. - let eventKey = generateKey(key.typeName, key.eventName) - - # Clean event's subscribers. - bus.events.withValue(eventKey, item): - item.subscribers.keepItIf(it.unique != key.unique) - - # Clean subscribers subscribed to all events. - bus.subscribers.keepItIf(it.unique != key.unique) - -proc emit[T](bus: AsyncEventBus, event: string, data: T, loc: ptr SrcLoc) = - let - eventKey = generateKey(T.name, event) - payload = - block: - var data = EventPayload[T](value: data, loc: loc) - cast[EventPayloadBase](data) - - # Used to capture the "subscriber" variable in the loops - # sugar.capture doesn't work in Nim <1.6 - proc triggerSubscriberCallback(subscriber: EventBusKey) = - callSoon(proc(udata: pointer) = - subscriber.cb(bus, event, subscriber, payload) - ) - - bus.events.withValue(eventKey, item): - # Schedule waiters which are waiting for the event ``event``. - for waiter in item.waiters: - var fut = cast[Future[T]](waiter) - fut.complete(data) - # Clear all the waiters. - item.waiters.setLen(0) - - # Schedule subscriber's callbacks, which are subscribed to the event. - for subscriber in item.subscribers: - triggerSubscriberCallback(subscriber) - - # Schedule waiters which are waiting all events - for waiter in bus.waiters: - waiter.complete(AwaitableEvent(eventName: event, payload: payload)) - # Clear all the waiters. - bus.waiters.setLen(0) - - # Schedule subscriber's callbacks which are subscribed to all events. - for subscriber in bus.subscribers: - triggerSubscriberCallback(subscriber) - -template emit*[T](bus: AsyncEventBus, event: string, data: T) {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue instead".} = - ## Emit new event ``event`` to the eventbus ``bus`` with payload ``data``. - emit(bus, event, data, getSrcLocation()) - -proc emitWait[T](bus: AsyncEventBus, event: string, data: T, - loc: ptr SrcLoc): Future[void] = - var retFuture = newFuture[void]("AsyncEventBus.emitWait") - proc continuation(udata: pointer) {.gcsafe.} = - if not(retFuture.finished()): - retFuture.complete() - emit(bus, event, data, loc) - callSoon(continuation) - return retFuture - -template emitWait*[T](bus: AsyncEventBus, event: string, - data: T): Future[void] {. - deprecated: "Implementation has unfixable flaws, please use " & - "AsyncEventQueue instead".} = - ## Emit new event ``event`` to the eventbus ``bus`` with payload ``data`` and - ## wait until all the subscribers/waiters will receive notification about - ## event. - emitWait(bus, event, data, getSrcLocation()) - proc `==`(a, b: EventQueueKey): bool {.borrow.} proc compact(ab: AsyncEventQueue) {.raises: [].} = From 8156e2997a12006b70e84a96f1c81b225eb04b93 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Fri, 10 Nov 2023 08:42:36 +0200 Subject: [PATCH 36/50] Fix not enough memory on i386. (#467) * Fix waitFor() should not exit earlier last callback will be scheduled. * Tune tests to use less memory. * Fix `testutils`. There is no more last poll() needed. * Update chronos/internal/asyncfutures.nim --------- Co-authored-by: Jacek Sieka --- chronos/internal/asyncfutures.nim | 10 +++++++--- tests/testfut.nim | 14 ++++++-------- tests/testhttpclient.nim | 4 ++-- tests/testproc.bat | 2 +- tests/testproc.nim | 4 ++-- tests/testproc.sh | 2 +- tests/testutils.nim | 5 ----- 7 files changed, 19 insertions(+), 22 deletions(-) diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index b144cea..c4a7374 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -560,9 +560,13 @@ proc waitFor*[T](fut: Future[T]): T {.raises: [CatchableError].} = ## **Blocks** the current thread until the specified future finishes and ## reads it, potentially raising an exception if the future failed or was ## cancelled. - while not(fut.finished()): - poll() - + var finished = false + # Ensure that callbacks currently scheduled on the future run before returning + proc continuation(udata: pointer) {.gcsafe.} = finished = true + if not(fut.finished()): + fut.addCallback(continuation) + while not(finished): + poll() fut.read() proc asyncSpawn*(future: Future[void]) = diff --git a/tests/testfut.nim b/tests/testfut.nim index fc9d482..1297dc4 100644 --- a/tests/testfut.nim +++ b/tests/testfut.nim @@ -54,7 +54,6 @@ suite "Future[T] behavior test suite": fut.addCallback proc(udata: pointer) = testResult &= "5" discard waitFor(fut) - poll() check: fut.finished @@ -80,7 +79,6 @@ suite "Future[T] behavior test suite": fut.addCallback cb5 fut.removeCallback cb3 discard waitFor(fut) - poll() check: fut.finished testResult == "1245" @@ -1260,12 +1258,12 @@ suite "Future[T] behavior test suite": (loc.procedure == procedure) check: - chk(loc10, "testfut.nim", 1227, "macroFuture") - chk(loc11, "testfut.nim", 1230, "") - chk(loc20, "testfut.nim", 1239, "template") - chk(loc21, "testfut.nim", 1242, "") - chk(loc30, "testfut.nim", 1236, "procedure") - chk(loc31, "testfut.nim", 1243, "") + chk(loc10, "testfut.nim", 1225, "macroFuture") + chk(loc11, "testfut.nim", 1228, "") + chk(loc20, "testfut.nim", 1237, "template") + chk(loc21, "testfut.nim", 1240, "") + chk(loc30, "testfut.nim", 1234, "procedure") + chk(loc31, "testfut.nim", 1241, "") asyncTest "withTimeout(fut) should wait cancellation test": proc futureNeverEnds(): Future[void] = diff --git a/tests/testhttpclient.nim b/tests/testhttpclient.nim index e10892e..f08b3c5 100644 --- a/tests/testhttpclient.nim +++ b/tests/testhttpclient.nim @@ -187,11 +187,11 @@ suite "HTTP client testing suite": let ResponseTests = [ (MethodGet, "/test/short_size_response", 65600, 1024, "SHORTSIZERESPONSE"), - (MethodGet, "/test/long_size_response", 262400, 1024, + (MethodGet, "/test/long_size_response", 131200, 1024, "LONGSIZERESPONSE"), (MethodGet, "/test/short_chunked_response", 65600, 1024, "SHORTCHUNKRESPONSE"), - (MethodGet, "/test/long_chunked_response", 262400, 1024, + (MethodGet, "/test/long_chunked_response", 131200, 1024, "LONGCHUNKRESPONSE") ] proc process(r: RequestFence): Future[HttpResponseRef] {. diff --git a/tests/testproc.bat b/tests/testproc.bat index 11b4047..0584039 100644 --- a/tests/testproc.bat +++ b/tests/testproc.bat @@ -34,7 +34,7 @@ ping -n 10 127.0.0.1 > NUL EXIT 0 :BIGDATA -FOR /L %%G IN (1, 1, 400000) DO ECHO ALICEWASBEGINNINGTOGETVERYTIREDOFSITTINGBYHERSISTERONTHEBANKANDO +FOR /L %%G IN (1, 1, 100000) DO ECHO ALICEWASBEGINNINGTOGETVERYTIREDOFSITTINGBYHERSISTERONTHEBANKANDO EXIT 0 :ENVTEST diff --git a/tests/testproc.nim b/tests/testproc.nim index 288ec18..588e308 100644 --- a/tests/testproc.nim +++ b/tests/testproc.nim @@ -214,9 +214,9 @@ suite "Asynchronous process management test suite": "tests/testproc.sh bigdata" let expect = when defined(windows): - 400_000 * (64 + 2) + 100_000 * (64 + 2) else: - 400_000 * (64 + 1) + 100_000 * (64 + 1) let process = await startProcess(command, options = options, stdoutHandle = AsyncProcess.Pipe, stderrHandle = AsyncProcess.Pipe) diff --git a/tests/testproc.sh b/tests/testproc.sh index c5e7e0a..e525da5 100755 --- a/tests/testproc.sh +++ b/tests/testproc.sh @@ -12,7 +12,7 @@ elif [ "$1" == "timeout2" ]; then elif [ "$1" == "timeout10" ]; then sleep 10 elif [ "$1" == "bigdata" ]; then - for i in {1..400000} + for i in {1..100000} do echo "ALICEWASBEGINNINGTOGETVERYTIREDOFSITTINGBYHERSISTERONTHEBANKANDO" done diff --git a/tests/testutils.nim b/tests/testutils.nim index fb5de50..49072eb 100644 --- a/tests/testutils.nim +++ b/tests/testutils.nim @@ -75,11 +75,6 @@ suite "Asynchronous utilities test suite": pendingFuturesCount() == 2'u waitFor fut - check: - getCount() == 1'u - pendingFuturesCount() == 1'u - - poll() check: getCount() == 0'u pendingFuturesCount() == 0'u From f0eb7a0ae9ef02fa9afdbb80ff4f0f4f42fe4dcc Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 13 Nov 2023 10:54:37 +0100 Subject: [PATCH 37/50] simplify tests (#469) * simplify tests `chronosPreviewV4` is obsolete * oops --- chronos.nimble | 10 +++------- nim.cfg | 1 + 2 files changed, 4 insertions(+), 7 deletions(-) create mode 100644 nim.cfg diff --git a/chronos.nimble b/chronos.nimble index e2fa998..667d1da 100644 --- a/chronos.nimble +++ b/chronos.nimble @@ -23,24 +23,20 @@ let testArguments = when defined(windows): [ "-d:debug -d:chronosDebug -d:useSysAssert -d:useGcAssert", - "-d:debug -d:chronosPreviewV4", "-d:release", - "-d:release -d:chronosPreviewV4" ] else: [ "-d:debug -d:chronosDebug -d:useSysAssert -d:useGcAssert", - "-d:debug -d:chronosPreviewV4", "-d:debug -d:chronosDebug -d:chronosEventEngine=poll -d:useSysAssert -d:useGcAssert", "-d:release", - "-d:release -d:chronosPreviewV4" ] -let styleCheckStyle = if (NimMajor, NimMinor) < (1, 6): "hint" else: "error" let cfg = - " --styleCheck:usages --styleCheck:" & styleCheckStyle & + " --styleCheck:usages --styleCheck:error" & (if verbose: "" else: " --verbosity:0 --hints:off") & - " --skipParentCfg --skipUserCfg --outdir:build --nimcache:build/nimcache -f" + " --skipParentCfg --skipUserCfg --outdir:build " & + quoteShell("--nimcache:build/nimcache/$projectName") proc build(args, path: string) = exec nimc & " " & lang & " " & cfg & " " & flags & " " & args & " " & path diff --git a/nim.cfg b/nim.cfg new file mode 100644 index 0000000..45d538b --- /dev/null +++ b/nim.cfg @@ -0,0 +1 @@ +nimcache = "build/nimcache/$projectName" From 0d55475c29f232b849c9d9456bcb21287b046cda Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 13 Nov 2023 10:56:19 +0100 Subject: [PATCH 38/50] `stew/results` -> `results` (#468) --- chronos.nimble | 1 + chronos/apps/http/httpclient.nim | 4 ++-- chronos/apps/http/httpcommon.nim | 2 +- chronos/apps/http/httpdebug.nim | 2 +- chronos/apps/http/httpserver.nim | 4 ++-- chronos/apps/http/multipart.nim | 4 ++-- chronos/asyncproc.nim | 2 +- chronos/handles.nim | 2 +- chronos/internal/asyncengine.nim | 2 +- chronos/osutils.nim | 4 ++-- chronos/selectors2.nim | 4 ++-- chronos/streams/boundstream.nim | 2 +- chronos/streams/chunkstream.nim | 2 +- chronos/threadsync.nim | 2 +- 14 files changed, 19 insertions(+), 18 deletions(-) diff --git a/chronos.nimble b/chronos.nimble index 667d1da..b2fdb3a 100644 --- a/chronos.nimble +++ b/chronos.nimble @@ -8,6 +8,7 @@ license = "MIT or Apache License 2.0" skipDirs = @["tests"] requires "nim >= 1.6.0", + "results", "stew", "bearssl", "httputils", diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 34089c7..83d1ddf 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -7,13 +7,13 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) import std/[uri, tables, sequtils] -import stew/[results, base10, base64, byteutils], httputils +import stew/[base10, base64, byteutils], httputils, results import ../../asyncloop, ../../asyncsync import ../../streams/[asyncstream, tlsstream, chunkstream, boundstream] import httptable, httpcommon, httpagent, httpbodyrw, multipart export results, asyncloop, asyncsync, asyncstream, tlsstream, chunkstream, boundstream, httptable, httpcommon, httpagent, httpbodyrw, multipart, - httputils, uri + httputils, uri, results export SocketFlags const diff --git a/chronos/apps/http/httpcommon.nim b/chronos/apps/http/httpcommon.nim index c01c1c3..da5e03f 100644 --- a/chronos/apps/http/httpcommon.nim +++ b/chronos/apps/http/httpcommon.nim @@ -7,7 +7,7 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) import std/[strutils, uri] -import stew/results, httputils +import results, httputils import ../../asyncloop, ../../asyncsync import ../../streams/[asyncstream, boundstream] export asyncloop, asyncsync, results, httputils, strutils diff --git a/chronos/apps/http/httpdebug.nim b/chronos/apps/http/httpdebug.nim index a1dc022..d343265 100644 --- a/chronos/apps/http/httpdebug.nim +++ b/chronos/apps/http/httpdebug.nim @@ -7,7 +7,7 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) import std/tables -import stew/results +import results import ../../timer import httpserver, shttpserver from httpclient import HttpClientScheme diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index 2ab5317..1e307a0 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -7,10 +7,10 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) import std/[tables, uri, strutils] -import stew/[results, base10], httputils +import stew/[base10], httputils, results import ../../asyncloop, ../../asyncsync import ../../streams/[asyncstream, boundstream, chunkstream] -import httptable, httpcommon, multipart +import "."/[httptable, httpcommon, multipart] export asyncloop, asyncsync, httptable, httpcommon, httputils, multipart, asyncstream, boundstream, chunkstream, uri, tables, results diff --git a/chronos/apps/http/multipart.nim b/chronos/apps/http/multipart.nim index 45506a2..b936996 100644 --- a/chronos/apps/http/multipart.nim +++ b/chronos/apps/http/multipart.nim @@ -8,10 +8,10 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) import std/[monotimes, strutils] -import stew/results, httputils +import results, httputils import ../../asyncloop import ../../streams/[asyncstream, boundstream, chunkstream] -import httptable, httpcommon, httpbodyrw +import "."/[httptable, httpcommon, httpbodyrw] export asyncloop, httptable, httpcommon, httpbodyrw, asyncstream, httputils const diff --git a/chronos/asyncproc.nim b/chronos/asyncproc.nim index 3e2df88..8615c57 100644 --- a/chronos/asyncproc.nim +++ b/chronos/asyncproc.nim @@ -13,7 +13,7 @@ import std/strtabs import "."/[config, asyncloop, handles, osdefs, osutils, oserrno], streams/asyncstream -import stew/[results, byteutils] +import stew/[byteutils], results from std/os import quoteShell, quoteShellWindows, quoteShellPosix, envPairs export strtabs, results diff --git a/chronos/handles.nim b/chronos/handles.nim index afa57fb..72b0751 100644 --- a/chronos/handles.nim +++ b/chronos/handles.nim @@ -10,7 +10,7 @@ {.push raises: [].} import "."/[asyncloop, osdefs, osutils] -import stew/results +import results from nativesockets import Domain, Protocol, SockType, toInt export Domain, Protocol, SockType, results diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim index 0a15799..d4e803c 100644 --- a/chronos/internal/asyncengine.nim +++ b/chronos/internal/asyncengine.nim @@ -12,7 +12,7 @@ from nativesockets import Port import std/[tables, heapqueue, deques] -import stew/results +import results import ".."/[config, futures, osdefs, oserrno, osutils, timer] import ./[asyncmacro, errors] diff --git a/chronos/osutils.nim b/chronos/osutils.nim index f9c09f2..d93c261 100644 --- a/chronos/osutils.nim +++ b/chronos/osutils.nim @@ -6,8 +6,8 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import stew/results -import osdefs, oserrno +import results +import "."/[osdefs, oserrno] export results diff --git a/chronos/selectors2.nim b/chronos/selectors2.nim index 5cb8a57..db8791a 100644 --- a/chronos/selectors2.nim +++ b/chronos/selectors2.nim @@ -31,8 +31,8 @@ # support - changes could potentially be backported to nim but are not # backwards-compatible. -import stew/results -import config, osdefs, osutils, oserrno +import results +import "."/[config, osdefs, osutils, oserrno] export results, oserrno when defined(nimdoc): diff --git a/chronos/streams/boundstream.nim b/chronos/streams/boundstream.nim index 73321eb..dbb36ef 100644 --- a/chronos/streams/boundstream.nim +++ b/chronos/streams/boundstream.nim @@ -14,7 +14,7 @@ ## ## For stream writing it means that you should write exactly bounded size ## of bytes. -import stew/results +import results import ../asyncloop, ../timer import asyncstream, ../transports/stream, ../transports/common export asyncloop, asyncstream, stream, timer, common diff --git a/chronos/streams/chunkstream.nim b/chronos/streams/chunkstream.nim index 729d8de..c0269a2 100644 --- a/chronos/streams/chunkstream.nim +++ b/chronos/streams/chunkstream.nim @@ -10,7 +10,7 @@ ## This module implements HTTP/1.1 chunked-encoded stream reading and writing. import ../asyncloop, ../timer import asyncstream, ../transports/stream, ../transports/common -import stew/results +import results export asyncloop, asyncstream, stream, timer, common, results const diff --git a/chronos/threadsync.nim b/chronos/threadsync.nim index d414181..bbff18b 100644 --- a/chronos/threadsync.nim +++ b/chronos/threadsync.nim @@ -8,7 +8,7 @@ # MIT license (LICENSE-MIT) ## This module implements some core async thread synchronization primitives. -import stew/results +import results import "."/[timer, asyncloop] export results From 9c93ab48deb0a0c5179b941cc036f373eb30ed6e Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Mon, 13 Nov 2023 13:14:21 +0200 Subject: [PATCH 39/50] Attempt to fix CI crash at Windows. (#465) * Attempt to fix CI crash at Windows. Remove all cast[string] and cast[seq[byte]] from the codebase. * Address review comments. --- chronos/internal/asyncengine.nim | 20 +++++++----- tests/testasyncstream.nim | 55 ++++++++++++++++---------------- tests/testdatagram.nim | 9 +++--- tests/testhttpclient.nim | 28 ++++++++-------- 4 files changed, 59 insertions(+), 53 deletions(-) diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim index d4e803c..578bfdf 100644 --- a/chronos/internal/asyncengine.nim +++ b/chronos/internal/asyncengine.nim @@ -1125,14 +1125,18 @@ proc addTimer*(at: uint64, cb: CallbackFunc, udata: pointer = nil) {. proc removeTimer*(at: Moment, cb: CallbackFunc, udata: pointer = nil) = ## Remove timer callback ``cb`` with absolute timestamp ``at`` from waiting ## queue. - let loop = getThreadDispatcher() - var list = cast[seq[TimerCallback]](loop.timers) - var index = -1 - for i in 0.. Date: Wed, 15 Nov 2023 09:06:37 +0100 Subject: [PATCH 40/50] move docs to docs (#466) * introduce user guide based on `mdbook` * set up structure for adding simple `chronos` usage examples * move most readme content to book * ci deploys book and api guide automatically * remove most of existing engine docs (obsolete) --- .appveyor.yml | 40 --- .github/workflows/ci.yml | 1 + .github/workflows/doc.yml | 56 ++-- .gitignore | 1 - .travis.yml | 27 -- README.md | 453 ++---------------------------- chronos.nim | 5 + chronos.nimble | 13 +- chronos/asyncloop.nim | 118 -------- chronos/internal/asyncengine.nim | 4 + docs/.gitignore | 1 + docs/book.toml | 20 ++ docs/examples/cancellation.nim | 21 ++ docs/examples/discards.nim | 28 ++ docs/examples/httpget.nim | 15 + docs/examples/nim.cfg | 1 + docs/examples/timeoutcomposed.nim | 25 ++ docs/examples/timeoutsimple.nim | 20 ++ docs/examples/twogets.nim | 24 ++ docs/open-in.css | 7 + docs/src/SUMMARY.md | 10 + docs/src/async_procs.md | 112 ++++++++ docs/src/concepts.md | 126 +++++++++ docs/src/error_handling.md | 134 +++++++++ docs/src/getting_started.md | 19 ++ docs/src/introduction.md | 32 +++ docs/src/porting.md | 54 ++++ docs/src/tips.md | 34 +++ docs/theme/highlight.js | 53 ++++ 29 files changed, 811 insertions(+), 643 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .travis.yml create mode 100644 docs/.gitignore create mode 100644 docs/book.toml create mode 100644 docs/examples/cancellation.nim create mode 100644 docs/examples/discards.nim create mode 100644 docs/examples/httpget.nim create mode 100644 docs/examples/nim.cfg create mode 100644 docs/examples/timeoutcomposed.nim create mode 100644 docs/examples/timeoutsimple.nim create mode 100644 docs/examples/twogets.nim create mode 100644 docs/open-in.css create mode 100644 docs/src/SUMMARY.md create mode 100644 docs/src/async_procs.md create mode 100644 docs/src/concepts.md create mode 100644 docs/src/error_handling.md create mode 100644 docs/src/getting_started.md create mode 100644 docs/src/introduction.md create mode 100644 docs/src/porting.md create mode 100644 docs/src/tips.md create mode 100644 docs/theme/highlight.js diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 768c261..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: '{build}' - -image: Visual Studio 2015 - -cache: -- NimBinaries - -matrix: - # We always want 32 and 64-bit compilation - fast_finish: false - -platform: - - x86 - - x64 - -# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" -clone_depth: 10 - -install: - # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin - - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH% - - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - # build nim from our own branch - this to avoid the day-to-day churn and - # regressions of the fast-paced Nim development while maintaining the - # flexibility to apply patches - - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries - - SET PATH=%CD%\Nim\bin;%PATH% - -build_script: - - cd C:\projects\%APPVEYOR_PROJECT_SLUG% - - nimble install -y --depsOnly - - nimble install -y libbacktrace - -test_script: - - nimble test - -deploy: off - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d2cace..e64f754 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,3 +165,4 @@ jobs: nimble install -y libbacktrace nimble test nimble test_libbacktrace + nimble examples diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 6e1510a..dc718f8 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -18,6 +18,26 @@ jobs: uses: actions/checkout@v3 with: submodules: true + - uses: actions-rs/install@v0.1 + with: + crate: mdbook + use-tool-cache: true + version: "0.4.35" + - uses: actions-rs/install@v0.1 + with: + crate: mdbook-toc + use-tool-cache: true + version: "0.14.1" + - uses: actions-rs/install@v0.1 + with: + crate: mdbook-open-on-gh + use-tool-cache: true + version: "2.4.1" + - uses: actions-rs/install@v0.1 + with: + crate: mdbook-admonish + use-tool-cache: true + version: "1.13.1" - uses: jiro4989/setup-nim-action@v1 with: @@ -28,35 +48,11 @@ jobs: nim --version nimble --version nimble install -dy - # nim doc can "fail", but the doc is still generated - nim doc --git.url:https://github.com/status-im/nim-chronos --git.commit:master --outdir:docs --project chronos || true + nimble docs || true - # check that the folder exists - ls docs - - - name: Clone the gh-pages branch - uses: actions/checkout@v3 + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 with: - repository: status-im/nim-chronos - ref: gh-pages - path: subdoc - submodules: true - fetch-depth: 0 - - - name: Commit & push - run: | - cd subdoc - - # Update / create this branch doc - rm -rf docs - mv ../docs . - - # Remove .idx files - # NOTE: git also uses idx files in his - # internal folder, hence the `*` instead of `.` - find * -name "*.idx" -delete - git add . - git config --global user.email "${{ github.actor }}@users.noreply.github.com" - git config --global user.name = "${{ github.actor }}" - git commit -a -m "update docs" - git push origin gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/book + force_orphan: true diff --git a/.gitignore b/.gitignore index b599536..c631551 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,3 @@ nimble.develop nimble.paths /build/ nimbledeps -/docs diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1a5bcd3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -language: c - -# https://docs.travis-ci.com/user/caching/ -cache: - directories: - - NimBinaries - -git: - # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" - depth: 10 - -os: - - linux - - osx - -install: - # build nim from our own branch - this to avoid the day-to-day churn and - # regressions of the fast-paced Nim development while maintaining the - # flexibility to apply patches - - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries - - export PATH="$PWD/Nim/bin:$PATH" - -script: - - nimble install -y - - nimble test - diff --git a/README.md b/README.md index c80f826..b3a80fe 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,11 @@ Chronos is an efficient [async/await](https://en.wikipedia.org/wiki/Async/await) * Synchronization primitivies like queues, events and locks * Cancellation * Efficient dispatch pipeline with excellent multi-platform support -* Exception effect support (see [exception effects](#exception-effects)) +* Exceptional error handling features, including `raises` tracking -## Installation +## Getting started -You can use Nim's official package manager Nimble to install Chronos: +Install `chronos` using `nimble`: ```text nimble install chronos @@ -30,6 +30,30 @@ or add a dependency to your `.nimble` file: requires "chronos" ``` +and start using it: + +```nim +import chronos/apps/http/httpclient + +proc retrievePage(uri: string): Future[string] {.async.} = + # Create a new HTTP session + let httpSession = HttpSessionRef.new() + try: + # Fetch page contents + let resp = await httpSession.fetch(parseUri(uri)) + # Convert response to a string, assuming its encoding matches the terminal! + bytesToString(resp.data) + finally: # Close the session + await noCancel(httpSession.closeWait()) + +echo waitFor retrievePage( + "https://raw.githubusercontent.com/status-im/nim-chronos/master/README.md") +``` + +## Documentation + +See the [user guide](https://status-im.github.io/nim-chronos/). + ## Projects using `chronos` * [libp2p](https://github.com/status-im/nim-libp2p) - Peer-to-Peer networking stack implemented in many languages @@ -42,426 +66,7 @@ requires "chronos" Submit a PR to add yours! -## Documentation - -### Concepts - -Chronos implements the async/await paradigm in a self-contained library using -the macro and closure iterator transformation features provided by Nim. - -The event loop is called a "dispatcher" and a single instance per thread is -created, as soon as one is needed. - -To trigger a dispatcher's processing step, we need to call `poll()` - either -directly or through a wrapper like `runForever()` or `waitFor()`. Each step -handles any file descriptors, timers and callbacks that are ready to be -processed. - -`Future` objects encapsulate the result of an `async` procedure upon successful -completion, and a list of callbacks to be scheduled after any type of -completion - be that success, failure or cancellation. - -(These explicit callbacks are rarely used outside Chronos, being replaced by -implicit ones generated by async procedure execution and `await` chaining.) - -Async procedures (those using the `{.async.}` pragma) return `Future` objects. - -Inside an async procedure, you can `await` the future returned by another async -procedure. At this point, control will be handled to the event loop until that -future is completed. - -Future completion is tested with `Future.finished()` and is defined as success, -failure or cancellation. This means that a future is either pending or completed. - -To differentiate between completion states, we have `Future.failed()` and -`Future.cancelled()`. - -### Dispatcher - -You can run the "dispatcher" event loop forever, with `runForever()` which is defined as: - -```nim -proc runForever*() = - while true: - poll() -``` - -You can also run it until a certain future is completed, with `waitFor()` which -will also call `Future.read()` on it: - -```nim -proc p(): Future[int] {.async.} = - await sleepAsync(100.milliseconds) - return 1 - -echo waitFor p() # prints "1" -``` - -`waitFor()` is defined like this: - -```nim -proc waitFor*[T](fut: Future[T]): T = - while not(fut.finished()): - poll() - return fut.read() -``` - -### Async procedures and methods - -The `{.async.}` pragma will transform a procedure (or a method) returning a -specialised `Future` type into a closure iterator. If there is no return type -specified, a `Future[void]` is returned. - -```nim -proc p() {.async.} = - await sleepAsync(100.milliseconds) - -echo p().type # prints "Future[system.void]" -``` - -Whenever `await` is encountered inside an async procedure, control is passed -back to the dispatcher for as many steps as it's necessary for the awaited -future to complete successfully, fail or be cancelled. `await` calls the -equivalent of `Future.read()` on the completed future and returns the -encapsulated value. - -```nim -proc p1() {.async.} = - await sleepAsync(1.seconds) - -proc p2() {.async.} = - await sleepAsync(1.seconds) - -proc p3() {.async.} = - let - fut1 = p1() - fut2 = p2() - # Just by executing the async procs, both resulting futures entered the - # dispatcher's queue and their "clocks" started ticking. - await fut1 - await fut2 - # Only one second passed while awaiting them both, not two. - -waitFor p3() -``` - -Don't let `await`'s behaviour of giving back control to the dispatcher surprise -you. If an async procedure modifies global state, and you can't predict when it -will start executing, the only way to avoid that state changing underneath your -feet, in a certain section, is to not use `await` in it. - -### Error handling - -Exceptions inheriting from [`CatchableError`](https://nim-lang.org/docs/system.html#CatchableError) -interrupt execution of the `async` procedure. The exception is placed in the -`Future.error` field while changing the status of the `Future` to `Failed` -and callbacks are scheduled. - -When a future is awaited, the exception is re-raised, traversing the `async` -execution chain until handled. - -```nim -proc p1() {.async.} = - await sleepAsync(1.seconds) - raise newException(ValueError, "ValueError inherits from CatchableError") - -proc p2() {.async.} = - await sleepAsync(1.seconds) - -proc p3() {.async.} = - let - fut1 = p1() - fut2 = p2() - await fut1 - echo "unreachable code here" - await fut2 - -# `waitFor()` would call `Future.read()` unconditionally, which would raise the -# exception in `Future.error`. -let fut3 = p3() -while not(fut3.finished()): - poll() - -echo "fut3.state = ", fut3.state # "Failed" -if fut3.failed(): - echo "p3() failed: ", fut3.error.name, ": ", fut3.error.msg - # prints "p3() failed: ValueError: ValueError inherits from CatchableError" -``` - -You can put the `await` in a `try` block, to deal with that exception sooner: - -```nim -proc p3() {.async.} = - let - fut1 = p1() - fut2 = p2() - try: - await fut1 - except CachableError: - echo "p1() failed: ", fut1.error.name, ": ", fut1.error.msg - echo "reachable code here" - await fut2 -``` - -Because `chronos` ensures that all exceptions are re-routed to the `Future`, -`poll` will not itself raise exceptions. - -`poll` may still panic / raise `Defect` if such are raised in user code due to -undefined behavior. - -#### Checked exceptions - -By specifying a `raises` list to an async procedure, you can check which -exceptions can be raised by it: - -```nim -proc p1(): Future[void] {.async: (raises: [IOError]).} = - assert not (compiles do: raise newException(ValueError, "uh-uh")) - raise newException(IOError, "works") # Or any child of IOError - -proc p2(): Future[void] {.async, (raises: [IOError]).} = - await p1() # Works, because await knows that p1 - # can only raise IOError -``` - -Under the hood, the return type of `p1` will be rewritten to an internal type -which will convey raises informations to `await`. - -#### The `Exception` type - -Exceptions deriving from `Exception` are not caught by default as these may -include `Defect` and other forms undefined or uncatchable behavior. - -Because exception effect tracking is turned on for `async` functions, this may -sometimes lead to compile errors around forward declarations, methods and -closures as Nim conservatively asssumes that any `Exception` might be raised -from those. - -Make sure to excplicitly annotate these with `{.raises.}`: - -```nim -# Forward declarations need to explicitly include a raises list: -proc myfunction() {.raises: [ValueError].} - -# ... as do `proc` types -type MyClosure = proc() {.raises: [ValueError].} - -proc myfunction() = - raise (ref ValueError)(msg: "Implementation here") - -let closure: MyClosure = myfunction -``` - -For compatibility, `async` functions can be instructed to handle `Exception` as -well, specifying `handleException: true`. `Exception` that is not a `Defect` and -not a `CatchableError` will then be caught and remapped to -`AsyncExceptionError`: - -```nim -proc raiseException() {.async: (handleException: true, raises: [AsyncExceptionError]).} = - raise (ref Exception)(msg: "Raising Exception is UB") - -proc callRaiseException() {.async: (raises: []).} = - try: - raiseException() - except AsyncExceptionError as exc: - # The original Exception is available from the `parent` field - echo exc.parent.msg -``` - -This mode can be enabled globally with `-d:chronosHandleException` as a help -when porting code to `chronos` but should generally be avoided as global -configuration settings may interfere with libraries that use `chronos` leading -to unexpected behavior. - -### Raw functions - -Raw functions are those that interact with `chronos` via the `Future` type but -whose body does not go through the async transformation. - -Such functions are created by adding `raw: true` to the `async` parameters: - -```nim -proc rawAsync(): Future[void] {.async: (raw: true).} = - let future = newFuture[void]("rawAsync") - future.complete() - return future -``` - -Raw functions must not raise exceptions directly - they are implicitly declared -as `raises: []` - instead they should store exceptions in the returned `Future`: - -```nim -proc rawFailure(): Future[void] {.async: (raw: true).} = - let future = newFuture[void]("rawAsync") - future.fail((ref ValueError)(msg: "Oh no!")) - return future -``` - -Raw functions can also use checked exceptions: - -```nim -proc rawAsyncRaises(): Future[void] {.async: (raw: true, raises: [IOError]).} = - let fut = newFuture[void]() - assert not (compiles do: fut.fail((ref ValueError)(msg: "uh-uh"))) - fut.fail((ref IOError)(msg: "IO")) - return fut -``` - -### Callbacks and closures - -Callback/closure types are declared using the `async` annotation as usual: - -```nim -type MyCallback = proc(): Future[void] {.async.} - -proc runCallback(cb: MyCallback) {.async: (raises: []).} = - try: - await cb() - except CatchableError: - discard # handle errors as usual -``` - -When calling a callback, it is important to remember that the given function -may raise and exceptions need to be handled. - -Checked exceptions can be used to limit the exceptions that a callback can -raise: - -```nim -type MyEasyCallback = proc: Future[void] {.async: (raises: []).} - -proc runCallback(cb: MyEasyCallback) {.async: (raises: [])} = - await cb() -``` - -### Platform independence - -Several functions in `chronos` are backed by the operating system, such as -waiting for network events, creating files and sockets etc. The specific -exceptions that are raised by the OS is platform-dependent, thus such functions -are declared as raising `CatchableError` but will in general raise something -more specific. In particular, it's possible that some functions that are -annotated as raising `CatchableError` only raise on _some_ platforms - in order -to work on all platforms, calling code must assume that they will raise even -when they don't seem to do so on one platform. - -### Cancellation support - -Any running `Future` can be cancelled. This can be used for timeouts, -to let a user cancel a running task, to start multiple futures in parallel -and cancel them as soon as one finishes, etc. - -```nim -import chronos/apps/http/httpclient - -proc cancellationExample() {.async.} = - # Simple cancellation - let future = sleepAsync(10.minutes) - future.cancelSoon() - # `cancelSoon` will not wait for the cancellation - # to be finished, so the Future could still be - # pending at this point. - - # Wait for cancellation - let future2 = sleepAsync(10.minutes) - await future2.cancelAndWait() - # Using `cancelAndWait`, we know that future2 isn't - # pending anymore. However, it could have completed - # before cancellation happened (in which case, it - # will hold a value) - - # Race between futures - proc retrievePage(uri: string): Future[string] {.async.} = - let httpSession = HttpSessionRef.new() - try: - let resp = await httpSession.fetch(parseUri(uri)) - return bytesToString(resp.data) - finally: - # be sure to always close the session - # `finally` will run also during cancellation - - # `noCancel` ensures that `closeWait` doesn't get cancelled - await noCancel(httpSession.closeWait()) - - let - futs = - @[ - retrievePage("https://duckduckgo.com/?q=chronos"), - retrievePage("https://www.google.fr/search?q=chronos") - ] - - let finishedFut = await one(futs) - for fut in futs: - if not fut.finished: - fut.cancelSoon() - echo "Result: ", await finishedFut - -waitFor(cancellationExample()) -``` - -Even if cancellation is initiated, it is not guaranteed that -the operation gets cancelled - the future might still be completed -or fail depending on the ordering of events and the specifics of -the operation. - -If the future indeed gets cancelled, `await` will raise a -`CancelledError` as is likely to happen in the following example: -```nim -proc c1 {.async.} = - echo "Before sleep" - try: - await sleepAsync(10.minutes) - echo "After sleep" # not reach due to cancellation - except CancelledError as exc: - echo "We got cancelled!" - raise exc - -proc c2 {.async.} = - await c1() - echo "Never reached, since the CancelledError got re-raised" - -let work = c2() -waitFor(work.cancelAndWait()) -``` - -The `CancelledError` will now travel up the stack like any other exception. -It can be caught and handled (for instance, freeing some resources) - -### Multiple async backend support - -Thanks to its powerful macro support, Nim allows `async`/`await` to be -implemented in libraries with only minimal support from the language - as such, -multiple `async` libraries exist, including `chronos` and `asyncdispatch`, and -more may come to be developed in the futures. - -Libraries built on top of `async`/`await` may wish to support multiple async -backends - the best way to do so is to create separate modules for each backend -that may be imported side-by-side - see [nim-metrics](https://github.com/status-im/nim-metrics/blob/master/metrics/) -for an example. - -An alternative way is to select backend using a global compile flag - this -method makes it diffucult to compose applications that use both backends as may -happen with transitive dependencies, but may be appropriate in some cases - -libraries choosing this path should call the flag `asyncBackend`, allowing -applications to choose the backend with `-d:asyncBackend=`. - -Known `async` backends include: - -* `chronos` - this library (`-d:asyncBackend=chronos`) -* `asyncdispatch` the standard library `asyncdispatch` [module](https://nim-lang.org/docs/asyncdispatch.html) (`-d:asyncBackend=asyncdispatch`) -* `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely - -``none`` can be used when a library supports both a synchronous and -asynchronous API, to disable the latter. - -### Compile-time configuration - -`chronos` contains several compile-time [configuration options](./chronos/config.nim) enabling stricter compile-time checks and debugging helpers whose runtime cost may be significant. - -Strictness options generally will become default in future chronos releases and allow adapting existing code without changing the new version - see the [`config.nim`](./chronos/config.nim) module for more information. - ## TODO - * Pipe/Subprocess Transports. * Multithreading Stream/Datagram servers ## Contributing @@ -470,10 +75,6 @@ When submitting pull requests, please add test cases for any new features or fix `chronos` follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/). -## Other resources - -* [Historical differences with asyncdispatch](https://github.com/status-im/nim-chronos/wiki/AsyncDispatch-comparison) - ## License Licensed and distributed under either of diff --git a/chronos.nim b/chronos.nim index 8295924..c044f42 100644 --- a/chronos.nim +++ b/chronos.nim @@ -5,5 +5,10 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +## `async`/`await` framework for [Nim](https://nim-lang.org) +## +## See https://status-im.github.io/nim-chronos/ for documentation + import chronos/[asyncloop, asyncsync, handles, transport, timer, debugutils] export asyncloop, asyncsync, handles, transport, timer, debugutils diff --git a/chronos.nimble b/chronos.nimble index b2fdb3a..e435883 100644 --- a/chronos.nimble +++ b/chronos.nimble @@ -14,7 +14,7 @@ requires "nim >= 1.6.0", "httputils", "unittest2" -import os +import os, strutils let nimc = getEnv("NIMC", "nim") # Which nim compiler to use let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js) @@ -46,12 +46,19 @@ proc run(args, path: string) = build args, path exec "build/" & path.splitPath[1] +task examples, "Build examples": + # Build book examples + for file in listFiles("docs/examples"): + if file.endsWith(".nim"): + build "", file + task test, "Run all tests": for args in testArguments: run args, "tests/testall" if (NimMajor, NimMinor) > (1, 6): run args & " --mm:refc", "tests/testall" + task test_libbacktrace, "test with libbacktrace": var allArgs = @[ "-d:release --debugger:native -d:chronosStackTrace -d:nimStackTraceOverride --import:libbacktrace", @@ -59,3 +66,7 @@ task test_libbacktrace, "test with libbacktrace": for args in allArgs: run args, "tests/testall" + +task docs, "Generate API documentation": + exec "mdbook build docs" + exec nimc & " doc " & "--git.url:https://github.com/status-im/nim-chronos --git.commit:master --outdir:docs/book/api --project chronos" diff --git a/chronos/asyncloop.nim b/chronos/asyncloop.nim index 428252c..7204226 100644 --- a/chronos/asyncloop.nim +++ b/chronos/asyncloop.nim @@ -10,124 +10,6 @@ {.push raises: [].} -## Chronos -## ************* -## -## This module implements asynchronous IO. This includes a dispatcher, -## a ``Future`` type implementation, and an ``async`` macro which allows -## asynchronous code to be written in a synchronous style with the ``await`` -## keyword. -## -## The dispatcher acts as a kind of event loop. You must call ``poll`` on it -## (or a function which does so for you such as ``waitFor`` or ``runForever``) -## in order to poll for any outstanding events. The underlying implementation -## is based on epoll on Linux, IO Completion Ports on Windows and select on -## other operating systems. -## -## The ``poll`` function will not, on its own, return any events. Instead -## an appropriate ``Future`` object will be completed. A ``Future`` is a -## type which holds a value which is not yet available, but which *may* be -## available in the future. You can check whether a future is finished -## by using the ``finished`` function. When a future is finished it means that -## either the value that it holds is now available or it holds an error instead. -## The latter situation occurs when the operation to complete a future fails -## with an exception. You can distinguish between the two situations with the -## ``failed`` function. -## -## Future objects can also store a callback procedure which will be called -## automatically once the future completes. -## -## Futures therefore can be thought of as an implementation of the proactor -## pattern. In this -## pattern you make a request for an action, and once that action is fulfilled -## a future is completed with the result of that action. Requests can be -## made by calling the appropriate functions. For example: calling the ``recv`` -## function will create a request for some data to be read from a socket. The -## future which the ``recv`` function returns will then complete once the -## requested amount of data is read **or** an exception occurs. -## -## Code to read some data from a socket may look something like this: -## -## .. code-block::nim -## var future = socket.recv(100) -## future.addCallback( -## proc () = -## echo(future.read) -## ) -## -## All asynchronous functions returning a ``Future`` will not block. They -## will not however return immediately. An asynchronous function will have -## code which will be executed before an asynchronous request is made, in most -## cases this code sets up the request. -## -## In the above example, the ``recv`` function will return a brand new -## ``Future`` instance once the request for data to be read from the socket -## is made. This ``Future`` instance will complete once the requested amount -## of data is read, in this case it is 100 bytes. The second line sets a -## callback on this future which will be called once the future completes. -## All the callback does is write the data stored in the future to ``stdout``. -## The ``read`` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. -## -## Asynchronous procedures -## ----------------------- -## -## Asynchronous procedures remove the pain of working with callbacks. They do -## this by allowing you to write asynchronous code the same way as you would -## write synchronous code. -## -## An asynchronous procedure is marked using the ``{.async.}`` pragma. -## When marking a procedure with the ``{.async.}`` pragma it must have a -## ``Future[T]`` return type or no return type at all. If you do not specify -## a return type then ``Future[void]`` is assumed. -## -## Inside asynchronous procedures ``await`` can be used to call any -## procedures which return a -## ``Future``; this includes asynchronous procedures. When a procedure is -## "awaited", the asynchronous procedure it is awaited in will -## suspend its execution -## until the awaited procedure's Future completes. At which point the -## asynchronous procedure will resume its execution. During the period -## when an asynchronous procedure is suspended other asynchronous procedures -## will be run by the dispatcher. -## -## The ``await`` call may be used in many contexts. It can be used on the right -## hand side of a variable declaration: ``var data = await socket.recv(100)``, -## in which case the variable will be set to the value of the future -## automatically. It can be used to await a ``Future`` object, and it can -## be used to await a procedure returning a ``Future[void]``: -## ``await socket.send("foobar")``. -## -## If an awaited future completes with an error, then ``await`` will re-raise -## this error. -## -## Handling Exceptions -## ------------------- -## -## The ``async`` procedures also offer support for the try statement. -## -## .. code-block:: Nim -## try: -## let data = await sock.recv(100) -## echo("Received ", data) -## except CancelledError as exc: -## # Handle exc -## -## Discarding futures -## ------------------ -## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the ``asyncSpawn`` procedure instead of the ``discard`` keyword. -## ``asyncSpawn`` will transform any exception thrown by the called procedure -## to a Defect -## -## Limitations/Bugs -## ---------------- -## -## * The effect system (``raises: []``) does not work with async procedures. - import ./internal/[asyncengine, asyncfutures, asyncmacro, errors] export asyncfutures, asyncengine, errors diff --git a/chronos/internal/asyncengine.nim b/chronos/internal/asyncengine.nim index 578bfdf..d794f72 100644 --- a/chronos/internal/asyncengine.nim +++ b/chronos/internal/asyncengine.nim @@ -10,6 +10,10 @@ {.push raises: [].} +## This module implements the core asynchronous engine / dispatcher. +## +## For more information, see the `Concepts` chapter of the guide. + from nativesockets import Port import std/[tables, heapqueue, deques] import results diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..7585238 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +book diff --git a/docs/book.toml b/docs/book.toml new file mode 100644 index 0000000..570b8f4 --- /dev/null +++ b/docs/book.toml @@ -0,0 +1,20 @@ +[book] +authors = ["Jacek Sieka"] +language = "en" +multilingual = false +src = "src" +title = "Chronos" + +[preprocessor.toc] +command = "mdbook-toc" +renderer = ["html"] +max-level = 2 + +[preprocessor.open-on-gh] +command = "mdbook-open-on-gh" +renderer = ["html"] + +[output.html] +git-repository-url = "https://github.com/status-im/nim-chronos/" +git-branch = "master" +additional-css = ["open-in.css"] diff --git a/docs/examples/cancellation.nim b/docs/examples/cancellation.nim new file mode 100644 index 0000000..5feec31 --- /dev/null +++ b/docs/examples/cancellation.nim @@ -0,0 +1,21 @@ +## Simple cancellation example + +import chronos + +proc someTask() {.async.} = await sleepAsync(10.minutes) + +proc cancellationExample() {.async.} = + # Start a task but don't wait for it to finish + let future = someTask() + future.cancelSoon() + # `cancelSoon` schedules but does not wait for the future to get cancelled - + # it might still be pending here + + let future2 = someTask() # Start another task concurrently + await future2.cancelAndWait() + # Using `cancelAndWait`, we can be sure that `future2` is either + # complete, failed or cancelled at this point. `future` could still be + # pending! + assert future2.finished() + +waitFor(cancellationExample()) diff --git a/docs/examples/discards.nim b/docs/examples/discards.nim new file mode 100644 index 0000000..990acfc --- /dev/null +++ b/docs/examples/discards.nim @@ -0,0 +1,28 @@ +## The peculiarities of `discard` in `async` procedures +import chronos + +proc failingOperation() {.async.} = + echo "Raising!" + raise (ref ValueError)(msg: "My error") + +proc myApp() {.async.} = + # This style of discard causes the `ValueError` to be discarded, hiding the + # failure of the operation - avoid! + discard failingOperation() + + proc runAsTask(fut: Future[void]): Future[void] {.async: (raises: []).} = + # runAsTask uses `raises: []` to ensure at compile-time that no exceptions + # escape it! + try: + await fut + except CatchableError as exc: + echo "The task failed! ", exc.msg + + # asyncSpawn ensures that errors don't leak unnoticed from tasks without + # blocking: + asyncSpawn runAsTask(failingOperation()) + + # If we didn't catch the exception with `runAsTask`, the program will crash: + asyncSpawn failingOperation() + +waitFor myApp() diff --git a/docs/examples/httpget.nim b/docs/examples/httpget.nim new file mode 100644 index 0000000..4ddf04a --- /dev/null +++ b/docs/examples/httpget.nim @@ -0,0 +1,15 @@ +import chronos/apps/http/httpclient + +proc retrievePage*(uri: string): Future[string] {.async.} = + # Create a new HTTP session + let httpSession = HttpSessionRef.new() + try: + # Fetch page contents + let resp = await httpSession.fetch(parseUri(uri)) + # Convert response to a string, assuming its encoding matches the terminal! + bytesToString(resp.data) + finally: # Close the session + await noCancel(httpSession.closeWait()) + +echo waitFor retrievePage( + "https://raw.githubusercontent.com/status-im/nim-chronos/master/README.md") diff --git a/docs/examples/nim.cfg b/docs/examples/nim.cfg new file mode 100644 index 0000000..80e5d9b --- /dev/null +++ b/docs/examples/nim.cfg @@ -0,0 +1 @@ +path = "../.." \ No newline at end of file diff --git a/docs/examples/timeoutcomposed.nim b/docs/examples/timeoutcomposed.nim new file mode 100644 index 0000000..8533af5 --- /dev/null +++ b/docs/examples/timeoutcomposed.nim @@ -0,0 +1,25 @@ +## Single timeout for several operations +import chronos + +proc shortTask {.async.} = + try: + await sleepAsync(1.seconds) + except CancelledError as exc: + echo "Short task was cancelled!" + raise exc # Propagate cancellation to the next operation + +proc composedTimeout() {.async.} = + let + # Common timout for several sub-tasks + timeout = sleepAsync(10.seconds) + + while not timeout.finished(): + let task = shortTask() # Start a task but don't `await` it + if (await race(task, timeout)) == task: + echo "Ran one more task" + else: + # This cancellation may or may not happen as task might have finished + # right at the timeout! + task.cancelSoon() + +waitFor composedTimeout() diff --git a/docs/examples/timeoutsimple.nim b/docs/examples/timeoutsimple.nim new file mode 100644 index 0000000..ce6a12a --- /dev/null +++ b/docs/examples/timeoutsimple.nim @@ -0,0 +1,20 @@ +## Simple timeouts +import chronos + +proc longTask {.async.} = + try: + await sleepAsync(10.minutes) + except CancelledError as exc: + echo "Long task was cancelled!" + raise exc # Propagate cancellation to the next operation + +proc simpleTimeout() {.async.} = + let + task = longTask() # Start a task but don't `await` it + + if not await task.withTimeout(1.seconds): + echo "Timeout reached - withTimeout should have cancelled the task" + else: + echo "Task completed" + +waitFor simpleTimeout() diff --git a/docs/examples/twogets.nim b/docs/examples/twogets.nim new file mode 100644 index 0000000..00ebab4 --- /dev/null +++ b/docs/examples/twogets.nim @@ -0,0 +1,24 @@ +## Make two http requests concurrently and output the one that wins + +import chronos +import ./httpget + +proc twoGets() {.async.} = + let + futs = @[ + # Both pages will start downloading concurrently... + httpget.retrievePage("https://duckduckgo.com/?q=chronos"), + httpget.retrievePage("https://www.google.fr/search?q=chronos") + ] + + # Wait for at least one request to finish.. + let winner = await one(futs) + # ..and cancel the others since we won't need them + for fut in futs: + # Trying to cancel an already-finished future is harmless + fut.cancelSoon() + + # An exception could be raised here if the winning request failed! + echo "Result: ", winner.read() + +waitFor(twoGets()) diff --git a/docs/open-in.css b/docs/open-in.css new file mode 100644 index 0000000..aeb951f --- /dev/null +++ b/docs/open-in.css @@ -0,0 +1,7 @@ +footer { + font-size: 0.8em; + text-align: center; + border-top: 1px solid black; + padding: 5px 0; +} + diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md new file mode 100644 index 0000000..186fadd --- /dev/null +++ b/docs/src/SUMMARY.md @@ -0,0 +1,10 @@ +- [Introduction](./introduction.md) +- [Getting started](./getting_started.md) + +# User guide + +- [Core concepts](./concepts.md) +- [`async` functions](async_procs.md) +- [Errors and exceptions](./error_handling.md) +- [Tips, tricks and best practices](./tips.md) +- [Porting code to `chronos`](./porting.md) diff --git a/docs/src/async_procs.md b/docs/src/async_procs.md new file mode 100644 index 0000000..ae8eb51 --- /dev/null +++ b/docs/src/async_procs.md @@ -0,0 +1,112 @@ +# Async procedures + + + +## The `async` pragma + +The `{.async.}` pragma will transform a procedure (or a method) returning a +`Future` into a closure iterator. If there is no return type specified, +`Future[void]` is returned. + +```nim +proc p() {.async.} = + await sleepAsync(100.milliseconds) + +echo p().type # prints "Future[system.void]" +``` + +## `await` keyword + +Whenever `await` is encountered inside an async procedure, control is given +back to the dispatcher for as many steps as it's necessary for the awaited +future to complete, fail or be cancelled. `await` calls the +equivalent of `Future.read()` on the completed future and returns the +encapsulated value. + +```nim +proc p1() {.async.} = + await sleepAsync(1.seconds) + +proc p2() {.async.} = + await sleepAsync(1.seconds) + +proc p3() {.async.} = + let + fut1 = p1() + fut2 = p2() + # Just by executing the async procs, both resulting futures entered the + # dispatcher queue and their "clocks" started ticking. + await fut1 + await fut2 + # Only one second passed while awaiting them both, not two. + +waitFor p3() +``` + +```admonition warning +Because `async` procedures are executed concurrently, they are subject to many +of the same risks that typically accompany multithreaded programming + +In particular, if two `async` procedures have access to the same mutable state, +the value before and after `await` might not be the same as the order of execution is not guaranteed! +``` + +## Raw functions + +Raw functions are those that interact with `chronos` via the `Future` type but +whose body does not go through the async transformation. + +Such functions are created by adding `raw: true` to the `async` parameters: + +```nim +proc rawAsync(): Future[void] {.async: (raw: true).} = + let fut = newFuture[void]("rawAsync") + fut.complete() + fut +``` + +Raw functions must not raise exceptions directly - they are implicitly declared +as `raises: []` - instead they should store exceptions in the returned `Future`: + +```nim +proc rawFailure(): Future[void] {.async: (raw: true).} = + let fut = newFuture[void]("rawAsync") + fut.fail((ref ValueError)(msg: "Oh no!")) + fut +``` + +Raw functions can also use checked exceptions: + +```nim +proc rawAsyncRaises(): Future[void] {.async: (raw: true, raises: [IOError]).} = + let fut = newFuture[void]() + assert not (compiles do: fut.fail((ref ValueError)(msg: "uh-uh"))) + fut.fail((ref IOError)(msg: "IO")) + fut +``` + +## Callbacks and closures + +Callback/closure types are declared using the `async` annotation as usual: + +```nim +type MyCallback = proc(): Future[void] {.async.} + +proc runCallback(cb: MyCallback) {.async: (raises: []).} = + try: + await cb() + except CatchableError: + discard # handle errors as usual +``` + +When calling a callback, it is important to remember that it may raise exceptions that need to be handled. + +Checked exceptions can be used to limit the exceptions that a callback can +raise: + +```nim +type MyEasyCallback = proc(): Future[void] {.async: (raises: []).} + +proc runCallback(cb: MyEasyCallback) {.async: (raises: [])} = + await cb() +``` diff --git a/docs/src/concepts.md b/docs/src/concepts.md new file mode 100644 index 0000000..fcc33af --- /dev/null +++ b/docs/src/concepts.md @@ -0,0 +1,126 @@ +# Concepts + + + +## The dispatcher + +Async/await programming relies on cooperative multitasking to coordinate the +concurrent execution of procedures, using event notifications from the operating system to resume execution. + +The event handler loop is called a "dispatcher" and a single instance per +thread is created, as soon as one is needed. + +Scheduling is done by calling [async procedures](./async_procs.md) that return +`Future` objects - each time a procedure is unable to make further +progress, for example because it's waiting for some data to arrive, it hands +control back to the dispatcher which ensures that the procedure is resumed when +ready. + +## The `Future` type + +`Future` objects encapsulate the outcome of executing an `async` procedure. The +`Future` may be `pending` meaning that the outcome is not yet known or +`finished` meaning that the return value is available, the operation failed +with an exception or was cancelled. + +Inside an async procedure, you can `await` the outcome of another async +procedure - if the `Future` representing that operation is still `pending`, a +callback representing where to resume execution will be added to it and the +dispatcher will be given back control to deal with other tasks. + +When a `Future` is `finished`, all its callbacks are scheduled to be run by +the dispatcher, thus continuing any operations that were waiting for an outcome. + +## The `poll` call + +To trigger the processing step of the dispatcher, we need to call `poll()` - +either directly or through a wrapper like `runForever()` or `waitFor()`. + +Each call to poll handles any file descriptors, timers and callbacks that are +ready to be processed. + +Using `waitFor`, the result of a single asynchronous operation can be obtained: + +```nim +proc myApp() {.async.} = + echo "Waiting for a second..." + await sleepAsync(1.seconds) + echo "done!" + +waitFor myApp() +``` + +It is also possible to keep running the event loop forever using `runForever`: + +```nim +proc myApp() {.async.} = + while true: + await sleepAsync(1.seconds) + echo "A bit more than a second passed!" + +let future = myApp() +runForever() +``` + +Such an application never terminates, thus it is rare that applications are +structured this way. + +```admonish warning +Both `waitFor` and `runForever` call `poll` which offers fine-grained +control over the event loop steps. + +Nested calls to `poll`, `waitFor` and `runForever` are not allowed. +``` + +## Cancellation + +Any pending `Future` can be cancelled. This can be used for timeouts, to start +multiple operations in parallel and cancel the rest as soon as one finishes, +to initiate the orderely shutdown of an application etc. + +```nim +{{#include ../examples/cancellation.nim}} +``` + +Even if cancellation is initiated, it is not guaranteed that the operation gets +cancelled - the future might still be completed or fail depending on the +order of events in the dispatcher and the specifics of the operation. + +If the future indeed gets cancelled, `await` will raise a +`CancelledError` as is likely to happen in the following example: + +```nim +proc c1 {.async.} = + echo "Before sleep" + try: + await sleepAsync(10.minutes) + echo "After sleep" # not reach due to cancellation + except CancelledError as exc: + echo "We got cancelled!" + # `CancelledError` is typically re-raised to notify the caller that the + # operation is being cancelled + raise exc + +proc c2 {.async.} = + await c1() + echo "Never reached, since the CancelledError got re-raised" + +let work = c2() +waitFor(work.cancelAndWait()) +``` + +The `CancelledError` will now travel up the stack like any other exception. +It can be caught and handled (for instance, freeing some resources) + +Cancelling an already-finished `Future` has no effect, as the following example +of downloading two web pages concurrently shows: + +```nim +{{#include ../examples/twogets.nim}} +``` + +## Compile-time configuration + +`chronos` contains several compile-time [configuration options](./chronos/config.nim) enabling stricter compile-time checks and debugging helpers whose runtime cost may be significant. + +Strictness options generally will become default in future chronos releases and allow adapting existing code without changing the new version - see the [`config.nim`](./chronos/config.nim) module for more information. diff --git a/docs/src/error_handling.md b/docs/src/error_handling.md new file mode 100644 index 0000000..be06a35 --- /dev/null +++ b/docs/src/error_handling.md @@ -0,0 +1,134 @@ +# Errors and exceptions + + + +## Exceptions + +Exceptions inheriting from [`CatchableError`](https://nim-lang.org/docs/system.html#CatchableError) +interrupt execution of an `async` procedure. The exception is placed in the +`Future.error` field while changing the status of the `Future` to `Failed` +and callbacks are scheduled. + +When a future is read or awaited the exception is re-raised, traversing the +`async` execution chain until handled. + +```nim +proc p1() {.async.} = + await sleepAsync(1.seconds) + raise newException(ValueError, "ValueError inherits from CatchableError") + +proc p2() {.async.} = + await sleepAsync(1.seconds) + +proc p3() {.async.} = + let + fut1 = p1() + fut2 = p2() + await fut1 + echo "unreachable code here" + await fut2 + +# `waitFor()` would call `Future.read()` unconditionally, which would raise the +# exception in `Future.error`. +let fut3 = p3() +while not(fut3.finished()): + poll() + +echo "fut3.state = ", fut3.state # "Failed" +if fut3.failed(): + echo "p3() failed: ", fut3.error.name, ": ", fut3.error.msg + # prints "p3() failed: ValueError: ValueError inherits from CatchableError" +``` + +You can put the `await` in a `try` block, to deal with that exception sooner: + +```nim +proc p3() {.async.} = + let + fut1 = p1() + fut2 = p2() + try: + await fut1 + except CachableError: + echo "p1() failed: ", fut1.error.name, ": ", fut1.error.msg + echo "reachable code here" + await fut2 +``` + +Because `chronos` ensures that all exceptions are re-routed to the `Future`, +`poll` will not itself raise exceptions. + +`poll` may still panic / raise `Defect` if such are raised in user code due to +undefined behavior. + +## Checked exceptions + +By specifying a `raises` list to an async procedure, you can check which +exceptions can be raised by it: + +```nim +proc p1(): Future[void] {.async: (raises: [IOError]).} = + assert not (compiles do: raise newException(ValueError, "uh-uh")) + raise newException(IOError, "works") # Or any child of IOError + +proc p2(): Future[void] {.async, (raises: [IOError]).} = + await p1() # Works, because await knows that p1 + # can only raise IOError +``` + +Under the hood, the return type of `p1` will be rewritten to an internal type +which will convey raises informations to `await`. + +```admonition note +Most `async` include `CancelledError` in the list of `raises`, indicating that +the operation they implement might get cancelled resulting in neither value nor +error! +``` + +## The `Exception` type + +Exceptions deriving from `Exception` are not caught by default as these may +include `Defect` and other forms undefined or uncatchable behavior. + +Because exception effect tracking is turned on for `async` functions, this may +sometimes lead to compile errors around forward declarations, methods and +closures as Nim conservatively asssumes that any `Exception` might be raised +from those. + +Make sure to excplicitly annotate these with `{.raises.}`: + +```nim +# Forward declarations need to explicitly include a raises list: +proc myfunction() {.raises: [ValueError].} + +# ... as do `proc` types +type MyClosure = proc() {.raises: [ValueError].} + +proc myfunction() = + raise (ref ValueError)(msg: "Implementation here") + +let closure: MyClosure = myfunction +``` + +For compatibility, `async` functions can be instructed to handle `Exception` as +well, specifying `handleException: true`. `Exception` that is not a `Defect` and +not a `CatchableError` will then be caught and remapped to +`AsyncExceptionError`: + +```nim +proc raiseException() {.async: (handleException: true, raises: [AsyncExceptionError]).} = + raise (ref Exception)(msg: "Raising Exception is UB") + +proc callRaiseException() {.async: (raises: []).} = + try: + raiseException() + except AsyncExceptionError as exc: + # The original Exception is available from the `parent` field + echo exc.parent.msg +``` + +This mode can be enabled globally with `-d:chronosHandleException` as a help +when porting code to `chronos` but should generally be avoided as global +configuration settings may interfere with libraries that use `chronos` leading +to unexpected behavior. + diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md new file mode 100644 index 0000000..809dbca --- /dev/null +++ b/docs/src/getting_started.md @@ -0,0 +1,19 @@ +## Getting started + +Install `chronos` using `nimble`: + +```text +nimble install chronos +``` + +or add a dependency to your `.nimble` file: + +```text +requires "chronos" +``` + +and start using it: + +```nim +{{#include ../examples/httpget.nim}} +``` diff --git a/docs/src/introduction.md b/docs/src/introduction.md new file mode 100644 index 0000000..9c2a308 --- /dev/null +++ b/docs/src/introduction.md @@ -0,0 +1,32 @@ +# Introduction + +Chronos implements the [async/await](https://en.wikipedia.org/wiki/Async/await) +paradigm in a self-contained library using macro and closure iterator +transformation features provided by Nim. + +Features include: + +* Asynchronous socket and process I/O +* HTTP server with SSL/TLS support out of the box (no OpenSSL needed) +* Synchronization primitivies like queues, events and locks +* Cancellation +* Efficient dispatch pipeline with excellent multi-platform support +* Exception [effect support](./guide.md#error-handling) + +## Platform support + +Several platforms are supported, with different backend [options](./concepts.md#compile-time-configuration): + +* Windows: [`IOCP`](https://learn.microsoft.com/en-us/windows/win32/fileio/i-o-completion-ports) +* Linux: [`epoll`](https://en.wikipedia.org/wiki/Epoll) / `poll` +* OSX / BSD: [`kqueue`](https://en.wikipedia.org/wiki/Kqueue) / `poll` +* Android / Emscripten / posix: `poll` + +## Examples + +Examples are available in the [`docs/examples/`](https://github.com/status-im/nim-chronos/docs/examples) folder. + +## API documentation + +This guide covers basic usage of chronos - for details, see the +[API reference](./api/chronos.html). diff --git a/docs/src/porting.md b/docs/src/porting.md new file mode 100644 index 0000000..519de64 --- /dev/null +++ b/docs/src/porting.md @@ -0,0 +1,54 @@ +# Porting code to `chronos` v4 + + + +Thanks to its macro support, Nim allows `async`/`await` to be implemented in +libraries with only minimal support from the language - as such, multiple +`async` libraries exist, including `chronos` and `asyncdispatch`, and more may +come to be developed in the futures. + +## Chronos v3 + +Chronos v4 introduces new features for IPv6, exception effects, a stand-alone +`Future` type as well as several other changes - when upgrading from chronos v3, +here are several things to consider: + +* Exception handling is now strict by default - see the [error handling](./error_handling.md) + chapter for how to deal with `raises` effects +* `AsyncEventBus` was removed - use `AsyncEventQueue` instead + +## `asyncdispatch` + +Projects written for `asyncdispatch` and `chronos` look similar but there are +several differences to be aware of: + +* `chronos` has its own dispatch loop - you can typically not mix `chronos` and + `asyncdispatch` in the same thread +* `import chronos` instead of `import asyncdispatch` +* cleanup is important - make sure to use `closeWait` to release any resources + you're using or file descript leaks and other +* cancellation support means that `CancelledError` may be raised from most + `{.async.}` functions +* Calling `yield` directly in tasks is not supported - instead, use `awaitne`. + +## Supporting multiple backends + +Libraries built on top of `async`/`await` may wish to support multiple async +backends - the best way to do so is to create separate modules for each backend +that may be imported side-by-side - see [nim-metrics](https://github.com/status-im/nim-metrics/blob/master/metrics/) +for an example. + +An alternative way is to select backend using a global compile flag - this +method makes it diffucult to compose applications that use both backends as may +happen with transitive dependencies, but may be appropriate in some cases - +libraries choosing this path should call the flag `asyncBackend`, allowing +applications to choose the backend with `-d:asyncBackend=`. + +Known `async` backends include: + +* `chronos` - this library (`-d:asyncBackend=chronos`) +* `asyncdispatch` the standard library `asyncdispatch` [module](https://nim-lang.org/docs/asyncdispatch.html) (`-d:asyncBackend=asyncdispatch`) +* `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely + +``none`` can be used when a library supports both a synchronous and +asynchronous API, to disable the latter. diff --git a/docs/src/tips.md b/docs/src/tips.md new file mode 100644 index 0000000..627e464 --- /dev/null +++ b/docs/src/tips.md @@ -0,0 +1,34 @@ +# Tips, tricks and best practices + +## Timeouts + +To prevent a single task from taking too long, `withTimeout` can be used: + +```nim +{{#include ../examples/timeoutsimple.nim}} +``` + +When several tasks should share a single timeout, a common timer can be created +with `sleepAsync`: + +```nim +{{#include ../examples/timeoutcomposed.nim}} +``` + +## `discard` + +When calling an asynchronous procedure without `await`, the operation is started +but its result is not processed until corresponding `Future` is `read`. + +It is therefore important to never `discard` futures directly - instead, one +can discard the result of awaiting the future or use `asyncSpawn` to monitor +the outcome of the future as if it were running in a separate thread. + +Similar to threads, tasks managed by `asyncSpawn` may causes the application to +crash if any exceptions leak out of it - use +[checked exceptions](./error_handling.md#checked-exceptions) to avoid this +problem. + +```nim +{{#include ../examples/discards.nim}} +``` diff --git a/docs/theme/highlight.js b/docs/theme/highlight.js new file mode 100644 index 0000000..3256c00 --- /dev/null +++ b/docs/theme/highlight.js @@ -0,0 +1,53 @@ +/* + Highlight.js 10.1.1 (93fd0d73) + License: BSD-3-Clause + Copyright (c) 2006-2020, Ivan Sagalaev +*/ +var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!Object.hasOwnProperty.call(n,r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}class n{constructor(e){void 0===e.data&&(e.data={}),this.data=e.data}ignoreMatch(){this.ignore=!0}}function t(e){return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'")}function r(e,...n){var t={};for(const n in e)t[n]=e[n];return n.forEach((function(e){for(const n in e)t[n]=e[n]})),t}function a(e){return e.nodeName.toLowerCase()}var i=Object.freeze({__proto__:null,escapeHTML:t,inherit:r,nodeStream:function(e){var n=[];return function e(t,r){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?r+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:r,node:i}),r=e(i,r),a(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:r,node:i}));return r}(e,0),n},mergeStreams:function(e,n,r){var i=0,s="",o=[];function l(){return e.length&&n.length?e[0].offset!==n[0].offset?e[0].offset"}function u(e){s+=""}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||n.length;){var g=l();if(s+=t(r.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+t(r.substr(i))}});const s="",o=e=>!!e.kind;class l{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=t(e)}openNode(e){if(!o(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){o(e)&&(this.buffer+=s)}value(){return this.buffer}span(e){this.buffer+=``}}class c{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){const n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){"string"!=typeof e&&e.children&&(e.children.every(e=>"string"==typeof e)?e.children=[e.children.join("")]:e.children.forEach(e=>{c._collapse(e)}))}}class u extends c{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){const t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new l(this,this.options).value()}finalize(){return!0}}function d(e){return e?"string"==typeof e?e:e.source:null}const g="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",h={begin:"\\\\[\\s\\S]",relevance:0},f={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[h]},p={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[h]},b={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,t={}){var a=r({className:"comment",begin:e,end:n,contains:[]},t);return a.contains.push(b),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):",relevance:0}),a},v=m("//","$"),x=m("/\\*","\\*/"),E=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:g,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",SHEBANG:(e={})=>{const n=/^#![ ]*\//;return e.binary&&(e.begin=function(...e){return e.map(e=>d(e)).join("")}(n,/.*\b/,e.binary,/\b.*/)),r({className:"meta",begin:n,end:/$/,relevance:0,"on:begin":(e,n)=>{0!==e.index&&n.ignoreMatch()}},e)},BACKSLASH_ESCAPE:h,APOS_STRING_MODE:f,QUOTE_STRING_MODE:p,PHRASAL_WORDS_MODE:b,COMMENT:m,C_LINE_COMMENT_MODE:v,C_BLOCK_COMMENT_MODE:x,HASH_COMMENT_MODE:E,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:g,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[h,{begin:/\[/,end:/\]/,relevance:0,contains:[h]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0},END_SAME_AS_BEGIN:function(e){return Object.assign(e,{"on:begin":(e,n)=>{n.data._beginMatch=e[1]},"on:end":(e,n)=>{n.data._beginMatch!==e[1]&&n.ignoreMatch()}})}}),N="of and for in not or if then".split(" ");function w(e,n){return n?+n:function(e){return N.includes(e.toLowerCase())}(e)?0:1}const R=t,y=r,{nodeStream:k,mergeStreams:O}=i,M=Symbol("nomatch");return function(t){var a=[],i={},s={},o=[],l=!0,c=/(^(<[^>]+>|\t|)+|\n)/gm,g="Could not find the language '{}', did you forget to load/include a language module?";const h={disableAutodetect:!0,name:"Plain text",contains:[]};var f={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:null,__emitter:u};function p(e){return f.noHighlightRe.test(e)}function b(e,n,t,r){var a={code:n,language:e};S("before:highlight",a);var i=a.result?a.result:m(a.language,a.code,t,r);return i.code=a.code,S("after:highlight",i),i}function m(e,t,a,s){var o=t;function c(e,n){var t=E.case_insensitive?n[0].toLowerCase():n[0];return Object.prototype.hasOwnProperty.call(e.keywords,t)&&e.keywords[t]}function u(){null!=y.subLanguage?function(){if(""!==A){var e=null;if("string"==typeof y.subLanguage){if(!i[y.subLanguage])return void O.addText(A);e=m(y.subLanguage,A,!0,k[y.subLanguage]),k[y.subLanguage]=e.top}else e=v(A,y.subLanguage.length?y.subLanguage:null);y.relevance>0&&(I+=e.relevance),O.addSublanguage(e.emitter,e.language)}}():function(){if(!y.keywords)return void O.addText(A);let e=0;y.keywordPatternRe.lastIndex=0;let n=y.keywordPatternRe.exec(A),t="";for(;n;){t+=A.substring(e,n.index);const r=c(y,n);if(r){const[e,a]=r;O.addText(t),t="",I+=a,O.addKeyword(n[0],e)}else t+=n[0];e=y.keywordPatternRe.lastIndex,n=y.keywordPatternRe.exec(A)}t+=A.substr(e),O.addText(t)}(),A=""}function h(e){return e.className&&O.openNode(e.className),y=Object.create(e,{parent:{value:y}})}function p(e){return 0===y.matcher.regexIndex?(A+=e[0],1):(L=!0,0)}var b={};function x(t,r){var i=r&&r[0];if(A+=t,null==i)return u(),0;if("begin"===b.type&&"end"===r.type&&b.index===r.index&&""===i){if(A+=o.slice(r.index,r.index+1),!l){const n=Error("0 width match regex");throw n.languageName=e,n.badRule=b.rule,n}return 1}if(b=r,"begin"===r.type)return function(e){var t=e[0],r=e.rule;const a=new n(r),i=[r.__beforeBegin,r["on:begin"]];for(const n of i)if(n&&(n(e,a),a.ignore))return p(t);return r&&r.endSameAsBegin&&(r.endRe=RegExp(t.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),r.skip?A+=t:(r.excludeBegin&&(A+=t),u(),r.returnBegin||r.excludeBegin||(A=t)),h(r),r.returnBegin?0:t.length}(r);if("illegal"===r.type&&!a){const e=Error('Illegal lexeme "'+i+'" for mode "'+(y.className||"")+'"');throw e.mode=y,e}if("end"===r.type){var s=function(e){var t=e[0],r=o.substr(e.index),a=function e(t,r,a){let i=function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(t.endRe,a);if(i){if(t["on:end"]){const e=new n(t);t["on:end"](r,e),e.ignore&&(i=!1)}if(i){for(;t.endsParent&&t.parent;)t=t.parent;return t}}if(t.endsWithParent)return e(t.parent,r,a)}(y,e,r);if(!a)return M;var i=y;i.skip?A+=t:(i.returnEnd||i.excludeEnd||(A+=t),u(),i.excludeEnd&&(A=t));do{y.className&&O.closeNode(),y.skip||y.subLanguage||(I+=y.relevance),y=y.parent}while(y!==a.parent);return a.starts&&(a.endSameAsBegin&&(a.starts.endRe=a.endRe),h(a.starts)),i.returnEnd?0:t.length}(r);if(s!==M)return s}if("illegal"===r.type&&""===i)return 1;if(B>1e5&&B>3*r.index)throw Error("potential infinite loop, way more iterations than matches");return A+=i,i.length}var E=T(e);if(!E)throw console.error(g.replace("{}",e)),Error('Unknown language: "'+e+'"');var _=function(e){function n(n,t){return RegExp(d(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class t{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);const e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n="|"){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i0&&(a+=n),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"===l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("===l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;const n=this.matcherRe.exec(e);if(!n)return null;const t=n.findIndex((e,n)=>n>0&&void 0!==e),r=this.matchIndexes[t];return n.splice(0,t),Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];const n=new t;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){const n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;const t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e,n){const t=e.input[e.index-1],r=e.input[e.index+e[0].length];"."!==t&&"."!==r||n.ignoreMatch()}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");return function t(s,o){const l=s;if(s.compiled)return l;s.compiled=!0,s.__beforeBegin=null,s.keywords=s.keywords||s.beginKeywords;let c=null;if("object"==typeof s.keywords&&(c=s.keywords.$pattern,delete s.keywords.$pattern),s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,w(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemes&&c)throw Error("ERR: Prefer `keywords.$pattern` to `mode.lexemes`, BOTH are not allowed. (see mode reference) ");return l.keywordPatternRe=n(s.lexemes||c||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__beforeBegin=i),s.begin||(s.begin=/\B|\b/),l.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(l.endRe=n(s.end)),l.terminator_end=d(s.end)||"",s.endsWithParent&&o.terminator_end&&(l.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(l.illegalRe=n(s.illegal)),void 0===s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return r(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?r(e,{starts:e.starts?r(e.starts):null}):Object.isFrozen(e)?r(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){t(e,l)})),s.starts&&t(s.starts,o),l.matcher=function(e){const n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(l),l}(e)}(E),N="",y=s||_,k={},O=new f.__emitter(f);!function(){for(var e=[],n=y;n!==E;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>O.openNode(e))}();var A="",I=0,S=0,B=0,L=!1;try{for(y.matcher.considerAll();;){B++,L?L=!1:(y.matcher.lastIndex=S,y.matcher.considerAll());const e=y.matcher.exec(o);if(!e)break;const n=x(o.substring(S,e.index),e);S=e.index+n}return x(o.substr(S)),O.closeAllNodes(),O.finalize(),N=O.toHTML(),{relevance:I,value:N,language:e,illegal:!1,emitter:O,top:y}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:o.slice(S-100,S+100),mode:n.mode},sofar:N,relevance:0,value:R(o),emitter:O};if(l)return{illegal:!1,relevance:0,value:R(o),emitter:O,language:e,top:y,errorRaised:n};throw n}}function v(e,n){n=n||f.languages||Object.keys(i);var t=function(e){const n={relevance:0,emitter:new f.__emitter(f),value:R(e),illegal:!1,top:h};return n.emitter.addText(e),n}(e),r=t;return n.filter(T).filter(I).forEach((function(n){var a=m(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function x(e){return f.tabReplace||f.useBR?e.replace(c,e=>"\n"===e?f.useBR?"
":e:f.tabReplace?e.replace(/\t/g,f.tabReplace):e):e}function E(e){let n=null;const t=function(e){var n=e.className+" ";n+=e.parentNode?e.parentNode.className:"";const t=f.languageDetectRe.exec(n);if(t){var r=T(t[1]);return r||(console.warn(g.replace("{}",t[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?t[1]:"no-highlight"}return n.split(/\s+/).find(e=>p(e)||T(e))}(e);if(p(t))return;S("before:highlightBlock",{block:e,language:t}),f.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n"):n=e;const r=n.textContent,a=t?b(t,r,!0):v(r),i=k(n);if(i.length){const e=document.createElement("div");e.innerHTML=a.value,a.value=O(i,k(e),r)}a.value=x(a.value),S("after:highlightBlock",{block:e,result:a}),e.innerHTML=a.value,e.className=function(e,n,t){var r=n?s[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,t,a.language),e.result={language:a.language,re:a.relevance,relavance:a.relevance},a.second_best&&(e.second_best={language:a.second_best.language,re:a.second_best.relevance,relavance:a.second_best.relevance})}const N=()=>{if(!N.called){N.called=!0;var e=document.querySelectorAll("pre code");a.forEach.call(e,E)}};function T(e){return e=(e||"").toLowerCase(),i[e]||i[s[e]]}function A(e,{languageName:n}){"string"==typeof e&&(e=[e]),e.forEach(e=>{s[e]=n})}function I(e){var n=T(e);return n&&!n.disableAutodetect}function S(e,n){var t=e;o.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(t,{highlight:b,highlightAuto:v,fixMarkup:x,highlightBlock:E,configure:function(e){f=y(f,e)},initHighlighting:N,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",N,!1)},registerLanguage:function(e,n){var r=null;try{r=n(t)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!l)throw n;console.error(n),r=h}r.name||(r.name=e),i[e]=r,r.rawDefinition=n.bind(null,t),r.aliases&&A(r.aliases,{languageName:e})},listLanguages:function(){return Object.keys(i)},getLanguage:T,registerAliases:A,requireLanguage:function(e){var n=T(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:I,inherit:y,addPlugin:function(e){o.push(e)}}),t.debugMode=function(){l=!1},t.safeMode=function(){l=!0},t.versionString="10.1.1";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(t,_),t}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs); +hljs.registerLanguage("apache",function(){"use strict";return function(e){var n={className:"number",begin:"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?"};return{name:"Apache config",aliases:["apacheconf"],case_insensitive:!0,contains:[e.HASH_COMMENT_MODE,{className:"section",begin:"",contains:[n,{className:"number",begin:":\\d{1,5}"},e.inherit(e.QUOTE_STRING_MODE,{relevance:0})]},{className:"attribute",begin:/\w+/,relevance:0,keywords:{nomarkup:"order deny allow setenv rewriterule rewriteengine rewritecond documentroot sethandler errordocument loadmodule options header listen serverroot servername"},starts:{end:/$/,relevance:0,keywords:{literal:"on off all deny allow"},contains:[{className:"meta",begin:"\\s\\[",end:"\\]$"},{className:"variable",begin:"[\\$%]\\{",end:"\\}",contains:["self",{className:"number",begin:"[\\$%]\\d+"}]},n,{className:"number",begin:"\\d+"},e.QUOTE_STRING_MODE]}}],illegal:/\S/}}}()); +hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const t={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},n={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,t]};t.contains.push(n);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]},i=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10}),c={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b-?[a-z\._]+\b/,keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[i,e.SHEBANG(),c,a,e.HASH_COMMENT_MODE,n,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}()); +hljs.registerLanguage("c-like",function(){"use strict";return function(e){function t(e){return"(?:"+e+")?"}var n="(decltype\\(auto\\)|"+t("[a-zA-Z_]\\w*::")+"[a-zA-Z_]\\w*"+t("<.*?>")+")",r={className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},a={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},e.END_SAME_AS_BEGIN({begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},i={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},s={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(a,{className:"meta-string"}),{className:"meta-string",begin:/<.*?>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},o={className:"title",begin:t("[a-zA-Z_]\\w*::")+e.IDENT_RE,relevance:0},c=t("[a-zA-Z_]\\w*::")+e.IDENT_RE+"\\s*\\(",l={keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary",literal:"true false nullptr NULL"},d=[r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,i,a],_={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:l,contains:d.concat([{begin:/\(/,end:/\)/,keywords:l,contains:d.concat(["self"]),relevance:0}]),relevance:0},u={className:"function",begin:"("+n+"[\\*&\\s]+)+"+c,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:l,illegal:/[^\w\s\*&:<>]/,contains:[{begin:"decltype\\(auto\\)",keywords:l,relevance:0},{begin:c,returnBegin:!0,contains:[o],relevance:0},{className:"params",begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,i,r,{begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,i,r]}]},r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s]};return{aliases:["c","cc","h","c++","h++","hpp","hh","hxx","cxx"],keywords:l,disableAutodetect:!0,illegal:"",keywords:l,contains:["self",r]},{begin:e.IDENT_RE+"::",keywords:l},{className:"class",beginKeywords:"class struct",end:/[{;:]/,contains:[{begin://,contains:["self"]},e.TITLE_MODE]}]),exports:{preprocessor:s,strings:a,keywords:l}}}}()); +hljs.registerLanguage("c",function(){"use strict";return function(e){var n=e.getLanguage("c-like").rawDefinition();return n.name="C",n.aliases=["c","h"],n}}()); +hljs.registerLanguage("coffeescript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);return function(r){var t={keyword:e.concat(["then","unless","until","loop","by","when","and","or","is","isnt","not"]).filter((e=>n=>!e.includes(n))(["var","const","let","function","static"])).join(" "),literal:n.concat(["yes","no","on","off"]).join(" "),built_in:a.concat(["npm","print"]).join(" ")},i="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/}/,keywords:t},o=[r.BINARY_NUMBER_MODE,r.inherit(r.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[r.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[r.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[r.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[r.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,r.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+i},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=o;var c=r.inherit(r.TITLE_MODE,{begin:i}),l={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:t,contains:["self"].concat(o)}]};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:t,illegal:/\/\*/,contains:o.concat([r.COMMENT("###","###"),r.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+i+"\\s*=\\s*(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[c,l]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:"(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[l]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[c]},c]},{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}}()); +hljs.registerLanguage("cpp",function(){"use strict";return function(e){var t=e.getLanguage("c-like").rawDefinition();return t.disableAutodetect=!1,t.name="C++",t.aliases=["cc","c++","h++","hpp","hh","hxx","cxx"],t}}()); +hljs.registerLanguage("csharp",function(){"use strict";return function(e){var n={keyword:"abstract as base bool break byte case catch char checked const continue decimal default delegate do double enum event explicit extern finally fixed float for foreach goto if implicit in int interface internal is lock long object operator out override params private protected public readonly ref sbyte sealed short sizeof stackalloc static string struct switch this try typeof uint ulong unchecked unsafe ushort using virtual void volatile while add alias ascending async await by descending dynamic equals from get global group into join let nameof on orderby partial remove select set value var when where yield",literal:"null false true"},i=e.inherit(e.TITLE_MODE,{begin:"[a-zA-Z](\\.?\\w)*"}),a={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},s={className:"string",begin:'@"',end:'"',contains:[{begin:'""'}]},t=e.inherit(s,{illegal:/\n/}),l={className:"subst",begin:"{",end:"}",keywords:n},r=e.inherit(l,{illegal:/\n/}),c={className:"string",begin:/\$"/,end:'"',illegal:/\n/,contains:[{begin:"{{"},{begin:"}}"},e.BACKSLASH_ESCAPE,r]},o={className:"string",begin:/\$@"/,end:'"',contains:[{begin:"{{"},{begin:"}}"},{begin:'""'},l]},g=e.inherit(o,{illegal:/\n/,contains:[{begin:"{{"},{begin:"}}"},{begin:'""'},r]});l.contains=[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.C_BLOCK_COMMENT_MODE],r.contains=[g,c,t,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.inherit(e.C_BLOCK_COMMENT_MODE,{illegal:/\n/})];var d={variants:[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},E={begin:"<",end:">",contains:[{beginKeywords:"in out"},i]},_=e.IDENT_RE+"(<"+e.IDENT_RE+"(\\s*,\\s*"+e.IDENT_RE+")*>)?(\\[\\])?",b={begin:"@"+e.IDENT_RE,relevance:0};return{name:"C#",aliases:["cs","c#"],keywords:n,illegal:/::/,contains:[e.COMMENT("///","$",{returnBegin:!0,contains:[{className:"doctag",variants:[{begin:"///",relevance:0},{begin:"\x3c!--|--\x3e"},{begin:""}]}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"meta",begin:"#",end:"$",keywords:{"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum"}},d,a,{beginKeywords:"class interface",end:/[{;=]/,illegal:/[^\s:,]/,contains:[{beginKeywords:"where class"},i,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"namespace",end:/[{;=]/,illegal:/[^\s:]/,contains:[i,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"meta",begin:"^\\s*\\[",excludeBegin:!0,end:"\\]",excludeEnd:!0,contains:[{className:"meta-string",begin:/"/,end:/"/}]},{beginKeywords:"new return throw await else",relevance:0},{className:"function",begin:"("+_+"\\s+)+"+e.IDENT_RE+"\\s*(\\<.+\\>)?\\s*\\(",returnBegin:!0,end:/\s*[{;=]/,excludeEnd:!0,keywords:n,contains:[{begin:e.IDENT_RE+"\\s*(\\<.+\\>)?\\s*\\(",returnBegin:!0,contains:[e.TITLE_MODE,E],relevance:0},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,relevance:0,contains:[d,a,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},b]}}}()); +hljs.registerLanguage("css",function(){"use strict";return function(e){var n={begin:/(?:[A-Z\_\.\-]+|--[a-zA-Z0-9_-]+)\s*:/,returnBegin:!0,end:";",endsWithParent:!0,contains:[{className:"attribute",begin:/\S/,end:":",excludeEnd:!0,starts:{endsWithParent:!0,excludeEnd:!0,contains:[{begin:/[\w-]+\(/,returnBegin:!0,contains:[{className:"built_in",begin:/[\w-]+/},{begin:/\(/,end:/\)/,contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.CSS_NUMBER_MODE]}]},e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_BLOCK_COMMENT_MODE,{className:"number",begin:"#[0-9A-Fa-f]+"},{className:"meta",begin:"!important"}]}}]};return{name:"CSS",case_insensitive:!0,illegal:/[=\/|'\$]/,contains:[e.C_BLOCK_COMMENT_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/},{className:"selector-class",begin:/\.[A-Za-z0-9_-]+/},{className:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},{className:"selector-pseudo",begin:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{begin:"@(page|font-face)",lexemes:"@[a-z-]+",keywords:"@page @font-face"},{begin:"@",end:"[{;]",illegal:/:/,returnBegin:!0,contains:[{className:"keyword",begin:/@\-?\w[\w]*(\-\w+)*/},{begin:/\s/,endsWithParent:!0,excludeEnd:!0,relevance:0,keywords:"and or not only",contains:[{begin:/[a-z-]+:/,className:"attribute"},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.CSS_NUMBER_MODE]}]},{className:"selector-tag",begin:"[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0},{begin:"{",end:"}",illegal:/\S/,contains:[e.C_BLOCK_COMMENT_MODE,n]}]}}}()); +hljs.registerLanguage("diff",function(){"use strict";return function(e){return{name:"Diff",aliases:["patch"],contains:[{className:"meta",relevance:10,variants:[{begin:/^@@ +\-\d+,\d+ +\+\d+,\d+ +@@$/},{begin:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{begin:/^\-\-\- +\d+,\d+ +\-\-\-\-$/}]},{className:"comment",variants:[{begin:/Index: /,end:/$/},{begin:/={3,}/,end:/$/},{begin:/^\-{3}/,end:/$/},{begin:/^\*{3} /,end:/$/},{begin:/^\+{3}/,end:/$/},{begin:/^\*{15}$/}]},{className:"addition",begin:"^\\+",end:"$"},{className:"deletion",begin:"^\\-",end:"$"},{className:"addition",begin:"^\\!",end:"$"}]}}}()); +hljs.registerLanguage("go",function(){"use strict";return function(e){var n={keyword:"break default func interface select case map struct chan else goto package switch const fallthrough if range type continue for import return var go defer bool byte complex64 complex128 float32 float64 int8 int16 int32 int64 string uint8 uint16 uint32 uint64 int uint uintptr rune",literal:"true false iota nil",built_in:"append cap close complex copy imag len make new panic print println real recover delete"};return{name:"Go",aliases:["golang"],keywords:n,illegal:"e(n)).join("")}return function(a){var s={className:"number",relevance:0,variants:[{begin:/([\+\-]+)?[\d]+_[\d_]+/},{begin:a.NUMBER_RE}]},i=a.COMMENT();i.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];var t={className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)}/}]},r={className:"literal",begin:/\bon|off|true|false|yes|no\b/},l={className:"string",contains:[a.BACKSLASH_ESCAPE],variants:[{begin:"'''",end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"'},{begin:"'",end:"'"}]},c={begin:/\[/,end:/\]/,contains:[i,r,t,l,s,"self"],relevance:0},g="("+[/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/].map(n=>e(n)).join("|")+")";return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/,contains:[i,{className:"section",begin:/\[+/,end:/\]+/},{begin:n(g,"(\\s*\\.\\s*",g,")*",n("(?=",/\s*=\s*[^#\s]/,")")),className:"attr",starts:{end:/$/,contains:[i,c,r,t,l,s]}}]}}}()); +hljs.registerLanguage("java",function(){"use strict";function e(e){return e?"string"==typeof e?e:e.source:null}function n(e){return a("(",e,")?")}function a(...n){return n.map(n=>e(n)).join("")}function s(...n){return"("+n.map(n=>e(n)).join("|")+")"}return function(e){var t="false synchronized int abstract float private char boolean var static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",i={className:"meta",begin:"@[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",contains:[{begin:/\(/,end:/\)/,contains:["self"]}]},r=e=>a("[",e,"]+([",e,"_]*[",e,"]+)?"),c={className:"number",variants:[{begin:`\\b(0[bB]${r("01")})[lL]?`},{begin:`\\b(0${r("0-7")})[dDfFlL]?`},{begin:a(/\b0[xX]/,s(a(r("a-fA-F0-9"),/\./,r("a-fA-F0-9")),a(r("a-fA-F0-9"),/\.?/),a(/\./,r("a-fA-F0-9"))),/([pP][+-]?(\d+))?/,/[fFdDlL]?/)},{begin:a(/\b/,s(a(/\d*\./,r("\\d")),r("\\d")),/[eE][+-]?[\d]+[dDfF]?/)},{begin:a(/\b/,r(/\d/),n(/\.?/),n(r(/\d/)),/[dDfFlL]?/)}],relevance:0};return{name:"Java",aliases:["jsp"],keywords:t,illegal:/<\/|#/,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/,relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"class",beginKeywords:"class interface",end:/[{;=]/,excludeEnd:!0,keywords:"class interface",illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"new throw return else",relevance:0},{className:"function",begin:"([À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*(<[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*(\\s*,\\s*[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*)*>)?\\s+)+"+e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:t,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"params",begin:/\(/,end:/\)/,keywords:t,relevance:0,contains:[i,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},c,i]}}}()); +hljs.registerLanguage("javascript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);function s(e){return r("(?=",e,")")}function r(...e){return e.map(e=>(function(e){return e?"string"==typeof e?e:e.source:null})(e)).join("")}return function(t){var i="[A-Za-z$_][0-9A-Za-z$_]*",c={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/},o={$pattern:"[A-Za-z$_][0-9A-Za-z$_]*",keyword:e.join(" "),literal:n.join(" "),built_in:a.join(" ")},l={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:t.C_NUMBER_RE+"n?"}],relevance:0},E={className:"subst",begin:"\\$\\{",end:"\\}",keywords:o,contains:[]},d={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[t.BACKSLASH_ESCAPE,E],subLanguage:"xml"}},g={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[t.BACKSLASH_ESCAPE,E],subLanguage:"css"}},u={className:"string",begin:"`",end:"`",contains:[t.BACKSLASH_ESCAPE,E]};E.contains=[t.APOS_STRING_MODE,t.QUOTE_STRING_MODE,d,g,u,l,t.REGEXP_MODE];var b=E.contains.concat([{begin:/\(/,end:/\)/,contains:["self"].concat(E.contains,[t.C_BLOCK_COMMENT_MODE,t.C_LINE_COMMENT_MODE])},t.C_BLOCK_COMMENT_MODE,t.C_LINE_COMMENT_MODE]),_={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:b};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:o,contains:[t.SHEBANG({binary:"node",relevance:5}),{className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},t.APOS_STRING_MODE,t.QUOTE_STRING_MODE,d,g,u,t.C_LINE_COMMENT_MODE,t.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+",contains:[{className:"type",begin:"\\{",end:"\\}",relevance:0},{className:"variable",begin:i+"(?=\\s*(-)|$)",endsParent:!0,relevance:0},{begin:/(?=[^\n])\s/,relevance:0}]}]}),t.C_BLOCK_COMMENT_MODE,l,{begin:r(/[{,\n]\s*/,s(r(/(((\/\/.*)|(\/\*(.|\n)*\*\/))\s*)*/,i+"\\s*:"))),relevance:0,contains:[{className:"attr",begin:i+s("\\s*:"),relevance:0}]},{begin:"("+t.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[t.C_LINE_COMMENT_MODE,t.C_BLOCK_COMMENT_MODE,t.REGEXP_MODE,{className:"function",begin:"(\\([^(]*(\\([^(]*(\\([^(]*\\))?\\))?\\)|"+t.UNDERSCORE_IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:t.UNDERSCORE_IDENT_RE},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:o,contains:b}]}]},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{variants:[{begin:"<>",end:""},{begin:c.begin,end:c.end}],subLanguage:"xml",contains:[{begin:c.begin,end:c.end,skip:!0,contains:["self"]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[t.inherit(t.TITLE_MODE,{begin:i}),_],illegal:/\[|%/},{begin:/\$[(.]/},t.METHOD_GUARD,{className:"class",beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends"},t.UNDERSCORE_TITLE_MODE]},{beginKeywords:"constructor",end:/\{/,excludeEnd:!0},{begin:"(get|set)\\s+(?="+i+"\\()",end:/{/,keywords:"get set",contains:[t.inherit(t.TITLE_MODE,{begin:i}),{begin:/\(\)/},_]}],illegal:/#(?!!)/}}}()); +hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}()); +hljs.registerLanguage("kotlin",function(){"use strict";return function(e){var n={keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit init interface annotation data sealed internal infix operator out by constructor super tailrec where const inner suspend typealias external expect actual trait volatile transient native default",built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",literal:"true false null"},a={className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"@"},i={className:"subst",begin:"\\${",end:"}",contains:[e.C_NUMBER_MODE]},s={className:"variable",begin:"\\$"+e.UNDERSCORE_IDENT_RE},t={className:"string",variants:[{begin:'"""',end:'"""(?=[^"])',contains:[s,i]},{begin:"'",end:"'",illegal:/\n/,contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"',illegal:/\n/,contains:[e.BACKSLASH_ESCAPE,s,i]}]};i.contains.push(t);var r={className:"meta",begin:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UNDERSCORE_IDENT_RE+")?"},l={className:"meta",begin:"@"+e.UNDERSCORE_IDENT_RE,contains:[{begin:/\(/,end:/\)/,contains:[e.inherit(t,{className:"meta-string"})]}]},c=e.COMMENT("/\\*","\\*/",{contains:[e.C_BLOCK_COMMENT_MODE]}),o={variants:[{className:"type",begin:e.UNDERSCORE_IDENT_RE},{begin:/\(/,end:/\)/,contains:[]}]},d=o;return d.variants[1].contains=[o],o.variants[1].contains=[d],{name:"Kotlin",aliases:["kt"],keywords:n,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,c,{className:"keyword",begin:/\b(break|continue|return|this)\b/,starts:{contains:[{className:"symbol",begin:/@\w+/}]}},a,r,l,{className:"function",beginKeywords:"fun",end:"[(]|$",returnBegin:!0,excludeEnd:!0,keywords:n,illegal:/fun\s+(<.*>)?[^\s\(]+(\s+[^\s\(]+)\s*=/,relevance:5,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"type",begin://,keywords:"reified",relevance:0},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:n,relevance:0,contains:[{begin:/:/,end:/[=,\/]/,endsWithParent:!0,contains:[o,e.C_LINE_COMMENT_MODE,c],relevance:0},e.C_LINE_COMMENT_MODE,c,r,l,t,e.C_NUMBER_MODE]},c]},{className:"class",beginKeywords:"class interface trait",end:/[:\{(]|$/,excludeEnd:!0,illegal:"extends implements",contains:[{beginKeywords:"public protected internal private constructor"},e.UNDERSCORE_TITLE_MODE,{className:"type",begin://,excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:/[,:]\s*/,end:/[<\(,]|$/,excludeBegin:!0,returnEnd:!0},r,l]},t,{className:"meta",begin:"^#!/usr/bin/env",end:"$",illegal:"\n"},{className:"number",begin:"\\b(0[bB]([01]+[01_]+[01]+|[01]+)|0[xX]([a-fA-F0-9]+[a-fA-F0-9_]+[a-fA-F0-9]+|[a-fA-F0-9]+)|(([\\d]+[\\d_]+[\\d]+|[\\d]+)(\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))?|\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))([eE][-+]?\\d+)?)[lLfF]?",relevance:0}]}}}()); +hljs.registerLanguage("less",function(){"use strict";return function(e){var n="([\\w-]+|@{[\\w-]+})",a=[],s=[],t=function(e){return{className:"string",begin:"~?"+e+".*?"+e}},r=function(e,n,a){return{className:e,begin:n,relevance:a}},i={begin:"\\(",end:"\\)",contains:s,relevance:0};s.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,t("'"),t('"'),e.CSS_NUMBER_MODE,{begin:"(url|data-uri)\\(",starts:{className:"string",end:"[\\)\\n]",excludeEnd:!0}},r("number","#[0-9A-Fa-f]+\\b"),i,r("variable","@@?[\\w-]+",10),r("variable","@{[\\w-]+}"),r("built_in","~?`[^`]*?`"),{className:"attribute",begin:"[\\w-]+\\s*:",end:":",returnBegin:!0,excludeEnd:!0},{className:"meta",begin:"!important"});var c=s.concat({begin:"{",end:"}",contains:a}),l={beginKeywords:"when",endsWithParent:!0,contains:[{beginKeywords:"and not"}].concat(s)},o={begin:n+"\\s*:",returnBegin:!0,end:"[;}]",relevance:0,contains:[{className:"attribute",begin:n,end:":",excludeEnd:!0,starts:{endsWithParent:!0,illegal:"[<=$]",relevance:0,contains:s}}]},g={className:"keyword",begin:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b",starts:{end:"[;{}]",returnEnd:!0,contains:s,relevance:0}},d={className:"variable",variants:[{begin:"@[\\w-]+\\s*:",relevance:15},{begin:"@[\\w-]+"}],starts:{end:"[;}]",returnEnd:!0,contains:c}},b={variants:[{begin:"[\\.#:&\\[>]",end:"[;{}]"},{begin:n,end:"{"}],returnBegin:!0,returnEnd:!0,illegal:"[<='$\"]",relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,l,r("keyword","all\\b"),r("variable","@{[\\w-]+}"),r("selector-tag",n+"%?",0),r("selector-id","#"+n),r("selector-class","\\."+n,0),r("selector-tag","&",0),{className:"selector-attr",begin:"\\[",end:"\\]"},{className:"selector-pseudo",begin:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{begin:"\\(",end:"\\)",contains:c},{begin:"!important"}]};return a.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,g,d,o,b),{name:"Less",case_insensitive:!0,illegal:"[=>'/<($\"]",contains:a}}}()); +hljs.registerLanguage("lua",function(){"use strict";return function(e){var t={begin:"\\[=*\\[",end:"\\]=*\\]",contains:["self"]},a=[e.COMMENT("--(?!\\[=*\\[)","$"),e.COMMENT("--\\[=*\\[","\\]=*\\]",{contains:[t],relevance:10})];return{name:"Lua",keywords:{$pattern:e.UNDERSCORE_IDENT_RE,literal:"true false nil",keyword:"and break do else elseif end for goto if in local not or repeat return then until while",built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall arg self coroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"},contains:a.concat([{className:"function",beginKeywords:"function",end:"\\)",contains:[e.inherit(e.TITLE_MODE,{begin:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{className:"params",begin:"\\(",endsWithParent:!0,contains:a}].concat(a)},e.C_NUMBER_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string",begin:"\\[=*\\[",end:"\\]=*\\]",contains:[t],relevance:5}])}}}()); +hljs.registerLanguage("makefile",function(){"use strict";return function(e){var i={className:"variable",variants:[{begin:"\\$\\("+e.UNDERSCORE_IDENT_RE+"\\)",contains:[e.BACKSLASH_ESCAPE]},{begin:/\$[@%`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:")",end:">",keywords:{name:"style"},contains:[c],starts:{end:"",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:")",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}()); +hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}()); +hljs.registerLanguage("nginx",function(){"use strict";return function(e){var n={className:"variable",variants:[{begin:/\$\d+/},{begin:/\$\{/,end:/}/},{begin:"[\\$\\@]"+e.UNDERSCORE_IDENT_RE}]},a={endsWithParent:!0,keywords:{$pattern:"[a-z/_]+",literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll"},relevance:0,illegal:"=>",contains:[e.HASH_COMMENT_MODE,{className:"string",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:/"/,end:/"/},{begin:/'/,end:/'/}]},{begin:"([a-z]+):/",end:"\\s",endsWithParent:!0,excludeEnd:!0,contains:[n]},{className:"regexp",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:"\\s\\^",end:"\\s|{|;",returnEnd:!0},{begin:"~\\*?\\s+",end:"\\s|{|;",returnEnd:!0},{begin:"\\*(\\.[a-z\\-]+)+"},{begin:"([a-z\\-]+\\.)+\\*"}]},{className:"number",begin:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{className:"number",begin:"\\b\\d+[kKmMgGdshdwy]*\\b",relevance:0},n]};return{name:"Nginx config",aliases:["nginxconf"],contains:[e.HASH_COMMENT_MODE,{begin:e.UNDERSCORE_IDENT_RE+"\\s+{",returnBegin:!0,end:"{",contains:[{className:"section",begin:e.UNDERSCORE_IDENT_RE}],relevance:0},{begin:e.UNDERSCORE_IDENT_RE+"\\s",end:";|{",returnBegin:!0,contains:[{className:"attribute",begin:e.UNDERSCORE_IDENT_RE,starts:a}],relevance:0}],illegal:"[^\\s\\}]"}}}()); +hljs.registerLanguage("objectivec",function(){"use strict";return function(e){var n=/[a-zA-Z@][a-zA-Z0-9_]*/,_={$pattern:n,keyword:"@interface @class @protocol @implementation"};return{name:"Objective-C",aliases:["mm","objc","obj-c"],keywords:{$pattern:n,keyword:"int float while char export sizeof typedef const struct for union unsigned long volatile static bool mutable if do return goto void enum else break extern asm case short default double register explicit signed typename this switch continue wchar_t inline readonly assign readwrite self @synchronized id typeof nonatomic super unichar IBOutlet IBAction strong weak copy in out inout bycopy byref oneway __strong __weak __block __autoreleasing @private @protected @public @try @property @end @throw @catch @finally @autoreleasepool @synthesize @dynamic @selector @optional @required @encode @package @import @defs @compatibility_alias __bridge __bridge_transfer __bridge_retained __bridge_retain __covariant __contravariant __kindof _Nonnull _Nullable _Null_unspecified __FUNCTION__ __PRETTY_FUNCTION__ __attribute__ getter setter retain unsafe_unretained nonnull nullable null_unspecified null_resettable class instancetype NS_DESIGNATED_INITIALIZER NS_UNAVAILABLE NS_REQUIRES_SUPER NS_RETURNS_INNER_POINTER NS_INLINE NS_AVAILABLE NS_DEPRECATED NS_ENUM NS_OPTIONS NS_SWIFT_UNAVAILABLE NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_END NS_REFINED_FOR_SWIFT NS_SWIFT_NAME NS_SWIFT_NOTHROW NS_DURING NS_HANDLER NS_ENDHANDLER NS_VALUERETURN NS_VOIDRETURN",literal:"false true FALSE TRUE nil YES NO NULL",built_in:"BOOL dispatch_once_t dispatch_queue_t dispatch_sync dispatch_async dispatch_once"},illegal:"/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"class",begin:"("+_.keyword.split(" ").join("|")+")\\b",end:"({|$)",excludeEnd:!0,keywords:_,contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"\\."+e.UNDERSCORE_IDENT_RE,relevance:0}]}}}()); +hljs.registerLanguage("perl",function(){"use strict";return function(e){var n={$pattern:/[\w.]+/,keyword:"getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qq fileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent shutdown dump chomp connect getsockname die socketpair close flock exists index shmget sub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedir ioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when"},t={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:n},s={begin:"->{",end:"}"},r={variants:[{begin:/\$\d/},{begin:/[\$%@](\^\w\b|#\w+(::\w+)*|{\w+}|\w+(::\w*)*)/},{begin:/[\$%@][^\s\w{]/,relevance:0}]},i=[e.BACKSLASH_ESCAPE,t,r],a=[r,e.HASH_COMMENT_MODE,e.COMMENT("^\\=\\w","\\=cut",{endsWithParent:!0}),s,{className:"string",contains:i,variants:[{begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[",end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*\\<",end:"\\>",relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE]},{begin:"{\\w+}",contains:[],relevance:0},{begin:"-?\\w+\\s*\\=\\>",contains:[],relevance:0}]},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",keywords:"split return print reverse grep",relevance:0,contains:[e.HASH_COMMENT_MODE,{className:"regexp",begin:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",relevance:10},{className:"regexp",begin:"(m|qr)?/",end:"/[a-z]*",contains:[e.BACKSLASH_ESCAPE],relevance:0}]},{className:"function",beginKeywords:"sub",end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE]},{begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$",subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}]}];return t.contains=a,s.contains=a,{name:"Perl",aliases:["pl","pm"],keywords:n,contains:a}}}()); +hljs.registerLanguage("php",function(){"use strict";return function(e){var r={begin:"\\$+[a-zA-Z_-ÿ][a-zA-Z0-9_-ÿ]*"},t={className:"meta",variants:[{begin:/<\?php/,relevance:10},{begin:/<\?[=]?/},{begin:/\?>/}]},a={className:"string",contains:[e.BACKSLASH_ESCAPE,t],variants:[{begin:'b"',end:'"'},{begin:"b'",end:"'"},e.inherit(e.APOS_STRING_MODE,{illegal:null}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null})]},n={variants:[e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]},i={keyword:"__CLASS__ __DIR__ __FILE__ __FUNCTION__ __LINE__ __METHOD__ __NAMESPACE__ __TRAIT__ die echo exit include include_once print require require_once array abstract and as binary bool boolean break callable case catch class clone const continue declare default do double else elseif empty enddeclare endfor endforeach endif endswitch endwhile eval extends final finally float for foreach from global goto if implements instanceof insteadof int integer interface isset iterable list new object or private protected public real return string switch throw trait try unset use var void while xor yield",literal:"false null true",built_in:"Error|0 AppendIterator ArgumentCountError ArithmeticError ArrayIterator ArrayObject AssertionError BadFunctionCallException BadMethodCallException CachingIterator CallbackFilterIterator CompileError Countable DirectoryIterator DivisionByZeroError DomainException EmptyIterator ErrorException Exception FilesystemIterator FilterIterator GlobIterator InfiniteIterator InvalidArgumentException IteratorIterator LengthException LimitIterator LogicException MultipleIterator NoRewindIterator OutOfBoundsException OutOfRangeException OuterIterator OverflowException ParentIterator ParseError RangeException RecursiveArrayIterator RecursiveCachingIterator RecursiveCallbackFilterIterator RecursiveDirectoryIterator RecursiveFilterIterator RecursiveIterator RecursiveIteratorIterator RecursiveRegexIterator RecursiveTreeIterator RegexIterator RuntimeException SeekableIterator SplDoublyLinkedList SplFileInfo SplFileObject SplFixedArray SplHeap SplMaxHeap SplMinHeap SplObjectStorage SplObserver SplObserver SplPriorityQueue SplQueue SplStack SplSubject SplSubject SplTempFileObject TypeError UnderflowException UnexpectedValueException ArrayAccess Closure Generator Iterator IteratorAggregate Serializable Throwable Traversable WeakReference Directory __PHP_Incomplete_Class parent php_user_filter self static stdClass"};return{aliases:["php","php3","php4","php5","php6","php7"],case_insensitive:!0,keywords:i,contains:[e.HASH_COMMENT_MODE,e.COMMENT("//","$",{contains:[t]}),e.COMMENT("/\\*","\\*/",{contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.COMMENT("__halt_compiler.+?;",!1,{endsWithParent:!0,keywords:"__halt_compiler"}),{className:"string",begin:/<<<['"]?\w+['"]?$/,end:/^\w+;?$/,contains:[e.BACKSLASH_ESCAPE,{className:"subst",variants:[{begin:/\$\w+/},{begin:/\{\$/,end:/\}/}]}]},t,{className:"keyword",begin:/\$this\b/},r,{begin:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{className:"function",beginKeywords:"fn function",end:/[;{]/,excludeEnd:!0,illegal:"[$%\\[]",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",begin:"\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0,keywords:i,contains:["self",r,e.C_BLOCK_COMMENT_MODE,a,n]}]},{className:"class",beginKeywords:"class interface",end:"{",excludeEnd:!0,illegal:/[:\(\$"]/,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"namespace",end:";",illegal:/[\.']/,contains:[e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"use",end:";",contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"=>"},a,n]}}}()); +hljs.registerLanguage("php-template",function(){"use strict";return function(n){return{name:"PHP template",subLanguage:"xml",contains:[{begin:/<\?(php|=)?/,end:/\?>/,subLanguage:"php",contains:[{begin:"/\\*",end:"\\*/",skip:!0},{begin:'b"',end:'"',skip:!0},{begin:"b'",end:"'",skip:!0},n.inherit(n.APOS_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0}),n.inherit(n.QUOTE_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0})]}]}}}()); +hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}()); +hljs.registerLanguage("properties",function(){"use strict";return function(e){var n="[ \\t\\f]*",t="("+n+"[:=]"+n+"|[ \\t\\f]+)",a="([^\\\\:= \\t\\f\\n]|\\\\.)+",s={end:t,relevance:0,starts:{className:"string",end:/$/,relevance:0,contains:[{begin:"\\\\\\n"}]}};return{name:".properties",case_insensitive:!0,illegal:/\S/,contains:[e.COMMENT("^\\s*[!#]","$"),{begin:"([^\\\\\\W:= \\t\\f\\n]|\\\\.)+"+t,returnBegin:!0,contains:[{className:"attr",begin:"([^\\\\\\W:= \\t\\f\\n]|\\\\.)+",endsParent:!0,relevance:0}],starts:s},{begin:a+t,returnBegin:!0,relevance:0,contains:[{className:"meta",begin:a,endsParent:!0,relevance:0}],starts:s},{className:"attr",relevance:0,begin:a+n+"$"}]}}}()); +hljs.registerLanguage("python",function(){"use strict";return function(e){var n={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10",built_in:"Ellipsis NotImplemented",literal:"False None True"},a={className:"meta",begin:/^(>>>|\.\.\.) /},i={className:"subst",begin:/\{/,end:/\}/,keywords:n,illegal:/#/},s={begin:/\{\{/,relevance:0},r={className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/(u|b)?r?'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(u|b)?r?"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(fr|rf|f)'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(fr|rf|f)"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(u|r|ur)'/,end:/'/,relevance:10},{begin:/(u|r|ur)"/,end:/"/,relevance:10},{begin:/(b|br)'/,end:/'/},{begin:/(b|br)"/,end:/"/},{begin:/(fr|rf|f)'/,end:/'/,contains:[e.BACKSLASH_ESCAPE,s,i]},{begin:/(fr|rf|f)"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,i]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},l={className:"number",relevance:0,variants:[{begin:e.BINARY_NUMBER_RE+"[lLjJ]?"},{begin:"\\b(0o[0-7]+)[lLjJ]?"},{begin:e.C_NUMBER_RE+"[lLjJ]?"}]},t={className:"params",variants:[{begin:/\(\s*\)/,skip:!0,className:null},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:["self",a,l,r,e.HASH_COMMENT_MODE]}]};return i.contains=[r,l,a],{name:"Python",aliases:["py","gyp","ipython"],keywords:n,illegal:/(<\/|->|\?)|=>/,contains:[a,l,{beginKeywords:"if",relevance:0},r,e.HASH_COMMENT_MODE,{variants:[{className:"function",beginKeywords:"def"},{className:"class",beginKeywords:"class"}],end:/:/,illegal:/[${=;\n,]/,contains:[e.UNDERSCORE_TITLE_MODE,t,{begin:/->/,endsWithParent:!0,keywords:"None"}]},{className:"meta",begin:/^[\t ]*@/,end:/$/},{begin:/\b(print|exec)\(/}]}}}()); +hljs.registerLanguage("python-repl",function(){"use strict";return function(n){return{aliases:["pycon"],contains:[{className:"meta",starts:{end:/ |$/,starts:{end:"$",subLanguage:"python"}},variants:[{begin:/^>>>(?=[ ]|$)/},{begin:/^\.\.\.(?=[ ]|$)/}]}]}}}()); +hljs.registerLanguage("ruby",function(){"use strict";return function(e){var n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",a={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},s={className:"doctag",begin:"@[A-Za-z]+"},i={begin:"#<",end:">"},r=[e.COMMENT("#","$",{contains:[s]}),e.COMMENT("^\\=begin","^\\=end",{contains:[s],relevance:10}),e.COMMENT("^__END__","\\n$")],c={className:"subst",begin:"#\\{",end:"}",keywords:a},t={className:"string",contains:[e.BACKSLASH_ESCAPE,c],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:"%[qQwWx]?\\(",end:"\\)"},{begin:"%[qQwWx]?\\[",end:"\\]"},{begin:"%[qQwWx]?{",end:"}"},{begin:"%[qQwWx]?<",end:">"},{begin:"%[qQwWx]?/",end:"/"},{begin:"%[qQwWx]?%",end:"%"},{begin:"%[qQwWx]?-",end:"-"},{begin:"%[qQwWx]?\\|",end:"\\|"},{begin:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{begin:/<<[-~]?'?(\w+)(?:.|\n)*?\n\s*\1\b/,returnBegin:!0,contains:[{begin:/<<[-~]?'?/},e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,contains:[e.BACKSLASH_ESCAPE,c]})]}]},b={className:"params",begin:"\\(",end:"\\)",endsParent:!0,keywords:a},d=[t,i,{className:"class",beginKeywords:"class module",end:"$|;",illegal:/=/,contains:[e.inherit(e.TITLE_MODE,{begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{begin:"<\\s*",contains:[{begin:"("+e.IDENT_RE+"::)?"+e.IDENT_RE}]}].concat(r)},{className:"function",beginKeywords:"def",end:"$|;",contains:[e.inherit(e.TITLE_MODE,{begin:n}),b].concat(r)},{begin:e.IDENT_RE+"::"},{className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"(\\!|\\?)?:",relevance:0},{className:"symbol",begin:":(?!\\s)",contains:[t,{begin:n}],relevance:0},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{className:"params",begin:/\|/,end:/\|/,keywords:a},{begin:"("+e.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[i,{className:"regexp",contains:[e.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{begin:"/",end:"/[a-z]*"},{begin:"%r{",end:"}[a-z]*"},{begin:"%r\\(",end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}]}].concat(r),relevance:0}].concat(r);c.contains=d,b.contains=d;var g=[{begin:/^\s*=>/,starts:{end:"$",contains:d}},{className:"meta",begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+>|(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>)",starts:{end:"$",contains:d}}];return{name:"Ruby",aliases:["rb","gemspec","podspec","thor","irb"],keywords:a,illegal:/\/\*/,contains:r.concat(g).concat(d)}}}()); +hljs.registerLanguage("rust",function(){"use strict";return function(e){var n="([ui](8|16|32|64|128|size)|f(32|64))?",t="drop i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize f32 f64 str char bool Box Option Result String Vec Copy Send Sized Sync Drop Fn FnMut FnOnce ToOwned Clone Debug PartialEq PartialOrd Eq Ord AsRef AsMut Into From Default Iterator Extend IntoIterator DoubleEndedIterator ExactSizeIterator SliceConcatExt ToString assert! assert_eq! bitflags! bytes! cfg! col! concat! concat_idents! debug_assert! debug_assert_eq! env! panic! file! format! format_args! include_bin! include_str! line! local_data_key! module_path! option_env! print! println! select! stringify! try! unimplemented! unreachable! vec! write! writeln! macro_rules! assert_ne! debug_assert_ne!";return{name:"Rust",aliases:["rs"],keywords:{$pattern:e.IDENT_RE+"!?",keyword:"abstract as async await become box break const continue crate do dyn else enum extern false final fn for if impl in let loop macro match mod move mut override priv pub ref return self Self static struct super trait true try type typeof unsafe unsized use virtual where while yield",literal:"true false Some None Ok Err",built_in:t},illegal:""}]}}}()); +hljs.registerLanguage("scss",function(){"use strict";return function(e){var t={className:"variable",begin:"(\\$[a-zA-Z-][a-zA-Z0-9_-]*)\\b"},i={className:"number",begin:"#[0-9A-Fa-f]+"};return e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_BLOCK_COMMENT_MODE,{name:"SCSS",case_insensitive:!0,illegal:"[=/|']",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"selector-id",begin:"\\#[A-Za-z0-9_-]+",relevance:0},{className:"selector-class",begin:"\\.[A-Za-z0-9_-]+",relevance:0},{className:"selector-attr",begin:"\\[",end:"\\]",illegal:"$"},{className:"selector-tag",begin:"\\b(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|command|datalist|dd|del|details|dfn|div|dl|dt|em|embed|fieldset|figcaption|figure|footer|form|frame|frameset|(h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|keygen|label|legend|li|link|map|mark|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|rp|rt|ruby|samp|script|section|select|small|span|strike|strong|style|sub|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)\\b",relevance:0},{className:"selector-pseudo",begin:":(visited|valid|root|right|required|read-write|read-only|out-range|optional|only-of-type|only-child|nth-of-type|nth-last-of-type|nth-last-child|nth-child|not|link|left|last-of-type|last-child|lang|invalid|indeterminate|in-range|hover|focus|first-of-type|first-line|first-letter|first-child|first|enabled|empty|disabled|default|checked|before|after|active)"},{className:"selector-pseudo",begin:"::(after|before|choices|first-letter|first-line|repeat-index|repeat-item|selection|value)"},t,{className:"attribute",begin:"\\b(src|z-index|word-wrap|word-spacing|word-break|width|widows|white-space|visibility|vertical-align|unicode-bidi|transition-timing-function|transition-property|transition-duration|transition-delay|transition|transform-style|transform-origin|transform|top|text-underline-position|text-transform|text-shadow|text-rendering|text-overflow|text-indent|text-decoration-style|text-decoration-line|text-decoration-color|text-decoration|text-align-last|text-align|tab-size|table-layout|right|resize|quotes|position|pointer-events|perspective-origin|perspective|page-break-inside|page-break-before|page-break-after|padding-top|padding-right|padding-left|padding-bottom|padding|overflow-y|overflow-x|overflow-wrap|overflow|outline-width|outline-style|outline-offset|outline-color|outline|orphans|order|opacity|object-position|object-fit|normal|none|nav-up|nav-right|nav-left|nav-index|nav-down|min-width|min-height|max-width|max-height|mask|marks|margin-top|margin-right|margin-left|margin-bottom|margin|list-style-type|list-style-position|list-style-image|list-style|line-height|letter-spacing|left|justify-content|initial|inherit|ime-mode|image-orientation|image-resolution|image-rendering|icon|hyphens|height|font-weight|font-variant-ligatures|font-variant|font-style|font-stretch|font-size-adjust|font-size|font-language-override|font-kerning|font-feature-settings|font-family|font|float|flex-wrap|flex-shrink|flex-grow|flex-flow|flex-direction|flex-basis|flex|filter|empty-cells|display|direction|cursor|counter-reset|counter-increment|content|column-width|column-span|column-rule-width|column-rule-style|column-rule-color|column-rule|column-gap|column-fill|column-count|columns|color|clip-path|clip|clear|caption-side|break-inside|break-before|break-after|box-sizing|box-shadow|box-decoration-break|bottom|border-width|border-top-width|border-top-style|border-top-right-radius|border-top-left-radius|border-top-color|border-top|border-style|border-spacing|border-right-width|border-right-style|border-right-color|border-right|border-radius|border-left-width|border-left-style|border-left-color|border-left|border-image-width|border-image-source|border-image-slice|border-image-repeat|border-image-outset|border-image|border-color|border-collapse|border-bottom-width|border-bottom-style|border-bottom-right-radius|border-bottom-left-radius|border-bottom-color|border-bottom|border|background-size|background-repeat|background-position|background-origin|background-image|background-color|background-clip|background-attachment|background-blend-mode|background|backface-visibility|auto|animation-timing-function|animation-play-state|animation-name|animation-iteration-count|animation-fill-mode|animation-duration|animation-direction|animation-delay|animation|align-self|align-items|align-content)\\b",illegal:"[^\\s]"},{begin:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b"},{begin:":",end:";",contains:[t,i,e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,{className:"meta",begin:"!important"}]},{begin:"@(page|font-face)",lexemes:"@[a-z-]+",keywords:"@page @font-face"},{begin:"@",end:"[{;]",returnBegin:!0,keywords:"and or not only",contains:[{begin:"@[a-z-]+",className:"keyword"},t,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,i,e.CSS_NUMBER_MODE]}]}}}()); +hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}()); +hljs.registerLanguage("sql",function(){"use strict";return function(e){var t=e.COMMENT("--","$");return{name:"SQL",case_insensitive:!0,illegal:/[<>{}*]/,contains:[{beginKeywords:"begin end start commit rollback savepoint lock alter create drop rename call delete do handler insert load replace select truncate update set show pragma grant merge describe use explain help declare prepare execute deallocate release unlock purge reset change stop analyze cache flush optimize repair kill install uninstall checksum restore check backup revoke comment values with",end:/;/,endsWithParent:!0,keywords:{$pattern:/[\w\.]+/,keyword:"as abort abs absolute acc acce accep accept access accessed accessible account acos action activate add addtime admin administer advanced advise aes_decrypt aes_encrypt after agent aggregate ali alia alias all allocate allow alter always analyze ancillary and anti any anydata anydataset anyschema anytype apply archive archived archivelog are as asc ascii asin assembly assertion associate asynchronous at atan atn2 attr attri attrib attribu attribut attribute attributes audit authenticated authentication authid authors auto autoallocate autodblink autoextend automatic availability avg backup badfile basicfile before begin beginning benchmark between bfile bfile_base big bigfile bin binary_double binary_float binlog bit_and bit_count bit_length bit_or bit_xor bitmap blob_base block blocksize body both bound bucket buffer_cache buffer_pool build bulk by byte byteordermark bytes cache caching call calling cancel capacity cascade cascaded case cast catalog category ceil ceiling chain change changed char_base char_length character_length characters characterset charindex charset charsetform charsetid check checksum checksum_agg child choose chr chunk class cleanup clear client clob clob_base clone close cluster_id cluster_probability cluster_set clustering coalesce coercibility col collate collation collect colu colum column column_value columns columns_updated comment commit compact compatibility compiled complete composite_limit compound compress compute concat concat_ws concurrent confirm conn connec connect connect_by_iscycle connect_by_isleaf connect_by_root connect_time connection consider consistent constant constraint constraints constructor container content contents context contributors controlfile conv convert convert_tz corr corr_k corr_s corresponding corruption cos cost count count_big counted covar_pop covar_samp cpu_per_call cpu_per_session crc32 create creation critical cross cube cume_dist curdate current current_date current_time current_timestamp current_user cursor curtime customdatum cycle data database databases datafile datafiles datalength date_add date_cache date_format date_sub dateadd datediff datefromparts datename datepart datetime2fromparts day day_to_second dayname dayofmonth dayofweek dayofyear days db_role_change dbtimezone ddl deallocate declare decode decompose decrement decrypt deduplicate def defa defau defaul default defaults deferred defi defin define degrees delayed delegate delete delete_all delimited demand dense_rank depth dequeue des_decrypt des_encrypt des_key_file desc descr descri describ describe descriptor deterministic diagnostics difference dimension direct_load directory disable disable_all disallow disassociate discardfile disconnect diskgroup distinct distinctrow distribute distributed div do document domain dotnet double downgrade drop dumpfile duplicate duration each edition editionable editions element ellipsis else elsif elt empty enable enable_all enclosed encode encoding encrypt end end-exec endian enforced engine engines enqueue enterprise entityescaping eomonth error errors escaped evalname evaluate event eventdata events except exception exceptions exchange exclude excluding execu execut execute exempt exists exit exp expire explain explode export export_set extended extent external external_1 external_2 externally extract failed failed_login_attempts failover failure far fast feature_set feature_value fetch field fields file file_name_convert filesystem_like_logging final finish first first_value fixed flash_cache flashback floor flush following follows for forall force foreign form forma format found found_rows freelist freelists freepools fresh from from_base64 from_days ftp full function general generated get get_format get_lock getdate getutcdate global global_name globally go goto grant grants greatest group group_concat group_id grouping grouping_id groups gtid_subtract guarantee guard handler hash hashkeys having hea head headi headin heading heap help hex hierarchy high high_priority hosts hour hours http id ident_current ident_incr ident_seed identified identity idle_time if ifnull ignore iif ilike ilm immediate import in include including increment index indexes indexing indextype indicator indices inet6_aton inet6_ntoa inet_aton inet_ntoa infile initial initialized initially initrans inmemory inner innodb input insert install instance instantiable instr interface interleaved intersect into invalidate invisible is is_free_lock is_ipv4 is_ipv4_compat is_not is_not_null is_used_lock isdate isnull isolation iterate java join json json_exists keep keep_duplicates key keys kill language large last last_day last_insert_id last_value lateral lax lcase lead leading least leaves left len lenght length less level levels library like like2 like4 likec limit lines link list listagg little ln load load_file lob lobs local localtime localtimestamp locate locator lock locked log log10 log2 logfile logfiles logging logical logical_reads_per_call logoff logon logs long loop low low_priority lower lpad lrtrim ltrim main make_set makedate maketime managed management manual map mapping mask master master_pos_wait match matched materialized max maxextents maximize maxinstances maxlen maxlogfiles maxloghistory maxlogmembers maxsize maxtrans md5 measures median medium member memcompress memory merge microsecond mid migration min minextents minimum mining minus minute minutes minvalue missing mod mode model modification modify module monitoring month months mount move movement multiset mutex name name_const names nan national native natural nav nchar nclob nested never new newline next nextval no no_write_to_binlog noarchivelog noaudit nobadfile nocheck nocompress nocopy nocycle nodelay nodiscardfile noentityescaping noguarantee nokeep nologfile nomapping nomaxvalue nominimize nominvalue nomonitoring none noneditionable nonschema noorder nopr nopro noprom nopromp noprompt norely noresetlogs noreverse normal norowdependencies noschemacheck noswitch not nothing notice notnull notrim novalidate now nowait nth_value nullif nulls num numb numbe nvarchar nvarchar2 object ocicoll ocidate ocidatetime ociduration ociinterval ociloblocator ocinumber ociref ocirefcursor ocirowid ocistring ocitype oct octet_length of off offline offset oid oidindex old on online only opaque open operations operator optimal optimize option optionally or oracle oracle_date oradata ord ordaudio orddicom orddoc order ordimage ordinality ordvideo organization orlany orlvary out outer outfile outline output over overflow overriding package pad parallel parallel_enable parameters parent parse partial partition partitions pascal passing password password_grace_time password_lock_time password_reuse_max password_reuse_time password_verify_function patch path patindex pctincrease pctthreshold pctused pctversion percent percent_rank percentile_cont percentile_disc performance period period_add period_diff permanent physical pi pipe pipelined pivot pluggable plugin policy position post_transaction pow power pragma prebuilt precedes preceding precision prediction prediction_cost prediction_details prediction_probability prediction_set prepare present preserve prior priority private private_sga privileges procedural procedure procedure_analyze processlist profiles project prompt protection public publishingservername purge quarter query quick quiesce quota quotename radians raise rand range rank raw read reads readsize rebuild record records recover recovery recursive recycle redo reduced ref reference referenced references referencing refresh regexp_like register regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy reject rekey relational relative relaylog release release_lock relies_on relocate rely rem remainder rename repair repeat replace replicate replication required reset resetlogs resize resource respect restore restricted result result_cache resumable resume retention return returning returns reuse reverse revoke right rlike role roles rollback rolling rollup round row row_count rowdependencies rowid rownum rows rtrim rules safe salt sample save savepoint sb1 sb2 sb4 scan schema schemacheck scn scope scroll sdo_georaster sdo_topo_geometry search sec_to_time second seconds section securefile security seed segment select self semi sequence sequential serializable server servererror session session_user sessions_per_user set sets settings sha sha1 sha2 share shared shared_pool short show shrink shutdown si_averagecolor si_colorhistogram si_featurelist si_positionalcolor si_stillimage si_texture siblings sid sign sin size size_t sizes skip slave sleep smalldatetimefromparts smallfile snapshot some soname sort soundex source space sparse spfile split sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_small_result sql_variant_property sqlcode sqldata sqlerror sqlname sqlstate sqrt square standalone standby start starting startup statement static statistics stats_binomial_test stats_crosstab stats_ks_test stats_mode stats_mw_test stats_one_way_anova stats_t_test_ stats_t_test_indep stats_t_test_one stats_t_test_paired stats_wsr_test status std stddev stddev_pop stddev_samp stdev stop storage store stored str str_to_date straight_join strcmp strict string struct stuff style subdate subpartition subpartitions substitutable substr substring subtime subtring_index subtype success sum suspend switch switchoffset switchover sync synchronous synonym sys sys_xmlagg sysasm sysaux sysdate sysdatetimeoffset sysdba sysoper system system_user sysutcdatetime table tables tablespace tablesample tan tdo template temporary terminated tertiary_weights test than then thread through tier ties time time_format time_zone timediff timefromparts timeout timestamp timestampadd timestampdiff timezone_abbr timezone_minute timezone_region to to_base64 to_date to_days to_seconds todatetimeoffset trace tracking transaction transactional translate translation treat trigger trigger_nestlevel triggers trim truncate try_cast try_convert try_parse type ub1 ub2 ub4 ucase unarchived unbounded uncompress under undo unhex unicode uniform uninstall union unique unix_timestamp unknown unlimited unlock unnest unpivot unrecoverable unsafe unsigned until untrusted unusable unused update updated upgrade upped upper upsert url urowid usable usage use use_stored_outlines user user_data user_resources users using utc_date utc_timestamp uuid uuid_short validate validate_password_strength validation valist value values var var_samp varcharc vari varia variab variabl variable variables variance varp varraw varrawc varray verify version versions view virtual visible void wait wallet warning warnings week weekday weekofyear wellformed when whene whenev wheneve whenever where while whitespace window with within without work wrapped xdb xml xmlagg xmlattributes xmlcast xmlcolattval xmlelement xmlexists xmlforest xmlindex xmlnamespaces xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltype xor year year_to_month years yearweek",literal:"true false null unknown",built_in:"array bigint binary bit blob bool boolean char character date dec decimal float int int8 integer interval number numeric real record serial serial8 smallint text time timestamp tinyint varchar varchar2 varying void"},contains:[{className:"string",begin:"'",end:"'",contains:[{begin:"''"}]},{className:"string",begin:'"',end:'"',contains:[{begin:'""'}]},{className:"string",begin:"`",end:"`"},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,t,e.HASH_COMMENT_MODE]},e.C_BLOCK_COMMENT_MODE,t,e.HASH_COMMENT_MODE]}}}()); +hljs.registerLanguage("swift",function(){"use strict";return function(e){var i={keyword:"#available #colorLiteral #column #else #elseif #endif #file #fileLiteral #function #if #imageLiteral #line #selector #sourceLocation _ __COLUMN__ __FILE__ __FUNCTION__ __LINE__ Any as as! as? associatedtype associativity break case catch class continue convenience default defer deinit didSet do dynamic dynamicType else enum extension fallthrough false fileprivate final for func get guard if import in indirect infix init inout internal is lazy left let mutating nil none nonmutating open operator optional override postfix precedence prefix private protocol Protocol public repeat required rethrows return right self Self set static struct subscript super switch throw throws true try try! try? Type typealias unowned var weak where while willSet",literal:"true false nil",built_in:"abs advance alignof alignofValue anyGenerator assert assertionFailure bridgeFromObjectiveC bridgeFromObjectiveCUnconditional bridgeToObjectiveC bridgeToObjectiveCUnconditional c compactMap contains count countElements countLeadingZeros debugPrint debugPrintln distance dropFirst dropLast dump encodeBitsAsWords enumerate equal fatalError filter find getBridgedObjectiveCType getVaList indices insertionSort isBridgedToObjectiveC isBridgedVerbatimToObjectiveC isUniquelyReferenced isUniquelyReferencedNonObjC join lazy lexicographicalCompare map max maxElement min minElement numericCast overlaps partition posix precondition preconditionFailure print println quickSort readLine reduce reflect reinterpretCast reverse roundUpToAlignment sizeof sizeofValue sort split startsWith stride strideof strideofValue swap toString transcode underestimateCount unsafeAddressOf unsafeBitCast unsafeDowncast unsafeUnwrap unsafeReflect withExtendedLifetime withObjectAtPlusZero withUnsafePointer withUnsafePointerToObject withUnsafeMutablePointer withUnsafeMutablePointers withUnsafePointer withUnsafePointers withVaList zip"},n=e.COMMENT("/\\*","\\*/",{contains:["self"]}),t={className:"subst",begin:/\\\(/,end:"\\)",keywords:i,contains:[]},a={className:"string",contains:[e.BACKSLASH_ESCAPE,t],variants:[{begin:/"""/,end:/"""/},{begin:/"/,end:/"/}]},r={className:"number",begin:"\\b([\\d_]+(\\.[\\deE_]+)?|0x[a-fA-F0-9_]+(\\.[a-fA-F0-9p_]+)?|0b[01_]+|0o[0-7_]+)\\b",relevance:0};return t.contains=[r],{name:"Swift",keywords:i,contains:[a,e.C_LINE_COMMENT_MODE,n,{className:"type",begin:"\\b[A-Z][\\wÀ-ʸ']*[!?]"},{className:"type",begin:"\\b[A-Z][\\wÀ-ʸ']*",relevance:0},r,{className:"function",beginKeywords:"func",end:"{",excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][0-9A-Za-z$_]*/}),{begin://},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:i,contains:["self",r,a,e.C_BLOCK_COMMENT_MODE,{begin:":"}],illegal:/["']/}],illegal:/\[|%/},{className:"class",beginKeywords:"struct protocol class extension enum",keywords:i,end:"\\{",excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/})]},{className:"meta",begin:"(@discardableResult|@warn_unused_result|@exported|@lazy|@noescape|@NSCopying|@NSManaged|@objc|@objcMembers|@convention|@required|@noreturn|@IBAction|@IBDesignable|@IBInspectable|@IBOutlet|@infix|@prefix|@postfix|@autoclosure|@testable|@available|@nonobjc|@NSApplicationMain|@UIApplicationMain|@dynamicMemberLookup|@propertyWrapper)\\b"},{beginKeywords:"import",end:/$/,contains:[e.C_LINE_COMMENT_MODE,n]}]}}}()); +hljs.registerLanguage("typescript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);return function(r){var t={$pattern:"[A-Za-z$_][0-9A-Za-z$_]*",keyword:e.concat(["type","namespace","typedef","interface","public","private","protected","implements","declare","abstract","readonly"]).join(" "),literal:n.join(" "),built_in:a.concat(["any","void","number","boolean","string","object","never","enum"]).join(" ")},s={className:"meta",begin:"@[A-Za-z$_][0-9A-Za-z$_]*"},i={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:r.C_NUMBER_RE+"n?"}],relevance:0},o={className:"subst",begin:"\\$\\{",end:"\\}",keywords:t,contains:[]},c={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[r.BACKSLASH_ESCAPE,o],subLanguage:"xml"}},l={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[r.BACKSLASH_ESCAPE,o],subLanguage:"css"}},E={className:"string",begin:"`",end:"`",contains:[r.BACKSLASH_ESCAPE,o]};o.contains=[r.APOS_STRING_MODE,r.QUOTE_STRING_MODE,c,l,E,i,r.REGEXP_MODE];var d={begin:"\\(",end:/\)/,keywords:t,contains:["self",r.QUOTE_STRING_MODE,r.APOS_STRING_MODE,r.NUMBER_MODE]},u={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,contains:[r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,s,d]};return{name:"TypeScript",aliases:["ts"],keywords:t,contains:[r.SHEBANG(),{className:"meta",begin:/^\s*['"]use strict['"]/},r.APOS_STRING_MODE,r.QUOTE_STRING_MODE,c,l,E,r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,i,{begin:"("+r.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,r.REGEXP_MODE,{className:"function",begin:"(\\([^(]*(\\([^(]*(\\([^(]*\\))?\\))?\\)|"+r.UNDERSCORE_IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:r.UNDERSCORE_IDENT_RE},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,contains:d.contains}]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/[\{;]/,excludeEnd:!0,keywords:t,contains:["self",r.inherit(r.TITLE_MODE,{begin:"[A-Za-z$_][0-9A-Za-z$_]*"}),u],illegal:/%/,relevance:0},{beginKeywords:"constructor",end:/[\{;]/,excludeEnd:!0,contains:["self",u]},{begin:/module\./,keywords:{built_in:"module"},relevance:0},{beginKeywords:"module",end:/\{/,excludeEnd:!0},{beginKeywords:"interface",end:/\{/,excludeEnd:!0,keywords:"interface extends"},{begin:/\$[(.]/},{begin:"\\."+r.IDENT_RE,relevance:0},s,d]}}}()); +hljs.registerLanguage("yaml",function(){"use strict";return function(e){var n="true false yes no null",a="[\\w#;/?:@&=+$,.~*\\'()[\\]]+",s={className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable",variants:[{begin:"{{",end:"}}"},{begin:"%{",end:"}"}]}]},i=e.inherit(s,{variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),l={end:",",endsWithParent:!0,excludeEnd:!0,contains:[],keywords:n,relevance:0},t={begin:"{",end:"}",contains:[l],illegal:"\\n",relevance:0},g={begin:"\\[",end:"\\]",contains:[l],illegal:"\\n",relevance:0},b=[{className:"attr",variants:[{begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---s*$",relevance:10},{className:"string",begin:"[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*"},{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:"!\\w+!"+a},{className:"type",begin:"!<"+a+">"},{className:"type",begin:"!"+a},{className:"type",begin:"!!"+a},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta",begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"\\-(?=[ ]|$)",relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{className:"number",begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"},{className:"number",begin:e.C_NUMBER_RE+"\\b"},t,g,s],c=[...b];return c.pop(),c.push(i),l.contains=c,{name:"YAML",case_insensitive:!0,aliases:["yml","YAML"],contains:b}}}()); +hljs.registerLanguage("armasm",function(){"use strict";return function(s){const e={variants:[s.COMMENT("^[ \\t]*(?=#)","$",{relevance:0,excludeBegin:!0}),s.COMMENT("[;@]","$",{relevance:0}),s.C_LINE_COMMENT_MODE,s.C_BLOCK_COMMENT_MODE]};return{name:"ARM Assembly",case_insensitive:!0,aliases:["arm"],keywords:{$pattern:"\\.?"+s.IDENT_RE,meta:".2byte .4byte .align .ascii .asciz .balign .byte .code .data .else .end .endif .endm .endr .equ .err .exitm .extern .global .hword .if .ifdef .ifndef .include .irp .long .macro .rept .req .section .set .skip .space .text .word .arm .thumb .code16 .code32 .force_thumb .thumb_func .ltorg ALIAS ALIGN ARM AREA ASSERT ATTR CN CODE CODE16 CODE32 COMMON CP DATA DCB DCD DCDU DCDO DCFD DCFDU DCI DCQ DCQU DCW DCWU DN ELIF ELSE END ENDFUNC ENDIF ENDP ENTRY EQU EXPORT EXPORTAS EXTERN FIELD FILL FUNCTION GBLA GBLL GBLS GET GLOBAL IF IMPORT INCBIN INCLUDE INFO KEEP LCLA LCLL LCLS LTORG MACRO MAP MEND MEXIT NOFP OPT PRESERVE8 PROC QN READONLY RELOC REQUIRE REQUIRE8 RLIST FN ROUT SETA SETL SETS SN SPACE SUBT THUMB THUMBX TTL WHILE WEND ",built_in:"r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 pc lr sp ip sl sb fp a1 a2 a3 a4 v1 v2 v3 v4 v5 v6 v7 v8 f0 f1 f2 f3 f4 f5 f6 f7 p0 p1 p2 p3 p4 p5 p6 p7 p8 p9 p10 p11 p12 p13 p14 p15 c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 q0 q1 q2 q3 q4 q5 q6 q7 q8 q9 q10 q11 q12 q13 q14 q15 cpsr_c cpsr_x cpsr_s cpsr_f cpsr_cx cpsr_cxs cpsr_xs cpsr_xsf cpsr_sf cpsr_cxsf spsr_c spsr_x spsr_s spsr_f spsr_cx spsr_cxs spsr_xs spsr_xsf spsr_sf spsr_cxsf s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 s16 s17 s18 s19 s20 s21 s22 s23 s24 s25 s26 s27 s28 s29 s30 s31 d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15 d16 d17 d18 d19 d20 d21 d22 d23 d24 d25 d26 d27 d28 d29 d30 d31 {PC} {VAR} {TRUE} {FALSE} {OPT} {CONFIG} {ENDIAN} {CODESIZE} {CPU} {FPU} {ARCHITECTURE} {PCSTOREOFFSET} {ARMASM_VERSION} {INTER} {ROPI} {RWPI} {SWST} {NOSWST} . @"},contains:[{className:"keyword",begin:"\\b(adc|(qd?|sh?|u[qh]?)?add(8|16)?|usada?8|(q|sh?|u[qh]?)?(as|sa)x|and|adrl?|sbc|rs[bc]|asr|b[lx]?|blx|bxj|cbn?z|tb[bh]|bic|bfc|bfi|[su]bfx|bkpt|cdp2?|clz|clrex|cmp|cmn|cpsi[ed]|cps|setend|dbg|dmb|dsb|eor|isb|it[te]{0,3}|lsl|lsr|ror|rrx|ldm(([id][ab])|f[ds])?|ldr((s|ex)?[bhd])?|movt?|mvn|mra|mar|mul|[us]mull|smul[bwt][bt]|smu[as]d|smmul|smmla|mla|umlaal|smlal?([wbt][bt]|d)|mls|smlsl?[ds]|smc|svc|sev|mia([bt]{2}|ph)?|mrr?c2?|mcrr2?|mrs|msr|orr|orn|pkh(tb|bt)|rbit|rev(16|sh)?|sel|[su]sat(16)?|nop|pop|push|rfe([id][ab])?|stm([id][ab])?|str(ex)?[bhd]?|(qd?)?sub|(sh?|q|u[qh]?)?sub(8|16)|[su]xt(a?h|a?b(16)?)|srs([id][ab])?|swpb?|swi|smi|tst|teq|wfe|wfi|yield)(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al|hs|lo)?[sptrx]?(?=\\s)"},e,s.QUOTE_STRING_MODE,{className:"string",begin:"'",end:"[^\\\\]'",relevance:0},{className:"title",begin:"\\|",end:"\\|",illegal:"\\n",relevance:0},{className:"number",variants:[{begin:"[#$=]?0x[0-9a-f]+"},{begin:"[#$=]?0b[01]+"},{begin:"[#$=]\\d+"},{begin:"\\b\\d+"}],relevance:0},{className:"symbol",variants:[{begin:"^[ \\t]*[a-z_\\.\\$][a-z0-9_\\.\\$]+:"},{begin:"^[a-z_\\.\\$][a-z0-9_\\.\\$]+"},{begin:"[=#]\\w+"}],relevance:0}]}}}()); +hljs.registerLanguage("d",function(){"use strict";return function(e){var a={$pattern:e.UNDERSCORE_IDENT_RE,keyword:"abstract alias align asm assert auto body break byte case cast catch class const continue debug default delete deprecated do else enum export extern final finally for foreach foreach_reverse|10 goto if immutable import in inout int interface invariant is lazy macro mixin module new nothrow out override package pragma private protected public pure ref return scope shared static struct super switch synchronized template this throw try typedef typeid typeof union unittest version void volatile while with __FILE__ __LINE__ __gshared|10 __thread __traits __DATE__ __EOF__ __TIME__ __TIMESTAMP__ __VENDOR__ __VERSION__",built_in:"bool cdouble cent cfloat char creal dchar delegate double dstring float function idouble ifloat ireal long real short string ubyte ucent uint ulong ushort wchar wstring",literal:"false null true"},d="((0|[1-9][\\d_]*)|0[bB][01_]+|0[xX]([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*))",n="\\\\(['\"\\?\\\\abfnrtv]|u[\\dA-Fa-f]{4}|[0-7]{1,3}|x[\\dA-Fa-f]{2}|U[\\dA-Fa-f]{8})|&[a-zA-Z\\d]{2,};",t={className:"number",begin:"\\b"+d+"(L|u|U|Lu|LU|uL|UL)?",relevance:0},_={className:"number",begin:"\\b(((0[xX](([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*)\\.([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*)|\\.?([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*))[pP][+-]?(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d))|((0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)(\\.\\d*|([eE][+-]?(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)))|\\d+\\.(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)|\\.(0|[1-9][\\d_]*)([eE][+-]?(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d))?))([fF]|L|i|[fF]i|Li)?|"+d+"(i|[fF]i|Li))",relevance:0},r={className:"string",begin:"'("+n+"|.)",end:"'",illegal:"."},i={className:"string",begin:'"',contains:[{begin:n,relevance:0}],end:'"[cwd]?'},s=e.COMMENT("\\/\\+","\\+\\/",{contains:["self"],relevance:10});return{name:"D",keywords:a,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,{className:"string",begin:'x"[\\da-fA-F\\s\\n\\r]*"[cwd]?',relevance:10},i,{className:"string",begin:'[rq]"',end:'"[cwd]?',relevance:5},{className:"string",begin:"`",end:"`[cwd]?"},{className:"string",begin:'q"\\{',end:'\\}"'},_,t,r,{className:"meta",begin:"^#!",end:"$",relevance:5},{className:"meta",begin:"#(line)",end:"$",relevance:5},{className:"keyword",begin:"@[a-zA-Z_][a-zA-Z_\\d]*"}]}}}()); +hljs.registerLanguage("handlebars",function(){"use strict";function e(...e){return e.map(e=>(function(e){return e?"string"==typeof e?e:e.source:null})(e)).join("")}return function(n){const a={"builtin-name":"action bindattr collection component concat debugger each each-in get hash if in input link-to loc log lookup mut outlet partial query-params render template textarea unbound unless view with yield"},t=/\[.*?\]/,s=/[^\s!"#%&'()*+,.\/;<=>@\[\\\]^`{|}~]+/,i=e("(",/'.*?'/,"|",/".*?"/,"|",t,"|",s,"|",/\.|\//,")+"),r=e("(",t,"|",s,")(?==)"),l={begin:i,lexemes:/[\w.\/]+/},c=n.inherit(l,{keywords:{literal:"true false undefined null"}}),o={begin:/\(/,end:/\)/},m={className:"attr",begin:r,relevance:0,starts:{begin:/=/,end:/=/,starts:{contains:[n.NUMBER_MODE,n.QUOTE_STRING_MODE,n.APOS_STRING_MODE,c,o]}}},d={contains:[n.NUMBER_MODE,n.QUOTE_STRING_MODE,n.APOS_STRING_MODE,{begin:/as\s+\|/,keywords:{keyword:"as"},end:/\|/,contains:[{begin:/\w+/}]},m,c,o],returnEnd:!0},g=n.inherit(l,{className:"name",keywords:a,starts:n.inherit(d,{end:/\)/})});o.contains=[g];const u=n.inherit(l,{keywords:a,className:"name",starts:n.inherit(d,{end:/}}/})}),b=n.inherit(l,{keywords:a,className:"name"}),h=n.inherit(l,{className:"name",keywords:a,starts:n.inherit(d,{end:/}}/})});return{name:"Handlebars",aliases:["hbs","html.hbs","html.handlebars","htmlbars"],case_insensitive:!0,subLanguage:"xml",contains:[{begin:/\\\{\{/,skip:!0},{begin:/\\\\(?=\{\{)/,skip:!0},n.COMMENT(/\{\{!--/,/--\}\}/),n.COMMENT(/\{\{!/,/\}\}/),{className:"template-tag",begin:/\{\{\{\{(?!\/)/,end:/\}\}\}\}/,contains:[u],starts:{end:/\{\{\{\{\//,returnEnd:!0,subLanguage:"xml"}},{className:"template-tag",begin:/\{\{\{\{\//,end:/\}\}\}\}/,contains:[b]},{className:"template-tag",begin:/\{\{#/,end:/\}\}/,contains:[u]},{className:"template-tag",begin:/\{\{(?=else\}\})/,end:/\}\}/,keywords:"else"},{className:"template-tag",begin:/\{\{\//,end:/\}\}/,contains:[b]},{className:"template-variable",begin:/\{\{\{/,end:/\}\}\}/,contains:[h]},{className:"template-variable",begin:/\{\{/,end:/\}\}/,contains:[h]}]}}}()); +hljs.registerLanguage("haskell",function(){"use strict";return function(e){var n={variants:[e.COMMENT("--","$"),e.COMMENT("{-","-}",{contains:["self"]})]},i={className:"meta",begin:"{-#",end:"#-}"},a={className:"meta",begin:"^#",end:"$"},s={className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},l={begin:"\\(",end:"\\)",illegal:'"',contains:[i,a,{className:"type",begin:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},e.inherit(e.TITLE_MODE,{begin:"[_a-z][\\w']*"}),n]};return{name:"Haskell",aliases:["hs"],keywords:"let in if then else case of where do module import hiding qualified type data newtype deriving class instance as default infix infixl infixr foreign export ccall stdcall cplusplus jvm dotnet safe unsafe family forall mdo proc rec",contains:[{beginKeywords:"module",end:"where",keywords:"module where",contains:[l,n],illegal:"\\W\\.|;"},{begin:"\\bimport\\b",end:"$",keywords:"import qualified as hiding",contains:[l,n],illegal:"\\W\\.|;"},{className:"class",begin:"^(\\s*)?(class|instance)\\b",end:"where",keywords:"class family instance where",contains:[s,l,n]},{className:"class",begin:"\\b(data|(new)?type)\\b",end:"$",keywords:"data family type newtype deriving",contains:[i,s,l,{begin:"{",end:"}",contains:l.contains},n]},{beginKeywords:"default",end:"$",contains:[s,l,n]},{beginKeywords:"infix infixl infixr",end:"$",contains:[e.C_NUMBER_MODE,n]},{begin:"\\bforeign\\b",end:"$",keywords:"foreign import export ccall stdcall cplusplus jvm dotnet safe unsafe",contains:[s,e.QUOTE_STRING_MODE,n]},{className:"meta",begin:"#!\\/usr\\/bin\\/env runhaskell",end:"$"},i,a,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,s,e.inherit(e.TITLE_MODE,{begin:"^[_a-z][\\w']*"}),n,{begin:"->|<-"}]}}}()); +hljs.registerLanguage("julia",function(){"use strict";return function(e){var r="[A-Za-z_\\u00A1-\\uFFFF][A-Za-z_0-9\\u00A1-\\uFFFF]*",t={$pattern:r,keyword:"in isa where baremodule begin break catch ccall const continue do else elseif end export false finally for function global if import importall let local macro module quote return true try using while type immutable abstract bitstype typealias ",literal:"true false ARGS C_NULL DevNull ENDIAN_BOM ENV I Inf Inf16 Inf32 Inf64 InsertionSort JULIA_HOME LOAD_PATH MergeSort NaN NaN16 NaN32 NaN64 PROGRAM_FILE QuickSort RoundDown RoundFromZero RoundNearest RoundNearestTiesAway RoundNearestTiesUp RoundToZero RoundUp STDERR STDIN STDOUT VERSION catalan e|0 eu|0 eulergamma golden im nothing pi γ π φ ",built_in:"ANY AbstractArray AbstractChannel AbstractFloat AbstractMatrix AbstractRNG AbstractSerializer AbstractSet AbstractSparseArray AbstractSparseMatrix AbstractSparseVector AbstractString AbstractUnitRange AbstractVecOrMat AbstractVector Any ArgumentError Array AssertionError Associative Base64DecodePipe Base64EncodePipe Bidiagonal BigFloat BigInt BitArray BitMatrix BitVector Bool BoundsError BufferStream CachingPool CapturedException CartesianIndex CartesianRange Cchar Cdouble Cfloat Channel Char Cint Cintmax_t Clong Clonglong ClusterManager Cmd CodeInfo Colon Complex Complex128 Complex32 Complex64 CompositeException Condition ConjArray ConjMatrix ConjVector Cptrdiff_t Cshort Csize_t Cssize_t Cstring Cuchar Cuint Cuintmax_t Culong Culonglong Cushort Cwchar_t Cwstring DataType Date DateFormat DateTime DenseArray DenseMatrix DenseVecOrMat DenseVector Diagonal Dict DimensionMismatch Dims DirectIndexString Display DivideError DomainError EOFError EachLine Enum Enumerate ErrorException Exception ExponentialBackOff Expr Factorization FileMonitor Float16 Float32 Float64 Function Future GlobalRef GotoNode HTML Hermitian IO IOBuffer IOContext IOStream IPAddr IPv4 IPv6 IndexCartesian IndexLinear IndexStyle InexactError InitError Int Int128 Int16 Int32 Int64 Int8 IntSet Integer InterruptException InvalidStateException Irrational KeyError LabelNode LinSpace LineNumberNode LoadError LowerTriangular MIME Matrix MersenneTwister Method MethodError MethodTable Module NTuple NewvarNode NullException Nullable Number ObjectIdDict OrdinalRange OutOfMemoryError OverflowError Pair ParseError PartialQuickSort PermutedDimsArray Pipe PollingFileWatcher ProcessExitedException Ptr QuoteNode RandomDevice Range RangeIndex Rational RawFD ReadOnlyMemoryError Real ReentrantLock Ref Regex RegexMatch RemoteChannel RemoteException RevString RoundingMode RowVector SSAValue SegmentationFault SerializationState Set SharedArray SharedMatrix SharedVector Signed SimpleVector Slot SlotNumber SparseMatrixCSC SparseVector StackFrame StackOverflowError StackTrace StepRange StepRangeLen StridedArray StridedMatrix StridedVecOrMat StridedVector String SubArray SubString SymTridiagonal Symbol Symmetric SystemError TCPSocket Task Text TextDisplay Timer Tridiagonal Tuple Type TypeError TypeMapEntry TypeMapLevel TypeName TypeVar TypedSlot UDPSocket UInt UInt128 UInt16 UInt32 UInt64 UInt8 UndefRefError UndefVarError UnicodeError UniformScaling Union UnionAll UnitRange Unsigned UpperTriangular Val Vararg VecElement VecOrMat Vector VersionNumber Void WeakKeyDict WeakRef WorkerConfig WorkerPool "},a={keywords:t,illegal:/<\//},n={className:"subst",begin:/\$\(/,end:/\)/,keywords:t},o={className:"variable",begin:"\\$"+r},i={className:"string",contains:[e.BACKSLASH_ESCAPE,n,o],variants:[{begin:/\w*"""/,end:/"""\w*/,relevance:10},{begin:/\w*"/,end:/"\w*/}]},l={className:"string",contains:[e.BACKSLASH_ESCAPE,n,o],begin:"`",end:"`"},s={className:"meta",begin:"@"+r};return a.name="Julia",a.contains=[{className:"number",begin:/(\b0x[\d_]*(\.[\d_]*)?|0x\.\d[\d_]*)p[-+]?\d+|\b0[box][a-fA-F0-9][a-fA-F0-9_]*|(\b\d[\d_]*(\.[\d_]*)?|\.\d[\d_]*)([eEfF][-+]?\d+)?/,relevance:0},{className:"string",begin:/'(.|\\[xXuU][a-zA-Z0-9]+)'/},i,l,s,{className:"comment",variants:[{begin:"#=",end:"=#",relevance:10},{begin:"#",end:"$"}]},e.HASH_COMMENT_MODE,{className:"keyword",begin:"\\b(((abstract|primitive)\\s+)type|(mutable\\s+)?struct)\\b"},{begin:/<:/}],n.contains=a.contains,a}}()); +hljs.registerLanguage("nim",function(){"use strict";return function(e){return{name:"Nim",aliases:["nim"],keywords:{keyword:"addr and as asm bind block break case cast const continue converter discard distinct div do elif else end enum except export finally for from func generic if import in include interface is isnot iterator let macro method mixin mod nil not notin object of or out proc ptr raise ref return shl shr static template try tuple type using var when while with without xor yield",literal:"shared guarded stdin stdout stderr result true false",built_in:"int int8 int16 int32 int64 uint uint8 uint16 uint32 uint64 float float32 float64 bool char string cstring pointer expr stmt void auto any range array openarray varargs seq set clong culong cchar cschar cshort cint csize clonglong cfloat cdouble clongdouble cuchar cushort cuint culonglong cstringarray semistatic"},contains:[{className:"meta",begin:/{\./,end:/\.}/,relevance:10},{className:"string",begin:/[a-zA-Z]\w*"/,end:/"/,contains:[{begin:/""/}]},{className:"string",begin:/([a-zA-Z]\w*)?"""/,end:/"""/},e.QUOTE_STRING_MODE,{className:"type",begin:/\b[A-Z]\w+\b/,relevance:0},{className:"number",relevance:0,variants:[{begin:/\b(0[xX][0-9a-fA-F][_0-9a-fA-F]*)('?[iIuU](8|16|32|64))?/},{begin:/\b(0o[0-7][_0-7]*)('?[iIuUfF](8|16|32|64))?/},{begin:/\b(0(b|B)[01][_01]*)('?[iIuUfF](8|16|32|64))?/},{begin:/\b(\d[_\d]*)('?[iIuUfF](8|16|32|64))?/}]},e.HASH_COMMENT_MODE]}}}()); +hljs.registerLanguage("r",function(){"use strict";return function(e){var n="([a-zA-Z]|\\.[a-zA-Z.])[a-zA-Z0-9._]*";return{name:"R",contains:[e.HASH_COMMENT_MODE,{begin:n,keywords:{$pattern:n,keyword:"function if in break next repeat else for return switch while try tryCatch stop warning require library attach detach source setMethod setGeneric setGroupGeneric setClass ...",literal:"NULL NA TRUE FALSE T F Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10"},relevance:0},{className:"number",begin:"0[xX][0-9a-fA-F]+[Li]?\\b",relevance:0},{className:"number",begin:"\\d+(?:[eE][+\\-]?\\d*)?L\\b",relevance:0},{className:"number",begin:"\\d+\\.(?!\\d)(?:i\\b)?",relevance:0},{className:"number",begin:"\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",relevance:0},{className:"number",begin:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",relevance:0},{begin:"`",end:"`",relevance:0},{className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:'"',end:'"'},{begin:"'",end:"'"}]}]}}}()); +hljs.registerLanguage("scala",function(){"use strict";return function(e){var n={className:"subst",variants:[{begin:"\\$[A-Za-z0-9_]+"},{begin:"\\${",end:"}"}]},a={className:"string",variants:[{begin:'"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:'"""',end:'"""',relevance:10},{begin:'[a-z]+"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE,n]},{className:"string",begin:'[a-z]+"""',end:'"""',contains:[n],relevance:10}]},s={className:"type",begin:"\\b[A-Z][A-Za-z0-9_]*",relevance:0},t={className:"title",begin:/[^0-9\n\t "'(),.`{}\[\]:;][^\n\t "'(),.`{}\[\]:;]+|[^0-9\n\t "'(),.`{}\[\]:;=]/,relevance:0},i={className:"class",beginKeywords:"class object trait type",end:/[:={\[\n;]/,excludeEnd:!0,contains:[{beginKeywords:"extends with",relevance:10},{begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0,relevance:0,contains:[s]},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,relevance:0,contains:[s]},t]},l={className:"function",beginKeywords:"def",end:/[:={\[(\n;]/,excludeEnd:!0,contains:[t]};return{name:"Scala",keywords:{literal:"true false null",keyword:"type yield lazy override def with val var sealed abstract private trait object if forSome for while throw finally protected extends import final return else break new catch super class case package default try this match continue throws implicit"},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,{className:"symbol",begin:"'\\w[\\w\\d_]*(?!')"},s,l,i,e.C_NUMBER_MODE,{className:"meta",begin:"@[A-Za-z]+"}]}}}()); +hljs.registerLanguage("x86asm",function(){"use strict";return function(s){return{name:"Intel x86 Assembly",case_insensitive:!0,keywords:{$pattern:"[.%]?"+s.IDENT_RE,keyword:"lock rep repe repz repne repnz xaquire xrelease bnd nobnd aaa aad aam aas adc add and arpl bb0_reset bb1_reset bound bsf bsr bswap bt btc btr bts call cbw cdq cdqe clc cld cli clts cmc cmp cmpsb cmpsd cmpsq cmpsw cmpxchg cmpxchg486 cmpxchg8b cmpxchg16b cpuid cpu_read cpu_write cqo cwd cwde daa das dec div dmint emms enter equ f2xm1 fabs fadd faddp fbld fbstp fchs fclex fcmovb fcmovbe fcmove fcmovnb fcmovnbe fcmovne fcmovnu fcmovu fcom fcomi fcomip fcomp fcompp fcos fdecstp fdisi fdiv fdivp fdivr fdivrp femms feni ffree ffreep fiadd ficom ficomp fidiv fidivr fild fimul fincstp finit fist fistp fisttp fisub fisubr fld fld1 fldcw fldenv fldl2e fldl2t fldlg2 fldln2 fldpi fldz fmul fmulp fnclex fndisi fneni fninit fnop fnsave fnstcw fnstenv fnstsw fpatan fprem fprem1 fptan frndint frstor fsave fscale fsetpm fsin fsincos fsqrt fst fstcw fstenv fstp fstsw fsub fsubp fsubr fsubrp ftst fucom fucomi fucomip fucomp fucompp fxam fxch fxtract fyl2x fyl2xp1 hlt ibts icebp idiv imul in inc incbin insb insd insw int int01 int1 int03 int3 into invd invpcid invlpg invlpga iret iretd iretq iretw jcxz jecxz jrcxz jmp jmpe lahf lar lds lea leave les lfence lfs lgdt lgs lidt lldt lmsw loadall loadall286 lodsb lodsd lodsq lodsw loop loope loopne loopnz loopz lsl lss ltr mfence monitor mov movd movq movsb movsd movsq movsw movsx movsxd movzx mul mwait neg nop not or out outsb outsd outsw packssdw packsswb packuswb paddb paddd paddsb paddsiw paddsw paddusb paddusw paddw pand pandn pause paveb pavgusb pcmpeqb pcmpeqd pcmpeqw pcmpgtb pcmpgtd pcmpgtw pdistib pf2id pfacc pfadd pfcmpeq pfcmpge pfcmpgt pfmax pfmin pfmul pfrcp pfrcpit1 pfrcpit2 pfrsqit1 pfrsqrt pfsub pfsubr pi2fd pmachriw pmaddwd pmagw pmulhriw pmulhrwa pmulhrwc pmulhw pmullw pmvgezb pmvlzb pmvnzb pmvzb pop popa popad popaw popf popfd popfq popfw por prefetch prefetchw pslld psllq psllw psrad psraw psrld psrlq psrlw psubb psubd psubsb psubsiw psubsw psubusb psubusw psubw punpckhbw punpckhdq punpckhwd punpcklbw punpckldq punpcklwd push pusha pushad pushaw pushf pushfd pushfq pushfw pxor rcl rcr rdshr rdmsr rdpmc rdtsc rdtscp ret retf retn rol ror rdm rsdc rsldt rsm rsts sahf sal salc sar sbb scasb scasd scasq scasw sfence sgdt shl shld shr shrd sidt sldt skinit smi smint smintold smsw stc std sti stosb stosd stosq stosw str sub svdc svldt svts swapgs syscall sysenter sysexit sysret test ud0 ud1 ud2b ud2 ud2a umov verr verw fwait wbinvd wrshr wrmsr xadd xbts xchg xlatb xlat xor cmove cmovz cmovne cmovnz cmova cmovnbe cmovae cmovnb cmovb cmovnae cmovbe cmovna cmovg cmovnle cmovge cmovnl cmovl cmovnge cmovle cmovng cmovc cmovnc cmovo cmovno cmovs cmovns cmovp cmovpe cmovnp cmovpo je jz jne jnz ja jnbe jae jnb jb jnae jbe jna jg jnle jge jnl jl jnge jle jng jc jnc jo jno js jns jpo jnp jpe jp sete setz setne setnz seta setnbe setae setnb setnc setb setnae setcset setbe setna setg setnle setge setnl setl setnge setle setng sets setns seto setno setpe setp setpo setnp addps addss andnps andps cmpeqps cmpeqss cmpleps cmpless cmpltps cmpltss cmpneqps cmpneqss cmpnleps cmpnless cmpnltps cmpnltss cmpordps cmpordss cmpunordps cmpunordss cmpps cmpss comiss cvtpi2ps cvtps2pi cvtsi2ss cvtss2si cvttps2pi cvttss2si divps divss ldmxcsr maxps maxss minps minss movaps movhps movlhps movlps movhlps movmskps movntps movss movups mulps mulss orps rcpps rcpss rsqrtps rsqrtss shufps sqrtps sqrtss stmxcsr subps subss ucomiss unpckhps unpcklps xorps fxrstor fxrstor64 fxsave fxsave64 xgetbv xsetbv xsave xsave64 xsaveopt xsaveopt64 xrstor xrstor64 prefetchnta prefetcht0 prefetcht1 prefetcht2 maskmovq movntq pavgb pavgw pextrw pinsrw pmaxsw pmaxub pminsw pminub pmovmskb pmulhuw psadbw pshufw pf2iw pfnacc pfpnacc pi2fw pswapd maskmovdqu clflush movntdq movnti movntpd movdqa movdqu movdq2q movq2dq paddq pmuludq pshufd pshufhw pshuflw pslldq psrldq psubq punpckhqdq punpcklqdq addpd addsd andnpd andpd cmpeqpd cmpeqsd cmplepd cmplesd cmpltpd cmpltsd cmpneqpd cmpneqsd cmpnlepd cmpnlesd cmpnltpd cmpnltsd cmpordpd cmpordsd cmpunordpd cmpunordsd cmppd comisd cvtdq2pd cvtdq2ps cvtpd2dq cvtpd2pi cvtpd2ps cvtpi2pd cvtps2dq cvtps2pd cvtsd2si cvtsd2ss cvtsi2sd cvtss2sd cvttpd2pi cvttpd2dq cvttps2dq cvttsd2si divpd divsd maxpd maxsd minpd minsd movapd movhpd movlpd movmskpd movupd mulpd mulsd orpd shufpd sqrtpd sqrtsd subpd subsd ucomisd unpckhpd unpcklpd xorpd addsubpd addsubps haddpd haddps hsubpd hsubps lddqu movddup movshdup movsldup clgi stgi vmcall vmclear vmfunc vmlaunch vmload vmmcall vmptrld vmptrst vmread vmresume vmrun vmsave vmwrite vmxoff vmxon invept invvpid pabsb pabsw pabsd palignr phaddw phaddd phaddsw phsubw phsubd phsubsw pmaddubsw pmulhrsw pshufb psignb psignw psignd extrq insertq movntsd movntss lzcnt blendpd blendps blendvpd blendvps dppd dpps extractps insertps movntdqa mpsadbw packusdw pblendvb pblendw pcmpeqq pextrb pextrd pextrq phminposuw pinsrb pinsrd pinsrq pmaxsb pmaxsd pmaxud pmaxuw pminsb pminsd pminud pminuw pmovsxbw pmovsxbd pmovsxbq pmovsxwd pmovsxwq pmovsxdq pmovzxbw pmovzxbd pmovzxbq pmovzxwd pmovzxwq pmovzxdq pmuldq pmulld ptest roundpd roundps roundsd roundss crc32 pcmpestri pcmpestrm pcmpistri pcmpistrm pcmpgtq popcnt getsec pfrcpv pfrsqrtv movbe aesenc aesenclast aesdec aesdeclast aesimc aeskeygenassist vaesenc vaesenclast vaesdec vaesdeclast vaesimc vaeskeygenassist vaddpd vaddps vaddsd vaddss vaddsubpd vaddsubps vandpd vandps vandnpd vandnps vblendpd vblendps vblendvpd vblendvps vbroadcastss vbroadcastsd vbroadcastf128 vcmpeq_ospd vcmpeqpd vcmplt_ospd vcmpltpd vcmple_ospd vcmplepd vcmpunord_qpd vcmpunordpd vcmpneq_uqpd vcmpneqpd vcmpnlt_uspd vcmpnltpd vcmpnle_uspd vcmpnlepd vcmpord_qpd vcmpordpd vcmpeq_uqpd vcmpnge_uspd vcmpngepd vcmpngt_uspd vcmpngtpd vcmpfalse_oqpd vcmpfalsepd vcmpneq_oqpd vcmpge_ospd vcmpgepd vcmpgt_ospd vcmpgtpd vcmptrue_uqpd vcmptruepd vcmplt_oqpd vcmple_oqpd vcmpunord_spd vcmpneq_uspd vcmpnlt_uqpd vcmpnle_uqpd vcmpord_spd vcmpeq_uspd vcmpnge_uqpd vcmpngt_uqpd vcmpfalse_ospd vcmpneq_ospd vcmpge_oqpd vcmpgt_oqpd vcmptrue_uspd vcmppd vcmpeq_osps vcmpeqps vcmplt_osps vcmpltps vcmple_osps vcmpleps vcmpunord_qps vcmpunordps vcmpneq_uqps vcmpneqps vcmpnlt_usps vcmpnltps vcmpnle_usps vcmpnleps vcmpord_qps vcmpordps vcmpeq_uqps vcmpnge_usps vcmpngeps vcmpngt_usps vcmpngtps vcmpfalse_oqps vcmpfalseps vcmpneq_oqps vcmpge_osps vcmpgeps vcmpgt_osps vcmpgtps vcmptrue_uqps vcmptrueps vcmplt_oqps vcmple_oqps vcmpunord_sps vcmpneq_usps vcmpnlt_uqps vcmpnle_uqps vcmpord_sps vcmpeq_usps vcmpnge_uqps vcmpngt_uqps vcmpfalse_osps vcmpneq_osps vcmpge_oqps vcmpgt_oqps vcmptrue_usps vcmpps vcmpeq_ossd vcmpeqsd vcmplt_ossd vcmpltsd vcmple_ossd vcmplesd vcmpunord_qsd vcmpunordsd vcmpneq_uqsd vcmpneqsd vcmpnlt_ussd vcmpnltsd vcmpnle_ussd vcmpnlesd vcmpord_qsd vcmpordsd vcmpeq_uqsd vcmpnge_ussd vcmpngesd vcmpngt_ussd vcmpngtsd vcmpfalse_oqsd vcmpfalsesd vcmpneq_oqsd vcmpge_ossd vcmpgesd vcmpgt_ossd vcmpgtsd vcmptrue_uqsd vcmptruesd vcmplt_oqsd vcmple_oqsd vcmpunord_ssd vcmpneq_ussd vcmpnlt_uqsd vcmpnle_uqsd vcmpord_ssd vcmpeq_ussd vcmpnge_uqsd vcmpngt_uqsd vcmpfalse_ossd vcmpneq_ossd vcmpge_oqsd vcmpgt_oqsd vcmptrue_ussd vcmpsd vcmpeq_osss vcmpeqss vcmplt_osss vcmpltss vcmple_osss vcmpless vcmpunord_qss vcmpunordss vcmpneq_uqss vcmpneqss vcmpnlt_usss vcmpnltss vcmpnle_usss vcmpnless vcmpord_qss vcmpordss vcmpeq_uqss vcmpnge_usss vcmpngess vcmpngt_usss vcmpngtss vcmpfalse_oqss vcmpfalsess vcmpneq_oqss vcmpge_osss vcmpgess vcmpgt_osss vcmpgtss vcmptrue_uqss vcmptruess vcmplt_oqss vcmple_oqss vcmpunord_sss vcmpneq_usss vcmpnlt_uqss vcmpnle_uqss vcmpord_sss vcmpeq_usss vcmpnge_uqss vcmpngt_uqss vcmpfalse_osss vcmpneq_osss vcmpge_oqss vcmpgt_oqss vcmptrue_usss vcmpss vcomisd vcomiss vcvtdq2pd vcvtdq2ps vcvtpd2dq vcvtpd2ps vcvtps2dq vcvtps2pd vcvtsd2si vcvtsd2ss vcvtsi2sd vcvtsi2ss vcvtss2sd vcvtss2si vcvttpd2dq vcvttps2dq vcvttsd2si vcvttss2si vdivpd vdivps vdivsd vdivss vdppd vdpps vextractf128 vextractps vhaddpd vhaddps vhsubpd vhsubps vinsertf128 vinsertps vlddqu vldqqu vldmxcsr vmaskmovdqu vmaskmovps vmaskmovpd vmaxpd vmaxps vmaxsd vmaxss vminpd vminps vminsd vminss vmovapd vmovaps vmovd vmovq vmovddup vmovdqa vmovqqa vmovdqu vmovqqu vmovhlps vmovhpd vmovhps vmovlhps vmovlpd vmovlps vmovmskpd vmovmskps vmovntdq vmovntqq vmovntdqa vmovntpd vmovntps vmovsd vmovshdup vmovsldup vmovss vmovupd vmovups vmpsadbw vmulpd vmulps vmulsd vmulss vorpd vorps vpabsb vpabsw vpabsd vpacksswb vpackssdw vpackuswb vpackusdw vpaddb vpaddw vpaddd vpaddq vpaddsb vpaddsw vpaddusb vpaddusw vpalignr vpand vpandn vpavgb vpavgw vpblendvb vpblendw vpcmpestri vpcmpestrm vpcmpistri vpcmpistrm vpcmpeqb vpcmpeqw vpcmpeqd vpcmpeqq vpcmpgtb vpcmpgtw vpcmpgtd vpcmpgtq vpermilpd vpermilps vperm2f128 vpextrb vpextrw vpextrd vpextrq vphaddw vphaddd vphaddsw vphminposuw vphsubw vphsubd vphsubsw vpinsrb vpinsrw vpinsrd vpinsrq vpmaddwd vpmaddubsw vpmaxsb vpmaxsw vpmaxsd vpmaxub vpmaxuw vpmaxud vpminsb vpminsw vpminsd vpminub vpminuw vpminud vpmovmskb vpmovsxbw vpmovsxbd vpmovsxbq vpmovsxwd vpmovsxwq vpmovsxdq vpmovzxbw vpmovzxbd vpmovzxbq vpmovzxwd vpmovzxwq vpmovzxdq vpmulhuw vpmulhrsw vpmulhw vpmullw vpmulld vpmuludq vpmuldq vpor vpsadbw vpshufb vpshufd vpshufhw vpshuflw vpsignb vpsignw vpsignd vpslldq vpsrldq vpsllw vpslld vpsllq vpsraw vpsrad vpsrlw vpsrld vpsrlq vptest vpsubb vpsubw vpsubd vpsubq vpsubsb vpsubsw vpsubusb vpsubusw vpunpckhbw vpunpckhwd vpunpckhdq vpunpckhqdq vpunpcklbw vpunpcklwd vpunpckldq vpunpcklqdq vpxor vrcpps vrcpss vrsqrtps vrsqrtss vroundpd vroundps vroundsd vroundss vshufpd vshufps vsqrtpd vsqrtps vsqrtsd vsqrtss vstmxcsr vsubpd vsubps vsubsd vsubss vtestps vtestpd vucomisd vucomiss vunpckhpd vunpckhps vunpcklpd vunpcklps vxorpd vxorps vzeroall vzeroupper pclmullqlqdq pclmulhqlqdq pclmullqhqdq pclmulhqhqdq pclmulqdq vpclmullqlqdq vpclmulhqlqdq vpclmullqhqdq vpclmulhqhqdq vpclmulqdq vfmadd132ps vfmadd132pd vfmadd312ps vfmadd312pd vfmadd213ps vfmadd213pd vfmadd123ps vfmadd123pd vfmadd231ps vfmadd231pd vfmadd321ps vfmadd321pd vfmaddsub132ps vfmaddsub132pd vfmaddsub312ps vfmaddsub312pd vfmaddsub213ps vfmaddsub213pd vfmaddsub123ps vfmaddsub123pd vfmaddsub231ps vfmaddsub231pd vfmaddsub321ps vfmaddsub321pd vfmsub132ps vfmsub132pd vfmsub312ps vfmsub312pd vfmsub213ps vfmsub213pd vfmsub123ps vfmsub123pd vfmsub231ps vfmsub231pd vfmsub321ps vfmsub321pd vfmsubadd132ps vfmsubadd132pd vfmsubadd312ps vfmsubadd312pd vfmsubadd213ps vfmsubadd213pd vfmsubadd123ps vfmsubadd123pd vfmsubadd231ps vfmsubadd231pd vfmsubadd321ps vfmsubadd321pd vfnmadd132ps vfnmadd132pd vfnmadd312ps vfnmadd312pd vfnmadd213ps vfnmadd213pd vfnmadd123ps vfnmadd123pd vfnmadd231ps vfnmadd231pd vfnmadd321ps vfnmadd321pd vfnmsub132ps vfnmsub132pd vfnmsub312ps vfnmsub312pd vfnmsub213ps vfnmsub213pd vfnmsub123ps vfnmsub123pd vfnmsub231ps vfnmsub231pd vfnmsub321ps vfnmsub321pd vfmadd132ss vfmadd132sd vfmadd312ss vfmadd312sd vfmadd213ss vfmadd213sd vfmadd123ss vfmadd123sd vfmadd231ss vfmadd231sd vfmadd321ss vfmadd321sd vfmsub132ss vfmsub132sd vfmsub312ss vfmsub312sd vfmsub213ss vfmsub213sd vfmsub123ss vfmsub123sd vfmsub231ss vfmsub231sd vfmsub321ss vfmsub321sd vfnmadd132ss vfnmadd132sd vfnmadd312ss vfnmadd312sd vfnmadd213ss vfnmadd213sd vfnmadd123ss vfnmadd123sd vfnmadd231ss vfnmadd231sd vfnmadd321ss vfnmadd321sd vfnmsub132ss vfnmsub132sd vfnmsub312ss vfnmsub312sd vfnmsub213ss vfnmsub213sd vfnmsub123ss vfnmsub123sd vfnmsub231ss vfnmsub231sd vfnmsub321ss vfnmsub321sd rdfsbase rdgsbase rdrand wrfsbase wrgsbase vcvtph2ps vcvtps2ph adcx adox rdseed clac stac xstore xcryptecb xcryptcbc xcryptctr xcryptcfb xcryptofb montmul xsha1 xsha256 llwpcb slwpcb lwpval lwpins vfmaddpd vfmaddps vfmaddsd vfmaddss vfmaddsubpd vfmaddsubps vfmsubaddpd vfmsubaddps vfmsubpd vfmsubps vfmsubsd vfmsubss vfnmaddpd vfnmaddps vfnmaddsd vfnmaddss vfnmsubpd vfnmsubps vfnmsubsd vfnmsubss vfrczpd vfrczps vfrczsd vfrczss vpcmov vpcomb vpcomd vpcomq vpcomub vpcomud vpcomuq vpcomuw vpcomw vphaddbd vphaddbq vphaddbw vphadddq vphaddubd vphaddubq vphaddubw vphaddudq vphadduwd vphadduwq vphaddwd vphaddwq vphsubbw vphsubdq vphsubwd vpmacsdd vpmacsdqh vpmacsdql vpmacssdd vpmacssdqh vpmacssdql vpmacsswd vpmacssww vpmacswd vpmacsww vpmadcsswd vpmadcswd vpperm vprotb vprotd vprotq vprotw vpshab vpshad vpshaq vpshaw vpshlb vpshld vpshlq vpshlw vbroadcasti128 vpblendd vpbroadcastb vpbroadcastw vpbroadcastd vpbroadcastq vpermd vpermpd vpermps vpermq vperm2i128 vextracti128 vinserti128 vpmaskmovd vpmaskmovq vpsllvd vpsllvq vpsravd vpsrlvd vpsrlvq vgatherdpd vgatherqpd vgatherdps vgatherqps vpgatherdd vpgatherqd vpgatherdq vpgatherqq xabort xbegin xend xtest andn bextr blci blcic blsi blsic blcfill blsfill blcmsk blsmsk blsr blcs bzhi mulx pdep pext rorx sarx shlx shrx tzcnt tzmsk t1mskc valignd valignq vblendmpd vblendmps vbroadcastf32x4 vbroadcastf64x4 vbroadcasti32x4 vbroadcasti64x4 vcompresspd vcompressps vcvtpd2udq vcvtps2udq vcvtsd2usi vcvtss2usi vcvttpd2udq vcvttps2udq vcvttsd2usi vcvttss2usi vcvtudq2pd vcvtudq2ps vcvtusi2sd vcvtusi2ss vexpandpd vexpandps vextractf32x4 vextractf64x4 vextracti32x4 vextracti64x4 vfixupimmpd vfixupimmps vfixupimmsd vfixupimmss vgetexppd vgetexpps vgetexpsd vgetexpss vgetmantpd vgetmantps vgetmantsd vgetmantss vinsertf32x4 vinsertf64x4 vinserti32x4 vinserti64x4 vmovdqa32 vmovdqa64 vmovdqu32 vmovdqu64 vpabsq vpandd vpandnd vpandnq vpandq vpblendmd vpblendmq vpcmpltd vpcmpled vpcmpneqd vpcmpnltd vpcmpnled vpcmpd vpcmpltq vpcmpleq vpcmpneqq vpcmpnltq vpcmpnleq vpcmpq vpcmpequd vpcmpltud vpcmpleud vpcmpnequd vpcmpnltud vpcmpnleud vpcmpud vpcmpequq vpcmpltuq vpcmpleuq vpcmpnequq vpcmpnltuq vpcmpnleuq vpcmpuq vpcompressd vpcompressq vpermi2d vpermi2pd vpermi2ps vpermi2q vpermt2d vpermt2pd vpermt2ps vpermt2q vpexpandd vpexpandq vpmaxsq vpmaxuq vpminsq vpminuq vpmovdb vpmovdw vpmovqb vpmovqd vpmovqw vpmovsdb vpmovsdw vpmovsqb vpmovsqd vpmovsqw vpmovusdb vpmovusdw vpmovusqb vpmovusqd vpmovusqw vpord vporq vprold vprolq vprolvd vprolvq vprord vprorq vprorvd vprorvq vpscatterdd vpscatterdq vpscatterqd vpscatterqq vpsraq vpsravq vpternlogd vpternlogq vptestmd vptestmq vptestnmd vptestnmq vpxord vpxorq vrcp14pd vrcp14ps vrcp14sd vrcp14ss vrndscalepd vrndscaleps vrndscalesd vrndscaless vrsqrt14pd vrsqrt14ps vrsqrt14sd vrsqrt14ss vscalefpd vscalefps vscalefsd vscalefss vscatterdpd vscatterdps vscatterqpd vscatterqps vshuff32x4 vshuff64x2 vshufi32x4 vshufi64x2 kandnw kandw kmovw knotw kortestw korw kshiftlw kshiftrw kunpckbw kxnorw kxorw vpbroadcastmb2q vpbroadcastmw2d vpconflictd vpconflictq vplzcntd vplzcntq vexp2pd vexp2ps vrcp28pd vrcp28ps vrcp28sd vrcp28ss vrsqrt28pd vrsqrt28ps vrsqrt28sd vrsqrt28ss vgatherpf0dpd vgatherpf0dps vgatherpf0qpd vgatherpf0qps vgatherpf1dpd vgatherpf1dps vgatherpf1qpd vgatherpf1qps vscatterpf0dpd vscatterpf0dps vscatterpf0qpd vscatterpf0qps vscatterpf1dpd vscatterpf1dps vscatterpf1qpd vscatterpf1qps prefetchwt1 bndmk bndcl bndcu bndcn bndmov bndldx bndstx sha1rnds4 sha1nexte sha1msg1 sha1msg2 sha256rnds2 sha256msg1 sha256msg2 hint_nop0 hint_nop1 hint_nop2 hint_nop3 hint_nop4 hint_nop5 hint_nop6 hint_nop7 hint_nop8 hint_nop9 hint_nop10 hint_nop11 hint_nop12 hint_nop13 hint_nop14 hint_nop15 hint_nop16 hint_nop17 hint_nop18 hint_nop19 hint_nop20 hint_nop21 hint_nop22 hint_nop23 hint_nop24 hint_nop25 hint_nop26 hint_nop27 hint_nop28 hint_nop29 hint_nop30 hint_nop31 hint_nop32 hint_nop33 hint_nop34 hint_nop35 hint_nop36 hint_nop37 hint_nop38 hint_nop39 hint_nop40 hint_nop41 hint_nop42 hint_nop43 hint_nop44 hint_nop45 hint_nop46 hint_nop47 hint_nop48 hint_nop49 hint_nop50 hint_nop51 hint_nop52 hint_nop53 hint_nop54 hint_nop55 hint_nop56 hint_nop57 hint_nop58 hint_nop59 hint_nop60 hint_nop61 hint_nop62 hint_nop63",built_in:"ip eip rip al ah bl bh cl ch dl dh sil dil bpl spl r8b r9b r10b r11b r12b r13b r14b r15b ax bx cx dx si di bp sp r8w r9w r10w r11w r12w r13w r14w r15w eax ebx ecx edx esi edi ebp esp eip r8d r9d r10d r11d r12d r13d r14d r15d rax rbx rcx rdx rsi rdi rbp rsp r8 r9 r10 r11 r12 r13 r14 r15 cs ds es fs gs ss st st0 st1 st2 st3 st4 st5 st6 st7 mm0 mm1 mm2 mm3 mm4 mm5 mm6 mm7 xmm0 xmm1 xmm2 xmm3 xmm4 xmm5 xmm6 xmm7 xmm8 xmm9 xmm10 xmm11 xmm12 xmm13 xmm14 xmm15 xmm16 xmm17 xmm18 xmm19 xmm20 xmm21 xmm22 xmm23 xmm24 xmm25 xmm26 xmm27 xmm28 xmm29 xmm30 xmm31 ymm0 ymm1 ymm2 ymm3 ymm4 ymm5 ymm6 ymm7 ymm8 ymm9 ymm10 ymm11 ymm12 ymm13 ymm14 ymm15 ymm16 ymm17 ymm18 ymm19 ymm20 ymm21 ymm22 ymm23 ymm24 ymm25 ymm26 ymm27 ymm28 ymm29 ymm30 ymm31 zmm0 zmm1 zmm2 zmm3 zmm4 zmm5 zmm6 zmm7 zmm8 zmm9 zmm10 zmm11 zmm12 zmm13 zmm14 zmm15 zmm16 zmm17 zmm18 zmm19 zmm20 zmm21 zmm22 zmm23 zmm24 zmm25 zmm26 zmm27 zmm28 zmm29 zmm30 zmm31 k0 k1 k2 k3 k4 k5 k6 k7 bnd0 bnd1 bnd2 bnd3 cr0 cr1 cr2 cr3 cr4 cr8 dr0 dr1 dr2 dr3 dr8 tr3 tr4 tr5 tr6 tr7 r0 r1 r2 r3 r4 r5 r6 r7 r0b r1b r2b r3b r4b r5b r6b r7b r0w r1w r2w r3w r4w r5w r6w r7w r0d r1d r2d r3d r4d r5d r6d r7d r0h r1h r2h r3h r0l r1l r2l r3l r4l r5l r6l r7l r8l r9l r10l r11l r12l r13l r14l r15l db dw dd dq dt ddq do dy dz resb resw resd resq rest resdq reso resy resz incbin equ times byte word dword qword nosplit rel abs seg wrt strict near far a32 ptr",meta:"%define %xdefine %+ %undef %defstr %deftok %assign %strcat %strlen %substr %rotate %elif %else %endif %if %ifmacro %ifctx %ifidn %ifidni %ifid %ifnum %ifstr %iftoken %ifempty %ifenv %error %warning %fatal %rep %endrep %include %push %pop %repl %pathsearch %depend %use %arg %stacksize %local %line %comment %endcomment .nolist __FILE__ __LINE__ __SECT__ __BITS__ __OUTPUT_FORMAT__ __DATE__ __TIME__ __DATE_NUM__ __TIME_NUM__ __UTC_DATE__ __UTC_TIME__ __UTC_DATE_NUM__ __UTC_TIME_NUM__ __PASS__ struc endstruc istruc at iend align alignb sectalign daz nodaz up down zero default option assume public bits use16 use32 use64 default section segment absolute extern global common cpu float __utf16__ __utf16le__ __utf16be__ __utf32__ __utf32le__ __utf32be__ __float8__ __float16__ __float32__ __float64__ __float80m__ __float80e__ __float128l__ __float128h__ __Infinity__ __QNaN__ __SNaN__ Inf NaN QNaN SNaN float8 float16 float32 float64 float80m float80e float128l float128h __FLOAT_DAZ__ __FLOAT_ROUND__ __FLOAT__"},contains:[s.COMMENT(";","$",{relevance:0}),{className:"number",variants:[{begin:"\\b(?:([0-9][0-9_]*)?\\.[0-9_]*(?:[eE][+-]?[0-9_]+)?|(0[Xx])?[0-9][0-9_]*\\.?[0-9_]*(?:[pP](?:[+-]?[0-9_]+)?)?)\\b",relevance:0},{begin:"\\$[0-9][0-9A-Fa-f]*",relevance:0},{begin:"\\b(?:[0-9A-Fa-f][0-9A-Fa-f_]*[Hh]|[0-9][0-9_]*[DdTt]?|[0-7][0-7_]*[QqOo]|[0-1][0-1_]*[BbYy])\\b"},{begin:"\\b(?:0[Xx][0-9A-Fa-f_]+|0[DdTt][0-9_]+|0[QqOo][0-7_]+|0[BbYy][0-1_]+)\\b"}]},s.QUOTE_STRING_MODE,{className:"string",variants:[{begin:"'",end:"[^\\\\]'"},{begin:"`",end:"[^\\\\]`"}],relevance:0},{className:"symbol",variants:[{begin:"^\\s*[A-Za-z._?][A-Za-z0-9_$#@~.?]*(:|\\s+label)"},{begin:"^\\s*%%[A-Za-z0-9_$#@~.?]*:"}],relevance:0},{className:"subst",begin:"%[0-9]+",relevance:0},{className:"subst",begin:"%!S+",relevance:0},{className:"meta",begin:/^\s*\.[\w_-]+/}]}}}()); \ No newline at end of file From f5ff9e32ca4bd45781426246e4fb6b9fcceff0c2 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 15 Nov 2023 09:38:48 +0100 Subject: [PATCH 41/50] introduce asyncraises in transports/asyncsync (#470) With these fixes, `transports`/`asyncsync` correctly propagate and document their raises information - generally, most transport functions (send etc) raise `TransportError` and `CancelledError` - `closeWait` is special in that it generally doesn't fail. This PR introduces the syntax `Future[void].Raises([types])` to create the `InternalRaisesFuture` type with the correct encoding for the types - this allows it to be used in user code while retaining the possibility to change the internal representation down the line. * introduce raising constraints on stream callbacks - these constraints now give a warning when called with a callback that can raise exceptions (raising callbacks would crash * fix fail and its tests, which wasn't always given a good generic match * work around nim bugs related to macro expansion of generic types * make sure transports raise only `TransportError`-derived exceptions (and `CancelledError`) --- chronos/asyncsync.nim | 126 ++--- chronos/internal/asyncfutures.nim | 56 +- chronos/internal/asyncmacro.nim | 17 +- chronos/internal/raisesfutures.nim | 82 ++- chronos/transports/common.nim | 6 +- chronos/transports/datagram.nim | 238 +++++++-- chronos/transports/stream.nim | 192 ++++--- tests/testasyncstream.nim | 827 ++++++++++++++++------------- tests/testbugs.nim | 23 +- tests/testdatagram.nim | 521 +++++++++--------- tests/testmacro.nim | 19 +- tests/testserver.nim | 43 +- tests/teststream.nim | 527 ++++++++++-------- 13 files changed, 1574 insertions(+), 1103 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index fa23471..9bab1fd 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -28,7 +28,7 @@ type ## is blocked in ``acquire()`` is being processed. locked: bool acquired: bool - waiters: seq[Future[void]] + waiters: seq[Future[void].Raising([CancelledError])] AsyncEvent* = ref object of RootRef ## A primitive event object. @@ -41,7 +41,7 @@ type ## state to be signaled, when event get fired, then all coroutines ## continue proceeds in order, they have entered waiting state. flag: bool - waiters: seq[Future[void]] + waiters: seq[Future[void].Raising([CancelledError])] AsyncQueue*[T] = ref object of RootRef ## A queue, useful for coordinating producer and consumer coroutines. @@ -50,8 +50,8 @@ type ## infinite. If it is an integer greater than ``0``, then "await put()" ## will block when the queue reaches ``maxsize``, until an item is ## removed by "await get()". - getters: seq[Future[void]] - putters: seq[Future[void]] + getters: seq[Future[void].Raising([CancelledError])] + putters: seq[Future[void].Raising([CancelledError])] queue: Deque[T] maxsize: int @@ -69,7 +69,7 @@ type EventQueueReader* = object key: EventQueueKey offset: int - waiter: Future[void] + waiter: Future[void].Raising([CancelledError]) overflow: bool AsyncEventQueue*[T] = ref object of RootObj @@ -90,17 +90,14 @@ proc newAsyncLock*(): AsyncLock = ## The ``release()`` procedure changes the state to unlocked and returns ## immediately. - # Workaround for callSoon() not worked correctly before - # getThreadDispatcher() call. - discard getThreadDispatcher() - AsyncLock(waiters: newSeq[Future[void]](), locked: false, acquired: false) + AsyncLock() proc wakeUpFirst(lock: AsyncLock): bool {.inline.} = ## Wake up the first waiter if it isn't done. var i = 0 var res = false while i < len(lock.waiters): - var waiter = lock.waiters[i] + let waiter = lock.waiters[i] inc(i) if not(waiter.finished()): waiter.complete() @@ -120,7 +117,7 @@ proc checkAll(lock: AsyncLock): bool {.inline.} = return false return true -proc acquire*(lock: AsyncLock) {.async.} = +proc acquire*(lock: AsyncLock) {.async: (raises: [CancelledError]).} = ## Acquire a lock ``lock``. ## ## This procedure blocks until the lock ``lock`` is unlocked, then sets it @@ -129,7 +126,7 @@ proc acquire*(lock: AsyncLock) {.async.} = lock.acquired = true lock.locked = true else: - var w = newFuture[void]("AsyncLock.acquire") + let w = Future[void].Raising([CancelledError]).init("AsyncLock.acquire") lock.waiters.add(w) await w lock.acquired = true @@ -165,13 +162,10 @@ proc newAsyncEvent*(): AsyncEvent = ## procedure and reset to `false` with the `clear()` procedure. ## The `wait()` procedure blocks until the flag is `true`. The flag is ## initially `false`. + AsyncEvent() - # Workaround for callSoon() not worked correctly before - # getThreadDispatcher() call. - discard getThreadDispatcher() - AsyncEvent(waiters: newSeq[Future[void]](), flag: false) - -proc wait*(event: AsyncEvent): Future[void] = +proc wait*(event: AsyncEvent): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Block until the internal flag of ``event`` is `true`. ## If the internal flag is `true` on entry, return immediately. Otherwise, ## block until another task calls `fire()` to set the flag to `true`, @@ -210,20 +204,15 @@ proc isSet*(event: AsyncEvent): bool = proc newAsyncQueue*[T](maxsize: int = 0): AsyncQueue[T] = ## Creates a new asynchronous queue ``AsyncQueue``. - # Workaround for callSoon() not worked correctly before - # getThreadDispatcher() call. - discard getThreadDispatcher() AsyncQueue[T]( - getters: newSeq[Future[void]](), - putters: newSeq[Future[void]](), queue: initDeque[T](), maxsize: maxsize ) -proc wakeupNext(waiters: var seq[Future[void]]) {.inline.} = +proc wakeupNext(waiters: var seq) {.inline.} = var i = 0 while i < len(waiters): - var waiter = waiters[i] + let waiter = waiters[i] inc(i) if not(waiter.finished()): @@ -250,6 +239,24 @@ proc empty*[T](aq: AsyncQueue[T]): bool {.inline.} = ## Return ``true`` if the queue is empty, ``false`` otherwise. (len(aq.queue) == 0) +proc addFirstImpl[T](aq: AsyncQueue[T], item: T) = + aq.queue.addFirst(item) + aq.getters.wakeupNext() + +proc addLastImpl[T](aq: AsyncQueue[T], item: T) = + aq.queue.addLast(item) + aq.getters.wakeupNext() + +proc popFirstImpl[T](aq: AsyncQueue[T]): T = + let res = aq.queue.popFirst() + aq.putters.wakeupNext() + res + +proc popLastImpl[T](aq: AsyncQueue[T]): T = + let res = aq.queue.popLast() + aq.putters.wakeupNext() + res + proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) {. raises: [AsyncQueueFullError].}= ## Put an item ``item`` to the beginning of the queue ``aq`` immediately. @@ -257,8 +264,7 @@ proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) {. ## If queue ``aq`` is full, then ``AsyncQueueFullError`` exception raised. if aq.full(): raise newException(AsyncQueueFullError, "AsyncQueue is full!") - aq.queue.addFirst(item) - aq.getters.wakeupNext() + aq.addFirstImpl(item) proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) {. raises: [AsyncQueueFullError].}= @@ -267,8 +273,7 @@ proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) {. ## If queue ``aq`` is full, then ``AsyncQueueFullError`` exception raised. if aq.full(): raise newException(AsyncQueueFullError, "AsyncQueue is full!") - aq.queue.addLast(item) - aq.getters.wakeupNext() + aq.addLastImpl(item) proc popFirstNoWait*[T](aq: AsyncQueue[T]): T {. raises: [AsyncQueueEmptyError].} = @@ -277,9 +282,7 @@ proc popFirstNoWait*[T](aq: AsyncQueue[T]): T {. ## If queue ``aq`` is empty, then ``AsyncQueueEmptyError`` exception raised. if aq.empty(): raise newException(AsyncQueueEmptyError, "AsyncQueue is empty!") - let res = aq.queue.popFirst() - aq.putters.wakeupNext() - res + aq.popFirstImpl() proc popLastNoWait*[T](aq: AsyncQueue[T]): T {. raises: [AsyncQueueEmptyError].} = @@ -288,65 +291,63 @@ proc popLastNoWait*[T](aq: AsyncQueue[T]): T {. ## If queue ``aq`` is empty, then ``AsyncQueueEmptyError`` exception raised. if aq.empty(): raise newException(AsyncQueueEmptyError, "AsyncQueue is empty!") - let res = aq.queue.popLast() - aq.putters.wakeupNext() - res + aq.popLastImpl() -proc addFirst*[T](aq: AsyncQueue[T], item: T) {.async.} = +proc addFirst*[T](aq: AsyncQueue[T], item: T) {.async: (raises: [CancelledError]).} = ## Put an ``item`` to the beginning of the queue ``aq``. If the queue is full, ## wait until a free slot is available before adding item. while aq.full(): - var putter = newFuture[void]("AsyncQueue.addFirst") + let putter = Future[void].Raising([CancelledError]).init("AsyncQueue.addFirst") aq.putters.add(putter) try: await putter - except CatchableError as exc: + except CancelledError as exc: if not(aq.full()) and not(putter.cancelled()): aq.putters.wakeupNext() raise exc - aq.addFirstNoWait(item) + aq.addFirstImpl(item) -proc addLast*[T](aq: AsyncQueue[T], item: T) {.async.} = +proc addLast*[T](aq: AsyncQueue[T], item: T) {.async: (raises: [CancelledError]).} = ## Put an ``item`` to the end of the queue ``aq``. If the queue is full, ## wait until a free slot is available before adding item. while aq.full(): - var putter = newFuture[void]("AsyncQueue.addLast") + let putter = Future[void].Raising([CancelledError]).init("AsyncQueue.addLast") aq.putters.add(putter) try: await putter - except CatchableError as exc: + except CancelledError as exc: if not(aq.full()) and not(putter.cancelled()): aq.putters.wakeupNext() raise exc - aq.addLastNoWait(item) + aq.addLastImpl(item) -proc popFirst*[T](aq: AsyncQueue[T]): Future[T] {.async.} = +proc popFirst*[T](aq: AsyncQueue[T]): Future[T] {.async: (raises: [CancelledError]).} = ## Remove and return an ``item`` from the beginning of the queue ``aq``. ## If the queue is empty, wait until an item is available. while aq.empty(): - var getter = newFuture[void]("AsyncQueue.popFirst") + let getter = Future[void].Raising([CancelledError]).init("AsyncQueue.popFirst") aq.getters.add(getter) try: await getter - except CatchableError as exc: + except CancelledError as exc: if not(aq.empty()) and not(getter.cancelled()): aq.getters.wakeupNext() raise exc - return aq.popFirstNoWait() + aq.popFirstImpl() -proc popLast*[T](aq: AsyncQueue[T]): Future[T] {.async.} = +proc popLast*[T](aq: AsyncQueue[T]): Future[T] {.async: (raises: [CancelledError]).} = ## Remove and return an ``item`` from the end of the queue ``aq``. ## If the queue is empty, wait until an item is available. while aq.empty(): - var getter = newFuture[void]("AsyncQueue.popLast") + let getter = Future[void].Raising([CancelledError]).init("AsyncQueue.popLast") aq.getters.add(getter) try: await getter - except CatchableError as exc: + except CancelledError as exc: if not(aq.empty()) and not(getter.cancelled()): aq.getters.wakeupNext() raise exc - return aq.popLastNoWait() + aq.popLastImpl() proc putNoWait*[T](aq: AsyncQueue[T], item: T) {. raises: [AsyncQueueFullError].} = @@ -358,11 +359,13 @@ proc getNoWait*[T](aq: AsyncQueue[T]): T {. ## Alias of ``popFirstNoWait()``. aq.popFirstNoWait() -proc put*[T](aq: AsyncQueue[T], item: T): Future[void] {.inline.} = +proc put*[T](aq: AsyncQueue[T], item: T): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Alias of ``addLast()``. aq.addLast(item) -proc get*[T](aq: AsyncQueue[T]): Future[T] {.inline.} = +proc get*[T](aq: AsyncQueue[T]): Future[T] {. + async: (raw: true, raises: [CancelledError]).} = ## Alias of ``popFirst()``. aq.popFirst() @@ -416,7 +419,7 @@ proc contains*[T](aq: AsyncQueue[T], item: T): bool {.inline.} = ## via the ``in`` operator. for e in aq.queue.items(): if e == item: return true - return false + false proc `$`*[T](aq: AsyncQueue[T]): string = ## Turn an async queue ``aq`` into its string representation. @@ -452,8 +455,7 @@ proc compact(ab: AsyncEventQueue) {.raises: [].} = else: ab.queue.clear() -proc getReaderIndex(ab: AsyncEventQueue, key: EventQueueKey): int {. - raises: [].} = +proc getReaderIndex(ab: AsyncEventQueue, key: EventQueueKey): int = for index, value in ab.readers.pairs(): if value.key == key: return index @@ -507,7 +509,7 @@ proc close*(ab: AsyncEventQueue) {.raises: [].} = ab.readers.reset() ab.queue.clear() -proc closeWait*(ab: AsyncEventQueue): Future[void] {.raises: [].} = +proc closeWait*(ab: AsyncEventQueue): Future[void] {.async: (raw: true, raises: []).} = let retFuture = newFuture[void]("AsyncEventQueue.closeWait()", {FutureFlag.OwnCancelSchedule}) proc continuation(udata: pointer) {.gcsafe.} = @@ -528,7 +530,7 @@ template readerOverflow*(ab: AsyncEventQueue, reader: EventQueueReader): bool = ab.limit + (reader.offset - ab.offset) <= len(ab.queue) -proc emit*[T](ab: AsyncEventQueue[T], data: T) {.raises: [].} = +proc emit*[T](ab: AsyncEventQueue[T], data: T) = if len(ab.readers) > 0: # We enqueue `data` only if there active reader present. var changesPresent = false @@ -565,7 +567,8 @@ proc emit*[T](ab: AsyncEventQueue[T], data: T) {.raises: [].} = proc waitEvents*[T](ab: AsyncEventQueue[T], key: EventQueueKey, - eventsCount = -1): Future[seq[T]] {.async.} = + eventsCount = -1): Future[seq[T]] {. + async: (raises: [AsyncEventQueueFullError, CancelledError]).} = ## Wait for events var events: seq[T] @@ -595,7 +598,8 @@ proc waitEvents*[T](ab: AsyncEventQueue[T], doAssert(length >= ab.readers[index].offset) if length == ab.readers[index].offset: # We are at the end of queue, it means that we should wait for new events. - let waitFuture = newFuture[void]("AsyncEventQueue.waitEvents") + let waitFuture = Future[void].Raising([CancelledError]).init( + "AsyncEventQueue.waitEvents") ab.readers[index].waiter = waitFuture resetFuture = true await waitFuture @@ -626,4 +630,4 @@ proc waitEvents*[T](ab: AsyncEventQueue[T], if (eventsCount <= 0) or (len(events) == eventsCount): break - return events + events diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index c4a7374..a36ff4a 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -16,7 +16,9 @@ import stew/base10 import ./[asyncengine, raisesfutures] import ../[config, futures] -export raisesfutures.InternalRaisesFuture +export + raisesfutures.Raising, raisesfutures.InternalRaisesFuture, + raisesfutures.init, raisesfutures.error, raisesfutures.readError when chronosStackTrace: import std/strutils @@ -109,7 +111,7 @@ template newInternalRaisesFuture*[T, E](fromProc: static[string] = ""): auto = ## that this future belongs to, is a good habit as it helps with debugging. newInternalRaisesFutureImpl[T, E](getSrcLocation(fromProc)) -template newFutureSeq*[A, B](fromProc: static[string] = ""): FutureSeq[A, B] = +template newFutureSeq*[A, B](fromProc: static[string] = ""): FutureSeq[A, B] {.deprecated.} = ## Create a new future which can hold/preserve GC sequence until future will ## not be completed. ## @@ -117,7 +119,7 @@ template newFutureSeq*[A, B](fromProc: static[string] = ""): FutureSeq[A, B] = ## that this future belongs to, is a good habit as it helps with debugging. newFutureSeqImpl[A, B](getSrcLocation(fromProc)) -template newFutureStr*[T](fromProc: static[string] = ""): FutureStr[T] = +template newFutureStr*[T](fromProc: static[string] = ""): FutureStr[T] {.deprecated.} = ## Create a new future which can hold/preserve GC string until future will ## not be completed. ## @@ -205,7 +207,8 @@ template complete*(future: Future[void]) = ## Completes a void ``future``. complete(future, getSrcLocation()) -proc fail(future: FutureBase, error: ref CatchableError, loc: ptr SrcLoc) = +proc failImpl( + future: FutureBase, error: ref CatchableError, loc: ptr SrcLoc) = if not(future.cancelled()): checkFinished(future, loc) future.internalError = error @@ -216,10 +219,16 @@ proc fail(future: FutureBase, error: ref CatchableError, loc: ptr SrcLoc) = getStackTrace(error) future.finish(FutureState.Failed) -template fail*( - future: FutureBase, error: ref CatchableError, warn: static bool = false) = +template fail*[T]( + future: Future[T], error: ref CatchableError, warn: static bool = false) = ## Completes ``future`` with ``error``. - fail(future, error, getSrcLocation()) + failImpl(future, error, getSrcLocation()) + +template fail*[T, E]( + future: InternalRaisesFuture[T, E], error: ref CatchableError, + warn: static bool = true) = + checkRaises(future, E, error, warn) + failImpl(future, error, getSrcLocation()) template newCancelledError(): ref CancelledError = (ref CancelledError)(msg: "Future operation cancelled!") @@ -377,8 +386,6 @@ proc futureContinue*(fut: FutureBase) {.raises: [], gcsafe.} = {.pop.} when chronosStackTrace: - import std/strutils - template getFilenameProcname(entry: StackTraceEntry): (string, string) = when compiles(entry.filenameStr) and compiles(entry.procnameStr): # We can't rely on "entry.filename" and "entry.procname" still being valid @@ -462,31 +469,36 @@ proc internalCheckComplete*(fut: FutureBase) {.raises: [CatchableError].} = injectStacktrace(fut.internalError) raise fut.internalError -macro internalCheckComplete*(f: InternalRaisesFuture): untyped = +macro internalCheckComplete*(fut: InternalRaisesFuture, raises: typed) = # For InternalRaisesFuture[void, (ValueError, OSError), will do: # {.cast(raises: [ValueError, OSError]).}: # if isNil(f.error): discard # else: raise f.error - let e = getTypeInst(f)[2] - let types = getType(e) + # TODO https://github.com/nim-lang/Nim/issues/22937 + # we cannot `getTypeInst` on the `fut` - when aliases are involved, the + # generics are lost - so instead, we pass the raises list explicitly + let types = getRaisesTypes(raises) if isNoRaises(types): return quote do: - if not(isNil(`f`.internalError)): - raiseAssert("Unhandled future exception: " & `f`.error.msg) + if not(isNil(`fut`.internalError)): + # This would indicate a bug in which `error` was set via the non-raising + # base type + raiseAssert("Error set on a non-raising future: " & `fut`.internalError.msg) expectKind(types, nnkBracketExpr) expectKind(types[0], nnkSym) + assert types[0].strVal == "tuple" let ifRaise = nnkIfExpr.newTree( nnkElifExpr.newTree( - quote do: isNil(`f`.internalError), + quote do: isNil(`fut`.internalError), quote do: discard ), nnkElseExpr.newTree( - nnkRaiseStmt.newNimNode(lineInfoFrom=f).add( - quote do: (`f`.internalError) + nnkRaiseStmt.newNimNode(lineInfoFrom=fut).add( + quote do: (`fut`.internalError) ) ) ) @@ -1118,7 +1130,7 @@ proc one*[F: SomeFuture](futs: varargs[F]): Future[F] {. return retFuture proc race*(futs: varargs[FutureBase]): Future[FutureBase] {. - async: (raw: true, raises: [CancelledError]).} = + async: (raw: true, raises: [ValueError, CancelledError]).} = ## Returns a future which will complete and return completed FutureBase, ## when one of the futures in ``futs`` will be completed, failed or canceled. ## @@ -1488,12 +1500,6 @@ when defined(windows): {.pop.} # Automatically deduced raises from here onwards -template fail*[T, E]( - future: InternalRaisesFuture[T, E], error: ref CatchableError, - warn: static bool = true) = - checkRaises(future, error, warn) - fail(future, error, getSrcLocation()) - proc waitFor*[T, E](fut: InternalRaisesFuture[T, E]): T = # {.raises: [E]} ## **Blocks** the current thread until the specified future finishes and ## reads it, potentially raising an exception if the future failed or was @@ -1512,7 +1518,7 @@ proc read*[T: not void, E](future: InternalRaisesFuture[T, E]): lent T = # {.rai # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") - internalCheckComplete(future) + internalCheckComplete(future, E) future.internalValue proc read*[E](future: InternalRaisesFuture[void, E]) = # {.raises: [E, CancelledError].} diff --git a/chronos/internal/asyncmacro.nim b/chronos/internal/asyncmacro.nim index 11daf33..88e11e3 100644 --- a/chronos/internal/asyncmacro.nim +++ b/chronos/internal/asyncmacro.nim @@ -497,7 +497,7 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = prc -template await*[T](f: Future[T]): untyped = +template await*[T](f: Future[T]): T = when declared(chronosInternalRetFuture): chronosInternalRetFuture.internalChild = f # `futureContinue` calls the iterator generated by the `async` @@ -512,6 +512,21 @@ template await*[T](f: Future[T]): untyped = else: unsupported "await is only available within {.async.}" +template await*[T, E](f: InternalRaisesFuture[T, E]): T = + when declared(chronosInternalRetFuture): + chronosInternalRetFuture.internalChild = f + # `futureContinue` calls the iterator generated by the `async` + # transformation - `yield` gives control back to `futureContinue` which is + # responsible for resuming execution once the yielded future is finished + yield chronosInternalRetFuture.internalChild + # `child` released by `futureContinue` + cast[type(f)](chronosInternalRetFuture.internalChild).internalCheckComplete(E) + + when T isnot void: + cast[type(f)](chronosInternalRetFuture.internalChild).value() + else: + unsupported "await is only available within {.async.}" + template awaitne*[T](f: Future[T]): Future[T] = when declared(chronosInternalRetFuture): chronosInternalRetFuture.internalChild = f diff --git a/chronos/internal/raisesfutures.nim b/chronos/internal/raisesfutures.nim index ad811f7..79384d2 100644 --- a/chronos/internal/raisesfutures.nim +++ b/chronos/internal/raisesfutures.nim @@ -1,5 +1,5 @@ import - std/macros, + std/[macros, sequtils], ../futures type @@ -18,6 +18,45 @@ proc makeNoRaises*(): NimNode {.compileTime.} = ident"void" +macro Raising*[T](F: typedesc[Future[T]], E: varargs[typedesc]): untyped = + ## Given a Future type instance, return a type storing `{.raises.}` + ## information + ## + ## Note; this type may change in the future + E.expectKind(nnkBracket) + + let raises = if E.len == 0: + makeNoRaises() + else: + nnkTupleConstr.newTree(E.mapIt(it)) + nnkBracketExpr.newTree( + ident "InternalRaisesFuture", + nnkDotExpr.newTree(F, ident"T"), + raises + ) + +template init*[T, E]( + F: type InternalRaisesFuture[T, E], fromProc: static[string] = ""): F = + ## Creates a new pending future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + let res = F() + internalInitFutureBase(res, getSrcLocation(fromProc), FutureState.Pending, {}) + res + +template init*[T, E]( + F: type InternalRaisesFuture[T, E], fromProc: static[string] = "", + flags: static[FutureFlags]): F = + ## Creates a new pending future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + let res = F() + internalInitFutureBase( + res, getSrcLocation(fromProc), FutureState.Pending, flags) + res + proc isNoRaises*(n: NimNode): bool {.compileTime.} = n.eqIdent("void") @@ -78,21 +117,15 @@ macro union*(tup0: typedesc[tuple], tup1: typedesc[tuple]): typedesc = if result.len == 0: result = makeNoRaises() -proc getRaises*(future: NimNode): NimNode {.compileTime.} = - # Given InternalRaisesFuture[T, (A, B, C)], returns (A, B, C) - let types = getType(getTypeInst(future)[2]) - if isNoRaises(types): - nnkBracketExpr.newTree(newEmptyNode()) - else: - expectKind(types, nnkBracketExpr) - expectKind(types[0], nnkSym) - assert types[0].strVal == "tuple" - assert types.len >= 1 - - types +proc getRaisesTypes*(raises: NimNode): NimNode = + let typ = getType(raises) + case typ.typeKind + of ntyTypeDesc: typ[1] + else: typ macro checkRaises*[T: CatchableError]( - future: InternalRaisesFuture, error: ref T, warn: static bool = true): untyped = + future: InternalRaisesFuture, raises: typed, error: ref T, + warn: static bool = true): untyped = ## Generate code that checks that the given error is compatible with the ## raises restrictions of `future`. ## @@ -100,11 +133,18 @@ macro checkRaises*[T: CatchableError]( ## information available at compile time - in particular, if the raises ## inherit from `error`, we end up with the equivalent of a downcast which ## raises a Defect if it fails. - let raises = getRaises(future) + let + raises = getRaisesTypes(raises) expectKind(getTypeInst(error), nnkRefTy) let toMatch = getTypeInst(error)[0] + + if isNoRaises(raises): + error( + "`fail`: `" & repr(toMatch) & "` incompatible with `raises: []`", future) + return + var typeChecker = ident"false" maybeChecker = ident"false" @@ -134,3 +174,15 @@ macro checkRaises*[T: CatchableError]( else: `warning` assert(`runtimeChecker`, `errorMsg`) + +proc error*[T](future: InternalRaisesFuture[T, void]): ref CatchableError {. + raises: [].} = + static: + warning("No exceptions possible with this operation, `error` always returns nil") + nil + +proc readError*[T](future: InternalRaisesFuture[T, void]): ref CatchableError {. + raises: [ValueError].} = + static: + warning("No exceptions possible with this operation, `readError` always raises") + raise newException(ValueError, "No error in future.") diff --git a/chronos/transports/common.nim b/chronos/transports/common.nim index d8263af..24f9852 100644 --- a/chronos/transports/common.nim +++ b/chronos/transports/common.nim @@ -113,6 +113,8 @@ type ## Transport's capability not supported exception TransportUseClosedError* = object of TransportError ## Usage after transport close exception + TransportUseEofError* = object of TransportError + ## Usage after transport half-close exception TransportTooManyError* = object of TransportError ## Too many open file descriptors exception TransportAbortedError* = object of TransportError @@ -567,11 +569,11 @@ template checkClosed*(t: untyped, future: untyped) = template checkWriteEof*(t: untyped, future: untyped) = if (WriteEof in (t).state): - future.fail(newException(TransportError, + future.fail(newException(TransportUseEofError, "Transport connection is already dropped!")) return future -template getError*(t: untyped): ref CatchableError = +template getError*(t: untyped): ref TransportError = var err = (t).error (t).error = nil err diff --git a/chronos/transports/datagram.nim b/chronos/transports/datagram.nim index aec18ae..30f872d 100644 --- a/chronos/transports/datagram.nim +++ b/chronos/transports/datagram.nim @@ -27,7 +27,10 @@ type DatagramCallback* = proc(transp: DatagramTransport, remote: TransportAddress): Future[void] {. - gcsafe, raises: [].} + async: (raises: []).} + + UnsafeDatagramCallback* = proc(transp: DatagramTransport, + remote: TransportAddress): Future[void] {.async.} DatagramTransport* = ref object of RootRef fd*: AsyncFD # File descriptor @@ -35,7 +38,7 @@ type flags: set[ServerFlags] # Flags buffer: seq[byte] # Reading buffer buflen: int # Reading buffer effective size - error: ref CatchableError # Current error + error: ref TransportError # Current error queue: Deque[GramVector] # Writer queue local: TransportAddress # Local address remote: TransportAddress # Remote address @@ -599,6 +602,41 @@ proc close*(transp: DatagramTransport) = transp.state.incl({WriteClosed, ReadClosed}) closeSocket(transp.fd, continuation) +proc newDatagramTransportCommon(cbproc: UnsafeDatagramCallback, + remote: TransportAddress, + local: TransportAddress, + sock: AsyncFD, + flags: set[ServerFlags], + udata: pointer, + child: DatagramTransport, + bufferSize: int, + ttl: int, + dualstack = DualStackType.Auto + ): DatagramTransport {. + raises: [TransportOsError].} = + ## Create new UDP datagram transport (IPv4). + ## + ## ``cbproc`` - callback which will be called, when new datagram received. + ## ``remote`` - bind transport to remote address (optional). + ## ``local`` - bind transport to local address (to serving incoming + ## datagrams, optional) + ## ``sock`` - application-driven socket to use. + ## ``flags`` - flags that will be applied to socket. + ## ``udata`` - custom argument which will be passed to ``cbproc``. + ## ``bufSize`` - size of internal buffer. + ## ``ttl`` - TTL for UDP datagram packet (only usable when flags has + ## ``Broadcast`` option). + + proc wrap(transp: DatagramTransport, + remote: TransportAddress) {.async: (raises: []).} = + try: + cbproc(transp, remote) + except CatchableError as exc: + raiseAssert "Unexpected exception from stream server cbproc: " & exc.msg + + newDatagramTransportCommon(wrap, remote, local, sock, flags, udata, child, + bufferSize, ttl, dualstack) + proc newDatagramTransport*(cbproc: DatagramCallback, remote: TransportAddress = AnyAddress, local: TransportAddress = AnyAddress, @@ -689,7 +727,102 @@ proc newDatagramTransport6*[T](cbproc: DatagramCallback, cast[pointer](udata), child, bufSize, ttl, dualstack) -proc join*(transp: DatagramTransport): Future[void] = +proc newDatagramTransport*(cbproc: UnsafeDatagramCallback, + remote: TransportAddress = AnyAddress, + local: TransportAddress = AnyAddress, + sock: AsyncFD = asyncInvalidSocket, + flags: set[ServerFlags] = {}, + udata: pointer = nil, + child: DatagramTransport = nil, + bufSize: int = DefaultDatagramBufferSize, + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. + raises: [TransportOsError], + deprecated: "Callback must not raise exceptions, annotate with {.async: (raises: []).}".} = + ## Create new UDP datagram transport (IPv4). + ## + ## ``cbproc`` - callback which will be called, when new datagram received. + ## ``remote`` - bind transport to remote address (optional). + ## ``local`` - bind transport to local address (to serving incoming + ## datagrams, optional) + ## ``sock`` - application-driven socket to use. + ## ``flags`` - flags that will be applied to socket. + ## ``udata`` - custom argument which will be passed to ``cbproc``. + ## ``bufSize`` - size of internal buffer. + ## ``ttl`` - TTL for UDP datagram packet (only usable when flags has + ## ``Broadcast`` option). + newDatagramTransportCommon(cbproc, remote, local, sock, flags, udata, child, + bufSize, ttl, dualstack) + +proc newDatagramTransport*[T](cbproc: UnsafeDatagramCallback, + udata: ref T, + remote: TransportAddress = AnyAddress, + local: TransportAddress = AnyAddress, + sock: AsyncFD = asyncInvalidSocket, + flags: set[ServerFlags] = {}, + child: DatagramTransport = nil, + bufSize: int = DefaultDatagramBufferSize, + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. + raises: [TransportOsError], + deprecated: "Callback must not raise exceptions, annotate with {.async: (raises: []).}".} = + var fflags = flags + {GCUserData} + GC_ref(udata) + newDatagramTransportCommon(cbproc, remote, local, sock, fflags, + cast[pointer](udata), child, bufSize, ttl, + dualstack) + +proc newDatagramTransport6*(cbproc: UnsafeDatagramCallback, + remote: TransportAddress = AnyAddress6, + local: TransportAddress = AnyAddress6, + sock: AsyncFD = asyncInvalidSocket, + flags: set[ServerFlags] = {}, + udata: pointer = nil, + child: DatagramTransport = nil, + bufSize: int = DefaultDatagramBufferSize, + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. + raises: [TransportOsError], + deprecated: "Callback must not raise exceptions, annotate with {.async: (raises: []).}".} = + ## Create new UDP datagram transport (IPv6). + ## + ## ``cbproc`` - callback which will be called, when new datagram received. + ## ``remote`` - bind transport to remote address (optional). + ## ``local`` - bind transport to local address (to serving incoming + ## datagrams, optional) + ## ``sock`` - application-driven socket to use. + ## ``flags`` - flags that will be applied to socket. + ## ``udata`` - custom argument which will be passed to ``cbproc``. + ## ``bufSize`` - size of internal buffer. + ## ``ttl`` - TTL for UDP datagram packet (only usable when flags has + ## ``Broadcast`` option). + newDatagramTransportCommon(cbproc, remote, local, sock, flags, udata, child, + bufSize, ttl, dualstack) + +proc newDatagramTransport6*[T](cbproc: UnsafeDatagramCallback, + udata: ref T, + remote: TransportAddress = AnyAddress6, + local: TransportAddress = AnyAddress6, + sock: AsyncFD = asyncInvalidSocket, + flags: set[ServerFlags] = {}, + child: DatagramTransport = nil, + bufSize: int = DefaultDatagramBufferSize, + ttl: int = 0, + dualstack = DualStackType.Auto + ): DatagramTransport {. + raises: [TransportOsError], + deprecated: "Callback must not raise exceptions, annotate with {.async: (raises: []).}".} = + var fflags = flags + {GCUserData} + GC_ref(udata) + newDatagramTransportCommon(cbproc, remote, local, sock, fflags, + cast[pointer](udata), child, bufSize, ttl, + dualstack) + +proc join*(transp: DatagramTransport): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Wait until the transport ``transp`` will be closed. var retFuture = newFuture[void]("datagram.transport.join") @@ -707,14 +840,15 @@ proc join*(transp: DatagramTransport): Future[void] = return retFuture -proc closeWait*(transp: DatagramTransport): Future[void] = +proc closeWait*(transp: DatagramTransport): Future[void] {. + async: (raw: true, raises: []).} = ## Close transport ``transp`` and release all resources. - const FutureName = "datagram.transport.closeWait" + let retFuture = newFuture[void]( + "datagram.transport.closeWait", {FutureFlag.OwnCancelSchedule}) if {ReadClosed, WriteClosed} * transp.state != {}: - return Future.completed(FutureName) - - let retFuture = newFuture[void](FutureName, {FutureFlag.OwnCancelSchedule}) + retFuture.complete() + return retFuture proc continuation(udata: pointer) {.gcsafe.} = retFuture.complete() @@ -733,7 +867,8 @@ proc closeWait*(transp: DatagramTransport): Future[void] = retFuture proc send*(transp: DatagramTransport, pbytes: pointer, - nbytes: int): Future[void] = + nbytes: int): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Send buffer with pointer ``pbytes`` and size ``nbytes`` using transport ## ``transp`` to remote destination address which was bounded on transport. var retFuture = newFuture[void]("datagram.transport.send(pointer)") @@ -751,22 +886,21 @@ proc send*(transp: DatagramTransport, pbytes: pointer, return retFuture proc send*(transp: DatagramTransport, msg: sink string, - msglen = -1): Future[void] = + msglen = -1): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Send string ``msg`` using transport ``transp`` to remote destination ## address which was bounded on transport. - var retFuture = newFutureStr[void]("datagram.transport.send(string)") + var retFuture = newFuture[void]("datagram.transport.send(string)") transp.checkClosed(retFuture) - when declared(shallowCopy): - if not(isLiteral(msg)): - shallowCopy(retFuture.gcholder, msg) - else: - retFuture.gcholder = msg - else: - retFuture.gcholder = msg + let length = if msglen <= 0: len(msg) else: msglen - let vector = GramVector(kind: WithoutAddress, buf: addr retFuture.gcholder[0], + var localCopy = msg + retFuture.addCallback(proc(_: pointer) = reset(localCopy)) + + let vector = GramVector(kind: WithoutAddress, buf: addr localCopy[0], buflen: length, - writer: cast[Future[void]](retFuture)) + writer: retFuture) + transp.queue.addLast(vector) if WritePaused in transp.state: let wres = transp.resumeWrite() @@ -775,22 +909,20 @@ proc send*(transp: DatagramTransport, msg: sink string, return retFuture proc send*[T](transp: DatagramTransport, msg: sink seq[T], - msglen = -1): Future[void] = + msglen = -1): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Send string ``msg`` using transport ``transp`` to remote destination ## address which was bounded on transport. - var retFuture = newFutureSeq[void, T]("datagram.transport.send(seq)") + var retFuture = newFuture[void]("datagram.transport.send(seq)") transp.checkClosed(retFuture) - when declared(shallowCopy): - if not(isLiteral(msg)): - shallowCopy(retFuture.gcholder, msg) - else: - retFuture.gcholder = msg - else: - retFuture.gcholder = msg + let length = if msglen <= 0: (len(msg) * sizeof(T)) else: (msglen * sizeof(T)) - let vector = GramVector(kind: WithoutAddress, buf: addr retFuture.gcholder[0], + var localCopy = msg + retFuture.addCallback(proc(_: pointer) = reset(localCopy)) + + let vector = GramVector(kind: WithoutAddress, buf: addr localCopy[0], buflen: length, - writer: cast[Future[void]](retFuture)) + writer: retFuture) transp.queue.addLast(vector) if WritePaused in transp.state: let wres = transp.resumeWrite() @@ -799,7 +931,8 @@ proc send*[T](transp: DatagramTransport, msg: sink seq[T], return retFuture proc sendTo*(transp: DatagramTransport, remote: TransportAddress, - pbytes: pointer, nbytes: int): Future[void] = + pbytes: pointer, nbytes: int): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Send buffer with pointer ``pbytes`` and size ``nbytes`` using transport ## ``transp`` to remote destination address ``remote``. var retFuture = newFuture[void]("datagram.transport.sendTo(pointer)") @@ -814,22 +947,20 @@ proc sendTo*(transp: DatagramTransport, remote: TransportAddress, return retFuture proc sendTo*(transp: DatagramTransport, remote: TransportAddress, - msg: sink string, msglen = -1): Future[void] = + msg: sink string, msglen = -1): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Send string ``msg`` using transport ``transp`` to remote destination ## address ``remote``. - var retFuture = newFutureStr[void]("datagram.transport.sendTo(string)") + var retFuture = newFuture[void]("datagram.transport.sendTo(string)") transp.checkClosed(retFuture) - when declared(shallowCopy): - if not(isLiteral(msg)): - shallowCopy(retFuture.gcholder, msg) - else: - retFuture.gcholder = msg - else: - retFuture.gcholder = msg + let length = if msglen <= 0: len(msg) else: msglen - let vector = GramVector(kind: WithAddress, buf: addr retFuture.gcholder[0], + var localCopy = msg + retFuture.addCallback(proc(_: pointer) = reset(localCopy)) + + let vector = GramVector(kind: WithAddress, buf: addr localCopy[0], buflen: length, - writer: cast[Future[void]](retFuture), + writer: retFuture, address: remote) transp.queue.addLast(vector) if WritePaused in transp.state: @@ -839,20 +970,17 @@ proc sendTo*(transp: DatagramTransport, remote: TransportAddress, return retFuture proc sendTo*[T](transp: DatagramTransport, remote: TransportAddress, - msg: sink seq[T], msglen = -1): Future[void] = + msg: sink seq[T], msglen = -1): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Send sequence ``msg`` using transport ``transp`` to remote destination ## address ``remote``. - var retFuture = newFutureSeq[void, T]("datagram.transport.sendTo(seq)") + var retFuture = newFuture[void]("datagram.transport.sendTo(seq)") transp.checkClosed(retFuture) - when declared(shallowCopy): - if not(isLiteral(msg)): - shallowCopy(retFuture.gcholder, msg) - else: - retFuture.gcholder = msg - else: - retFuture.gcholder = msg let length = if msglen <= 0: (len(msg) * sizeof(T)) else: (msglen * sizeof(T)) - let vector = GramVector(kind: WithAddress, buf: addr retFuture.gcholder[0], + var localCopy = msg + retFuture.addCallback(proc(_: pointer) = reset(localCopy)) + + let vector = GramVector(kind: WithAddress, buf: addr localCopy[0], buflen: length, writer: cast[Future[void]](retFuture), address: remote) @@ -864,7 +992,7 @@ proc sendTo*[T](transp: DatagramTransport, remote: TransportAddress, return retFuture proc peekMessage*(transp: DatagramTransport, msg: var seq[byte], - msglen: var int) {.raises: [CatchableError].} = + msglen: var int) {.raises: [TransportError].} = ## Get access to internal message buffer and length of incoming datagram. if ReadError in transp.state: transp.state.excl(ReadError) @@ -876,7 +1004,7 @@ proc peekMessage*(transp: DatagramTransport, msg: var seq[byte], msglen = transp.buflen proc getMessage*(transp: DatagramTransport): seq[byte] {. - raises: [CatchableError].} = + raises: [TransportError].} = ## Copy data from internal message buffer and return result. var default: seq[byte] if ReadError in transp.state: diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 7471a44..bdcb8d7 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -58,6 +58,8 @@ type done: bool] {. gcsafe, raises: [].} + ReaderFuture = Future[void].Raising([TransportError, CancelledError]) + const StreamTransportTrackerName* = "stream.transport" StreamServerTrackerName* = "stream.server" @@ -68,10 +70,10 @@ when defined(windows): StreamTransport* = ref object of RootRef fd*: AsyncFD # File descriptor state: set[TransportState] # Current Transport state - reader: Future[void] # Current reader Future + reader: ReaderFuture # Current reader Future buffer: seq[byte] # Reading buffer offset: int # Reading buffer offset - error: ref CatchableError # Current error + error: ref TransportError # Current error queue: Deque[StreamVector] # Writer queue future: Future[void] # Stream life future # Windows specific part @@ -87,18 +89,18 @@ when defined(windows): local: TransportAddress # Local address remote: TransportAddress # Remote address of TransportKind.Pipe: - todo1: int + discard of TransportKind.File: - todo2: int + discard else: type StreamTransport* = ref object of RootRef fd*: AsyncFD # File descriptor state: set[TransportState] # Current Transport state - reader: Future[void] # Current reader Future + reader: ReaderFuture # Current reader Future buffer: seq[byte] # Reading buffer offset: int # Reading buffer offset - error: ref CatchableError # Current error + error: ref TransportError # Current error queue: Deque[StreamVector] # Writer queue future: Future[void] # Stream life future case kind*: TransportKind @@ -107,18 +109,23 @@ else: local: TransportAddress # Local address remote: TransportAddress # Remote address of TransportKind.Pipe: - todo1: int + discard of TransportKind.File: - todo2: int + discard type StreamCallback* = proc(server: StreamServer, - client: StreamTransport): Future[void] {. - gcsafe, raises: [].} + client: StreamTransport) {.async: (raises: []).} ## New remote client connection callback ## ``server`` - StreamServer object. ## ``client`` - accepted client transport. + UnsafeStreamCallback* = proc(server: StreamServer, + client: StreamTransport) {.async.} + ## Connection callback that doesn't check for exceptions at compile time + ## ``server`` - StreamServer object. + ## ``client`` - accepted client transport. + TransportInitCallback* = proc(server: StreamServer, fd: AsyncFD): StreamTransport {. gcsafe, raises: [].} @@ -199,7 +206,7 @@ proc completePendingWriteQueue(queue: var Deque[StreamVector], vector.writer.complete(v) proc failPendingWriteQueue(queue: var Deque[StreamVector], - error: ref CatchableError) {.inline.} = + error: ref TransportError) {.inline.} = while len(queue) > 0: var vector = queue.popFirst() if not(vector.writer.finished()): @@ -640,7 +647,8 @@ when defined(windows): localAddress = TransportAddress(), flags: set[SocketFlags] = {}, dualstack = DualStackType.Auto - ): Future[StreamTransport] = + ): Future[StreamTransport] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Open new connection to remote peer with address ``address`` and create ## new transport object ``StreamTransport`` for established connection. ## ``bufferSize`` is size of internal buffer for transport. @@ -1031,7 +1039,8 @@ when defined(windows): server.aovl.data.cb(addr server.aovl) ok() - proc accept*(server: StreamServer): Future[StreamTransport] = + proc accept*(server: StreamServer): Future[StreamTransport] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = var retFuture = newFuture[StreamTransport]("stream.server.accept") doAssert(server.status != ServerStatus.Running, @@ -1472,7 +1481,8 @@ else: localAddress = TransportAddress(), flags: set[SocketFlags] = {}, dualstack = DualStackType.Auto, - ): Future[StreamTransport] = + ): Future[StreamTransport] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Open new connection to remote peer with address ``address`` and create ## new transport object ``StreamTransport`` for established connection. ## ``bufferSize`` - size of internal buffer for transport. @@ -1658,7 +1668,8 @@ else: transp.state.excl(WritePaused) ok() - proc accept*(server: StreamServer): Future[StreamTransport] = + proc accept*(server: StreamServer): Future[StreamTransport] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = var retFuture = newFuture[StreamTransport]("stream.server.accept") doAssert(server.status != ServerStatus.Running, @@ -1762,7 +1773,8 @@ proc stop*(server: StreamServer) {.raises: [TransportOsError].} = let res = stop2(server) if res.isErr(): raiseTransportOsError(res.error()) -proc join*(server: StreamServer): Future[void] = +proc join*(server: StreamServer): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Waits until ``server`` is not closed. var retFuture = newFuture[void]("stream.transport.server.join") @@ -1785,7 +1797,8 @@ proc connect*(address: TransportAddress, flags: set[TransportFlags], localAddress = TransportAddress(), dualstack = DualStackType.Auto - ): Future[StreamTransport] = + ): Future[StreamTransport] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = # Retro compatibility with TransportFlags var mappedFlags: set[SocketFlags] if TcpNoDelay in flags: mappedFlags.incl(SocketFlags.TcpNoDelay) @@ -1817,7 +1830,8 @@ proc close*(server: StreamServer) = else: server.sock.closeSocket(continuation) -proc closeWait*(server: StreamServer): Future[void] = +proc closeWait*(server: StreamServer): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Close server ``server`` and release all resources. server.close() server.join() @@ -2066,6 +2080,7 @@ proc createStreamServer*(host: TransportAddress, sres proc createStreamServer*(host: TransportAddress, + cbproc: UnsafeStreamCallback, flags: set[ServerFlags] = {}, sock: AsyncFD = asyncInvalidSocket, backlog: int = DefaultBacklogSize, @@ -2074,8 +2089,30 @@ proc createStreamServer*(host: TransportAddress, init: TransportInitCallback = nil, udata: pointer = nil, dualstack = DualStackType.Auto): StreamServer {. - raises: [CatchableError].} = - createStreamServer(host, nil, flags, sock, backlog, bufferSize, + raises: [TransportOsError], + deprecated: "Callback must not raise exceptions, annotate with {.async: (raises: []).}".} = + proc wrap(server: StreamServer, + client: StreamTransport) {.async: (raises: []).} = + try: + cbproc(server, client) + except CatchableError as exc: + raiseAssert "Unexpected exception from stream server cbproc: " & exc.msg + + createStreamServer( + host, wrap, flags, sock, backlog, bufferSize, child, init, udata, + dualstack) + +proc createStreamServer*(host: TransportAddress, + flags: set[ServerFlags] = {}, + sock: AsyncFD = asyncInvalidSocket, + backlog: int = DefaultBacklogSize, + bufferSize: int = DefaultStreamBufferSize, + child: StreamServer = nil, + init: TransportInitCallback = nil, + udata: pointer = nil, + dualstack = DualStackType.Auto): StreamServer {. + raises: [TransportOsError].} = + createStreamServer(host, StreamCallback(nil), flags, sock, backlog, bufferSize, child, init, cast[pointer](udata), dualstack) proc createStreamServer*[T](host: TransportAddress, @@ -2088,7 +2125,24 @@ proc createStreamServer*[T](host: TransportAddress, child: StreamServer = nil, init: TransportInitCallback = nil, dualstack = DualStackType.Auto): StreamServer {. - raises: [CatchableError].} = + raises: [TransportOsError].} = + var fflags = flags + {GCUserData} + GC_ref(udata) + createStreamServer(host, cbproc, fflags, sock, backlog, bufferSize, + child, init, cast[pointer](udata), dualstack) + +proc createStreamServer*[T](host: TransportAddress, + cbproc: UnsafeStreamCallback, + flags: set[ServerFlags] = {}, + udata: ref T, + sock: AsyncFD = asyncInvalidSocket, + backlog: int = DefaultBacklogSize, + bufferSize: int = DefaultStreamBufferSize, + child: StreamServer = nil, + init: TransportInitCallback = nil, + dualstack = DualStackType.Auto): StreamServer {. + raises: [TransportOsError], + deprecated: "Callback must not raise exceptions, annotate with {.async: (raises: []).}".} = var fflags = flags + {GCUserData} GC_ref(udata) createStreamServer(host, cbproc, fflags, sock, backlog, bufferSize, @@ -2103,10 +2157,10 @@ proc createStreamServer*[T](host: TransportAddress, child: StreamServer = nil, init: TransportInitCallback = nil, dualstack = DualStackType.Auto): StreamServer {. - raises: [CatchableError].} = + raises: [TransportOsError].} = var fflags = flags + {GCUserData} GC_ref(udata) - createStreamServer(host, nil, fflags, sock, backlog, bufferSize, + createStreamServer(host, StreamCallback(nil), fflags, sock, backlog, bufferSize, child, init, cast[pointer](udata), dualstack) proc getUserData*[T](server: StreamServer): T {.inline.} = @@ -2157,7 +2211,8 @@ template fastWrite(transp: auto, pbytes: var ptr byte, rbytes: var int, return retFuture proc write*(transp: StreamTransport, pbytes: pointer, - nbytes: int): Future[int] = + nbytes: int): Future[int] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Write data from buffer ``pbytes`` with size ``nbytes`` using transport ## ``transp``. var retFuture = newFuture[int]("stream.transport.write(pointer)") @@ -2179,9 +2234,10 @@ proc write*(transp: StreamTransport, pbytes: pointer, return retFuture proc write*(transp: StreamTransport, msg: sink string, - msglen = -1): Future[int] = + msglen = -1): Future[int] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Write data from string ``msg`` using transport ``transp``. - var retFuture = newFutureStr[int]("stream.transport.write(string)") + var retFuture = newFuture[int]("stream.transport.write(string)") transp.checkClosed(retFuture) transp.checkWriteEof(retFuture) @@ -2197,17 +2253,10 @@ proc write*(transp: StreamTransport, msg: sink string, let written = nbytes - rbytes # In case fastWrite wrote some - pbytes = - when declared(shallowCopy): - if not(isLiteral(msg)): - shallowCopy(retFuture.gcholder, msg) - cast[ptr byte](addr retFuture.gcholder[written]) - else: - retFuture.gcholder = msg[written ..< nbytes] - cast[ptr byte](addr retFuture.gcholder[0]) - else: - retFuture.gcholder = msg[written ..< nbytes] - cast[ptr byte](addr retFuture.gcholder[0]) + var localCopy = msg + retFuture.addCallback(proc(_: pointer) = reset(localCopy)) + + pbytes = cast[ptr byte](addr localCopy[written]) var vector = StreamVector(kind: DataBuffer, writer: retFuture, buf: pbytes, buflen: rbytes, size: nbytes) @@ -2218,9 +2267,10 @@ proc write*(transp: StreamTransport, msg: sink string, return retFuture proc write*[T](transp: StreamTransport, msg: sink seq[T], - msglen = -1): Future[int] = + msglen = -1): Future[int] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Write sequence ``msg`` using transport ``transp``. - var retFuture = newFutureSeq[int, T]("stream.transport.write(seq)") + var retFuture = newFuture[int]("stream.transport.write(seq)") transp.checkClosed(retFuture) transp.checkWriteEof(retFuture) @@ -2236,17 +2286,10 @@ proc write*[T](transp: StreamTransport, msg: sink seq[T], let written = nbytes - rbytes # In case fastWrite wrote some - pbytes = - when declared(shallowCopy): - if not(isLiteral(msg)): - shallowCopy(retFuture.gcholder, msg) - cast[ptr byte](addr retFuture.gcholder[written]) - else: - retFuture.gcholder = msg[written ..< nbytes] - cast[ptr byte](addr retFuture.gcholder[0]) - else: - retFuture.gcholder = msg[written ..< nbytes] - cast[ptr byte](addr retFuture.gcholder[0]) + var localCopy = msg + retFuture.addCallback(proc(_: pointer) = reset(localCopy)) + + pbytes = cast[ptr byte](addr localCopy[written]) var vector = StreamVector(kind: DataBuffer, writer: retFuture, buf: pbytes, buflen: rbytes, size: nbytes) @@ -2257,7 +2300,8 @@ proc write*[T](transp: StreamTransport, msg: sink seq[T], return retFuture proc writeFile*(transp: StreamTransport, handle: int, - offset: uint = 0, size: int = 0): Future[int] = + offset: uint = 0, size: int = 0): Future[int] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Write data from file descriptor ``handle`` to transport ``transp``. ## ## You can specify starting ``offset`` in opened file and number of bytes @@ -2304,7 +2348,7 @@ template readLoop(name, body: untyped): untyped = break else: checkPending(transp) - var fut = newFuture[void](name) + let fut = ReaderFuture.init(name) transp.reader = fut let res = resumeRead(transp) if res.isErr(): @@ -2328,7 +2372,8 @@ template readLoop(name, body: untyped): untyped = await fut proc readExactly*(transp: StreamTransport, pbytes: pointer, - nbytes: int) {.async.} = + nbytes: int) {. + async: (raises: [TransportError, CancelledError]).} = ## Read exactly ``nbytes`` bytes from transport ``transp`` and store it to ## ``pbytes``. ``pbytes`` must not be ``nil`` pointer and ``nbytes`` should ## be Natural. @@ -2357,7 +2402,8 @@ proc readExactly*(transp: StreamTransport, pbytes: pointer, (consumed: count, done: index == nbytes) proc readOnce*(transp: StreamTransport, pbytes: pointer, - nbytes: int): Future[int] {.async.} = + nbytes: int): Future[int] {. + async: (raises: [TransportError, CancelledError]).} = ## Perform one read operation on transport ``transp``. ## ## If internal buffer is not empty, ``nbytes`` bytes will be transferred from @@ -2376,7 +2422,8 @@ proc readOnce*(transp: StreamTransport, pbytes: pointer, return count proc readUntil*(transp: StreamTransport, pbytes: pointer, nbytes: int, - sep: seq[byte]): Future[int] {.async.} = + sep: seq[byte]): Future[int] {. + async: (raises: [TransportError, CancelledError]).} = ## Read data from the transport ``transp`` until separator ``sep`` is found. ## ## On success, the data and separator will be removed from the internal @@ -2428,7 +2475,8 @@ proc readUntil*(transp: StreamTransport, pbytes: pointer, nbytes: int, return k proc readLine*(transp: StreamTransport, limit = 0, - sep = "\r\n"): Future[string] {.async.} = + sep = "\r\n"): Future[string] {. + async: (raises: [TransportError, CancelledError]).} = ## Read one line from transport ``transp``, where "line" is a sequence of ## bytes ending with ``sep`` (default is "\r\n"). ## @@ -2470,7 +2518,8 @@ proc readLine*(transp: StreamTransport, limit = 0, (index, (state == len(sep)) or (lim == len(result))) -proc read*(transp: StreamTransport): Future[seq[byte]] {.async.} = +proc read*(transp: StreamTransport): Future[seq[byte]] {. + async: (raises: [TransportError, CancelledError]).} = ## Read all bytes from transport ``transp``. ## ## This procedure allocates buffer seq[byte] and return it as result. @@ -2481,7 +2530,8 @@ proc read*(transp: StreamTransport): Future[seq[byte]] {.async.} = result.add(transp.buffer.toOpenArray(0, transp.offset - 1)) (transp.offset, false) -proc read*(transp: StreamTransport, n: int): Future[seq[byte]] {.async.} = +proc read*(transp: StreamTransport, n: int): Future[seq[byte]] {. + async: (raises: [TransportError, CancelledError]).} = ## Read all bytes (n <= 0) or exactly `n` bytes from transport ``transp``. ## ## This procedure allocates buffer seq[byte] and return it as result. @@ -2496,7 +2546,8 @@ proc read*(transp: StreamTransport, n: int): Future[seq[byte]] {.async.} = result.add(transp.buffer.toOpenArray(0, count - 1)) (count, len(result) == n) -proc consume*(transp: StreamTransport): Future[int] {.async.} = +proc consume*(transp: StreamTransport): Future[int] {. + async: (raises: [TransportError, CancelledError]).} = ## Consume all bytes from transport ``transp`` and discard it. ## ## Return number of bytes actually consumed and discarded. @@ -2507,7 +2558,8 @@ proc consume*(transp: StreamTransport): Future[int] {.async.} = result += transp.offset (transp.offset, false) -proc consume*(transp: StreamTransport, n: int): Future[int] {.async.} = +proc consume*(transp: StreamTransport, n: int): Future[int] {. + async: (raises: [TransportError, CancelledError]).} = ## Consume all bytes (n <= 0) or ``n`` bytes from transport ``transp`` and ## discard it. ## @@ -2524,7 +2576,8 @@ proc consume*(transp: StreamTransport, n: int): Future[int] {.async.} = (count, result == n) proc readMessage*(transp: StreamTransport, - predicate: ReadMessagePredicate) {.async.} = + predicate: ReadMessagePredicate) {. + async: (raises: [TransportError, CancelledError]).} = ## Read all bytes from transport ``transp`` until ``predicate`` callback ## will not be satisfied. ## @@ -2547,7 +2600,8 @@ proc readMessage*(transp: StreamTransport, else: predicate(transp.buffer.toOpenArray(0, transp.offset - 1)) -proc join*(transp: StreamTransport): Future[void] = +proc join*(transp: StreamTransport): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Wait until ``transp`` will not be closed. var retFuture = newFuture[void]("stream.transport.join") @@ -2606,14 +2660,15 @@ proc close*(transp: StreamTransport) = elif transp.kind == TransportKind.Socket: closeSocket(transp.fd, continuation) -proc closeWait*(transp: StreamTransport): Future[void] = +proc closeWait*(transp: StreamTransport): Future[void] {. + async: (raw: true, raises: []).} = ## Close and frees resources of transport ``transp``. - const FutureName = "stream.transport.closeWait" + let retFuture = newFuture[void]( + "stream.transport.closeWait", {FutureFlag.OwnCancelSchedule}) if {ReadClosed, WriteClosed} * transp.state != {}: - return Future.completed(FutureName) - - let retFuture = newFuture[void](FutureName, {FutureFlag.OwnCancelSchedule}) + retFuture.complete() + return retFuture proc continuation(udata: pointer) {.gcsafe.} = retFuture.complete() @@ -2631,7 +2686,8 @@ proc closeWait*(transp: StreamTransport): Future[void] = retFuture.cancelCallback = cancellation retFuture -proc shutdownWait*(transp: StreamTransport): Future[void] = +proc shutdownWait*(transp: StreamTransport): Future[void] {. + async: (raw: true, raises: [TransportError, CancelledError]).} = ## Perform graceful shutdown of TCP connection backed by transport ``transp``. doAssert(transp.kind == TransportKind.Socket) let retFuture = newFuture[void]("stream.transport.shutdown") diff --git a/tests/testasyncstream.nim b/tests/testasyncstream.nim index 86b7357..bd0207f 100644 --- a/tests/testasyncstream.nim +++ b/tests/testasyncstream.nim @@ -87,14 +87,17 @@ suite "AsyncStream test suite": test "AsyncStream(StreamTransport) readExactly() test": proc testReadExactly(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - await wstream.write("000000000011111111112222222222") - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + await wstream.write("000000000011111111112222222222") + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var buffer = newSeq[byte](10) var server = createStreamServer(initTAddress("127.0.0.1:0"), @@ -117,14 +120,17 @@ suite "AsyncStream test suite": test "AsyncStream(StreamTransport) readUntil() test": proc testReadUntil(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - await wstream.write("0000000000NNz1111111111NNz2222222222NNz") - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + await wstream.write("0000000000NNz1111111111NNz2222222222NNz") + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var buffer = newSeq[byte](13) var sep = @[byte('N'), byte('N'), byte('z')] @@ -155,14 +161,17 @@ suite "AsyncStream test suite": test "AsyncStream(StreamTransport) readLine() test": proc testReadLine(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - await wstream.write("0000000000\r\n1111111111\r\n2222222222\r\n") - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + await wstream.write("0000000000\r\n1111111111\r\n2222222222\r\n") + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -184,14 +193,17 @@ suite "AsyncStream test suite": test "AsyncStream(StreamTransport) read() test": proc testRead(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - await wstream.write("000000000011111111112222222222") - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + await wstream.write("000000000011111111112222222222") + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -211,14 +223,17 @@ suite "AsyncStream test suite": test "AsyncStream(StreamTransport) consume() test": proc testConsume(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - await wstream.write("0000000000111111111122222222223333333333") - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + await wstream.write("0000000000111111111122222222223333333333") + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -247,26 +262,29 @@ suite "AsyncStream test suite": test "AsyncStream(AsyncStream) readExactly() test": proc testReadExactly2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) - var s1 = "00000" - var s2 = "11111" - var s3 = "22222" - await wstream2.write("00000") - await wstream2.write(addr s1[0], len(s1)) - await wstream2.write("11111") - await wstream2.write(s2.toBytes()) - await wstream2.write("22222") - await wstream2.write(addr s3[0], len(s3)) + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) + var s1 = "00000" + var s2 = "11111" + var s3 = "22222" + await wstream2.write("00000") + await wstream2.write(addr s1[0], len(s1)) + await wstream2.write("11111") + await wstream2.write(s2.toBytes()) + await wstream2.write("22222") + await wstream2.write(addr s3[0], len(s3)) - await wstream2.finish() - await wstream.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + await wstream2.finish() + await wstream.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var buffer = newSeq[byte](10) var server = createStreamServer(initTAddress("127.0.0.1:0"), @@ -299,25 +317,28 @@ suite "AsyncStream test suite": test "AsyncStream(AsyncStream) readUntil() test": proc testReadUntil2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) - var s1 = "00000NNz" - var s2 = "11111NNz" - var s3 = "22222NNz" - await wstream2.write("00000") - await wstream2.write(addr s1[0], len(s1)) - await wstream2.write("11111") - await wstream2.write(s2) - await wstream2.write("22222") - await wstream2.write(s3.toBytes()) - await wstream2.finish() - await wstream.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) + var s1 = "00000NNz" + var s2 = "11111NNz" + var s3 = "22222NNz" + await wstream2.write("00000") + await wstream2.write(addr s1[0], len(s1)) + await wstream2.write("11111") + await wstream2.write(s2) + await wstream2.write("22222") + await wstream2.write(s3.toBytes()) + await wstream2.finish() + await wstream.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var buffer = newSeq[byte](13) var sep = @[byte('N'), byte('N'), byte('z')] @@ -358,22 +379,25 @@ suite "AsyncStream test suite": test "AsyncStream(AsyncStream) readLine() test": proc testReadLine2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) - await wstream2.write("00000") - await wstream2.write("00000\r\n") - await wstream2.write("11111") - await wstream2.write("11111\r\n") - await wstream2.write("22222") - await wstream2.write("22222\r\n") - await wstream2.finish() - await wstream.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) + await wstream2.write("00000") + await wstream2.write("00000\r\n") + await wstream2.write("11111") + await wstream2.write("11111\r\n") + await wstream2.write("22222") + await wstream2.write("22222\r\n") + await wstream2.finish() + await wstream.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -405,21 +429,24 @@ suite "AsyncStream test suite": test "AsyncStream(AsyncStream) read() test": proc testRead2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) - var s2 = "1111111111" - var s3 = "2222222222" - await wstream2.write("0000000000") - await wstream2.write(s2) - await wstream2.write(s3.toBytes()) - await wstream2.finish() - await wstream.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) + var s2 = "1111111111" + var s3 = "2222222222" + await wstream2.write("0000000000") + await wstream2.write(s2) + await wstream2.write(s3.toBytes()) + await wstream2.finish() + await wstream.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -446,31 +473,34 @@ suite "AsyncStream test suite": test "AsyncStream(AsyncStream) consume() test": proc testConsume2(): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - const - S4 = @[byte('3'), byte('3'), byte('3'), byte('3'), byte('3')] - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) + transp: StreamTransport) {.async: (raises: []).} = + try: + const + S4 = @[byte('3'), byte('3'), byte('3'), byte('3'), byte('3')] + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) - var s1 = "00000" - var s2 = "11111".toBytes() - var s3 = "22222" + var s1 = "00000" + var s2 = "11111".toBytes() + var s3 = "22222" - await wstream2.write("00000") - await wstream2.write(s1) - await wstream2.write("11111") - await wstream2.write(s2) - await wstream2.write("22222") - await wstream2.write(addr s3[0], len(s3)) - await wstream2.write("33333") - await wstream2.write(S4) - await wstream2.finish() - await wstream.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + await wstream2.write("00000") + await wstream2.write(s1) + await wstream2.write("11111") + await wstream2.write(s2) + await wstream2.write("22222") + await wstream2.write(addr s3[0], len(s3)) + await wstream2.write("33333") + await wstream2.write(S4) + await wstream2.finish() + await wstream.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -511,27 +541,30 @@ suite "AsyncStream test suite": message = createBigMessage("ABCDEFGHIJKLMNOP", size) proc processClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wbstream = newBoundedStreamWriter(wstream, uint64(size)) + transp: StreamTransport) {.async: (raises: []).} = try: - check wbstream.atEof() == false - await wbstream.write(message) - check wbstream.atEof() == false - await wbstream.finish() - check wbstream.atEof() == true - expect AsyncStreamWriteEOFError: + var wstream = newAsyncStreamWriter(transp) + var wbstream = newBoundedStreamWriter(wstream, uint64(size)) + try: + check wbstream.atEof() == false await wbstream.write(message) - expect AsyncStreamWriteEOFError: - await wbstream.write(message) - expect AsyncStreamWriteEOFError: - await wbstream.write(message) - check wbstream.atEof() == true - await wbstream.closeWait() - check wbstream.atEof() == true - finally: - await wstream.closeWait() - await transp.closeWait() + check wbstream.atEof() == false + await wbstream.finish() + check wbstream.atEof() == true + expect AsyncStreamWriteEOFError: + await wbstream.write(message) + expect AsyncStreamWriteEOFError: + await wbstream.write(message) + expect AsyncStreamWriteEOFError: + await wbstream.write(message) + check wbstream.atEof() == true + await wbstream.closeWait() + check wbstream.atEof() == true + finally: + await wstream.closeWait() + await transp.closeWait() + except CatchableError as exc: + raiseAssert exc.msg let flags = {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay} var server = createStreamServer(initTAddress("127.0.0.1:0"), @@ -580,15 +613,18 @@ suite "ChunkedStream test suite": ] proc checkVector(inputstr: string): Future[string] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var data = inputstr - await wstream.write(data) - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var data = inputstr + await wstream.write(data) + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -630,15 +666,18 @@ suite "ChunkedStream test suite": ] proc checkVector(inputstr: string): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var data = inputstr - await wstream.write(data) - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var data = inputstr + await wstream.write(data) + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var res = false var server = createStreamServer(initTAddress("127.0.0.1:0"), @@ -713,14 +752,17 @@ suite "ChunkedStream test suite": test "ChunkedStream too big chunk header test": proc checkTooBigChunkHeader(inputstr: seq[byte]): Future[bool] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - await wstream.write(inputstr) - await wstream.finish() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + await wstream.write(inputstr) + await wstream.finish() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -751,23 +793,26 @@ suite "ChunkedStream test suite": proc checkVector(inputstr: seq[byte], chunkSize: int): Future[seq[byte]] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) - var data = inputstr - var offset = 0 - while true: - if len(data) == offset: - break - let toWrite = min(chunkSize, len(data) - offset) - await wstream2.write(addr data[offset], toWrite) - offset = offset + toWrite - await wstream2.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) + var data = inputstr + var offset = 0 + while true: + if len(data) == offset: + break + let toWrite = min(chunkSize, len(data) - offset) + await wstream2.write(addr data[offset], toWrite) + offset = offset + toWrite + await wstream2.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -796,23 +841,26 @@ suite "ChunkedStream test suite": writeChunkSize: int, readChunkSize: int): Future[seq[byte]] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newChunkedStreamWriter(wstream) - var data = inputstr - var offset = 0 - while true: - if len(data) == offset: - break - let toWrite = min(writeChunkSize, len(data) - offset) - await wstream2.write(addr data[offset], toWrite) - offset = offset + toWrite - await wstream2.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newChunkedStreamWriter(wstream) + var data = inputstr + var offset = 0 + while true: + if len(data) == offset: + break + let toWrite = min(writeChunkSize, len(data) - offset) + await wstream2.write(addr data[offset], toWrite) + offset = offset + toWrite + await wstream2.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -849,30 +897,33 @@ suite "TLSStream test suite": const HttpHeadersMark = @[byte(0x0D), byte(0x0A), byte(0x0D), byte(0x0A)] test "Simple HTTPS connection": proc headerClient(address: TransportAddress, - name: string): Future[bool] {.async.} = - var mark = "HTTP/1.1 " - var buffer = newSeq[byte](8192) - var transp = await connect(address) - var reader = newAsyncStreamReader(transp) - var writer = newAsyncStreamWriter(transp) - var tlsstream = newTLSClientAsyncStream(reader, writer, name) - await tlsstream.writer.write("GET / HTTP/1.1\r\nHost: " & name & - "\r\nConnection: close\r\n\r\n") - var readFut = tlsstream.reader.readUntil(addr buffer[0], len(buffer), - HttpHeadersMark) - let res = await withTimeout(readFut, 5.seconds) - if res: - var length = readFut.read() - buffer.setLen(length) - if len(buffer) > len(mark): - if equalMem(addr buffer[0], addr mark[0], len(mark)): - result = true + name: string): Future[bool] {.async: (raises: []).} = + try: + var mark = "HTTP/1.1 " + var buffer = newSeq[byte](8192) + var transp = await connect(address) + var reader = newAsyncStreamReader(transp) + var writer = newAsyncStreamWriter(transp) + var tlsstream = newTLSClientAsyncStream(reader, writer, name) + await tlsstream.writer.write("GET / HTTP/1.1\r\nHost: " & name & + "\r\nConnection: close\r\n\r\n") + var readFut = tlsstream.reader.readUntil(addr buffer[0], len(buffer), + HttpHeadersMark) + let res = await withTimeout(readFut, 5.seconds) + if res: + var length = readFut.read() + buffer.setLen(length) + if len(buffer) > len(mark): + if equalMem(addr buffer[0], addr mark[0], len(mark)): + result = true - await tlsstream.reader.closeWait() - await tlsstream.writer.closeWait() - await reader.closeWait() - await writer.closeWait() - await transp.closeWait() + await tlsstream.reader.closeWait() + await tlsstream.writer.closeWait() + await reader.closeWait() + await writer.closeWait() + await transp.closeWait() + except CatchableError as exc: + raiseAssert exc.msg let res = waitFor(headerClient(resolveTAddress("www.google.com:443")[0], "www.google.com")) @@ -884,20 +935,23 @@ suite "TLSStream test suite": let testMessage = "TEST MESSAGE" proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var reader = newAsyncStreamReader(transp) - var writer = newAsyncStreamWriter(transp) - var sstream = newTLSServerAsyncStream(reader, writer, key, cert) - await handshake(sstream) - await sstream.writer.write(testMessage & "\r\n") - await sstream.writer.finish() - await sstream.writer.closeWait() - await sstream.reader.closeWait() - await reader.closeWait() - await writer.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var reader = newAsyncStreamReader(transp) + var writer = newAsyncStreamWriter(transp) + var sstream = newTLSServerAsyncStream(reader, writer, key, cert) + await handshake(sstream) + await sstream.writer.write(testMessage & "\r\n") + await sstream.writer.finish() + await sstream.writer.closeWait() + await sstream.reader.closeWait() + await reader.closeWait() + await writer.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg key = TLSPrivateKey.init(pemkey) cert = TLSCertificate.init(pemcert) @@ -931,20 +985,23 @@ suite "TLSStream test suite": let trustAnchors = TrustAnchorStore.new(SelfSignedTrustAnchors) proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var reader = newAsyncStreamReader(transp) - var writer = newAsyncStreamWriter(transp) - var sstream = newTLSServerAsyncStream(reader, writer, key, cert) - await handshake(sstream) - await sstream.writer.write(testMessage & "\r\n") - await sstream.writer.finish() - await sstream.writer.closeWait() - await sstream.reader.closeWait() - await reader.closeWait() - await writer.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var reader = newAsyncStreamReader(transp) + var writer = newAsyncStreamWriter(transp) + var sstream = newTLSServerAsyncStream(reader, writer, key, cert) + await handshake(sstream) + await sstream.writer.write(testMessage & "\r\n") + await sstream.writer.finish() + await sstream.writer.closeWait() + await sstream.reader.closeWait() + await reader.closeWait() + await writer.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -988,46 +1045,49 @@ suite "BoundedStream test suite": var clientRes = false proc processClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - case btest - of BoundaryRead: - await wstream.write(message) - await wstream.write(boundary) - await wstream.finish() - await wstream.closeWait() - clientRes = true - of BoundaryDouble: - await wstream.write(message) - await wstream.write(boundary) - await wstream.write(message) - await wstream.finish() - await wstream.closeWait() - clientRes = true - of BoundarySize: - var ncmessage = message - ncmessage.setLen(len(message) - 2) - await wstream.write(ncmessage) - await wstream.write(@[0x2D'u8, 0x2D'u8]) - await wstream.finish() - await wstream.closeWait() - clientRes = true - of BoundaryIncomplete: - var ncmessage = message - ncmessage.setLen(len(message) - 2) - await wstream.write(ncmessage) - await wstream.finish() - await wstream.closeWait() - clientRes = true - of BoundaryEmpty: - await wstream.write(boundary) - await wstream.finish() - await wstream.closeWait() - clientRes = true + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + case btest + of BoundaryRead: + await wstream.write(message) + await wstream.write(boundary) + await wstream.finish() + await wstream.closeWait() + clientRes = true + of BoundaryDouble: + await wstream.write(message) + await wstream.write(boundary) + await wstream.write(message) + await wstream.finish() + await wstream.closeWait() + clientRes = true + of BoundarySize: + var ncmessage = message + ncmessage.setLen(len(message) - 2) + await wstream.write(ncmessage) + await wstream.write(@[0x2D'u8, 0x2D'u8]) + await wstream.finish() + await wstream.closeWait() + clientRes = true + of BoundaryIncomplete: + var ncmessage = message + ncmessage.setLen(len(message) - 2) + await wstream.write(ncmessage) + await wstream.finish() + await wstream.closeWait() + clientRes = true + of BoundaryEmpty: + await wstream.write(boundary) + await wstream.finish() + await wstream.closeWait() + clientRes = true - await transp.closeWait() - server.stop() - server.close() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var res = false let flags = {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay} @@ -1090,60 +1150,63 @@ suite "BoundedStream test suite": message.add(messagePart) proc processClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wbstream = newBoundedStreamWriter(wstream, uint64(size), - comparison = cmp) - case stest - of SizeReadWrite: - for i in 0 ..< 10: - await wbstream.write(messagePart) - await wbstream.finish() - await wbstream.closeWait() - clientRes = true - of SizeOverflow: - for i in 0 ..< 10: - await wbstream.write(messagePart) - try: - await wbstream.write(messagePart) - except BoundedStreamOverflowError: + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wbstream = newBoundedStreamWriter(wstream, uint64(size), + comparison = cmp) + case stest + of SizeReadWrite: + for i in 0 ..< 10: + await wbstream.write(messagePart) + await wbstream.finish() + await wbstream.closeWait() clientRes = true - await wbstream.closeWait() - of SizeIncomplete: - for i in 0 ..< 9: - await wbstream.write(messagePart) - case cmp - of BoundCmp.Equal: + of SizeOverflow: + for i in 0 ..< 10: + await wbstream.write(messagePart) try: - await wbstream.finish() - except BoundedStreamIncompleteError: + await wbstream.write(messagePart) + except BoundedStreamOverflowError: clientRes = true - of BoundCmp.LessOrEqual: - try: - await wbstream.finish() - clientRes = true - except BoundedStreamIncompleteError: - discard - await wbstream.closeWait() - of SizeEmpty: - case cmp - of BoundCmp.Equal: - try: - await wbstream.finish() - except BoundedStreamIncompleteError: - clientRes = true - of BoundCmp.LessOrEqual: - try: - await wbstream.finish() - clientRes = true - except BoundedStreamIncompleteError: - discard - await wbstream.closeWait() + await wbstream.closeWait() + of SizeIncomplete: + for i in 0 ..< 9: + await wbstream.write(messagePart) + case cmp + of BoundCmp.Equal: + try: + await wbstream.finish() + except BoundedStreamIncompleteError: + clientRes = true + of BoundCmp.LessOrEqual: + try: + await wbstream.finish() + clientRes = true + except BoundedStreamIncompleteError: + discard + await wbstream.closeWait() + of SizeEmpty: + case cmp + of BoundCmp.Equal: + try: + await wbstream.finish() + except BoundedStreamIncompleteError: + clientRes = true + of BoundCmp.LessOrEqual: + try: + await wbstream.finish() + clientRes = true + except BoundedStreamIncompleteError: + discard + await wbstream.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg let flags = {ServerFlags.ReuseAddr, ServerFlags.TcpNoDelay} var server = createStreamServer(initTAddress("127.0.0.1:0"), @@ -1243,23 +1306,26 @@ suite "BoundedStream test suite": writeChunkSize: int, readChunkSize: int): Future[seq[byte]] {.async.} = proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newBoundedStreamWriter(wstream, uint64(len(inputstr))) - var data = inputstr - var offset = 0 - while true: - if len(data) == offset: - break - let toWrite = min(writeChunkSize, len(data) - offset) - await wstream2.write(addr data[offset], toWrite) - offset = offset + toWrite - await wstream2.finish() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newBoundedStreamWriter(wstream, uint64(len(inputstr))) + var data = inputstr + var offset = 0 + while true: + if len(data) == offset: + break + let toWrite = min(writeChunkSize, len(data) - offset) + await wstream2.write(addr data[offset], toWrite) + offset = offset + toWrite + await wstream2.finish() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) @@ -1293,17 +1359,20 @@ suite "BoundedStream test suite": proc checkEmptyStreams(): Future[bool] {.async.} = var writer1Res = false proc serveClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var wstream = newAsyncStreamWriter(transp) - var wstream2 = newBoundedStreamWriter(wstream, 0'u64) - await wstream2.finish() - let res = wstream2.atEof() - await wstream2.closeWait() - await wstream.closeWait() - await transp.closeWait() - server.stop() - server.close() - writer1Res = res + transp: StreamTransport) {.async: (raises: []).} = + try: + var wstream = newAsyncStreamWriter(transp) + var wstream2 = newBoundedStreamWriter(wstream, 0'u64) + await wstream2.finish() + let res = wstream2.atEof() + await wstream2.closeWait() + await wstream.closeWait() + await transp.closeWait() + server.stop() + server.close() + writer1Res = res + except CatchableError as exc: + raiseAssert exc.msg var server = createStreamServer(initTAddress("127.0.0.1:0"), serveClient, {ReuseAddr}) diff --git a/tests/testbugs.nim b/tests/testbugs.nim index 1f2a932..fc4af3a 100644 --- a/tests/testbugs.nim +++ b/tests/testbugs.nim @@ -21,16 +21,19 @@ suite "Asynchronous issues test suite": test: string proc udp4DataAvailable(transp: DatagramTransport, - remote: TransportAddress) {.async, gcsafe.} = - var udata = getUserData[CustomData](transp) - var expect = TEST_MSG - var data: seq[byte] - var datalen: int - transp.peekMessage(data, datalen) - if udata.test == "CHECK" and datalen == MSG_LEN and - equalMem(addr data[0], addr expect[0], datalen): - udata.test = "OK" - transp.close() + remote: TransportAddress) {.async: (raises: []).} = + try: + var udata = getUserData[CustomData](transp) + var expect = TEST_MSG + var data: seq[byte] + var datalen: int + transp.peekMessage(data, datalen) + if udata.test == "CHECK" and datalen == MSG_LEN and + equalMem(addr data[0], addr expect[0], datalen): + udata.test = "OK" + transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc issue6(): Future[bool] {.async.} = var myself = initTAddress("127.0.0.1:" & $HELLO_PORT) diff --git a/tests/testdatagram.nim b/tests/testdatagram.nim index c941761..bd33ef3 100644 --- a/tests/testdatagram.nim +++ b/tests/testdatagram.nim @@ -30,286 +30,319 @@ suite "Datagram Transport test suite": " clients x " & $MessagesCount & " messages)" proc client1(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("REQUEST"): - var numstr = data[7..^1] - var num = parseInt(numstr) - var ans = "ANSWER" & $num - await transp.sendTo(raddr, addr ans[0], len(ans)) + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("REQUEST"): + var numstr = data[7..^1] + var num = parseInt(numstr) + var ans = "ANSWER" & $num + await transp.sendTo(raddr, addr ans[0], len(ans)) + else: + var err = "ERROR" + await transp.sendTo(raddr, addr err[0], len(err)) else: - var err = "ERROR" - await transp.sendTo(raddr, addr err[0], len(err)) - else: - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client2(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == TestsCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == TestsCount: + transp.close() + else: + var ta = initTAddress("127.0.0.1:33336") + var req = "REQUEST" & $counterPtr[] + await transp.sendTo(ta, addr req[0], len(req)) else: - var ta = initTAddress("127.0.0.1:33336") - var req = "REQUEST" & $counterPtr[] - await transp.sendTo(ta, addr req[0], len(req)) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client3(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == TestsCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == TestsCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + await transp.send(addr req[0], len(req)) else: - var req = "REQUEST" & $counterPtr[] - await transp.send(addr req[0], len(req)) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client4(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == MessagesCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == MessagesCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + await transp.send(addr req[0], len(req)) else: - var req = "REQUEST" & $counterPtr[] - await transp.send(addr req[0], len(req)) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client5(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == MessagesCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == MessagesCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + await transp.sendTo(raddr, addr req[0], len(req)) else: - var req = "REQUEST" & $counterPtr[] - await transp.sendTo(raddr, addr req[0], len(req)) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client6(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("REQUEST"): - var numstr = data[7..^1] - var num = parseInt(numstr) - var ans = "ANSWER" & $num - await transp.sendTo(raddr, ans) + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("REQUEST"): + var numstr = data[7..^1] + var num = parseInt(numstr) + var ans = "ANSWER" & $num + await transp.sendTo(raddr, ans) + else: + var err = "ERROR" + await transp.sendTo(raddr, err) else: - var err = "ERROR" - await transp.sendTo(raddr, err) - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + ## Read operation failed with error + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client7(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == TestsCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == TestsCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + await transp.sendTo(raddr, req) else: - var req = "REQUEST" & $counterPtr[] - await transp.sendTo(raddr, req) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client8(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == TestsCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == TestsCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + await transp.send(req) else: - var req = "REQUEST" & $counterPtr[] - await transp.send(req) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client9(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("REQUEST"): - var numstr = data[7..^1] - var num = parseInt(numstr) - var ans = "ANSWER" & $num - var ansseq = newSeq[byte](len(ans)) - copyMem(addr ansseq[0], addr ans[0], len(ans)) - await transp.sendTo(raddr, ansseq) + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("REQUEST"): + var numstr = data[7..^1] + var num = parseInt(numstr) + var ans = "ANSWER" & $num + var ansseq = newSeq[byte](len(ans)) + copyMem(addr ansseq[0], addr ans[0], len(ans)) + await transp.sendTo(raddr, ansseq) + else: + var err = "ERROR" + var errseq = newSeq[byte](len(err)) + copyMem(addr errseq[0], addr err[0], len(err)) + await transp.sendTo(raddr, errseq) else: - var err = "ERROR" - var errseq = newSeq[byte](len(err)) - copyMem(addr errseq[0], addr err[0], len(err)) - await transp.sendTo(raddr, errseq) - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + ## Read operation failed with error + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client10(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == TestsCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == TestsCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + var reqseq = newSeq[byte](len(req)) + copyMem(addr reqseq[0], addr req[0], len(req)) + await transp.sendTo(raddr, reqseq) else: - var req = "REQUEST" & $counterPtr[] - var reqseq = newSeq[byte](len(req)) - copyMem(addr reqseq[0], addr req[0], len(req)) - await transp.sendTo(raddr, reqseq) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc client11(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var pbytes = transp.getMessage() - var nbytes = len(pbytes) - if nbytes > 0: - var data = newString(nbytes + 1) - copyMem(addr data[0], addr pbytes[0], nbytes) - data.setLen(nbytes) - if data.startsWith("ANSWER"): - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = counterPtr[] + 1 - if counterPtr[] == TestsCount: - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var pbytes = transp.getMessage() + var nbytes = len(pbytes) + if nbytes > 0: + var data = newString(nbytes + 1) + copyMem(addr data[0], addr pbytes[0], nbytes) + data.setLen(nbytes) + if data.startsWith("ANSWER"): + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = counterPtr[] + 1 + if counterPtr[] == TestsCount: + transp.close() + else: + var req = "REQUEST" & $counterPtr[] + var reqseq = newSeq[byte](len(req)) + copyMem(addr reqseq[0], addr req[0], len(req)) + await transp.send(reqseq) else: - var req = "REQUEST" & $counterPtr[] - var reqseq = newSeq[byte](len(req)) - copyMem(addr reqseq[0], addr req[0], len(req)) - await transp.send(reqseq) + var counterPtr = cast[ptr int](transp.udata) + counterPtr[] = -1 + transp.close() else: + ## Read operation failed with error var counterPtr = cast[ptr int](transp.udata) counterPtr[] = -1 transp.close() - else: - ## Read operation failed with error - var counterPtr = cast[ptr int](transp.udata) - counterPtr[] = -1 - transp.close() + except CatchableError as exc: + raiseAssert exc.msg proc testPointerSendTo(): Future[int] {.async.} = ## sendTo(pointer) test @@ -439,7 +472,7 @@ suite "Datagram Transport test suite": var ta = initTAddress("127.0.0.1:0") var counter = 0 proc clientMark(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = + raddr: TransportAddress): Future[void] {.async: (raises: []).} = counter = 1 transp.close() var dgram1 = newDatagramTransport(client1, local = ta) @@ -457,7 +490,7 @@ suite "Datagram Transport test suite": proc testTransportClose(): Future[bool] {.async.} = var ta = initTAddress("127.0.0.1:45000") proc clientMark(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = + raddr: TransportAddress): Future[void] {.async: (raises: []).} = discard var dgram = newDatagramTransport(clientMark, local = ta) dgram.close() @@ -473,12 +506,15 @@ suite "Datagram Transport test suite": var bta = initTAddress("255.255.255.255:45010") var res = 0 proc clientMark(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var bmsg = transp.getMessage() - var smsg = string.fromBytes(bmsg) - if smsg == expectMessage: - inc(res) - transp.close() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var bmsg = transp.getMessage() + var smsg = string.fromBytes(bmsg) + if smsg == expectMessage: + inc(res) + transp.close() + except CatchableError as exc: + raiseAssert exc.msg var dgram1 = newDatagramTransport(clientMark, local = ta1, flags = {Broadcast}, ttl = 2) await dgram1.sendTo(bta, expectMessage) @@ -493,15 +529,19 @@ suite "Datagram Transport test suite": var event = newAsyncEvent() proc clientMark1(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var bmsg = transp.getMessage() - var smsg = string.fromBytes(bmsg) - if smsg == expectStr: - inc(res) - event.fire() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var bmsg = transp.getMessage() + var smsg = string.fromBytes(bmsg) + if smsg == expectStr: + inc(res) + event.fire() + except CatchableError as exc: + raiseAssert exc.msg + proc clientMark2(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = + raddr: TransportAddress): Future[void] {.async: (raises: []).} = discard var dgram1 = newDatagramTransport(clientMark1, local = ta) @@ -544,15 +584,18 @@ suite "Datagram Transport test suite": res = 0 proc process1(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = - var bmsg = transp.getMessage() - var smsg = string.fromBytes(bmsg) - if smsg == expectStr: - inc(res) - event.fire() + raddr: TransportAddress): Future[void] {.async: (raises: []).} = + try: + var bmsg = transp.getMessage() + var smsg = string.fromBytes(bmsg) + if smsg == expectStr: + inc(res) + event.fire() + except CatchableError as exc: + raiseAssert exc.msg proc process2(transp: DatagramTransport, - raddr: TransportAddress): Future[void] {.async.} = + raddr: TransportAddress): Future[void] {.async: (raises: []).} = discard let diff --git a/tests/testmacro.nim b/tests/testmacro.nim index 1361193..0133793 100644 --- a/tests/testmacro.nim +++ b/tests/testmacro.nim @@ -459,20 +459,31 @@ suite "Exceptions tracking": check waitFor(test1()) == 12 proc test2: Future[int] {.async: (raw: true, raises: [IOError, OSError]).} = + checkNotCompiles: + result.fail(newException(ValueError, "fail")) + result = newFuture[int]() result.fail(newException(IOError, "fail")) - result.fail(newException(OSError, "fail")) - checkNotCompiles: - result.fail(newException(ValueError, "fail")) proc test3: Future[void] {.async: (raw: true, raises: []).} = + result = newFuture[void]() checkNotCompiles: result.fail(newException(ValueError, "fail")) - + result.complete() # Inheritance proc test4: Future[void] {.async: (raw: true, raises: [CatchableError]).} = + result = newFuture[void]() result.fail(newException(IOError, "fail")) + check: + waitFor(test1()) == 12 + expect(IOError): + discard waitFor(test2()) + + waitFor(test3()) + expect(IOError): + waitFor(test4()) + test "or errors": proc testit {.async: (raises: [ValueError]).} = raise (ref ValueError)() diff --git a/tests/testserver.nim b/tests/testserver.nim index a63c9df..280148c 100644 --- a/tests/testserver.nim +++ b/tests/testserver.nim @@ -27,29 +27,36 @@ suite "Server's test suite": checkLeaks() proc serveStreamClient(server: StreamServer, - transp: StreamTransport) {.async.} = + transp: StreamTransport) {.async: (raises: []).} = discard proc serveCustomStreamClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var cserver = cast[CustomServer](server) - var ctransp = cast[CustomTransport](transp) - cserver.test1 = "CONNECTION" - cserver.test2 = ctransp.test - cserver.test3 = await transp.readLine() - var answer = "ANSWER\r\n" - discard await transp.write(answer) - transp.close() - await transp.join() + transp: StreamTransport) {.async: (raises: []).} = + try: + var cserver = cast[CustomServer](server) + var ctransp = cast[CustomTransport](transp) + cserver.test1 = "CONNECTION" + cserver.test2 = ctransp.test + cserver.test3 = await transp.readLine() + var answer = "ANSWER\r\n" + discard await transp.write(answer) + transp.close() + await transp.join() + except CatchableError as exc: + raiseAssert exc.msg + proc serveUdataStreamClient(server: StreamServer, - transp: StreamTransport) {.async.} = - var udata = getUserData[CustomData](server) - var line = await transp.readLine() - var msg = line & udata.test & "\r\n" - discard await transp.write(msg) - transp.close() - await transp.join() + transp: StreamTransport) {.async: (raises: []).} = + try: + var udata = getUserData[CustomData](server) + var line = await transp.readLine() + var msg = line & udata.test & "\r\n" + discard await transp.write(msg) + transp.close() + await transp.join() + except CatchableError as exc: + raiseAssert exc.msg proc customServerTransport(server: StreamServer, fd: AsyncFD): StreamTransport = diff --git a/tests/teststream.nim b/tests/teststream.nim index b042792..fb5534b 100644 --- a/tests/teststream.nim +++ b/tests/teststream.nim @@ -55,124 +55,148 @@ suite "Stream Transport test suite": for i in 0 ..< len(result): result[i] = byte(message[i mod len(message)]) - proc serveClient1(server: StreamServer, transp: StreamTransport) {.async.} = - while not transp.atEof(): - var data = await transp.readLine() - if len(data) == 0: - doAssert(transp.atEof()) - break - doAssert(data.startsWith("REQUEST")) - var numstr = data[7..^1] - var num = parseInt(numstr) - var ans = "ANSWER" & $num & "\r\n" - var res = await transp.write(cast[pointer](addr ans[0]), len(ans)) - doAssert(res == len(ans)) - transp.close() - await transp.join() + proc serveClient1(server: StreamServer, transp: StreamTransport) {. + async: (raises: []).} = + try: + while not transp.atEof(): + var data = await transp.readLine() + if len(data) == 0: + doAssert(transp.atEof()) + break + doAssert(data.startsWith("REQUEST")) + var numstr = data[7..^1] + var num = parseInt(numstr) + var ans = "ANSWER" & $num & "\r\n" + var res = await transp.write(cast[pointer](addr ans[0]), len(ans)) + doAssert(res == len(ans)) + transp.close() + await transp.join() + except CatchableError as exc: + raiseAssert exc.msg - proc serveClient2(server: StreamServer, transp: StreamTransport) {.async.} = - var buffer: array[20, char] - var check = "REQUEST" - while not transp.atEof(): - zeroMem(addr buffer[0], MessageSize) - try: - await transp.readExactly(addr buffer[0], MessageSize) - except TransportIncompleteError: - break - doAssert(equalMem(addr buffer[0], addr check[0], len(check))) - var numstr = "" - var i = 7 - while i < MessageSize and (buffer[i] in {'0'..'9'}): - numstr.add(buffer[i]) - inc(i) - var num = parseInt(numstr) - var ans = "ANSWER" & $num - zeroMem(addr buffer[0], MessageSize) - copyMem(addr buffer[0], addr ans[0], len(ans)) - var res = await transp.write(cast[pointer](addr buffer[0]), MessageSize) - doAssert(res == MessageSize) - transp.close() - await transp.join() + proc serveClient2(server: StreamServer, transp: StreamTransport) {. + async: (raises: []).} = + try: + var buffer: array[20, char] + var check = "REQUEST" + while not transp.atEof(): + zeroMem(addr buffer[0], MessageSize) + try: + await transp.readExactly(addr buffer[0], MessageSize) + except TransportIncompleteError: + break + doAssert(equalMem(addr buffer[0], addr check[0], len(check))) + var numstr = "" + var i = 7 + while i < MessageSize and (buffer[i] in {'0'..'9'}): + numstr.add(buffer[i]) + inc(i) + var num = parseInt(numstr) + var ans = "ANSWER" & $num + zeroMem(addr buffer[0], MessageSize) + copyMem(addr buffer[0], addr ans[0], len(ans)) + var res = await transp.write(cast[pointer](addr buffer[0]), MessageSize) + doAssert(res == MessageSize) + transp.close() + await transp.join() + except CatchableError as exc: + raiseAssert exc.msg - proc serveClient3(server: StreamServer, transp: StreamTransport) {.async.} = - var buffer: array[20, char] - var check = "REQUEST" - var suffixStr = "SUFFIX" - var suffix = newSeq[byte](6) - copyMem(addr suffix[0], addr suffixStr[0], len(suffixStr)) - var counter = MessagesCount - while counter > 0: - zeroMem(addr buffer[0], MessageSize) - var res = await transp.readUntil(addr buffer[0], MessageSize, suffix) - doAssert(equalMem(addr buffer[0], addr check[0], len(check))) - var numstr = "" - var i = 7 - while i < MessageSize and (buffer[i] in {'0'..'9'}): - numstr.add(buffer[i]) - inc(i) - var num = parseInt(numstr) - doAssert(len(numstr) < 8) - var ans = "ANSWER" & $num & "SUFFIX" - zeroMem(addr buffer[0], MessageSize) - copyMem(addr buffer[0], addr ans[0], len(ans)) - res = await transp.write(cast[pointer](addr buffer[0]), len(ans)) - doAssert(res == len(ans)) - dec(counter) - transp.close() - await transp.join() + proc serveClient3(server: StreamServer, transp: StreamTransport) {. + async: (raises: []).} = + try: + var buffer: array[20, char] + var check = "REQUEST" + var suffixStr = "SUFFIX" + var suffix = newSeq[byte](6) + copyMem(addr suffix[0], addr suffixStr[0], len(suffixStr)) + var counter = MessagesCount + while counter > 0: + zeroMem(addr buffer[0], MessageSize) + var res = await transp.readUntil(addr buffer[0], MessageSize, suffix) + doAssert(equalMem(addr buffer[0], addr check[0], len(check))) + var numstr = "" + var i = 7 + while i < MessageSize and (buffer[i] in {'0'..'9'}): + numstr.add(buffer[i]) + inc(i) + var num = parseInt(numstr) + doAssert(len(numstr) < 8) + var ans = "ANSWER" & $num & "SUFFIX" + zeroMem(addr buffer[0], MessageSize) + copyMem(addr buffer[0], addr ans[0], len(ans)) + res = await transp.write(cast[pointer](addr buffer[0]), len(ans)) + doAssert(res == len(ans)) + dec(counter) + transp.close() + await transp.join() + except CatchableError as exc: + raiseAssert exc.msg - proc serveClient4(server: StreamServer, transp: StreamTransport) {.async.} = - var pathname = await transp.readLine() - var size = await transp.readLine() - var sizeNum = parseInt(size) - doAssert(sizeNum >= 0) - var rbuffer = newSeq[byte](sizeNum) - await transp.readExactly(addr rbuffer[0], sizeNum) - var lbuffer = readFile(pathname) - doAssert(len(lbuffer) == sizeNum) - doAssert(equalMem(addr rbuffer[0], addr lbuffer[0], sizeNum)) - var answer = "OK\r\n" - var res = await transp.write(cast[pointer](addr answer[0]), len(answer)) - doAssert(res == len(answer)) - transp.close() - await transp.join() + proc serveClient4(server: StreamServer, transp: StreamTransport) {. + async: (raises: []).} = + try: + var pathname = await transp.readLine() + var size = await transp.readLine() + var sizeNum = parseInt(size) + doAssert(sizeNum >= 0) + var rbuffer = newSeq[byte](sizeNum) + await transp.readExactly(addr rbuffer[0], sizeNum) + var lbuffer = readFile(pathname) + doAssert(len(lbuffer) == sizeNum) + doAssert(equalMem(addr rbuffer[0], addr lbuffer[0], sizeNum)) + var answer = "OK\r\n" + var res = await transp.write(cast[pointer](addr answer[0]), len(answer)) + doAssert(res == len(answer)) + transp.close() + await transp.join() + except CatchableError as exc: + raiseAssert exc.msg - proc serveClient7(server: StreamServer, transp: StreamTransport) {.async.} = - var answer = "DONE\r\n" - var expect = "" - var line = await transp.readLine() - doAssert(len(line) == BigMessageCount * len(BigMessagePattern)) - for i in 0.. Date: Fri, 17 Nov 2023 13:45:17 +0100 Subject: [PATCH 42/50] dedicated exceptions for `Future.read` failures (#474) Dedicated exceptions for `read` failures reduce the risk of mixing up "user" exceptions with those of Future itself. The risk still exists, if the user allows a chronos exception to bubble up explicitly. Because `await` structurally guarantees that the Future is not `pending` at the time of `read`, it does not raise this new exception. * introduce `FuturePendingError` and `FutureCompletedError` when `read`:ing a future of uncertain state * fix `waitFor` / `read` to return `lent` values * simplify code generation for `void`-returning async procs * document `Raising` type helper --- chronos/futures.nim | 22 +-- chronos/internal/asyncfutures.nim | 221 ++++++++++++++++++++---------- chronos/internal/asyncmacro.nim | 177 ++++++++++++++---------- docs/src/async_procs.md | 18 ++- docs/src/concepts.md | 20 ++- docs/src/error_handling.md | 15 ++ docs/src/porting.md | 9 +- 7 files changed, 315 insertions(+), 167 deletions(-) diff --git a/chronos/futures.nim b/chronos/futures.nim index 0af635f..6fb9592 100644 --- a/chronos/futures.nim +++ b/chronos/futures.nim @@ -73,10 +73,15 @@ type cause*: FutureBase FutureError* = object of CatchableError + future*: FutureBase CancelledError* = object of FutureError ## Exception raised when accessing the value of a cancelled future +func raiseFutureDefect(msg: static string, fut: FutureBase) {. + noinline, noreturn.} = + raise (ref FutureDefect)(msg: msg, cause: fut) + when chronosFutureId: var currentID* {.threadvar.}: uint template id*(fut: FutureBase): uint = fut.internalId @@ -202,13 +207,11 @@ func value*[T: not void](future: Future[T]): lent T = ## Return the value in a completed future - raises Defect when ## `fut.completed()` is `false`. ## - ## See `read` for a version that raises an catchable error when future + ## See `read` for a version that raises a catchable error when future ## has not completed. when chronosStrictFutureAccess: if not future.completed(): - raise (ref FutureDefect)( - msg: "Future not completed while accessing value", - cause: future) + raiseFutureDefect("Future not completed while accessing value", future) future.internalValue @@ -216,13 +219,11 @@ func value*(future: Future[void]) = ## Return the value in a completed future - raises Defect when ## `fut.completed()` is `false`. ## - ## See `read` for a version that raises an catchable error when future + ## See `read` for a version that raises a catchable error when future ## has not completed. when chronosStrictFutureAccess: if not future.completed(): - raise (ref FutureDefect)( - msg: "Future not completed while accessing value", - cause: future) + raiseFutureDefect("Future not completed while accessing value", future) func error*(future: FutureBase): ref CatchableError = ## Return the error of `future`, or `nil` if future did not fail. @@ -231,9 +232,8 @@ func error*(future: FutureBase): ref CatchableError = ## future has not failed. when chronosStrictFutureAccess: if not future.failed() and not future.cancelled(): - raise (ref FutureDefect)( - msg: "Future not failed/cancelled while accessing error", - cause: future) + raiseFutureDefect( + "Future not failed/cancelled while accessing error", future) future.internalError diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index a36ff4a..f60b2d9 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -8,6 +8,9 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) +## Features and utilities for `Future` that integrate it with the dispatcher +## and the rest of the async machinery + {.push raises: [].} import std/[sequtils, macros] @@ -45,15 +48,28 @@ func `[]`*(loc: array[LocationKind, ptr SrcLoc], v: int): ptr SrcLoc {. type FutureStr*[T] = ref object of Future[T] - ## Future to hold GC strings + ## Deprecated gcholder*: string FutureSeq*[A, B] = ref object of Future[A] - ## Future to hold GC seqs + ## Deprecated gcholder*: seq[B] + FuturePendingError* = object of FutureError + ## Error raised when trying to `read` a Future that is still pending + FutureCompletedError* = object of FutureError + ## Error raised when trying access the error of a completed Future + SomeFuture = Future|InternalRaisesFuture +func raiseFuturePendingError(fut: FutureBase) {. + noinline, noreturn, raises: FuturePendingError.} = + raise (ref FuturePendingError)(msg: "Future is still pending", future: fut) +func raiseFutureCompletedError(fut: FutureBase) {. + noinline, noreturn, raises: FutureCompletedError.} = + raise (ref FutureCompletedError)( + msg: "Future is completed, cannot read error", future: fut) + # Backwards compatibility for old FutureState name template Finished* {.deprecated: "Use Completed instead".} = Completed template Finished*(T: type FutureState): FutureState {. @@ -479,6 +495,10 @@ macro internalCheckComplete*(fut: InternalRaisesFuture, raises: typed) = # generics are lost - so instead, we pass the raises list explicitly let types = getRaisesTypes(raises) + types.copyLineInfo(raises) + for t in types: + t.copyLineInfo(raises) + if isNoRaises(types): return quote do: if not(isNil(`fut`.internalError)): @@ -497,8 +517,8 @@ macro internalCheckComplete*(fut: InternalRaisesFuture, raises: typed) = quote do: discard ), nnkElseExpr.newTree( - nnkRaiseStmt.newNimNode(lineInfoFrom=fut).add( - quote do: (`fut`.internalError) + nnkRaiseStmt.newTree( + nnkDotExpr.newTree(fut, ident "internalError") ) ) ) @@ -520,39 +540,51 @@ macro internalCheckComplete*(fut: InternalRaisesFuture, raises: typed) = ifRaise ) -proc read*[T: not void](future: Future[T] ): lent T {.raises: [CatchableError].} = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if not future.finished(): - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") +proc readFinished[T: not void](fut: Future[T]): lent T {. + raises: [CatchableError].} = + # Read a future that is known to be finished, avoiding the extra exception + # effect. + internalCheckComplete(fut) + fut.internalValue - internalCheckComplete(future) - future.internalValue - -proc read*(future: Future[void] ) {.raises: [CatchableError].} = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. +proc read*[T: not void](fut: Future[T] ): lent T {.raises: [CatchableError].} = + ## Retrieves the value of `fut`. ## - ## If the result of the future is an error then that error will be raised. - if future.finished(): - internalCheckComplete(future) - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - -proc readError*(future: FutureBase): ref CatchableError {.raises: [ValueError].} = - ## Retrieves the exception stored in ``future``. + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. ## - ## An ``ValueError`` exception will be thrown if no exception exists - ## in the specified Future. - if not(isNil(future.error)): - return future.error - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "No error in future.") + ## If the future is still pending, `FuturePendingError` will be raised. + if not fut.finished(): + raiseFuturePendingError(fut) + + fut.readFinished() + +proc read*(fut: Future[void]) {.raises: [CatchableError].} = + ## Checks that `fut` completed. + ## + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. + ## + ## If the future is still pending, `FuturePendingError` will be raised. + if not fut.finished(): + raiseFuturePendingError(fut) + + internalCheckComplete(fut) + +proc readError*(fut: FutureBase): ref CatchableError {.raises: [FutureError].} = + ## Retrieves the exception of the failed or cancelled `fut`. + ## + ## If the future was completed with a value, `FutureCompletedError` will be + ## raised. + ## + ## If the future is still pending, `FuturePendingError` will be raised. + if not fut.finished(): + raiseFuturePendingError(fut) + + if isNil(fut.error): + raiseFutureCompletedError(fut) + + fut.error template taskFutureLocation(future: FutureBase): string = let loc = future.location[LocationKind.Create] @@ -568,18 +600,46 @@ template taskErrorMessage(future: FutureBase): string = template taskCancelMessage(future: FutureBase): string = "Asynchronous task " & taskFutureLocation(future) & " was cancelled!" -proc waitFor*[T](fut: Future[T]): T {.raises: [CatchableError].} = - ## **Blocks** the current thread until the specified future finishes and - ## reads it, potentially raising an exception if the future failed or was - ## cancelled. - var finished = false - # Ensure that callbacks currently scheduled on the future run before returning - proc continuation(udata: pointer) {.gcsafe.} = finished = true +proc pollFor[F: Future | InternalRaisesFuture](fut: F): F {.raises: [].} = + # Blocks the current thread of execution until `fut` has finished, returning + # the given future. + # + # Must not be called recursively (from inside `async` procedures). + # + # See alse `awaitne`. if not(fut.finished()): + var finished = false + # Ensure that callbacks currently scheduled on the future run before returning + proc continuation(udata: pointer) {.gcsafe.} = finished = true fut.addCallback(continuation) + while not(finished): poll() - fut.read() + + fut + +proc waitFor*[T: not void](fut: Future[T]): lent T {.raises: [CatchableError].} = + ## Blocks the current thread of execution until `fut` has finished, returning + ## its value. + ## + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. + ## + ## Must not be called recursively (from inside `async` procedures). + ## + ## See also `await`, `Future.read` + pollFor(fut).readFinished() + +proc waitFor*(fut: Future[void]) {.raises: [CatchableError].} = + ## Blocks the current thread of execution until `fut` has finished. + ## + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. + ## + ## Must not be called recursively (from inside `async` procedures). + ## + ## See also `await`, `Future.read` + pollFor(fut).internalCheckComplete() proc asyncSpawn*(future: Future[void]) = ## Spawns a new concurrent async task. @@ -943,7 +1003,7 @@ proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] {. retFuture -template cancelAndWait*(future: FutureBase): Future[void] = +template cancelAndWait*(future: FutureBase): Future[void].Raising([CancelledError]) = ## Cancel ``future``. cancelAndWait(future, getSrcLocation()) @@ -1500,37 +1560,56 @@ when defined(windows): {.pop.} # Automatically deduced raises from here onwards -proc waitFor*[T, E](fut: InternalRaisesFuture[T, E]): T = # {.raises: [E]} - ## **Blocks** the current thread until the specified future finishes and - ## reads it, potentially raising an exception if the future failed or was - ## cancelled. - while not(fut.finished()): - poll() +proc readFinished[T: not void; E](fut: InternalRaisesFuture[T, E]): lent T = + internalCheckComplete(fut, E) + fut.internalValue - fut.read() - -proc read*[T: not void, E](future: InternalRaisesFuture[T, E]): lent T = # {.raises: [E, ValueError].} - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. +proc read*[T: not void, E](fut: InternalRaisesFuture[T, E]): lent T = # {.raises: [E, FuturePendingError].} + ## Retrieves the value of `fut`. ## - ## If the result of the future is an error then that error will be raised. - if not future.finished(): - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - - internalCheckComplete(future, E) - future.internalValue - -proc read*[E](future: InternalRaisesFuture[void, E]) = # {.raises: [E, CancelledError].} - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. ## - ## If the result of the future is an error then that error will be raised. - if future.finished(): - internalCheckComplete(future) - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") + ## If the future is still pending, `FuturePendingError` will be raised. + if not fut.finished(): + raiseFuturePendingError(fut) + + fut.readFinished() + +proc read*[E](fut: InternalRaisesFuture[void, E]) = # {.raises: [E].} + ## Checks that `fut` completed. + ## + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. + ## + ## If the future is still pending, `FuturePendingError` will be raised. + if not fut.finished(): + raiseFuturePendingError(fut) + + internalCheckComplete(fut, E) + +proc waitFor*[T: not void; E](fut: InternalRaisesFuture[T, E]): lent T = # {.raises: [E]} + ## Blocks the current thread of execution until `fut` has finished, returning + ## its value. + ## + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. + ## + ## Must not be called recursively (from inside `async` procedures). + ## + ## See also `await`, `Future.read` + pollFor(fut).readFinished() + +proc waitFor*[E](fut: InternalRaisesFuture[void, E]) = # {.raises: [E]} + ## Blocks the current thread of execution until `fut` has finished. + ## + ## If the future failed or was cancelled, the corresponding exception will be + ## raised. + ## + ## Must not be called recursively (from inside `async` procedures). + ## + ## See also `await`, `Future.read` + pollFor(fut).internalCheckComplete(E) proc `or`*[T, Y, E1, E2]( fut1: InternalRaisesFuture[T, E1], diff --git a/chronos/internal/asyncmacro.nim b/chronos/internal/asyncmacro.nim index 88e11e3..079e3bb 100644 --- a/chronos/internal/asyncmacro.nim +++ b/chronos/internal/asyncmacro.nim @@ -13,14 +13,14 @@ import ../[futures, config], ./raisesfutures -proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} = +proc processBody(node, setResultSym: NimNode): NimNode {.compileTime.} = case node.kind of nnkReturnStmt: # `return ...` -> `setResult(...); return` let res = newNimNode(nnkStmtList, node) if node[0].kind != nnkEmpty: - res.add newCall(setResultSym, processBody(node[0], setResultSym, baseType)) + res.add newCall(setResultSym, processBody(node[0], setResultSym)) res.add newNimNode(nnkReturnStmt, node).add(newEmptyNode()) res @@ -29,8 +29,14 @@ proc processBody(node, setResultSym, baseType: NimNode): NimNode {.compileTime.} # the Future we inject node else: + if node.kind == nnkYieldStmt: + # asyncdispatch allows `yield` but this breaks cancellation + warning( + "`yield` in async procedures not supported - use `awaitne` instead", + node) + for i in 0 ..< node.len: - node[i] = processBody(node[i], setResultSym, baseType) + node[i] = processBody(node[i], setResultSym) node proc wrapInTryFinally( @@ -179,7 +185,7 @@ proc getName(node: NimNode): string {.compileTime.} = macro unsupported(s: static[string]): untyped = error s -proc params2(someProc: NimNode): NimNode = +proc params2(someProc: NimNode): NimNode {.compileTime.} = # until https://github.com/nim-lang/Nim/pull/19563 is available if someProc.kind == nnkProcTy: someProc[0] @@ -275,6 +281,10 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = returnType[1] let + # When the base type is known to be void (and not generic), we can simplify + # code generation - however, in the case of generic async procedures it + # could still end up being void, meaning void detection needs to happen + # post-macro-expansion. baseTypeIsVoid = baseType.eqIdent("void") (raw, raises, handleException) = decodeParams(params) internalFutureType = @@ -295,7 +305,7 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = prc.params2[0] = internalReturnType - if prc.kind notin {nnkProcTy, nnkLambda}: # TODO: Nim bug? + if prc.kind notin {nnkProcTy, nnkLambda}: prc.addPragma(newColonExpr(ident "stackTrace", ident "off")) # The proc itself doesn't raise @@ -326,63 +336,57 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = prc.body ) - when chronosDumpAsync: - echo repr prc - - return prc - - if prc.kind in {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo} and + elif prc.kind in {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo} and not isEmpty(prc.body): - # don't do anything with forward bodies (empty) let - prcName = prc.name.getName setResultSym = ident "setResult" - procBody = prc.body.processBody(setResultSym, baseType) - internalFutureSym = ident "chronosInternalRetFuture" - castFutureSym = nnkCast.newTree(internalFutureType, internalFutureSym) + procBody = prc.body.processBody(setResultSym) resultIdent = ident "result" - - resultDecl = nnkWhenStmt.newTree( - # when `baseType` is void: - nnkElifExpr.newTree( - nnkInfix.newTree(ident "is", baseType, ident "void"), - quote do: - template result: auto {.used.} = - {.fatal: "You should not reference the `result` variable inside" & - " a void async proc".} - ), - # else: - nnkElseExpr.newTree( - newStmtList( - quote do: {.push warning[resultshadowed]: off.}, - # var result {.used.}: `baseType` - # In the proc body, result may or may not end up being used - # depending on how the body is written - with implicit returns / - # expressions in particular, it is likely but not guaranteed that - # it is not used. Ideally, we would avoid emitting it in this - # case to avoid the default initializaiton. {.used.} typically - # works better than {.push.} which has a tendency to leak out of - # scope. - # TODO figure out if there's a way to detect `result` usage in - # the proc body _after_ template exapnsion, and therefore - # avoid creating this variable - one option is to create an - # addtional when branch witha fake `result` and check - # `compiles(procBody)` - this is not without cost though - nnkVarSection.newTree(nnkIdentDefs.newTree( - nnkPragmaExpr.newTree( - resultIdent, - nnkPragma.newTree(ident "used")), - baseType, newEmptyNode()) - ), - quote do: {.pop.}, + fakeResult = quote do: + template result: auto {.used.} = + {.fatal: "You should not reference the `result` variable inside" & + " a void async proc".} + resultDecl = + if baseTypeIsVoid: fakeResult + else: nnkWhenStmt.newTree( + # when `baseType` is void: + nnkElifExpr.newTree( + nnkInfix.newTree(ident "is", baseType, ident "void"), + fakeResult + ), + # else: + nnkElseExpr.newTree( + newStmtList( + quote do: {.push warning[resultshadowed]: off.}, + # var result {.used.}: `baseType` + # In the proc body, result may or may not end up being used + # depending on how the body is written - with implicit returns / + # expressions in particular, it is likely but not guaranteed that + # it is not used. Ideally, we would avoid emitting it in this + # case to avoid the default initializaiton. {.used.} typically + # works better than {.push.} which has a tendency to leak out of + # scope. + # TODO figure out if there's a way to detect `result` usage in + # the proc body _after_ template exapnsion, and therefore + # avoid creating this variable - one option is to create an + # addtional when branch witha fake `result` and check + # `compiles(procBody)` - this is not without cost though + nnkVarSection.newTree(nnkIdentDefs.newTree( + nnkPragmaExpr.newTree( + resultIdent, + nnkPragma.newTree(ident "used")), + baseType, newEmptyNode()) + ), + quote do: {.pop.}, + ) ) ) - ) - # generates: + # ```nim # template `setResultSym`(code: untyped) {.used.} = # when typeof(code) is void: code # else: `resultIdent` = code + # ``` # # this is useful to handle implicit returns, but also # to bind the `result` to the one we declare here @@ -415,6 +419,8 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = ) ) + internalFutureSym = ident "chronosInternalRetFuture" + castFutureSym = nnkCast.newTree(internalFutureType, internalFutureSym) # Wrapping in try/finally ensures that early returns are handled properly # and that `defer` is processed in the right scope completeDecl = wrapInTryFinally( @@ -429,18 +435,13 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = internalFutureParameter = nnkIdentDefs.newTree( internalFutureSym, newIdentNode("FutureBase"), newEmptyNode()) + prcName = prc.name.getName iteratorNameSym = genSym(nskIterator, $prcName) closureIterator = newProc( iteratorNameSym, [newIdentNode("FutureBase"), internalFutureParameter], closureBody, nnkIteratorDef) - outerProcBody = newNimNode(nnkStmtList, prc.body) - - # Copy comment for nimdoc - if prc.body.len > 0 and prc.body[0].kind == nnkCommentStmt: - outerProcBody.add(prc.body[0]) - iteratorNameSym.copyLineInfo(prc) closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom=prc.body) @@ -455,39 +456,56 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = nnkBracket.newTree() )) + # The body of the original procedure (now moved to the iterator) is replaced + # with: + # + # ```nim + # let resultFuture = newFuture[T]() + # resultFuture.internalClosure = `iteratorNameSym` + # futureContinue(resultFuture) + # return resultFuture + # ``` + # + # Declared at the end to be sure that the closure doesn't reference it, + # avoid cyclic ref (#203) + # + # Do not change this code to `quote do` version because `instantiationInfo` + # will be broken for `newFuture()` call. + + let + outerProcBody = newNimNode(nnkStmtList, prc.body) + + # Copy comment for nimdoc + if prc.body.len > 0 and prc.body[0].kind == nnkCommentStmt: + outerProcBody.add(prc.body[0]) + outerProcBody.add(closureIterator) - # -> let resultFuture = newInternalRaisesFuture[T, E]() - # declared at the end to be sure that the closure - # doesn't reference it, avoid cyclic ref (#203) let retFutureSym = ident "resultFuture" newFutProc = if raises == nil: - newTree(nnkBracketExpr, ident "newFuture", baseType) + nnkBracketExpr.newTree(ident "newFuture", baseType) else: - newTree(nnkBracketExpr, ident "newInternalRaisesFuture", baseType, raises) + nnkBracketExpr.newTree(ident "newInternalRaisesFuture", baseType, raises) + retFutureSym.copyLineInfo(prc) - # Do not change this code to `quote do` version because `instantiationInfo` - # will be broken for `newFuture()` call. outerProcBody.add( newLetStmt( retFutureSym, newCall(newFutProc, newLit(prcName)) ) ) - # -> resultFuture.internalClosure = iterator + outerProcBody.add( newAssignment( newDotExpr(retFutureSym, newIdentNode("internalClosure")), iteratorNameSym) ) - # -> futureContinue(resultFuture)) outerProcBody.add( newCall(newIdentNode("futureContinue"), retFutureSym) ) - # -> return resultFuture outerProcBody.add newNimNode(nnkReturnStmt, prc.body[^1]).add(retFutureSym) prc.body = outerProcBody @@ -498,6 +516,13 @@ proc asyncSingleProc(prc, params: NimNode): NimNode {.compileTime.} = prc template await*[T](f: Future[T]): T = + ## Ensure that the given `Future` is finished, then return its value. + ## + ## If the `Future` failed or was cancelled, the corresponding exception will + ## be raised instead. + ## + ## If the `Future` is pending, execution of the current `async` procedure + ## will be suspended until the `Future` is finished. when declared(chronosInternalRetFuture): chronosInternalRetFuture.internalChild = f # `futureContinue` calls the iterator generated by the `async` @@ -512,18 +537,26 @@ template await*[T](f: Future[T]): T = else: unsupported "await is only available within {.async.}" -template await*[T, E](f: InternalRaisesFuture[T, E]): T = +template await*[T, E](fut: InternalRaisesFuture[T, E]): T = + ## Ensure that the given `Future` is finished, then return its value. + ## + ## If the `Future` failed or was cancelled, the corresponding exception will + ## be raised instead. + ## + ## If the `Future` is pending, execution of the current `async` procedure + ## will be suspended until the `Future` is finished. when declared(chronosInternalRetFuture): - chronosInternalRetFuture.internalChild = f + chronosInternalRetFuture.internalChild = fut # `futureContinue` calls the iterator generated by the `async` # transformation - `yield` gives control back to `futureContinue` which is # responsible for resuming execution once the yielded future is finished yield chronosInternalRetFuture.internalChild # `child` released by `futureContinue` - cast[type(f)](chronosInternalRetFuture.internalChild).internalCheckComplete(E) + cast[type(fut)]( + chronosInternalRetFuture.internalChild).internalCheckComplete(E) when T isnot void: - cast[type(f)](chronosInternalRetFuture.internalChild).value() + cast[type(fut)](chronosInternalRetFuture.internalChild).value() else: unsupported "await is only available within {.async.}" diff --git a/docs/src/async_procs.md b/docs/src/async_procs.md index ae8eb51..648f19b 100644 --- a/docs/src/async_procs.md +++ b/docs/src/async_procs.md @@ -1,5 +1,13 @@ # Async procedures +Async procedures are those that interact with `chronos` to cooperatively +suspend and resume their execution depending on the completion of other +async procedures which themselves may be waiting for I/O to complete, timers to +expire or tasks running on other threads to complete. + +Async procedures are marked with the `{.async.}` pragma and return a `Future` +indicating the state of the operation. + ## The `async` pragma @@ -20,8 +28,8 @@ echo p().type # prints "Future[system.void]" Whenever `await` is encountered inside an async procedure, control is given back to the dispatcher for as many steps as it's necessary for the awaited future to complete, fail or be cancelled. `await` calls the -equivalent of `Future.read()` on the completed future and returns the -encapsulated value. +equivalent of `Future.read()` on the completed future to return the +encapsulated value when the operation finishes. ```nim proc p1() {.async.} = @@ -51,10 +59,10 @@ In particular, if two `async` procedures have access to the same mutable state, the value before and after `await` might not be the same as the order of execution is not guaranteed! ``` -## Raw functions +## Raw procedures -Raw functions are those that interact with `chronos` via the `Future` type but -whose body does not go through the async transformation. +Raw async procedures are those that interact with `chronos` via the `Future` +type but whose body does not go through the async transformation. Such functions are created by adding `raw: true` to the `async` parameters: diff --git a/docs/src/concepts.md b/docs/src/concepts.md index fcc33af..0469b8b 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -1,12 +1,13 @@ # Concepts +Async/await is a programming model that relies on cooperative multitasking to +coordinate the concurrent execution of procedures, using event notifications +from the operating system or other treads to resume execution. + ## The dispatcher -Async/await programming relies on cooperative multitasking to coordinate the -concurrent execution of procedures, using event notifications from the operating system to resume execution. - The event handler loop is called a "dispatcher" and a single instance per thread is created, as soon as one is needed. @@ -16,6 +17,9 @@ progress, for example because it's waiting for some data to arrive, it hands control back to the dispatcher which ensures that the procedure is resumed when ready. +A single thread, and thus a single dispatcher, is typically able to handle +thousands of concurrent in-progress requests. + ## The `Future` type `Future` objects encapsulate the outcome of executing an `async` procedure. The @@ -69,13 +73,14 @@ structured this way. Both `waitFor` and `runForever` call `poll` which offers fine-grained control over the event loop steps. -Nested calls to `poll`, `waitFor` and `runForever` are not allowed. +Nested calls to `poll` - directly or indirectly via `waitFor` and `runForever` +are not allowed. ``` ## Cancellation Any pending `Future` can be cancelled. This can be used for timeouts, to start -multiple operations in parallel and cancel the rest as soon as one finishes, +multiple parallel operations and cancel the rest as soon as one finishes, to initiate the orderely shutdown of an application etc. ```nim @@ -110,7 +115,10 @@ waitFor(work.cancelAndWait()) ``` The `CancelledError` will now travel up the stack like any other exception. -It can be caught and handled (for instance, freeing some resources) +It can be caught for instance to free some resources and is then typically +re-raised for the whole chain operations to get cancelled. + +Alternatively, the cancellation request can be translated to a regular outcome of the operation - for example, a `read` operation might return an empty result. Cancelling an already-finished `Future` has no effect, as the following example of downloading two web pages concurrently shows: diff --git a/docs/src/error_handling.md b/docs/src/error_handling.md index be06a35..54c1236 100644 --- a/docs/src/error_handling.md +++ b/docs/src/error_handling.md @@ -85,6 +85,21 @@ the operation they implement might get cancelled resulting in neither value nor error! ``` +When using checked exceptions, the `Future` type is modified to include +`raises` information - it can be constructed with the `Raising` helper: + +```nim +# Create a variable of the type that will be returned by a an async function +# raising `[CancelledError]`: +var fut: Future[int].Raising([CancelledError]) +``` + +```admonition note +`Raising` creates a specialization of `InternalRaisesFuture` type - as the name +suggests, this is an internal type whose implementation details are likely to +change in future `chronos` versions. +``` + ## The `Exception` type Exceptions deriving from `Exception` are not caught by default as these may diff --git a/docs/src/porting.md b/docs/src/porting.md index 519de64..1bdffe2 100644 --- a/docs/src/porting.md +++ b/docs/src/porting.md @@ -16,20 +16,25 @@ here are several things to consider: * Exception handling is now strict by default - see the [error handling](./error_handling.md) chapter for how to deal with `raises` effects * `AsyncEventBus` was removed - use `AsyncEventQueue` instead +* `Future.value` and `Future.error` panic when accessed in the wrong state +* `Future.read` and `Future.readError` raise `FutureError` instead of + `ValueError` when accessed in the wrong state ## `asyncdispatch` -Projects written for `asyncdispatch` and `chronos` look similar but there are +Code written for `asyncdispatch` and `chronos` looks similar but there are several differences to be aware of: * `chronos` has its own dispatch loop - you can typically not mix `chronos` and `asyncdispatch` in the same thread * `import chronos` instead of `import asyncdispatch` * cleanup is important - make sure to use `closeWait` to release any resources - you're using or file descript leaks and other + you're using or file descriptor and other leaks will ensue * cancellation support means that `CancelledError` may be raised from most `{.async.}` functions * Calling `yield` directly in tasks is not supported - instead, use `awaitne`. +* `asyncSpawn` is used instead of `asyncCheck` - note that exceptions raised + in tasks that are `asyncSpawn`:ed cause panic ## Supporting multiple backends From 0b136b33c8b8d8ee09777f31cf6fb53362a741f4 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Sat, 18 Nov 2023 00:18:09 +0200 Subject: [PATCH 43/50] Asyncstreams asyncraises. (#472) * Fix transports addresses functions should not return so many exceptions. * Add raising `Defect` functions to AsyncQueue. * Add raises/asyncraises into async streams. * Remove `Safe` primitives. Make AsyncStreamError to be ancestor of AsyncError. Make AsyncStreamReader/Writer loops requirement to not raise any exceptions * Remove `par` fields. * Remove `par` fields from TLSStream. * Attempt to lower memory usage. --- chronos/asyncsync.nim | 47 +++++--- chronos/streams/asyncstream.nim | 207 ++++++++++++++------------------ chronos/streams/boundstream.nim | 22 +++- chronos/streams/chunkstream.nim | 17 ++- chronos/streams/tlsstream.nim | 114 ++++++++++-------- chronos/transports/stream.nim | 8 +- 6 files changed, 217 insertions(+), 198 deletions(-) diff --git a/chronos/asyncsync.nim b/chronos/asyncsync.nim index 9bab1fd..f77d5fe 100644 --- a/chronos/asyncsync.nim +++ b/chronos/asyncsync.nim @@ -165,7 +165,7 @@ proc newAsyncEvent*(): AsyncEvent = AsyncEvent() proc wait*(event: AsyncEvent): Future[void] {. - async: (raw: true, raises: [CancelledError]).} = + async: (raw: true, raises: [CancelledError]).} = ## Block until the internal flag of ``event`` is `true`. ## If the internal flag is `true` on entry, return immediately. Otherwise, ## block until another task calls `fire()` to set the flag to `true`, @@ -258,7 +258,7 @@ proc popLastImpl[T](aq: AsyncQueue[T]): T = res proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) {. - raises: [AsyncQueueFullError].}= + raises: [AsyncQueueFullError].} = ## Put an item ``item`` to the beginning of the queue ``aq`` immediately. ## ## If queue ``aq`` is full, then ``AsyncQueueFullError`` exception raised. @@ -267,7 +267,7 @@ proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) {. aq.addFirstImpl(item) proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) {. - raises: [AsyncQueueFullError].}= + raises: [AsyncQueueFullError].} = ## Put an item ``item`` at the end of the queue ``aq`` immediately. ## ## If queue ``aq`` is full, then ``AsyncQueueFullError`` exception raised. @@ -276,7 +276,7 @@ proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) {. aq.addLastImpl(item) proc popFirstNoWait*[T](aq: AsyncQueue[T]): T {. - raises: [AsyncQueueEmptyError].} = + raises: [AsyncQueueEmptyError].} = ## Get an item from the beginning of the queue ``aq`` immediately. ## ## If queue ``aq`` is empty, then ``AsyncQueueEmptyError`` exception raised. @@ -285,7 +285,7 @@ proc popFirstNoWait*[T](aq: AsyncQueue[T]): T {. aq.popFirstImpl() proc popLastNoWait*[T](aq: AsyncQueue[T]): T {. - raises: [AsyncQueueEmptyError].} = + raises: [AsyncQueueEmptyError].} = ## Get an item from the end of the queue ``aq`` immediately. ## ## If queue ``aq`` is empty, then ``AsyncQueueEmptyError`` exception raised. @@ -293,11 +293,13 @@ proc popLastNoWait*[T](aq: AsyncQueue[T]): T {. raise newException(AsyncQueueEmptyError, "AsyncQueue is empty!") aq.popLastImpl() -proc addFirst*[T](aq: AsyncQueue[T], item: T) {.async: (raises: [CancelledError]).} = +proc addFirst*[T](aq: AsyncQueue[T], item: T) {. + async: (raises: [CancelledError]).} = ## Put an ``item`` to the beginning of the queue ``aq``. If the queue is full, ## wait until a free slot is available before adding item. while aq.full(): - let putter = Future[void].Raising([CancelledError]).init("AsyncQueue.addFirst") + let putter = + Future[void].Raising([CancelledError]).init("AsyncQueue.addFirst") aq.putters.add(putter) try: await putter @@ -307,11 +309,13 @@ proc addFirst*[T](aq: AsyncQueue[T], item: T) {.async: (raises: [CancelledError] raise exc aq.addFirstImpl(item) -proc addLast*[T](aq: AsyncQueue[T], item: T) {.async: (raises: [CancelledError]).} = +proc addLast*[T](aq: AsyncQueue[T], item: T) {. + async: (raises: [CancelledError]).} = ## Put an ``item`` to the end of the queue ``aq``. If the queue is full, ## wait until a free slot is available before adding item. while aq.full(): - let putter = Future[void].Raising([CancelledError]).init("AsyncQueue.addLast") + let putter = + Future[void].Raising([CancelledError]).init("AsyncQueue.addLast") aq.putters.add(putter) try: await putter @@ -321,11 +325,13 @@ proc addLast*[T](aq: AsyncQueue[T], item: T) {.async: (raises: [CancelledError]) raise exc aq.addLastImpl(item) -proc popFirst*[T](aq: AsyncQueue[T]): Future[T] {.async: (raises: [CancelledError]).} = +proc popFirst*[T](aq: AsyncQueue[T]): Future[T] {. + async: (raises: [CancelledError]).} = ## Remove and return an ``item`` from the beginning of the queue ``aq``. ## If the queue is empty, wait until an item is available. while aq.empty(): - let getter = Future[void].Raising([CancelledError]).init("AsyncQueue.popFirst") + let getter = + Future[void].Raising([CancelledError]).init("AsyncQueue.popFirst") aq.getters.add(getter) try: await getter @@ -335,11 +341,13 @@ proc popFirst*[T](aq: AsyncQueue[T]): Future[T] {.async: (raises: [CancelledErro raise exc aq.popFirstImpl() -proc popLast*[T](aq: AsyncQueue[T]): Future[T] {.async: (raises: [CancelledError]).} = +proc popLast*[T](aq: AsyncQueue[T]): Future[T] {. + async: (raises: [CancelledError]).} = ## Remove and return an ``item`` from the end of the queue ``aq``. ## If the queue is empty, wait until an item is available. while aq.empty(): - let getter = Future[void].Raising([CancelledError]).init("AsyncQueue.popLast") + let getter = + Future[void].Raising([CancelledError]).init("AsyncQueue.popLast") aq.getters.add(getter) try: await getter @@ -350,22 +358,22 @@ proc popLast*[T](aq: AsyncQueue[T]): Future[T] {.async: (raises: [CancelledError aq.popLastImpl() proc putNoWait*[T](aq: AsyncQueue[T], item: T) {. - raises: [AsyncQueueFullError].} = + raises: [AsyncQueueFullError].} = ## Alias of ``addLastNoWait()``. aq.addLastNoWait(item) proc getNoWait*[T](aq: AsyncQueue[T]): T {. - raises: [AsyncQueueEmptyError].} = + raises: [AsyncQueueEmptyError].} = ## Alias of ``popFirstNoWait()``. aq.popFirstNoWait() proc put*[T](aq: AsyncQueue[T], item: T): Future[void] {. - async: (raw: true, raises: [CancelledError]).} = + async: (raw: true, raises: [CancelledError]).} = ## Alias of ``addLast()``. aq.addLast(item) proc get*[T](aq: AsyncQueue[T]): Future[T] {. - async: (raw: true, raises: [CancelledError]).} = + async: (raw: true, raises: [CancelledError]).} = ## Alias of ``popFirst()``. aq.popFirst() @@ -509,7 +517,8 @@ proc close*(ab: AsyncEventQueue) {.raises: [].} = ab.readers.reset() ab.queue.clear() -proc closeWait*(ab: AsyncEventQueue): Future[void] {.async: (raw: true, raises: []).} = +proc closeWait*(ab: AsyncEventQueue): Future[void] {. + async: (raw: true, raises: []).} = let retFuture = newFuture[void]("AsyncEventQueue.closeWait()", {FutureFlag.OwnCancelSchedule}) proc continuation(udata: pointer) {.gcsafe.} = @@ -568,7 +577,7 @@ proc emit*[T](ab: AsyncEventQueue[T], data: T) = proc waitEvents*[T](ab: AsyncEventQueue[T], key: EventQueueKey, eventsCount = -1): Future[seq[T]] {. - async: (raises: [AsyncEventQueueFullError, CancelledError]).} = + async: (raises: [AsyncEventQueueFullError, CancelledError]).} = ## Wait for events var events: seq[T] diff --git a/chronos/streams/asyncstream.nim b/chronos/streams/asyncstream.nim index 4698e83..a521084 100644 --- a/chronos/streams/asyncstream.nim +++ b/chronos/streams/asyncstream.nim @@ -24,15 +24,13 @@ const ## AsyncStreamWriter leaks tracker name type - AsyncStreamError* = object of CatchableError + AsyncStreamError* = object of AsyncError AsyncStreamIncorrectDefect* = object of Defect AsyncStreamIncompleteError* = object of AsyncStreamError AsyncStreamLimitError* = object of AsyncStreamError AsyncStreamUseClosedError* = object of AsyncStreamError AsyncStreamReadError* = object of AsyncStreamError - par*: ref CatchableError AsyncStreamWriteError* = object of AsyncStreamError - par*: ref CatchableError AsyncStreamWriteEOFError* = object of AsyncStreamWriteError AsyncBuffer* = object @@ -53,7 +51,7 @@ type dataStr*: string size*: int offset*: int - future*: Future[void] + future*: Future[void].Raising([CancelledError, AsyncStreamError]) AsyncStreamState* = enum Running, ## Stream is online and working @@ -64,10 +62,10 @@ type Closed ## Stream was closed StreamReaderLoop* = proc (stream: AsyncStreamReader): Future[void] {. - gcsafe, raises: [].} + async: (raises: []).} ## Main read loop for read streams. StreamWriterLoop* = proc (stream: AsyncStreamWriter): Future[void] {. - gcsafe, raises: [].} + async: (raises: []).} ## Main write loop for write streams. AsyncStreamReader* = ref object of RootRef @@ -124,12 +122,12 @@ proc `[]`*(sb: AsyncBuffer, index: int): byte {.inline.} = proc update*(sb: var AsyncBuffer, size: int) {.inline.} = sb.offset += size -proc wait*(sb: var AsyncBuffer): Future[void] = +template wait*(sb: var AsyncBuffer): untyped = sb.events[0].clear() sb.events[1].fire() sb.events[0].wait() -proc transfer*(sb: var AsyncBuffer): Future[void] = +template transfer*(sb: var AsyncBuffer): untyped = sb.events[1].clear() sb.events[0].fire() sb.events[1].wait() @@ -150,7 +148,8 @@ proc copyData*(sb: AsyncBuffer, dest: pointer, offset, length: int) {.inline.} = unsafeAddr sb.buffer[0], length) proc upload*(sb: ptr AsyncBuffer, pbytes: ptr byte, - nbytes: int): Future[void] {.async.} = + nbytes: int): Future[void] {. + async: (raises: [CancelledError]).} = ## You can upload any amount of bytes to the buffer. If size of internal ## buffer is not enough to fit all the data at once, data will be uploaded ## via chunks of size up to internal buffer size. @@ -186,18 +185,20 @@ template copyOut*(dest: pointer, item: WriteItem, length: int) = elif item.kind == String: copyMem(dest, unsafeAddr item.dataStr[item.offset], length) -proc newAsyncStreamReadError(p: ref CatchableError): ref AsyncStreamReadError {. - noinline.} = +proc newAsyncStreamReadError( + p: ref TransportError + ): ref AsyncStreamReadError {.noinline.} = var w = newException(AsyncStreamReadError, "Read stream failed") w.msg = w.msg & ", originated from [" & $p.name & "] " & p.msg - w.par = p + w.parent = p w -proc newAsyncStreamWriteError(p: ref CatchableError): ref AsyncStreamWriteError {. - noinline.} = +proc newAsyncStreamWriteError( + p: ref TransportError + ): ref AsyncStreamWriteError {.noinline.} = var w = newException(AsyncStreamWriteError, "Write stream failed") w.msg = w.msg & ", originated from [" & $p.name & "] " & p.msg - w.par = p + w.parent = p w proc newAsyncStreamIncompleteError*(): ref AsyncStreamIncompleteError {. @@ -344,7 +345,8 @@ template readLoop(body: untyped): untyped = await rstream.buffer.wait() proc readExactly*(rstream: AsyncStreamReader, pbytes: pointer, - nbytes: int) {.async.} = + nbytes: int) {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Read exactly ``nbytes`` bytes from read-only stream ``rstream`` and store ## it to ``pbytes``. ## @@ -365,7 +367,7 @@ proc readExactly*(rstream: AsyncStreamReader, pbytes: pointer, raise exc except TransportIncompleteError: raise newAsyncStreamIncompleteError() - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -384,7 +386,8 @@ proc readExactly*(rstream: AsyncStreamReader, pbytes: pointer, (consumed: count, done: index == nbytes) proc readOnce*(rstream: AsyncStreamReader, pbytes: pointer, - nbytes: int): Future[int] {.async.} = + nbytes: int): Future[int] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Perform one read operation on read-only stream ``rstream``. ## ## If internal buffer is not empty, ``nbytes`` bytes will be transferred from @@ -398,7 +401,7 @@ proc readOnce*(rstream: AsyncStreamReader, pbytes: pointer, return await readOnce(rstream.tsource, pbytes, nbytes) except CancelledError as exc: raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -415,7 +418,8 @@ proc readOnce*(rstream: AsyncStreamReader, pbytes: pointer, return count proc readUntil*(rstream: AsyncStreamReader, pbytes: pointer, nbytes: int, - sep: seq[byte]): Future[int] {.async.} = + sep: seq[byte]): Future[int] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Read data from the read-only stream ``rstream`` until separator ``sep`` is ## found. ## @@ -446,7 +450,7 @@ proc readUntil*(rstream: AsyncStreamReader, pbytes: pointer, nbytes: int, raise newAsyncStreamIncompleteError() except TransportLimitError: raise newAsyncStreamLimitError() - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -476,7 +480,8 @@ proc readUntil*(rstream: AsyncStreamReader, pbytes: pointer, nbytes: int, return k proc readLine*(rstream: AsyncStreamReader, limit = 0, - sep = "\r\n"): Future[string] {.async.} = + sep = "\r\n"): Future[string] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Read one line from read-only stream ``rstream``, where ``"line"`` is a ## sequence of bytes ending with ``sep`` (default is ``"\r\n"``). ## @@ -495,7 +500,7 @@ proc readLine*(rstream: AsyncStreamReader, limit = 0, return await readLine(rstream.tsource, limit, sep) except CancelledError as exc: raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -530,7 +535,8 @@ proc readLine*(rstream: AsyncStreamReader, limit = 0, (index, (state == len(sep)) or (lim == len(res))) return res -proc read*(rstream: AsyncStreamReader): Future[seq[byte]] {.async.} = +proc read*(rstream: AsyncStreamReader): Future[seq[byte]] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Read all bytes from read-only stream ``rstream``. ## ## This procedure allocates buffer seq[byte] and return it as result. @@ -543,7 +549,7 @@ proc read*(rstream: AsyncStreamReader): Future[seq[byte]] {.async.} = raise exc except TransportLimitError: raise newAsyncStreamLimitError() - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -559,7 +565,8 @@ proc read*(rstream: AsyncStreamReader): Future[seq[byte]] {.async.} = (count, false) return res -proc read*(rstream: AsyncStreamReader, n: int): Future[seq[byte]] {.async.} = +proc read*(rstream: AsyncStreamReader, n: int): Future[seq[byte]] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Read all bytes (n <= 0) or exactly `n` bytes from read-only stream ## ``rstream``. ## @@ -571,7 +578,7 @@ proc read*(rstream: AsyncStreamReader, n: int): Future[seq[byte]] {.async.} = return await read(rstream.tsource, n) except CancelledError as exc: raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -590,7 +597,8 @@ proc read*(rstream: AsyncStreamReader, n: int): Future[seq[byte]] {.async.} = (count, len(res) == n) return res -proc consume*(rstream: AsyncStreamReader): Future[int] {.async.} = +proc consume*(rstream: AsyncStreamReader): Future[int] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Consume (discard) all bytes from read-only stream ``rstream``. ## ## Return number of bytes actually consumed (discarded). @@ -603,7 +611,7 @@ proc consume*(rstream: AsyncStreamReader): Future[int] {.async.} = raise exc except TransportLimitError: raise newAsyncStreamLimitError() - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -618,7 +626,8 @@ proc consume*(rstream: AsyncStreamReader): Future[int] {.async.} = (rstream.buffer.dataLen(), false) return res -proc consume*(rstream: AsyncStreamReader, n: int): Future[int] {.async.} = +proc consume*(rstream: AsyncStreamReader, n: int): Future[int] {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Consume (discard) all bytes (n <= 0) or ``n`` bytes from read-only stream ## ``rstream``. ## @@ -632,7 +641,7 @@ proc consume*(rstream: AsyncStreamReader, n: int): Future[int] {.async.} = raise exc except TransportLimitError: raise newAsyncStreamLimitError() - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -652,7 +661,7 @@ proc consume*(rstream: AsyncStreamReader, n: int): Future[int] {.async.} = return res proc readMessage*(rstream: AsyncStreamReader, pred: ReadMessagePredicate) {. - async.} = + async: (raises: [CancelledError, AsyncStreamError]).} = ## Read all bytes from stream ``rstream`` until ``predicate`` callback ## will not be satisfied. ## @@ -673,7 +682,7 @@ proc readMessage*(rstream: AsyncStreamReader, pred: ReadMessagePredicate) {. await readMessage(rstream.tsource, pred) except CancelledError as exc: raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamReadError(exc) else: if isNil(rstream.readerLoop): @@ -691,7 +700,8 @@ proc readMessage*(rstream: AsyncStreamReader, pred: ReadMessagePredicate) {. pred(rstream.buffer.buffer.toOpenArray(0, count - 1)) proc write*(wstream: AsyncStreamWriter, pbytes: pointer, - nbytes: int) {.async.} = + nbytes: int) {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Write sequence of bytes pointed by ``pbytes`` of length ``nbytes`` to ## writer stream ``wstream``. ## @@ -708,9 +718,7 @@ proc write*(wstream: AsyncStreamWriter, pbytes: pointer, res = await write(wstream.tsource, pbytes, nbytes) except CancelledError as exc: raise exc - except AsyncStreamError as exc: - raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamWriteError(exc) if res != nbytes: raise newAsyncStreamIncompleteError() @@ -720,23 +728,17 @@ proc write*(wstream: AsyncStreamWriter, pbytes: pointer, await write(wstream.wsource, pbytes, nbytes) wstream.bytesCount = wstream.bytesCount + uint64(nbytes) else: - var item = WriteItem(kind: Pointer) - item.dataPtr = pbytes - item.size = nbytes - item.future = newFuture[void]("async.stream.write(pointer)") - try: - await wstream.queue.put(item) - await item.future - wstream.bytesCount = wstream.bytesCount + uint64(item.size) - except CancelledError as exc: - raise exc - except AsyncStreamError as exc: - raise exc - except CatchableError as exc: - raise newAsyncStreamWriteError(exc) + let item = WriteItem( + kind: Pointer, dataPtr: pbytes, size: nbytes, + future: Future[void].Raising([CancelledError, AsyncStreamError]) + .init("async.stream.write(pointer)")) + await wstream.queue.put(item) + await item.future + wstream.bytesCount = wstream.bytesCount + uint64(item.size) proc write*(wstream: AsyncStreamWriter, sbytes: sink seq[byte], - msglen = -1) {.async.} = + msglen = -1) {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Write sequence of bytes ``sbytes`` of length ``msglen`` to writer ## stream ``wstream``. ## @@ -758,7 +760,7 @@ proc write*(wstream: AsyncStreamWriter, sbytes: sink seq[byte], res = await write(wstream.tsource, sbytes, length) except CancelledError as exc: raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamWriteError(exc) if res != length: raise newAsyncStreamIncompleteError() @@ -768,29 +770,17 @@ proc write*(wstream: AsyncStreamWriter, sbytes: sink seq[byte], await write(wstream.wsource, sbytes, length) wstream.bytesCount = wstream.bytesCount + uint64(length) else: - var item = WriteItem(kind: Sequence) - when declared(shallowCopy): - if not(isLiteral(sbytes)): - shallowCopy(item.dataSeq, sbytes) - else: - item.dataSeq = sbytes - else: - item.dataSeq = sbytes - item.size = length - item.future = newFuture[void]("async.stream.write(seq)") - try: - await wstream.queue.put(item) - await item.future - wstream.bytesCount = wstream.bytesCount + uint64(item.size) - except CancelledError as exc: - raise exc - except AsyncStreamError as exc: - raise exc - except CatchableError as exc: - raise newAsyncStreamWriteError(exc) + let item = WriteItem( + kind: Sequence, dataSeq: move(sbytes), size: length, + future: Future[void].Raising([CancelledError, AsyncStreamError]) + .init("async.stream.write(seq)")) + await wstream.queue.put(item) + await item.future + wstream.bytesCount = wstream.bytesCount + uint64(item.size) proc write*(wstream: AsyncStreamWriter, sbytes: sink string, - msglen = -1) {.async.} = + msglen = -1) {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Write string ``sbytes`` of length ``msglen`` to writer stream ``wstream``. ## ## String ``sbytes`` must not be zero-length. @@ -811,7 +801,7 @@ proc write*(wstream: AsyncStreamWriter, sbytes: sink string, res = await write(wstream.tsource, sbytes, length) except CancelledError as exc: raise exc - except CatchableError as exc: + except TransportError as exc: raise newAsyncStreamWriteError(exc) if res != length: raise newAsyncStreamIncompleteError() @@ -821,28 +811,16 @@ proc write*(wstream: AsyncStreamWriter, sbytes: sink string, await write(wstream.wsource, sbytes, length) wstream.bytesCount = wstream.bytesCount + uint64(length) else: - var item = WriteItem(kind: String) - when declared(shallowCopy): - if not(isLiteral(sbytes)): - shallowCopy(item.dataStr, sbytes) - else: - item.dataStr = sbytes - else: - item.dataStr = sbytes - item.size = length - item.future = newFuture[void]("async.stream.write(string)") - try: - await wstream.queue.put(item) - await item.future - wstream.bytesCount = wstream.bytesCount + uint64(item.size) - except CancelledError as exc: - raise exc - except AsyncStreamError as exc: - raise exc - except CatchableError as exc: - raise newAsyncStreamWriteError(exc) + let item = WriteItem( + kind: String, dataStr: move(sbytes), size: length, + future: Future[void].Raising([CancelledError, AsyncStreamError]) + .init("async.stream.write(string)")) + await wstream.queue.put(item) + await item.future + wstream.bytesCount = wstream.bytesCount + uint64(item.size) -proc finish*(wstream: AsyncStreamWriter) {.async.} = +proc finish*(wstream: AsyncStreamWriter) {. + async: (raises: [CancelledError, AsyncStreamError]).} = ## Finish write stream ``wstream``. checkStreamClosed(wstream) # For AsyncStreamWriter Finished state could be set manually or by stream's @@ -852,20 +830,15 @@ proc finish*(wstream: AsyncStreamWriter) {.async.} = if isNil(wstream.writerLoop): await wstream.wsource.finish() else: - var item = WriteItem(kind: Pointer) - item.size = 0 - item.future = newFuture[void]("async.stream.finish") - try: - await wstream.queue.put(item) - await item.future - except CancelledError as exc: - raise exc - except AsyncStreamError as exc: - raise exc - except CatchableError as exc: - raise newAsyncStreamWriteError(exc) + let item = WriteItem( + kind: Pointer, size: 0, + future: Future[void].Raising([CancelledError, AsyncStreamError]) + .init("async.stream.finish")) + await wstream.queue.put(item) + await item.future -proc join*(rw: AsyncStreamRW): Future[void] = +proc join*(rw: AsyncStreamRW): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Get Future[void] which will be completed when stream become finished or ## closed. when rw is AsyncStreamReader: @@ -924,7 +897,8 @@ proc close*(rw: AsyncStreamRW) = rw.future.addCallback(continuation) rw.future.cancelSoon() -proc closeWait*(rw: AsyncStreamRW): Future[void] = +proc closeWait*(rw: AsyncStreamRW): Future[void] {. + async: (raw: true, raises: []).} = ## Close and frees resources of stream ``rw``. const FutureName = when rw is AsyncStreamReader: @@ -932,25 +906,20 @@ proc closeWait*(rw: AsyncStreamRW): Future[void] = else: "async.stream.writer.closeWait" - if rw.closed(): - return Future.completed(FutureName) + let retFuture = Future[void].Raising([]).init(FutureName) - let retFuture = newFuture[void](FutureName, {FutureFlag.OwnCancelSchedule}) + if rw.closed(): + retFuture.complete() + return retFuture proc continuation(udata: pointer) {.gcsafe, raises:[].} = retFuture.complete() - proc cancellation(udata: pointer) {.gcsafe, raises:[].} = - # We are not going to change the state of `retFuture` to cancelled, so we - # will prevent the entire sequence of Futures from being cancelled. - discard - rw.close() if rw.future.finished(): retFuture.complete() else: rw.future.addCallback(continuation, cast[pointer](retFuture)) - retFuture.cancelCallback = cancellation retFuture proc startReader(rstream: AsyncStreamReader) = diff --git a/chronos/streams/boundstream.nim b/chronos/streams/boundstream.nim index dbb36ef..ce69571 100644 --- a/chronos/streams/boundstream.nim +++ b/chronos/streams/boundstream.nim @@ -14,6 +14,9 @@ ## ## For stream writing it means that you should write exactly bounded size ## of bytes. + +{.push raises: [].} + import results import ../asyncloop, ../timer import asyncstream, ../transports/stream, ../transports/common @@ -52,7 +55,8 @@ template newBoundedStreamOverflowError(): ref BoundedStreamOverflowError = newException(BoundedStreamOverflowError, "Stream boundary exceeded") proc readUntilBoundary(rstream: AsyncStreamReader, pbytes: pointer, - nbytes: int, sep: seq[byte]): Future[int] {.async.} = + nbytes: int, sep: seq[byte]): Future[int] {. + async: (raises: [CancelledError, AsyncStreamError]).} = doAssert(not(isNil(pbytes)), "pbytes must not be nil") doAssert(nbytes >= 0, "nbytes must be non-negative value") checkStreamClosed(rstream) @@ -96,7 +100,7 @@ func endsWith(s, suffix: openArray[byte]): bool = inc(i) if i >= len(suffix): return true -proc boundedReadLoop(stream: AsyncStreamReader) {.async.} = +proc boundedReadLoop(stream: AsyncStreamReader) {.async: (raises: []).} = var rstream = BoundedStreamReader(stream) rstream.state = AsyncStreamState.Running var buffer = newSeq[byte](rstream.buffer.bufferLen()) @@ -186,12 +190,16 @@ proc boundedReadLoop(stream: AsyncStreamReader) {.async.} = break of AsyncStreamState.Finished: # Send `EOF` state to the consumer and wait until it will be received. - await rstream.buffer.transfer() + try: + await rstream.buffer.transfer() + except CancelledError: + rstream.state = AsyncStreamState.Error + rstream.error = newBoundedStreamIncompleteError() break of AsyncStreamState.Closing, AsyncStreamState.Closed: break -proc boundedWriteLoop(stream: AsyncStreamWriter) {.async.} = +proc boundedWriteLoop(stream: AsyncStreamWriter) {.async: (raises: []).} = var error: ref AsyncStreamError var wstream = BoundedStreamWriter(stream) @@ -255,7 +263,11 @@ proc boundedWriteLoop(stream: AsyncStreamWriter) {.async.} = doAssert(not(isNil(error))) while not(wstream.queue.empty()): - let item = wstream.queue.popFirstNoWait() + let item = + try: + wstream.queue.popFirstNoWait() + except AsyncQueueEmptyError: + raiseAssert "AsyncQueue should not be empty at this moment" if not(item.future.finished()): item.future.fail(error) diff --git a/chronos/streams/chunkstream.nim b/chronos/streams/chunkstream.nim index c0269a2..7739207 100644 --- a/chronos/streams/chunkstream.nim +++ b/chronos/streams/chunkstream.nim @@ -8,6 +8,9 @@ # MIT license (LICENSE-MIT) ## This module implements HTTP/1.1 chunked-encoded stream reading and writing. + +{.push raises: [].} + import ../asyncloop, ../timer import asyncstream, ../transports/stream, ../transports/common import results @@ -95,7 +98,7 @@ proc setChunkSize(buffer: var openArray[byte], length: int64): int = buffer[c + 1] = byte(0x0A) (c + 2) -proc chunkedReadLoop(stream: AsyncStreamReader) {.async.} = +proc chunkedReadLoop(stream: AsyncStreamReader) {.async: (raises: []).} = var rstream = ChunkedStreamReader(stream) var buffer = newSeq[byte](MaxChunkHeaderSize) rstream.state = AsyncStreamState.Running @@ -156,6 +159,10 @@ proc chunkedReadLoop(stream: AsyncStreamReader) {.async.} = if rstream.state == AsyncStreamState.Running: rstream.state = AsyncStreamState.Error rstream.error = exc + except AsyncStreamError as exc: + if rstream.state == AsyncStreamState.Running: + rstream.state = AsyncStreamState.Error + rstream.error = exc if rstream.state != AsyncStreamState.Running: # We need to notify consumer about error/close, but we do not care about @@ -163,7 +170,7 @@ proc chunkedReadLoop(stream: AsyncStreamReader) {.async.} = rstream.buffer.forget() break -proc chunkedWriteLoop(stream: AsyncStreamWriter) {.async.} = +proc chunkedWriteLoop(stream: AsyncStreamWriter) {.async: (raises: []).} = var wstream = ChunkedStreamWriter(stream) var buffer: array[16, byte] var error: ref AsyncStreamError @@ -220,7 +227,11 @@ proc chunkedWriteLoop(stream: AsyncStreamWriter) {.async.} = if not(item.future.finished()): item.future.fail(error) while not(wstream.queue.empty()): - let pitem = wstream.queue.popFirstNoWait() + let pitem = + try: + wstream.queue.popFirstNoWait() + except AsyncQueueEmptyError: + raiseAssert "AsyncQueue should not be empty at this moment" if not(pitem.future.finished()): pitem.future.fail(error) break diff --git a/chronos/streams/tlsstream.nim b/chronos/streams/tlsstream.nim index 0c8efb9..26f2bab 100644 --- a/chronos/streams/tlsstream.nim +++ b/chronos/streams/tlsstream.nim @@ -9,6 +9,9 @@ ## This module implements Transport Layer Security (TLS) stream. This module ## uses sources of BearSSL by Thomas Pornin. + +{.push raises: [].} + import bearssl/[brssl, ec, errors, pem, rsa, ssl, x509], bearssl/certs/cacert @@ -71,7 +74,7 @@ type scontext: ptr SslServerContext stream*: TLSAsyncStream handshaked*: bool - handshakeFut*: Future[void] + handshakeFut*: Future[void].Raising([CancelledError, AsyncStreamError]) TLSStreamReader* = ref object of AsyncStreamReader case kind: TLSStreamKind @@ -81,7 +84,7 @@ type scontext: ptr SslServerContext stream*: TLSAsyncStream handshaked*: bool - handshakeFut*: Future[void] + handshakeFut*: Future[void].Raising([CancelledError, AsyncStreamError]) TLSAsyncStream* = ref object of RootRef xwc*: X509NoanchorContext @@ -91,7 +94,7 @@ type x509*: X509MinimalContext reader*: TLSStreamReader writer*: TLSStreamWriter - mainLoop*: Future[void] + mainLoop*: Future[void].Raising([]) trustAnchors: TrustAnchorStore SomeTLSStreamType* = TLSStreamReader|TLSStreamWriter|TLSAsyncStream @@ -101,9 +104,7 @@ type TLSStreamHandshakeError* = object of TLSStreamError TLSStreamInitError* = object of TLSStreamError TLSStreamReadError* = object of TLSStreamError - par*: ref AsyncStreamError TLSStreamWriteError* = object of TLSStreamError - par*: ref AsyncStreamError TLSStreamProtocolError* = object of TLSStreamError errCode*: int @@ -111,7 +112,7 @@ proc newTLSStreamWriteError(p: ref AsyncStreamError): ref TLSStreamWriteError {. noinline.} = var w = newException(TLSStreamWriteError, "Write stream failed") w.msg = w.msg & ", originated from [" & $p.name & "] " & p.msg - w.par = p + w.parent = p w template newTLSStreamProtocolImpl[T](message: T): ref TLSStreamProtocolError = @@ -137,7 +138,8 @@ template newTLSUnexpectedProtocolError(): ref TLSStreamProtocolError = proc newTLSStreamProtocolError[T](message: T): ref TLSStreamProtocolError = newTLSStreamProtocolImpl(message) -proc raiseTLSStreamProtocolError[T](message: T) {.noreturn, noinline.} = +proc raiseTLSStreamProtocolError[T](message: T) {. + noreturn, noinline, raises: [TLSStreamProtocolError].} = raise newTLSStreamProtocolImpl(message) proc new*(T: typedesc[TrustAnchorStore], @@ -150,7 +152,8 @@ proc new*(T: typedesc[TrustAnchorStore], TrustAnchorStore(anchors: res) proc tlsWriteRec(engine: ptr SslEngineContext, - writer: TLSStreamWriter): Future[TLSResult] {.async.} = + writer: TLSStreamWriter): Future[TLSResult] {. + async: (raises: []).} = try: var length = 0'u var buf = sslEngineSendrecBuf(engine[], length) @@ -168,7 +171,8 @@ proc tlsWriteRec(engine: ptr SslEngineContext, TLSResult.Stopped proc tlsWriteApp(engine: ptr SslEngineContext, - writer: TLSStreamWriter): Future[TLSResult] {.async.} = + writer: TLSStreamWriter): Future[TLSResult] {. + async: (raises: []).} = try: var item = await writer.queue.get() if item.size > 0: @@ -192,7 +196,10 @@ proc tlsWriteApp(engine: ptr SslEngineContext, # only part of item and adjust offset. item.offset = item.offset + int(length) item.size = item.size - int(length) - writer.queue.addFirstNoWait(item) + try: + writer.queue.addFirstNoWait(item) + except AsyncQueueFullError: + raiseAssert "AsyncQueue should not be full at this moment" sslEngineSendappAck(engine[], length) TLSResult.Success else: @@ -205,7 +212,8 @@ proc tlsWriteApp(engine: ptr SslEngineContext, TLSResult.Stopped proc tlsReadRec(engine: ptr SslEngineContext, - reader: TLSStreamReader): Future[TLSResult] {.async.} = + reader: TLSStreamReader): Future[TLSResult] {. + async: (raises: []).} = try: var length = 0'u var buf = sslEngineRecvrecBuf(engine[], length) @@ -226,7 +234,8 @@ proc tlsReadRec(engine: ptr SslEngineContext, TLSResult.Stopped proc tlsReadApp(engine: ptr SslEngineContext, - reader: TLSStreamReader): Future[TLSResult] {.async.} = + reader: TLSStreamReader): Future[TLSResult] {. + async: (raises: []).} = try: var length = 0'u var buf = sslEngineRecvappBuf(engine[], length) @@ -240,7 +249,7 @@ proc tlsReadApp(engine: ptr SslEngineContext, template readAndReset(fut: untyped) = if fut.finished(): - let res = fut.read() + let res = fut.value() case res of TLSResult.Success, TLSResult.WriteEof, TLSResult.Stopped: fut = nil @@ -256,18 +265,6 @@ template readAndReset(fut: untyped) = loopState = AsyncStreamState.Finished break -proc cancelAndWait*(a, b, c, d: Future[TLSResult]): Future[void] = - var waiting: seq[FutureBase] - if not(isNil(a)) and not(a.finished()): - waiting.add(a.cancelAndWait()) - if not(isNil(b)) and not(b.finished()): - waiting.add(b.cancelAndWait()) - if not(isNil(c)) and not(c.finished()): - waiting.add(c.cancelAndWait()) - if not(isNil(d)) and not(d.finished()): - waiting.add(d.cancelAndWait()) - allFutures(waiting) - proc dumpState*(state: cuint): string = var res = "" if (state and SSL_CLOSED) == SSL_CLOSED: @@ -287,10 +284,10 @@ proc dumpState*(state: cuint): string = res.add("SSL_RECVAPP") "{" & res & "}" -proc tlsLoop*(stream: TLSAsyncStream) {.async.} = +proc tlsLoop*(stream: TLSAsyncStream) {.async: (raises: []).} = var - sendRecFut, sendAppFut: Future[TLSResult] - recvRecFut, recvAppFut: Future[TLSResult] + sendRecFut, sendAppFut: Future[TLSResult].Raising([]) + recvRecFut, recvAppFut: Future[TLSResult].Raising([]) let engine = case stream.reader.kind @@ -302,7 +299,7 @@ proc tlsLoop*(stream: TLSAsyncStream) {.async.} = var loopState = AsyncStreamState.Running while true: - var waiting: seq[Future[TLSResult]] + var waiting: seq[Future[TLSResult].Raising([])] var state = sslEngineCurrentState(engine[]) if (state and SSL_CLOSED) == SSL_CLOSED: @@ -353,6 +350,8 @@ proc tlsLoop*(stream: TLSAsyncStream) {.async.} = if len(waiting) > 0: try: discard await one(waiting) + except ValueError: + raiseAssert "array should not be empty at this moment" except CancelledError: if loopState == AsyncStreamState.Running: loopState = AsyncStreamState.Stopped @@ -360,8 +359,18 @@ proc tlsLoop*(stream: TLSAsyncStream) {.async.} = if loopState != AsyncStreamState.Running: break - # Cancelling and waiting all the pending operations - await cancelAndWait(sendRecFut, sendAppFut, recvRecFut, recvAppFut) + # Cancelling and waiting and all the pending operations + var pending: seq[FutureBase] + if not(isNil(sendRecFut)) and not(sendRecFut.finished()): + pending.add(sendRecFut.cancelAndWait()) + if not(isNil(sendAppFut)) and not(sendAppFut.finished()): + pending.add(sendAppFut.cancelAndWait()) + if not(isNil(recvRecFut)) and not(recvRecFut.finished()): + pending.add(recvRecFut.cancelAndWait()) + if not(isNil(recvAppFut)) and not(recvAppFut.finished()): + pending.add(recvAppFut.cancelAndWait()) + await noCancel(allFutures(pending)) + # Calculating error let error = case loopState @@ -395,7 +404,11 @@ proc tlsLoop*(stream: TLSAsyncStream) {.async.} = if not(isNil(error)): # Completing all pending writes while(not(stream.writer.queue.empty())): - let item = stream.writer.queue.popFirstNoWait() + let item = + try: + stream.writer.queue.popFirstNoWait() + except AsyncQueueEmptyError: + raiseAssert "AsyncQueue should not be empty at this moment" if not(item.future.finished()): item.future.fail(error) # Completing handshake @@ -415,18 +428,18 @@ proc tlsLoop*(stream: TLSAsyncStream) {.async.} = # Completing readers stream.reader.buffer.forget() -proc tlsWriteLoop(stream: AsyncStreamWriter) {.async.} = +proc tlsWriteLoop(stream: AsyncStreamWriter) {.async: (raises: []).} = var wstream = TLSStreamWriter(stream) wstream.state = AsyncStreamState.Running - await sleepAsync(0.milliseconds) + await noCancel(sleepAsync(0.milliseconds)) if isNil(wstream.stream.mainLoop): wstream.stream.mainLoop = tlsLoop(wstream.stream) await wstream.stream.mainLoop -proc tlsReadLoop(stream: AsyncStreamReader) {.async.} = +proc tlsReadLoop(stream: AsyncStreamReader) {.async: (raises: []).} = var rstream = TLSStreamReader(stream) rstream.state = AsyncStreamState.Running - await sleepAsync(0.milliseconds) + await noCancel(sleepAsync(0.milliseconds)) if isNil(rstream.stream.mainLoop): rstream.stream.mainLoop = tlsLoop(rstream.stream) await rstream.stream.mainLoop @@ -451,7 +464,7 @@ proc newTLSClientAsyncStream*( maxVersion = TLSVersion.TLS12, flags: set[TLSFlags] = {}, trustAnchors: SomeTrustAnchorType = MozillaTrustAnchors - ): TLSAsyncStream = + ): TLSAsyncStream {.raises: [TLSStreamInitError].} = ## Create new TLS asynchronous stream for outbound (client) connections ## using reading stream ``rsource`` and writing stream ``wsource``. ## @@ -541,7 +554,8 @@ proc newTLSServerAsyncStream*(rsource: AsyncStreamReader, minVersion = TLSVersion.TLS11, maxVersion = TLSVersion.TLS12, cache: TLSSessionCache = nil, - flags: set[TLSFlags] = {}): TLSAsyncStream = + flags: set[TLSFlags] = {}): TLSAsyncStream {. + raises: [TLSStreamInitError, TLSStreamProtocolError].} = ## Create new TLS asynchronous stream for inbound (server) connections ## using reading stream ``rsource`` and writing stream ``wsource``. ## @@ -609,10 +623,8 @@ proc newTLSServerAsyncStream*(rsource: AsyncStreamReader, if err == 0: raise newException(TLSStreamInitError, "Could not initialize TLS layer") - init(AsyncStreamWriter(res.writer), wsource, tlsWriteLoop, - bufferSize) - init(AsyncStreamReader(res.reader), rsource, tlsReadLoop, - bufferSize) + init(AsyncStreamWriter(res.writer), wsource, tlsWriteLoop, bufferSize) + init(AsyncStreamReader(res.reader), rsource, tlsReadLoop, bufferSize) res proc copyKey(src: RsaPrivateKey): TLSPrivateKey = @@ -653,7 +665,8 @@ proc copyKey(src: EcPrivateKey): TLSPrivateKey = res.eckey.curve = src.curve res -proc init*(tt: typedesc[TLSPrivateKey], data: openArray[byte]): TLSPrivateKey = +proc init*(tt: typedesc[TLSPrivateKey], data: openArray[byte]): TLSPrivateKey {. + raises: [TLSStreamProtocolError].} = ## Initialize TLS private key from array of bytes ``data``. ## ## This procedure initializes private key using raw, DER-encoded format, @@ -676,7 +689,8 @@ proc init*(tt: typedesc[TLSPrivateKey], data: openArray[byte]): TLSPrivateKey = raiseTLSStreamProtocolError("Unknown key type (" & $keyType & ")") res -proc pemDecode*(data: openArray[char]): seq[PEMElement] = +proc pemDecode*(data: openArray[char]): seq[PEMElement] {. + raises: [TLSStreamProtocolError].} = ## Decode PEM encoded string and get array of binary blobs. if len(data) == 0: raiseTLSStreamProtocolError("Empty PEM message") @@ -717,7 +731,8 @@ proc pemDecode*(data: openArray[char]): seq[PEMElement] = raiseTLSStreamProtocolError("Invalid PEM encoding") res -proc init*(tt: typedesc[TLSPrivateKey], data: openArray[char]): TLSPrivateKey = +proc init*(tt: typedesc[TLSPrivateKey], data: openArray[char]): TLSPrivateKey {. + raises: [TLSStreamProtocolError].} = ## Initialize TLS private key from string ``data``. ## ## This procedure initializes private key using unencrypted PKCS#8 PEM @@ -735,7 +750,8 @@ proc init*(tt: typedesc[TLSPrivateKey], data: openArray[char]): TLSPrivateKey = res proc init*(tt: typedesc[TLSCertificate], - data: openArray[char]): TLSCertificate = + data: openArray[char]): TLSCertificate {. + raises: [TLSStreamProtocolError].} = ## Initialize TLS certificates from string ``data``. ## ## This procedure initializes array of certificates from PEM encoded string. @@ -770,9 +786,11 @@ proc init*(tt: typedesc[TLSSessionCache], size: int = 4096): TLSSessionCache = sslSessionCacheLruInit(addr res.context, addr res.storage[0], rsize) res -proc handshake*(rws: SomeTLSStreamType): Future[void] = +proc handshake*(rws: SomeTLSStreamType): Future[void] {. + async: (raw: true, raises: [CancelledError, AsyncStreamError]).} = ## Wait until initial TLS handshake will be successfully performed. - var retFuture = newFuture[void]("tlsstream.handshake") + let retFuture = Future[void].Raising([CancelledError, AsyncStreamError]) + .init("tlsstream.handshake") when rws is TLSStreamReader: if rws.handshaked: retFuture.complete() diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index bdcb8d7..f2e7a58 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -140,7 +140,7 @@ type # transport for new client proc remoteAddress*(transp: StreamTransport): TransportAddress {. - raises: [TransportAbortedError, TransportTooManyError, TransportOsError].} = + raises: [TransportOsError].} = ## Returns ``transp`` remote socket address. doAssert(transp.kind == TransportKind.Socket, "Socket transport required!") if transp.remote.family == AddressFamily.None: @@ -148,12 +148,12 @@ proc remoteAddress*(transp: StreamTransport): TransportAddress {. var slen = SockLen(sizeof(saddr)) if getpeername(SocketHandle(transp.fd), cast[ptr SockAddr](addr saddr), addr slen) != 0: - raiseTransportError(osLastError()) + raiseTransportOsError(osLastError()) fromSAddr(addr saddr, slen, transp.remote) transp.remote proc localAddress*(transp: StreamTransport): TransportAddress {. - raises: [TransportAbortedError, TransportTooManyError, TransportOsError].} = + raises: [TransportOsError].} = ## Returns ``transp`` local socket address. doAssert(transp.kind == TransportKind.Socket, "Socket transport required!") if transp.local.family == AddressFamily.None: @@ -161,7 +161,7 @@ proc localAddress*(transp: StreamTransport): TransportAddress {. var slen = SockLen(sizeof(saddr)) if getsockname(SocketHandle(transp.fd), cast[ptr SockAddr](addr saddr), addr slen) != 0: - raiseTransportError(osLastError()) + raiseTransportOsError(osLastError()) fromSAddr(addr saddr, slen, transp.local) transp.local From f03cdfcc409efdb66801ddd230bba9c4614b62f3 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sun, 19 Nov 2023 18:29:09 +0100 Subject: [PATCH 44/50] futures: sinkify (#475) This avoids copies here and there throughout the pipeline - ie `copyString` and friends can often be avoided when moving things into and out of futures Annoyingly, one has to sprinkle the codebase liberally with `sink` and `move` for the pipeline to work well - sink stuff _generally_ works better in orc/arc Looking at nim 1.6/refc, sink + local variable + move generates the best code: msg directly: ```nim T1_ = (*colonenv_).msg1; (*colonenv_).msg1 = copyStringRC1(msg); ``` local copy without move: ```nim T60_ = (*colonenv_).localCopy1; (*colonenv_).localCopy1 = copyStringRC1(msg); ``` local copy with move: ```nim asgnRef((void**) (&(*colonenv_).localCopy1), msg); ``` Annoyingly, sink is also broken for refc+literals as it tries to changes the refcount of the literal as part of the move (which shouldn't be happening, but here we are), so we have to use a hack to find literals and avoid moving them. --- chronos/config.nim | 37 +++++++++++++++++++++++++++++++ chronos/internal/asyncfutures.nim | 6 ++--- chronos/internal/asyncmacro.nim | 2 +- chronos/streams/tlsstream.nim | 8 +++---- chronos/transports/common.nim | 16 ------------- chronos/transports/datagram.nim | 10 ++++----- chronos/transports/stream.nim | 22 +++++++++--------- tests/testfut.nim | 6 +++++ 8 files changed, 67 insertions(+), 40 deletions(-) diff --git a/chronos/config.nim b/chronos/config.nim index 4055361..21c3132 100644 --- a/chronos/config.nim +++ b/chronos/config.nim @@ -101,3 +101,40 @@ when defined(debug) or defined(chronosConfig): printOption("chronosEventEngine", chronosEventEngine) printOption("chronosEventsCount", chronosEventsCount) printOption("chronosInitialSize", chronosInitialSize) + + +# In nim 1.6, `sink` + local variable + `move` generates the best code for +# moving a proc parameter into a closure - this only works for closure +# procedures however - in closure iterators, the parameter is always copied +# into the closure (!) meaning that non-raw `{.async.}` functions always carry +# this overhead, sink or no. See usages of chronosMoveSink for examples. +# In addition, we need to work around https://github.com/nim-lang/Nim/issues/22175 +# which has not been backported to 1.6. +# Long story short, the workaround is not needed in non-raw {.async.} because +# a copy of the literal is always made. +# TODO review the above for 2.0 / 2.0+refc +type + SeqHeader = object + length, reserved: int + +proc isLiteral(s: string): bool {.inline.} = + when defined(gcOrc) or defined(gcArc): + false + else: + s.len > 0 and (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0 + +proc isLiteral[T](s: seq[T]): bool {.inline.} = + when defined(gcOrc) or defined(gcArc): + false + else: + s.len > 0 and (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0 + +template chronosMoveSink*(val: auto): untyped = + bind isLiteral + when not (defined(gcOrc) or defined(gcArc)) and val is seq|string: + if isLiteral(val): + val + else: + move(val) + else: + move(val) diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index f60b2d9..6a6dbb2 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -202,14 +202,14 @@ proc finish(fut: FutureBase, state: FutureState) = when chronosFutureTracking: scheduleDestructor(fut) -proc complete[T](future: Future[T], val: T, loc: ptr SrcLoc) = +proc complete[T](future: Future[T], val: sink T, loc: ptr SrcLoc) = if not(future.cancelled()): checkFinished(future, loc) doAssert(isNil(future.internalError)) - future.internalValue = val + future.internalValue = chronosMoveSink(val) future.finish(FutureState.Completed) -template complete*[T](future: Future[T], val: T) = +template complete*[T](future: Future[T], val: sink T) = ## Completes ``future`` with value ``val``. complete(future, val, getSrcLocation()) diff --git a/chronos/internal/asyncmacro.nim b/chronos/internal/asyncmacro.nim index 079e3bb..4e9b8d4 100644 --- a/chronos/internal/asyncmacro.nim +++ b/chronos/internal/asyncmacro.nim @@ -157,7 +157,7 @@ proc wrapInTryFinally( newCall(ident "complete", fut) ), nnkElseExpr.newTree( - newCall(ident "complete", fut, ident "result") + newCall(ident "complete", fut, newCall(ident "move", ident "result")) ) ) ) diff --git a/chronos/streams/tlsstream.nim b/chronos/streams/tlsstream.nim index 26f2bab..12ea6d3 100644 --- a/chronos/streams/tlsstream.nim +++ b/chronos/streams/tlsstream.nim @@ -15,7 +15,7 @@ import bearssl/[brssl, ec, errors, pem, rsa, ssl, x509], bearssl/certs/cacert -import ../asyncloop, ../timer, ../asyncsync +import ".."/[asyncloop, asyncsync, config, timer] import asyncstream, ../transports/stream, ../transports/common export asyncloop, asyncsync, timer, asyncstream @@ -62,7 +62,7 @@ type PEMContext = ref object data: seq[byte] - + TrustAnchorStore* = ref object anchors: seq[X509TrustAnchor] @@ -158,7 +158,7 @@ proc tlsWriteRec(engine: ptr SslEngineContext, var length = 0'u var buf = sslEngineSendrecBuf(engine[], length) doAssert(length != 0 and not isNil(buf)) - await writer.wsource.write(buf, int(length)) + await writer.wsource.write(chronosMoveSink(buf), int(length)) sslEngineSendrecAck(engine[], length) TLSResult.Success except AsyncStreamError as exc: @@ -481,7 +481,7 @@ proc newTLSClientAsyncStream*( ## ``minVersion`` of bigger then ``maxVersion`` you will get an error. ## ## ``flags`` - custom TLS connection flags. - ## + ## ## ``trustAnchors`` - use this if you want to use certificate trust ## anchors other than the default Mozilla trust anchors. If you pass ## a ``TrustAnchorStore`` you should reuse the same instance for diff --git a/chronos/transports/common.nim b/chronos/transports/common.nim index 24f9852..ba7568a 100644 --- a/chronos/transports/common.nim +++ b/chronos/transports/common.nim @@ -596,22 +596,6 @@ proc raiseTransportOsError*(err: OSErrorCode) {. ## Raises transport specific OS error. raise getTransportOsError(err) -type - SeqHeader = object - length, reserved: int - -proc isLiteral*(s: string): bool {.inline.} = - when defined(gcOrc) or defined(gcArc): - false - else: - (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0 - -proc isLiteral*[T](s: seq[T]): bool {.inline.} = - when defined(gcOrc) or defined(gcArc): - false - else: - (cast[ptr SeqHeader](s).reserved and (1 shl (sizeof(int) * 8 - 2))) != 0 - template getTransportTooManyError*( code = OSErrorCode(0) ): ref TransportTooManyError = diff --git a/chronos/transports/datagram.nim b/chronos/transports/datagram.nim index 30f872d..fed15d3 100644 --- a/chronos/transports/datagram.nim +++ b/chronos/transports/datagram.nim @@ -11,7 +11,7 @@ import std/deques when not(defined(windows)): import ".."/selectors2 -import ".."/[asyncloop, osdefs, oserrno, osutils, handles] +import ".."/[asyncloop, config, osdefs, oserrno, osutils, handles] import "."/common type @@ -894,7 +894,7 @@ proc send*(transp: DatagramTransport, msg: sink string, transp.checkClosed(retFuture) let length = if msglen <= 0: len(msg) else: msglen - var localCopy = msg + var localCopy = chronosMoveSink(msg) retFuture.addCallback(proc(_: pointer) = reset(localCopy)) let vector = GramVector(kind: WithoutAddress, buf: addr localCopy[0], @@ -917,7 +917,7 @@ proc send*[T](transp: DatagramTransport, msg: sink seq[T], transp.checkClosed(retFuture) let length = if msglen <= 0: (len(msg) * sizeof(T)) else: (msglen * sizeof(T)) - var localCopy = msg + var localCopy = chronosMoveSink(msg) retFuture.addCallback(proc(_: pointer) = reset(localCopy)) let vector = GramVector(kind: WithoutAddress, buf: addr localCopy[0], @@ -955,7 +955,7 @@ proc sendTo*(transp: DatagramTransport, remote: TransportAddress, transp.checkClosed(retFuture) let length = if msglen <= 0: len(msg) else: msglen - var localCopy = msg + var localCopy = chronosMoveSink(msg) retFuture.addCallback(proc(_: pointer) = reset(localCopy)) let vector = GramVector(kind: WithAddress, buf: addr localCopy[0], @@ -977,7 +977,7 @@ proc sendTo*[T](transp: DatagramTransport, remote: TransportAddress, var retFuture = newFuture[void]("datagram.transport.sendTo(seq)") transp.checkClosed(retFuture) let length = if msglen <= 0: (len(msg) * sizeof(T)) else: (msglen * sizeof(T)) - var localCopy = msg + var localCopy = chronosMoveSink(msg) retFuture.addCallback(proc(_: pointer) = reset(localCopy)) let vector = GramVector(kind: WithAddress, buf: addr localCopy[0], diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index f2e7a58..58aabc3 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -10,8 +10,9 @@ {.push raises: [].} import std/deques -import ".."/[asyncloop, handles, osdefs, osutils, oserrno] -import common +import stew/ptrops +import ".."/[asyncloop, config, handles, osdefs, osutils, oserrno] +import ./common type VectorKind = enum @@ -770,7 +771,7 @@ when defined(windows): # Continue only if `retFuture` is not cancelled. if not(retFuture.finished()): let - pipeSuffix = $cast[cstring](unsafeAddr address.address_un[0]) + pipeSuffix = $cast[cstring](baseAddr address.address_un) pipeAsciiName = PipeHeaderName & pipeSuffix[1 .. ^1] pipeName = toWideString(pipeAsciiName).valueOr: retFuture.fail(getTransportOsError(error)) @@ -806,7 +807,7 @@ when defined(windows): proc createAcceptPipe(server: StreamServer): Result[AsyncFD, OSErrorCode] = let - pipeSuffix = $cast[cstring](addr server.local.address_un) + pipeSuffix = $cast[cstring](baseAddr server.local.address_un) pipeName = ? toWideString(PipeHeaderName & pipeSuffix) openMode = if FirstPipe notin server.flags: @@ -878,7 +879,7 @@ when defined(windows): if server.status notin {ServerStatus.Stopped, ServerStatus.Closed}: server.apending = true let - pipeSuffix = $cast[cstring](addr server.local.address_un) + pipeSuffix = $cast[cstring](baseAddr server.local.address_un) pipeAsciiName = PipeHeaderName & pipeSuffix pipeName = toWideString(pipeAsciiName).valueOr: raiseOsDefect(error, "acceptPipeLoop(): Unable to create name " & @@ -2011,7 +2012,7 @@ proc createStreamServer*(host: TransportAddress, elif host.family in {AddressFamily.Unix}: # We do not care about result here, because if file cannot be removed, # `bindSocket` will return EADDRINUSE. - discard osdefs.unlink(cast[cstring](unsafeAddr host.address_un[0])) + discard osdefs.unlink(cast[cstring](baseAddr host.address_un)) host.toSAddr(saddr, slen) if osdefs.bindSocket(SocketHandle(serverSocket), @@ -2240,12 +2241,11 @@ proc write*(transp: StreamTransport, msg: sink string, var retFuture = newFuture[int]("stream.transport.write(string)") transp.checkClosed(retFuture) transp.checkWriteEof(retFuture) - let nbytes = if msglen <= 0: len(msg) else: msglen var - pbytes = cast[ptr byte](unsafeAddr msg[0]) + pbytes = cast[ptr byte](baseAddr msg) rbytes = nbytes fastWrite(transp, pbytes, rbytes, nbytes) @@ -2253,7 +2253,7 @@ proc write*(transp: StreamTransport, msg: sink string, let written = nbytes - rbytes # In case fastWrite wrote some - var localCopy = msg + var localCopy = chronosMoveSink(msg) retFuture.addCallback(proc(_: pointer) = reset(localCopy)) pbytes = cast[ptr byte](addr localCopy[written]) @@ -2278,7 +2278,7 @@ proc write*[T](transp: StreamTransport, msg: sink seq[T], nbytes = if msglen <= 0: (len(msg) * sizeof(T)) else: (msglen * sizeof(T)) var - pbytes = cast[ptr byte](unsafeAddr msg[0]) + pbytes = cast[ptr byte](baseAddr msg) rbytes = nbytes fastWrite(transp, pbytes, rbytes, nbytes) @@ -2286,7 +2286,7 @@ proc write*[T](transp: StreamTransport, msg: sink seq[T], let written = nbytes - rbytes # In case fastWrite wrote some - var localCopy = msg + var localCopy = chronosMoveSink(msg) retFuture.addCallback(proc(_: pointer) = reset(localCopy)) pbytes = cast[ptr byte](addr localCopy[written]) diff --git a/tests/testfut.nim b/tests/testfut.nim index 1297dc4..367b5d0 100644 --- a/tests/testfut.nim +++ b/tests/testfut.nim @@ -1997,3 +1997,9 @@ suite "Future[T] behavior test suite": check: future1.cancelled() == true future2.cancelled() == true + test "Sink with literals": + # https://github.com/nim-lang/Nim/issues/22175 + let fut = newFuture[string]() + fut.complete("test") + check: + fut.value() == "test" From fa0bf405e64b1c2c3693110aaff1881a535e5fab Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 20 Nov 2023 11:04:28 +0100 Subject: [PATCH 45/50] varargs overloads (#477) * varargs overloads for convenience and compatibility * no parameterless varargs calls with generic overloads --- chronos/internal/asyncfutures.nim | 38 +++++++++++++++++++++++++++---- tests/testfut.nim | 13 +++++++---- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index 6a6dbb2..ba7eaf0 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -1094,10 +1094,18 @@ proc allFutures*[T](futs: varargs[Future[T]]): Future[void] {. ## ## On cancel all the awaited futures ``futs`` WILL NOT BE cancelled. # Because we can't capture varargs[T] in closures we need to create copy. - var nfuts: seq[FutureBase] - for future in futs: - nfuts.add(future) - allFutures(nfuts) + allFutures(futs.mapIt(FutureBase(it))) + +proc allFutures*[T, E](futs: varargs[InternalRaisesFuture[T, E]]): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = + ## Returns a future which will complete only when all futures in ``futs`` + ## will be completed, failed or canceled. + ## + ## If the argument is empty, the returned future COMPLETES immediately. + ## + ## On cancel all the awaited futures ``futs`` WILL NOT BE cancelled. + # Because we can't capture varargs[T] in closures we need to create copy. + allFutures(futs.mapIt(FutureBase(it))) proc allFinished*[F: SomeFuture](futs: varargs[F]): Future[seq[F]] {. async: (raw: true, raises: [CancelledError]).} = @@ -1239,6 +1247,28 @@ proc race*(futs: varargs[FutureBase]): Future[FutureBase] {. return retFuture +proc race*[T](futs: varargs[Future[T]]): Future[FutureBase] {. + async: (raw: true, raises: [ValueError, CancelledError]).} = + ## Returns a future which will complete only when all futures in ``futs`` + ## will be completed, failed or canceled. + ## + ## If the argument is empty, the returned future COMPLETES immediately. + ## + ## On cancel all the awaited futures ``futs`` WILL NOT BE cancelled. + # Because we can't capture varargs[T] in closures we need to create copy. + race(futs.mapIt(FutureBase(it))) + +proc race*[T, E](futs: varargs[InternalRaisesFuture[T, E]]): Future[FutureBase] {. + async: (raw: true, raises: [ValueError, CancelledError]).} = + ## Returns a future which will complete only when all futures in ``futs`` + ## will be completed, failed or canceled. + ## + ## If the argument is empty, the returned future COMPLETES immediately. + ## + ## On cancel all the awaited futures ``futs`` WILL NOT BE cancelled. + # Because we can't capture varargs[T] in closures we need to create copy. + race(futs.mapIt(FutureBase(it))) + when (chronosEventEngine in ["epoll", "kqueue"]) or defined(windows): import std/os diff --git a/tests/testfut.nim b/tests/testfut.nim index 367b5d0..fc2401d 100644 --- a/tests/testfut.nim +++ b/tests/testfut.nim @@ -1314,12 +1314,17 @@ suite "Future[T] behavior test suite": test "race(zero) test": var tseq = newSeq[FutureBase]() var fut1 = race(tseq) - var fut2 = race() - var fut3 = race([]) + check: + # https://github.com/nim-lang/Nim/issues/22964 + not compiles(block: + var fut2 = race()) + not compiles(block: + var fut3 = race([])) + check: fut1.failed() - fut2.failed() - fut3.failed() + # fut2.failed() + # fut3.failed() asyncTest "race(varargs) test": proc vlient1() {.async.} = From b18d471629357ac7a07aed47d74e15fc9c71c664 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Tue, 21 Nov 2023 12:01:44 +0200 Subject: [PATCH 46/50] Asyncraises HTTP client/server. (#476) * Fixes. * Make httpcommon no-raises. * Make httpbodyrw no-raises. * Make multipart no-raises. * Make httpdebug no-raises. * Make httpagent no-raises. * Make httpclient no-raises. * Make httpserver/shttpserver no-raises. * fix prepend/remove when E is noraises --------- Co-authored-by: Jacek Sieka --- chronos/apps/http/httpagent.nim | 3 + chronos/apps/http/httpbodyrw.nim | 12 +- chronos/apps/http/httpclient.nim | 274 +++++++++++++++------------- chronos/apps/http/httpcommon.nim | 51 +++--- chronos/apps/http/httpdebug.nim | 5 +- chronos/apps/http/httpserver.nim | 280 ++++++++++++++++------------- chronos/apps/http/multipart.nim | 103 ++++++----- chronos/apps/http/shttpserver.nim | 51 +++--- chronos/internal/asyncfutures.nim | 4 +- chronos/internal/raisesfutures.nim | 25 ++- chronos/transports/stream.nim | 15 +- 11 files changed, 469 insertions(+), 354 deletions(-) diff --git a/chronos/apps/http/httpagent.nim b/chronos/apps/http/httpagent.nim index c8cac48..36d13f2 100644 --- a/chronos/apps/http/httpagent.nim +++ b/chronos/apps/http/httpagent.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import strutils const diff --git a/chronos/apps/http/httpbodyrw.nim b/chronos/apps/http/httpbodyrw.nim index bb28ea6..c9ac899 100644 --- a/chronos/apps/http/httpbodyrw.nim +++ b/chronos/apps/http/httpbodyrw.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import ../../asyncloop, ../../asyncsync import ../../streams/[asyncstream, boundstream] import httpcommon @@ -36,7 +39,7 @@ proc newHttpBodyReader*(streams: varargs[AsyncStreamReader]): HttpBodyReader = trackCounter(HttpBodyReaderTrackerName) res -proc closeWait*(bstream: HttpBodyReader) {.async.} = +proc closeWait*(bstream: HttpBodyReader) {.async: (raises: []).} = ## Close and free resource allocated by body reader. if bstream.bstate == HttpState.Alive: bstream.bstate = HttpState.Closing @@ -61,7 +64,7 @@ proc newHttpBodyWriter*(streams: varargs[AsyncStreamWriter]): HttpBodyWriter = trackCounter(HttpBodyWriterTrackerName) res -proc closeWait*(bstream: HttpBodyWriter) {.async.} = +proc closeWait*(bstream: HttpBodyWriter) {.async: (raises: []).} = ## Close and free all the resources allocated by body writer. if bstream.bstate == HttpState.Alive: bstream.bstate = HttpState.Closing @@ -73,7 +76,7 @@ proc closeWait*(bstream: HttpBodyWriter) {.async.} = bstream.bstate = HttpState.Closed untrackCounter(HttpBodyWriterTrackerName) -proc hasOverflow*(bstream: HttpBodyReader): bool {.raises: [].} = +proc hasOverflow*(bstream: HttpBodyReader): bool = if len(bstream.streams) == 1: # If HttpBodyReader has only one stream it has ``BoundedStreamReader``, in # such case its impossible to get more bytes then expected amount. @@ -89,6 +92,5 @@ proc hasOverflow*(bstream: HttpBodyReader): bool {.raises: [].} = else: false -proc closed*(bstream: HttpBodyReader | HttpBodyWriter): bool {. - raises: [].} = +proc closed*(bstream: HttpBodyReader | HttpBodyWriter): bool = bstream.bstate != HttpState.Alive diff --git a/chronos/apps/http/httpclient.nim b/chronos/apps/http/httpclient.nim index 83d1ddf..5f4bd71 100644 --- a/chronos/apps/http/httpclient.nim +++ b/chronos/apps/http/httpclient.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import std/[uri, tables, sequtils] import stew/[base10, base64, byteutils], httputils, results import ../../asyncloop, ../../asyncsync @@ -120,7 +123,7 @@ type headersTimeout*: Duration idleTimeout: Duration idlePeriod: Duration - watcherFut: Future[void] + watcherFut: Future[void].Raising([]) connectionBufferSize*: int maxConnections*: int connectionsCount*: int @@ -253,7 +256,7 @@ template isIdle(conn: HttpClientConnectionRef, timestamp: Moment, timeout: Duration): bool = (timestamp - conn.timestamp) >= timeout -proc sessionWatcher(session: HttpSessionRef) {.async.} +proc sessionWatcher(session: HttpSessionRef) {.async: (raises: []).} proc new*(t: typedesc[HttpSessionRef], flags: HttpClientFlags = {}, @@ -265,8 +268,7 @@ proc new*(t: typedesc[HttpSessionRef], idleTimeout = HttpConnectionIdleTimeout, idlePeriod = HttpConnectionCheckPeriod, socketFlags: set[SocketFlags] = {}, - dualstack = DualStackType.Auto): HttpSessionRef {. - raises: [] .} = + dualstack = DualStackType.Auto): HttpSessionRef = ## Create new HTTP session object. ## ## ``maxRedirections`` - maximum number of HTTP 3xx redirections @@ -292,10 +294,10 @@ proc new*(t: typedesc[HttpSessionRef], if HttpClientFlag.Http11Pipeline in flags: sessionWatcher(res) else: - newFuture[void]("session.watcher.placeholder") + Future[void].Raising([]).init("session.watcher.placeholder") res -proc getTLSFlags(flags: HttpClientFlags): set[TLSFlags] {.raises: [] .} = +proc getTLSFlags(flags: HttpClientFlags): set[TLSFlags] = var res: set[TLSFlags] if HttpClientFlag.NoVerifyHost in flags: res.incl(TLSFlags.NoVerifyHost) @@ -306,7 +308,7 @@ proc getTLSFlags(flags: HttpClientFlags): set[TLSFlags] {.raises: [] .} = proc getHttpAddress*( url: Uri, flags: HttpClientFlags = {} - ): HttpAddressResult {.raises: [].} = + ): HttpAddressResult = let scheme = if len(url.scheme) == 0: @@ -370,24 +372,23 @@ proc getHttpAddress*( proc getHttpAddress*( url: string, flags: HttpClientFlags = {} - ): HttpAddressResult {.raises: [].} = + ): HttpAddressResult = getHttpAddress(parseUri(url), flags) proc getHttpAddress*( session: HttpSessionRef, url: Uri - ): HttpAddressResult {.raises: [].} = + ): HttpAddressResult = getHttpAddress(url, session.flags) proc getHttpAddress*( session: HttpSessionRef, url: string - ): HttpAddressResult {.raises: [].} = + ): HttpAddressResult = ## Create new HTTP address using URL string ``url`` and . getHttpAddress(parseUri(url), session.flags) -proc getAddress*(session: HttpSessionRef, url: Uri): HttpResult[HttpAddress] {. - raises: [] .} = +proc getAddress*(session: HttpSessionRef, url: Uri): HttpResult[HttpAddress] = let scheme = if len(url.scheme) == 0: HttpClientScheme.NonSecure @@ -451,13 +452,13 @@ proc getAddress*(session: HttpSessionRef, url: Uri): HttpResult[HttpAddress] {. addresses: addresses)) proc getAddress*(session: HttpSessionRef, - url: string): HttpResult[HttpAddress] {.raises: [].} = + url: string): HttpResult[HttpAddress] = ## Create new HTTP address using URL string ``url`` and . session.getAddress(parseUri(url)) proc getAddress*(address: TransportAddress, ctype: HttpClientScheme = HttpClientScheme.NonSecure, - queryString: string = "/"): HttpAddress {.raises: [].} = + queryString: string = "/"): HttpAddress = ## Create new HTTP address using Transport address ``address``, connection ## type ``ctype`` and query string ``queryString``. let uri = parseUri(queryString) @@ -540,8 +541,12 @@ proc getUniqueConnectionId(session: HttpSessionRef): uint64 = inc(session.counter) session.counter -proc new(t: typedesc[HttpClientConnectionRef], session: HttpSessionRef, - ha: HttpAddress, transp: StreamTransport): HttpClientConnectionRef = +proc new( + t: typedesc[HttpClientConnectionRef], + session: HttpSessionRef, + ha: HttpAddress, + transp: StreamTransport + ): Result[HttpClientConnectionRef, string] = case ha.scheme of HttpClientScheme.NonSecure: let res = HttpClientConnectionRef( @@ -554,44 +559,48 @@ proc new(t: typedesc[HttpClientConnectionRef], session: HttpSessionRef, remoteHostname: ha.id ) trackCounter(HttpClientConnectionTrackerName) - res + ok(res) of HttpClientScheme.Secure: - let treader = newAsyncStreamReader(transp) - let twriter = newAsyncStreamWriter(transp) - let tls = newTLSClientAsyncStream(treader, twriter, ha.hostname, - flags = session.flags.getTLSFlags()) - let res = HttpClientConnectionRef( - id: session.getUniqueConnectionId(), - kind: HttpClientScheme.Secure, - transp: transp, - treader: treader, - twriter: twriter, - reader: tls.reader, - writer: tls.writer, - tls: tls, - state: HttpClientConnectionState.Connecting, - remoteHostname: ha.id - ) - trackCounter(HttpClientConnectionTrackerName) - res + let + treader = newAsyncStreamReader(transp) + twriter = newAsyncStreamWriter(transp) + tls = + try: + newTLSClientAsyncStream(treader, twriter, ha.hostname, + flags = session.flags.getTLSFlags()) + except TLSStreamInitError as exc: + return err(exc.msg) -proc setError(request: HttpClientRequestRef, error: ref HttpError) {. - raises: [] .} = + res = HttpClientConnectionRef( + id: session.getUniqueConnectionId(), + kind: HttpClientScheme.Secure, + transp: transp, + treader: treader, + twriter: twriter, + reader: tls.reader, + writer: tls.writer, + tls: tls, + state: HttpClientConnectionState.Connecting, + remoteHostname: ha.id + ) + trackCounter(HttpClientConnectionTrackerName) + ok(res) + +proc setError(request: HttpClientRequestRef, error: ref HttpError) = request.error = error request.state = HttpReqRespState.Error if not(isNil(request.connection)): request.connection.state = HttpClientConnectionState.Error request.connection.error = error -proc setError(response: HttpClientResponseRef, error: ref HttpError) {. - raises: [] .} = +proc setError(response: HttpClientResponseRef, error: ref HttpError) = response.error = error response.state = HttpReqRespState.Error if not(isNil(response.connection)): response.connection.state = HttpClientConnectionState.Error response.connection.error = error -proc closeWait(conn: HttpClientConnectionRef) {.async.} = +proc closeWait(conn: HttpClientConnectionRef) {.async: (raises: []).} = ## Close HttpClientConnectionRef instance ``conn`` and free all the resources. if conn.state notin {HttpClientConnectionState.Closing, HttpClientConnectionState.Closed}: @@ -613,7 +622,8 @@ proc closeWait(conn: HttpClientConnectionRef) {.async.} = untrackCounter(HttpClientConnectionTrackerName) proc connect(session: HttpSessionRef, - ha: HttpAddress): Future[HttpClientConnectionRef] {.async.} = + ha: HttpAddress): Future[HttpClientConnectionRef] {. + async: (raises: [CancelledError, HttpConnectionError]).} = ## Establish new connection with remote server using ``url`` and ``flags``. ## On success returns ``HttpClientConnectionRef`` object. var lastError = "" @@ -627,12 +637,14 @@ proc connect(session: HttpSessionRef, dualstack = session.dualstack) except CancelledError as exc: raise exc - except CatchableError: + except TransportError: nil if not(isNil(transp)): let conn = block: - let res = HttpClientConnectionRef.new(session, ha, transp) + let res = HttpClientConnectionRef.new(session, ha, transp).valueOr: + raiseHttpConnectionError( + "Could not connect to remote host, reason: " & error) if res.kind == HttpClientScheme.Secure: try: await res.tls.handshake() @@ -662,7 +674,7 @@ proc connect(session: HttpSessionRef, raiseHttpConnectionError("Could not connect to remote host") proc removeConnection(session: HttpSessionRef, - conn: HttpClientConnectionRef) {.async.} = + conn: HttpClientConnectionRef) {.async: (raises: []).} = let removeHost = block: var res = false @@ -686,7 +698,8 @@ proc acquireConnection( session: HttpSessionRef, ha: HttpAddress, flags: set[HttpClientRequestFlag] - ): Future[HttpClientConnectionRef] {.async.} = + ): Future[HttpClientConnectionRef] {. + async: (raises: [CancelledError, HttpConnectionError]).} = ## Obtain connection from ``session`` or establish a new one. var default: seq[HttpClientConnectionRef] let timestamp = Moment.now() @@ -710,10 +723,11 @@ proc acquireConnection( inc(session.connectionsCount) connection.setTimestamp(timestamp) connection.setDuration() - return connection + connection proc releaseConnection(session: HttpSessionRef, - connection: HttpClientConnectionRef) {.async.} = + connection: HttpClientConnectionRef) {. + async: (raises: []).} = ## Return connection back to the ``session``. let removeConnection = if HttpClientFlag.Http11Pipeline notin session.flags: @@ -751,7 +765,7 @@ proc releaseConnection(session: HttpSessionRef, HttpClientConnectionFlag.Response, HttpClientConnectionFlag.NoBody}) -proc releaseConnection(request: HttpClientRequestRef) {.async.} = +proc releaseConnection(request: HttpClientRequestRef) {.async: (raises: []).} = let session = request.session connection = request.connection @@ -763,7 +777,8 @@ proc releaseConnection(request: HttpClientRequestRef) {.async.} = if HttpClientConnectionFlag.Response notin connection.flags: await session.releaseConnection(connection) -proc releaseConnection(response: HttpClientResponseRef) {.async.} = +proc releaseConnection(response: HttpClientResponseRef) {. + async: (raises: []).} = let session = response.session connection = response.connection @@ -775,7 +790,7 @@ proc releaseConnection(response: HttpClientResponseRef) {.async.} = if HttpClientConnectionFlag.Request notin connection.flags: await session.releaseConnection(connection) -proc closeWait*(session: HttpSessionRef) {.async.} = +proc closeWait*(session: HttpSessionRef) {.async: (raises: []).} = ## Closes HTTP session object. ## ## This closes all the connections opened to remote servers. @@ -788,7 +803,7 @@ proc closeWait*(session: HttpSessionRef) {.async.} = pending.add(closeWait(conn)) await noCancel(allFutures(pending)) -proc sessionWatcher(session: HttpSessionRef) {.async.} = +proc sessionWatcher(session: HttpSessionRef) {.async: (raises: []).} = while true: let firstBreak = try: @@ -819,18 +834,19 @@ proc sessionWatcher(session: HttpSessionRef) {.async.} = var pending: seq[Future[void]] let secondBreak = try: - pending = idleConnections.mapIt(it.closeWait()) + for conn in idleConnections: + pending.add(conn.closeWait()) await allFutures(pending) false except CancelledError: # We still want to close connections to avoid socket leaks. - await allFutures(pending) + await noCancel(allFutures(pending)) true if secondBreak: break -proc closeWait*(request: HttpClientRequestRef) {.async.} = +proc closeWait*(request: HttpClientRequestRef) {.async: (raises: []).} = var pending: seq[FutureBase] if request.state notin {HttpReqRespState.Closing, HttpReqRespState.Closed}: request.state = HttpReqRespState.Closing @@ -845,7 +861,7 @@ proc closeWait*(request: HttpClientRequestRef) {.async.} = request.state = HttpReqRespState.Closed untrackCounter(HttpClientRequestTrackerName) -proc closeWait*(response: HttpClientResponseRef) {.async.} = +proc closeWait*(response: HttpClientResponseRef) {.async: (raises: []).} = var pending: seq[FutureBase] if response.state notin {HttpReqRespState.Closing, HttpReqRespState.Closed}: response.state = HttpReqRespState.Closing @@ -860,8 +876,10 @@ proc closeWait*(response: HttpClientResponseRef) {.async.} = response.state = HttpReqRespState.Closed untrackCounter(HttpClientResponseTrackerName) -proc prepareResponse(request: HttpClientRequestRef, data: openArray[byte] - ): HttpResult[HttpClientResponseRef] {.raises: [] .} = +proc prepareResponse( + request: HttpClientRequestRef, + data: openArray[byte] + ): HttpResult[HttpClientResponseRef] = ## Process response headers. let resp = parseResponse(data, false) if resp.failed(): @@ -972,7 +990,7 @@ proc prepareResponse(request: HttpClientRequestRef, data: openArray[byte] ok(res) proc getResponse(req: HttpClientRequestRef): Future[HttpClientResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = var buffer: array[HttpMaxHeadersSize, byte] let timestamp = Moment.now() req.connection.setTimestamp(timestamp) @@ -984,8 +1002,9 @@ proc getResponse(req: HttpClientRequestRef): Future[HttpClientResponseRef] {. req.session.headersTimeout) except AsyncTimeoutError: raiseHttpReadError("Reading response headers timed out") - except AsyncStreamError: - raiseHttpReadError("Could not read response headers") + except AsyncStreamError as exc: + raiseHttpReadError( + "Could not read response headers, reason: " & $exc.msg) let response = prepareResponse(req, buffer.toOpenArray(0, bytesRead - 1)) if response.isErr(): @@ -999,8 +1018,7 @@ proc new*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [], - body: openArray[byte] = []): HttpClientRequestRef {. - raises: [].} = + body: openArray[byte] = []): HttpClientRequestRef = let res = HttpClientRequestRef( state: HttpReqRespState.Ready, session: session, meth: meth, version: version, flags: flags, headers: HttpTable.init(headers), @@ -1014,8 +1032,7 @@ proc new*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [], - body: openArray[byte] = []): HttpResult[HttpClientRequestRef] {. - raises: [].} = + body: openArray[byte] = []): HttpResult[HttpClientRequestRef] = let address = ? session.getAddress(parseUri(url)) let res = HttpClientRequestRef( state: HttpReqRespState.Ready, session: session, meth: meth, @@ -1029,14 +1046,14 @@ proc get*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, url: string, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [] - ): HttpResult[HttpClientRequestRef] {.raises: [].} = + ): HttpResult[HttpClientRequestRef] = HttpClientRequestRef.new(session, url, MethodGet, version, flags, headers) proc get*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, ha: HttpAddress, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [] - ): HttpClientRequestRef {.raises: [].} = + ): HttpClientRequestRef = HttpClientRequestRef.new(session, ha, MethodGet, version, flags, headers) proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, @@ -1044,7 +1061,7 @@ proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [], body: openArray[byte] = [] - ): HttpResult[HttpClientRequestRef] {.raises: [].} = + ): HttpResult[HttpClientRequestRef] = HttpClientRequestRef.new(session, url, MethodPost, version, flags, headers, body) @@ -1052,8 +1069,7 @@ proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, url: string, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [], - body: openArray[char] = []): HttpResult[HttpClientRequestRef] {. - raises: [].} = + body: openArray[char] = []): HttpResult[HttpClientRequestRef] = HttpClientRequestRef.new(session, url, MethodPost, version, flags, headers, body.toOpenArrayByte(0, len(body) - 1)) @@ -1061,8 +1077,7 @@ proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, ha: HttpAddress, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [], - body: openArray[byte] = []): HttpClientRequestRef {. - raises: [].} = + body: openArray[byte] = []): HttpClientRequestRef = HttpClientRequestRef.new(session, ha, MethodPost, version, flags, headers, body) @@ -1070,13 +1085,11 @@ proc post*(t: typedesc[HttpClientRequestRef], session: HttpSessionRef, ha: HttpAddress, version: HttpVersion = HttpVersion11, flags: set[HttpClientRequestFlag] = {}, headers: openArray[HttpHeaderTuple] = [], - body: openArray[char] = []): HttpClientRequestRef {. - raises: [].} = + body: openArray[char] = []): HttpClientRequestRef = HttpClientRequestRef.new(session, ha, MethodPost, version, flags, headers, body.toOpenArrayByte(0, len(body) - 1)) -proc prepareRequest(request: HttpClientRequestRef): string {. - raises: [].} = +proc prepareRequest(request: HttpClientRequestRef): string = template hasChunkedEncoding(request: HttpClientRequestRef): bool = toLowerAscii(request.headers.getString(TransferEncodingHeader)) == "chunked" @@ -1151,7 +1164,7 @@ proc prepareRequest(request: HttpClientRequestRef): string {. res proc send*(request: HttpClientRequestRef): Future[HttpClientResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = doAssert(request.state == HttpReqRespState.Ready, "Request's state is " & $request.state) let connection = @@ -1184,25 +1197,24 @@ proc send*(request: HttpClientRequestRef): Future[HttpClientResponseRef] {. request.setDuration() request.setError(newHttpInterruptError()) raise exc - except AsyncStreamError: + except AsyncStreamError as exc: request.setDuration() - let error = newHttpWriteError("Could not send request headers") + let error = newHttpWriteError( + "Could not send request headers, reason: " & $exc.msg) request.setError(error) raise error - let resp = - try: - await request.getResponse() - except CancelledError as exc: - request.setError(newHttpInterruptError()) - raise exc - except HttpError as exc: - request.setError(exc) - raise exc - return resp + try: + await request.getResponse() + except CancelledError as exc: + request.setError(newHttpInterruptError()) + raise exc + except HttpError as exc: + request.setError(exc) + raise exc proc open*(request: HttpClientRequestRef): Future[HttpBodyWriter] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = ## Start sending request's headers and return `HttpBodyWriter`, which can be ## used to send request's body. doAssert(request.state == HttpReqRespState.Ready, @@ -1232,8 +1244,9 @@ proc open*(request: HttpClientRequestRef): Future[HttpBodyWriter] {. request.setDuration() request.setError(newHttpInterruptError()) raise exc - except AsyncStreamError: - let error = newHttpWriteError("Could not send request headers") + except AsyncStreamError as exc: + let error = newHttpWriteError( + "Could not send request headers, reason: " & $exc.msg) request.setDuration() request.setError(error) raise error @@ -1255,10 +1268,10 @@ proc open*(request: HttpClientRequestRef): Future[HttpBodyWriter] {. request.writer = writer request.state = HttpReqRespState.Open request.connection.state = HttpClientConnectionState.RequestBodySending - return writer + writer proc finish*(request: HttpClientRequestRef): Future[HttpClientResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = ## Finish sending request and receive response. doAssert(not(isNil(request.connection)), "Request missing connection instance") @@ -1295,7 +1308,8 @@ proc getNewLocation*(resp: HttpClientResponseRef): HttpResult[HttpAddress] = else: err("Location header is missing") -proc getBodyReader*(response: HttpClientResponseRef): HttpBodyReader = +proc getBodyReader*(response: HttpClientResponseRef): HttpBodyReader {. + raises: [HttpUseClosedError].} = ## Returns stream's reader instance which can be used to read response's body. ## ## Streams which was obtained using this procedure must be closed to avoid @@ -1324,7 +1338,8 @@ proc getBodyReader*(response: HttpClientResponseRef): HttpBodyReader = response.reader = reader response.reader -proc finish*(response: HttpClientResponseRef) {.async.} = +proc finish*(response: HttpClientResponseRef) {. + async: (raises: [HttpUseClosedError]).} = ## Finish receiving response. ## ## Because ``finish()`` returns nothing, this operation become NOP for @@ -1343,7 +1358,7 @@ proc finish*(response: HttpClientResponseRef) {.async.} = response.setDuration() proc getBodyBytes*(response: HttpClientResponseRef): Future[seq[byte]] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = ## Read all bytes from response ``response``. ## ## Note: This procedure performs automatic finishing for ``response``. @@ -1353,21 +1368,22 @@ proc getBodyBytes*(response: HttpClientResponseRef): Future[seq[byte]] {. await reader.closeWait() reader = nil await response.finish() - return data + data except CancelledError as exc: if not(isNil(reader)): await reader.closeWait() response.setError(newHttpInterruptError()) raise exc - except AsyncStreamError: + except AsyncStreamError as exc: + let error = newHttpReadError("Could not read response, reason: " & $exc.msg) if not(isNil(reader)): await reader.closeWait() - let error = newHttpReadError("Could not read response") response.setError(error) raise error proc getBodyBytes*(response: HttpClientResponseRef, - nbytes: int): Future[seq[byte]] {.async.} = + nbytes: int): Future[seq[byte]] {. + async: (raises: [CancelledError, HttpError]).} = ## Read all bytes (nbytes <= 0) or exactly `nbytes` bytes from response ## ``response``. ## @@ -1378,20 +1394,21 @@ proc getBodyBytes*(response: HttpClientResponseRef, await reader.closeWait() reader = nil await response.finish() - return data + data except CancelledError as exc: if not(isNil(reader)): await reader.closeWait() response.setError(newHttpInterruptError()) raise exc - except AsyncStreamError: + except AsyncStreamError as exc: + let error = newHttpReadError("Could not read response, reason: " & $exc.msg) if not(isNil(reader)): await reader.closeWait() - let error = newHttpReadError("Could not read response") response.setError(error) raise error -proc consumeBody*(response: HttpClientResponseRef): Future[int] {.async.} = +proc consumeBody*(response: HttpClientResponseRef): Future[int] {. + async: (raises: [CancelledError, HttpError]).} = ## Consume/discard response and return number of bytes consumed. ## ## Note: This procedure performs automatic finishing for ``response``. @@ -1401,16 +1418,17 @@ proc consumeBody*(response: HttpClientResponseRef): Future[int] {.async.} = await reader.closeWait() reader = nil await response.finish() - return res + res except CancelledError as exc: if not(isNil(reader)): await reader.closeWait() response.setError(newHttpInterruptError()) raise exc - except AsyncStreamError: + except AsyncStreamError as exc: + let error = newHttpReadError( + "Could not consume response, reason: " & $exc.msg) if not(isNil(reader)): await reader.closeWait() - let error = newHttpReadError("Could not read response") response.setError(error) raise error @@ -1460,7 +1478,7 @@ proc redirect*(request: HttpClientRequestRef, ok(res) proc fetch*(request: HttpClientRequestRef): Future[HttpResponseTuple] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = var response: HttpClientResponseRef try: response = await request.send() @@ -1468,7 +1486,7 @@ proc fetch*(request: HttpClientRequestRef): Future[HttpResponseTuple] {. let status = response.status await response.closeWait() response = nil - return (status, buffer) + (status, buffer) except HttpError as exc: if not(isNil(response)): await response.closeWait() raise exc @@ -1477,7 +1495,7 @@ proc fetch*(request: HttpClientRequestRef): Future[HttpResponseTuple] {. raise exc proc fetch*(session: HttpSessionRef, url: Uri): Future[HttpResponseTuple] {. - async.} = + async: (raises: [CancelledError, HttpError]).} = ## Fetch resource pointed by ``url`` using HTTP GET method and ``session`` ## parameters. ## @@ -1519,28 +1537,34 @@ proc fetch*(session: HttpSessionRef, url: Uri): Future[HttpResponseTuple] {. request = redirect redirect = nil else: - let data = await response.getBodyBytes() - let code = response.status + let + data = await response.getBodyBytes() + code = response.status await response.closeWait() response = nil await request.closeWait() request = nil return (code, data) except CancelledError as exc: - if not(isNil(response)): await closeWait(response) - if not(isNil(request)): await closeWait(request) - if not(isNil(redirect)): await closeWait(redirect) + var pending: seq[Future[void]] + if not(isNil(response)): pending.add(closeWait(response)) + if not(isNil(request)): pending.add(closeWait(request)) + if not(isNil(redirect)): pending.add(closeWait(redirect)) + await noCancel(allFutures(pending)) raise exc except HttpError as exc: - if not(isNil(response)): await closeWait(response) - if not(isNil(request)): await closeWait(request) - if not(isNil(redirect)): await closeWait(redirect) + var pending: seq[Future[void]] + if not(isNil(response)): pending.add(closeWait(response)) + if not(isNil(request)): pending.add(closeWait(request)) + if not(isNil(redirect)): pending.add(closeWait(redirect)) + await noCancel(allFutures(pending)) raise exc proc getServerSentEvents*( response: HttpClientResponseRef, maxEventSize: int = -1 - ): Future[seq[ServerSentEvent]] {.async.} = + ): Future[seq[ServerSentEvent]] {. + async: (raises: [CancelledError, HttpError]).} = ## Read number of server-sent events (SSE) from HTTP response ``response``. ## ## ``maxEventSize`` - maximum size of events chunk in one message, use @@ -1628,8 +1652,14 @@ proc getServerSentEvents*( (i, false) - await reader.readMessage(predicate) + try: + await reader.readMessage(predicate) + except CancelledError as exc: + raise exc + except AsyncStreamError as exc: + raiseHttpReadError($exc.msg) + if not isNil(error): raise error - else: - return res + + res diff --git a/chronos/apps/http/httpcommon.nim b/chronos/apps/http/httpcommon.nim index da5e03f..d2148fb 100644 --- a/chronos/apps/http/httpcommon.nim +++ b/chronos/apps/http/httpcommon.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import std/[strutils, uri] import results, httputils import ../../asyncloop, ../../asyncsync @@ -49,7 +52,7 @@ type HttpResultCode*[T] = Result[T, HttpCode] HttpDefect* = object of Defect - HttpError* = object of CatchableError + HttpError* = object of AsyncError HttpCriticalError* = object of HttpError code*: HttpCode HttpRecoverableError* = object of HttpError @@ -124,35 +127,43 @@ func toString*(error: HttpAddressErrorType): string = of HttpAddressErrorType.NoAddressResolved: "No address has been resolved" -proc raiseHttpCriticalError*(msg: string, - code = Http400) {.noinline, noreturn.} = +proc raiseHttpCriticalError*(msg: string, code = Http400) {. + noinline, noreturn, raises: [HttpCriticalError].} = raise (ref HttpCriticalError)(code: code, msg: msg) -proc raiseHttpDisconnectError*() {.noinline, noreturn.} = +proc raiseHttpDisconnectError*() {. + noinline, noreturn, raises: [HttpDisconnectError].} = raise (ref HttpDisconnectError)(msg: "Remote peer disconnected") proc raiseHttpDefect*(msg: string) {.noinline, noreturn.} = raise (ref HttpDefect)(msg: msg) -proc raiseHttpConnectionError*(msg: string) {.noinline, noreturn.} = +proc raiseHttpConnectionError*(msg: string) {. + noinline, noreturn, raises: [HttpConnectionError].} = raise (ref HttpConnectionError)(msg: msg) -proc raiseHttpInterruptError*() {.noinline, noreturn.} = +proc raiseHttpInterruptError*() {. + noinline, noreturn, raises: [HttpInterruptError].} = raise (ref HttpInterruptError)(msg: "Connection was interrupted") -proc raiseHttpReadError*(msg: string) {.noinline, noreturn.} = +proc raiseHttpReadError*(msg: string) {. + noinline, noreturn, raises: [HttpReadError].} = raise (ref HttpReadError)(msg: msg) -proc raiseHttpProtocolError*(msg: string) {.noinline, noreturn.} = +proc raiseHttpProtocolError*(msg: string) {. + noinline, noreturn, raises: [HttpProtocolError].} = raise (ref HttpProtocolError)(msg: msg) -proc raiseHttpWriteError*(msg: string) {.noinline, noreturn.} = +proc raiseHttpWriteError*(msg: string) {. + noinline, noreturn, raises: [HttpWriteError].} = raise (ref HttpWriteError)(msg: msg) -proc raiseHttpRedirectError*(msg: string) {.noinline, noreturn.} = +proc raiseHttpRedirectError*(msg: string) {. + noinline, noreturn, raises: [HttpRedirectError].} = raise (ref HttpRedirectError)(msg: msg) -proc raiseHttpAddressError*(msg: string) {.noinline, noreturn.} = +proc raiseHttpAddressError*(msg: string) {. + noinline, noreturn, raises: [HttpAddressError].} = raise (ref HttpAddressError)(msg: msg) template newHttpInterruptError*(): ref HttpInterruptError = @@ -168,8 +179,7 @@ template newHttpUseClosedError*(): ref HttpUseClosedError = newException(HttpUseClosedError, "Connection was already closed") iterator queryParams*(query: string, - flags: set[QueryParamsFlag] = {}): KeyValueTuple {. - raises: [].} = + flags: set[QueryParamsFlag] = {}): KeyValueTuple = ## Iterate over url-encoded query string. for pair in query.split('&'): let items = pair.split('=', maxsplit = 1) @@ -182,9 +192,9 @@ iterator queryParams*(query: string, else: yield (decodeUrl(k), decodeUrl(v)) -func getTransferEncoding*(ch: openArray[string]): HttpResult[ - set[TransferEncodingFlags]] {. - raises: [].} = +func getTransferEncoding*( + ch: openArray[string] + ): HttpResult[set[TransferEncodingFlags]] = ## Parse value of multiple HTTP headers ``Transfer-Encoding`` and return ## it as set of ``TransferEncodingFlags``. var res: set[TransferEncodingFlags] = {} @@ -213,9 +223,9 @@ func getTransferEncoding*(ch: openArray[string]): HttpResult[ return err("Incorrect Transfer-Encoding value") ok(res) -func getContentEncoding*(ch: openArray[string]): HttpResult[ - set[ContentEncodingFlags]] {. - raises: [].} = +func getContentEncoding*( + ch: openArray[string] + ): HttpResult[set[ContentEncodingFlags]] = ## Parse value of multiple HTTP headers ``Content-Encoding`` and return ## it as set of ``ContentEncodingFlags``. var res: set[ContentEncodingFlags] = {} @@ -244,8 +254,7 @@ func getContentEncoding*(ch: openArray[string]): HttpResult[ return err("Incorrect Content-Encoding value") ok(res) -func getContentType*(ch: openArray[string]): HttpResult[ContentTypeData] {. - raises: [].} = +func getContentType*(ch: openArray[string]): HttpResult[ContentTypeData] = ## Check and prepare value of ``Content-Type`` header. if len(ch) == 0: err("No Content-Type values found") diff --git a/chronos/apps/http/httpdebug.nim b/chronos/apps/http/httpdebug.nim index d343265..7d52575 100644 --- a/chronos/apps/http/httpdebug.nim +++ b/chronos/apps/http/httpdebug.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import std/tables import results import ../../timer @@ -16,8 +19,6 @@ from ../../osdefs import SocketHandle from ../../transports/common import TransportAddress, ServerFlags export HttpClientScheme, SocketHandle, TransportAddress, ServerFlags, HttpState -{.push raises: [].} - type ConnectionType* {.pure.} = enum NonSecure, Secure diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index 1e307a0..e8326cc 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import std/[tables, uri, strutils] import stew/[base10], httputils, results import ../../asyncloop, ../../asyncsync @@ -67,11 +70,11 @@ type HttpConnectionCallback* = proc(server: HttpServerRef, transp: StreamTransport): Future[HttpConnectionRef] {. - gcsafe, raises: [].} + async: (raises: [CancelledError, HttpConnectionError]), gcsafe.} HttpCloseConnectionCallback* = proc(connection: HttpConnectionRef): Future[void] {. - gcsafe, raises: [].} + async: (raises: []), gcsafe.} HttpConnectionHolder* = object of RootObj connection*: HttpConnectionRef @@ -94,7 +97,7 @@ type flags*: set[HttpServerFlags] socketFlags*: set[ServerFlags] connections*: OrderedTable[string, HttpConnectionHolderRef] - acceptLoop*: Future[void] + acceptLoop*: Future[void].Raising([]) lifetime*: Future[void] headersTimeout*: Duration bufferSize*: int @@ -157,13 +160,11 @@ type proc init(htype: typedesc[HttpProcessError], error: HttpServerError, exc: ref CatchableError, remote: Opt[TransportAddress], - code: HttpCode): HttpProcessError {. - raises: [].} = + code: HttpCode): HttpProcessError = HttpProcessError(kind: error, exc: exc, remote: remote, code: code) proc init(htype: typedesc[HttpProcessError], - error: HttpServerError): HttpProcessError {. - raises: [].} = + error: HttpServerError): HttpProcessError = HttpProcessError(kind: error) proc new(htype: typedesc[HttpConnectionHolderRef], server: HttpServerRef, @@ -176,8 +177,8 @@ proc new(htype: typedesc[HttpConnectionHolderRef], server: HttpServerRef, proc error*(e: HttpProcessError): HttpServerError = e.kind proc createConnection(server: HttpServerRef, - transp: StreamTransport): Future[HttpConnectionRef] {. - gcsafe.} + transp: StreamTransport): Future[HttpConnectionRef] {. + async: (raises: [CancelledError, HttpConnectionError]).} proc new*(htype: typedesc[HttpServerRef], address: TransportAddress, @@ -192,8 +193,7 @@ proc new*(htype: typedesc[HttpServerRef], httpHeadersTimeout = 10.seconds, maxHeadersSize: int = 8192, maxRequestBodySize: int = 1_048_576, - dualstack = DualStackType.Auto): HttpResult[HttpServerRef] {. - raises: [].} = + dualstack = DualStackType.Auto): HttpResult[HttpServerRef] = let serverUri = if len(serverUri.hostname) > 0: @@ -210,8 +210,6 @@ proc new*(htype: typedesc[HttpServerRef], backlog = backlogSize, dualstack = dualstack) except TransportOsError as exc: return err(exc.msg) - except CatchableError as exc: - return err(exc.msg) var res = HttpServerRef( address: serverInstance.localAddress(), @@ -259,13 +257,13 @@ proc getResponseFlags(req: HttpRequestRef): set[HttpResponseFlags] = else: defaultFlags -proc getResponseVersion(reqFence: RequestFence): HttpVersion {.raises: [].} = +proc getResponseVersion(reqFence: RequestFence): HttpVersion = if reqFence.isErr(): HttpVersion11 else: reqFence.get().version -proc getResponse*(req: HttpRequestRef): HttpResponseRef {.raises: [].} = +proc getResponse*(req: HttpRequestRef): HttpResponseRef = if req.response.isNone(): var resp = HttpResponseRef( status: Http200, @@ -286,30 +284,29 @@ proc getHostname*(server: HttpServerRef): string = else: server.baseUri.hostname -proc defaultResponse*(): HttpResponseRef {.raises: [].} = +proc defaultResponse*(): HttpResponseRef = ## Create an empty response to return when request processor got no request. HttpResponseRef(state: HttpResponseState.Default, version: HttpVersion11) -proc dumbResponse*(): HttpResponseRef {.raises: [], +proc dumbResponse*(): HttpResponseRef {. deprecated: "Please use defaultResponse() instead".} = ## Create an empty response to return when request processor got no request. defaultResponse() -proc getId(transp: StreamTransport): Result[string, string] {.inline.} = +proc getId(transp: StreamTransport): Result[string, string] {.inline.} = ## Returns string unique transport's identifier as string. try: ok($transp.remoteAddress() & "_" & $transp.localAddress()) except TransportOsError as exc: err($exc.msg) -proc hasBody*(request: HttpRequestRef): bool {.raises: [].} = +proc hasBody*(request: HttpRequestRef): bool = ## Returns ``true`` if request has body. request.requestFlags * {HttpRequestFlags.BoundBody, HttpRequestFlags.UnboundBody} != {} proc prepareRequest(conn: HttpConnectionRef, - req: HttpRequestHeader): HttpResultCode[HttpRequestRef] {. - raises: [].}= + req: HttpRequestHeader): HttpResultCode[HttpRequestRef] = var request = HttpRequestRef(connection: conn, state: HttpState.Alive) if req.version notin {HttpVersion10, HttpVersion11}: @@ -450,7 +447,8 @@ proc getBodyReader*(request: HttpRequestRef): HttpResult[HttpBodyReader] = else: err("Request do not have body available") -proc handleExpect*(request: HttpRequestRef) {.async.} = +proc handleExpect*(request: HttpRequestRef) {. + async: (raises: [CancelledError, HttpError]).} = ## Handle expectation for ``Expect`` header. ## https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect if HttpServerFlags.NoExpectHandler notin request.connection.server.flags: @@ -461,10 +459,12 @@ proc handleExpect*(request: HttpRequestRef) {.async.} = await request.connection.writer.write(message) except CancelledError as exc: raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: - raiseHttpCriticalError("Unable to send `100-continue` response") + except AsyncStreamError as exc: + raiseHttpCriticalError( + "Unable to send `100-continue` response, reason: " & $exc.msg) -proc getBody*(request: HttpRequestRef): Future[seq[byte]] {.async.} = +proc getBody*(request: HttpRequestRef): Future[seq[byte]] {. + async: (raises: [CancelledError, HttpError]).} = ## Obtain request's body as sequence of bytes. let bodyReader = request.getBodyReader() if bodyReader.isErr(): @@ -486,12 +486,18 @@ proc getBody*(request: HttpRequestRef): Future[seq[byte]] {.async.} = if not(isNil(reader)): await reader.closeWait() raise exc - except AsyncStreamError: + except HttpError as exc: if not(isNil(reader)): await reader.closeWait() - raiseHttpCriticalError("Unable to read request's body") + raise exc + except AsyncStreamError as exc: + let msg = "Unable to read request's body, reason: " & $exc.msg + if not(isNil(reader)): + await reader.closeWait() + raiseHttpCriticalError(msg) -proc consumeBody*(request: HttpRequestRef): Future[void] {.async.} = +proc consumeBody*(request: HttpRequestRef): Future[void] {. + async: (raises: [CancelledError, HttpError]).} = ## Consume/discard request's body. let bodyReader = request.getBodyReader() if bodyReader.isErr(): @@ -513,10 +519,15 @@ proc consumeBody*(request: HttpRequestRef): Future[void] {.async.} = if not(isNil(reader)): await reader.closeWait() raise exc - except AsyncStreamError: + except HttpError as exc: if not(isNil(reader)): await reader.closeWait() - raiseHttpCriticalError("Unable to read request's body") + raise exc + except AsyncStreamError as exc: + let msg = "Unable to consume request's body, reason: " & $exc.msg + if not(isNil(reader)): + await reader.closeWait() + raiseHttpCriticalError(msg) proc getAcceptInfo*(request: HttpRequestRef): Result[AcceptInfo, cstring] = ## Returns value of `Accept` header as `AcceptInfo` object. @@ -636,7 +647,8 @@ proc preferredContentType*(request: HttpRequestRef, proc sendErrorResponse(conn: HttpConnectionRef, version: HttpVersion, code: HttpCode, keepAlive = true, datatype = "text/text", - databody = "") {.async.} = + databody = "") {. + async: (raises: [CancelledError]).} = var answer = $version & " " & $code & "\r\n" answer.add(DateHeader) answer.add(": ") @@ -664,7 +676,7 @@ proc sendErrorResponse(conn: HttpConnectionRef, version: HttpVersion, await conn.writer.write(answer) except CancelledError as exc: raise exc - except CatchableError: + except AsyncStreamError: # We ignore errors here, because we indicating error already. discard @@ -672,7 +684,7 @@ proc sendErrorResponse( conn: HttpConnectionRef, reqFence: RequestFence, respError: HttpProcessError - ): Future[HttpProcessExitType] {.async.} = + ): Future[HttpProcessExitType] {.async: (raises: []).} = let version = getResponseVersion(reqFence) try: if reqFence.isOk(): @@ -694,14 +706,12 @@ proc sendErrorResponse( HttpProcessExitType.Graceful except CancelledError: HttpProcessExitType.Immediate - except CatchableError: - HttpProcessExitType.Immediate proc sendDefaultResponse( conn: HttpConnectionRef, reqFence: RequestFence, response: HttpResponseRef - ): Future[HttpProcessExitType] {.async.} = + ): Future[HttpProcessExitType] {.async: (raises: []).} = let version = getResponseVersion(reqFence) keepConnection = @@ -772,7 +782,8 @@ proc sendDefaultResponse( except CatchableError: HttpProcessExitType.Immediate -proc getRequest(conn: HttpConnectionRef): Future[HttpRequestRef] {.async.} = +proc getRequest(conn: HttpConnectionRef): Future[HttpRequestRef] {. + async: (raises: [CancelledError, HttpError]).} = try: conn.buffer.setLen(conn.server.maxHeadersSize) let res = await conn.reader.readUntil(addr conn.buffer[0], len(conn.buffer), @@ -787,10 +798,10 @@ proc getRequest(conn: HttpConnectionRef): Future[HttpRequestRef] {.async.} = raiseHttpCriticalError("Invalid request received", res.error) else: return res.get() - except AsyncStreamIncompleteError, AsyncStreamReadError: - raiseHttpDisconnectError() except AsyncStreamLimitError: raiseHttpCriticalError("Maximum size of request headers reached", Http431) + except AsyncStreamError: + raiseHttpDisconnectError() proc init*(value: var HttpConnection, server: HttpServerRef, transp: StreamTransport) = @@ -803,7 +814,7 @@ proc init*(value: var HttpConnection, server: HttpServerRef, mainWriter: newAsyncStreamWriter(transp) ) -proc closeUnsecureConnection(conn: HttpConnectionRef) {.async.} = +proc closeUnsecureConnection(conn: HttpConnectionRef) {.async: (raises: []).} = if conn.state == HttpState.Alive: conn.state = HttpState.Closing var pending: seq[Future[void]] @@ -826,14 +837,20 @@ proc new(ht: typedesc[HttpConnectionRef], server: HttpServerRef, trackCounter(HttpServerUnsecureConnectionTrackerName) res -proc gracefulCloseWait*(conn: HttpConnectionRef) {.async.} = - await noCancel(conn.transp.shutdownWait()) +proc gracefulCloseWait*(conn: HttpConnectionRef) {.async: (raises: []).} = + try: + await noCancel(conn.transp.shutdownWait()) + except TransportError: + # We try to gracefully close connection, so we ignore any errors here, + # because right after this operation we closing connection. + discard await conn.closeCb(conn) -proc closeWait*(conn: HttpConnectionRef): Future[void] = +proc closeWait*(conn: HttpConnectionRef): Future[void] {. + async: (raw: true, raises: []).} = conn.closeCb(conn) -proc closeWait*(req: HttpRequestRef) {.async.} = +proc closeWait*(req: HttpRequestRef) {.async: (raises: []).} = if req.state == HttpState.Alive: if req.response.isSome(): req.state = HttpState.Closing @@ -847,8 +864,8 @@ proc closeWait*(req: HttpRequestRef) {.async.} = proc createConnection(server: HttpServerRef, transp: StreamTransport): Future[HttpConnectionRef] {. - async.} = - return HttpConnectionRef.new(server, transp) + async: (raises: [CancelledError, HttpConnectionError]).} = + HttpConnectionRef.new(server, transp) proc `keepalive=`*(resp: HttpResponseRef, value: bool) = doAssert(resp.state == HttpResponseState.Empty) @@ -857,25 +874,23 @@ proc `keepalive=`*(resp: HttpResponseRef, value: bool) = else: resp.flags.excl(HttpResponseFlags.KeepAlive) -proc keepalive*(resp: HttpResponseRef): bool {.raises: [].} = +proc keepalive*(resp: HttpResponseRef): bool = HttpResponseFlags.KeepAlive in resp.flags -proc getRemoteAddress(transp: StreamTransport): Opt[TransportAddress] {. - raises: [].} = +proc getRemoteAddress(transp: StreamTransport): Opt[TransportAddress] = if isNil(transp): return Opt.none(TransportAddress) try: Opt.some(transp.remoteAddress()) - except CatchableError: + except TransportOsError: Opt.none(TransportAddress) -proc getRemoteAddress(connection: HttpConnectionRef): Opt[TransportAddress] {. - raises: [].} = +proc getRemoteAddress(connection: HttpConnectionRef): Opt[TransportAddress] = if isNil(connection): return Opt.none(TransportAddress) getRemoteAddress(connection.transp) proc getResponseFence*(connection: HttpConnectionRef, reqFence: RequestFence): Future[ResponseFence] {. - async.} = + async: (raises: []).} = try: let res = await connection.server.processCallback(reqFence) ResponseFence.ok(res) @@ -897,7 +912,7 @@ proc getResponseFence*(connection: HttpConnectionRef, proc getResponseFence*(server: HttpServerRef, connFence: ConnectionFence): Future[ResponseFence] {. - async.} = + async: (raises: []).} = doAssert(connFence.isErr()) try: let @@ -922,7 +937,7 @@ proc getResponseFence*(server: HttpServerRef, proc getRequestFence*(server: HttpServerRef, connection: HttpConnectionRef): Future[RequestFence] {. - async.} = + async: (raises: []).} = try: let res = if server.headersTimeout.isInfinite(): @@ -956,7 +971,7 @@ proc getRequestFence*(server: HttpServerRef, proc getConnectionFence*(server: HttpServerRef, transp: StreamTransport): Future[ConnectionFence] {. - async.} = + async: (raises: []).} = try: let res = await server.createConnCallback(server, transp) ConnectionFence.ok(res) @@ -975,7 +990,8 @@ proc getConnectionFence*(server: HttpServerRef, proc processRequest(server: HttpServerRef, connection: HttpConnectionRef, - connId: string): Future[HttpProcessExitType] {.async.} = + connId: string): Future[HttpProcessExitType] {. + async: (raises: []).} = let requestFence = await getRequestFence(server, connection) if requestFence.isErr(): case requestFence.error.kind @@ -1005,7 +1021,7 @@ proc processRequest(server: HttpServerRef, res -proc processLoop(holder: HttpConnectionHolderRef) {.async.} = +proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} = let server = holder.server transp = holder.transp @@ -1042,7 +1058,7 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async.} = server.connections.del(connectionId) -proc acceptClientLoop(server: HttpServerRef) {.async.} = +proc acceptClientLoop(server: HttpServerRef) {.async: (raises: []).} = var runLoop = true while runLoop: try: @@ -1067,7 +1083,7 @@ proc acceptClientLoop(server: HttpServerRef) {.async.} = # Critical, cancellation or unexpected error runLoop = false -proc state*(server: HttpServerRef): HttpServerState {.raises: [].} = +proc state*(server: HttpServerRef): HttpServerState = ## Returns current HTTP server's state. if server.lifetime.finished(): ServerClosed @@ -1085,12 +1101,12 @@ proc start*(server: HttpServerRef) = if server.state == ServerStopped: server.acceptLoop = acceptClientLoop(server) -proc stop*(server: HttpServerRef) {.async.} = +proc stop*(server: HttpServerRef) {.async: (raises: []).} = ## Stop HTTP server from accepting new connections. if server.state == ServerRunning: await server.acceptLoop.cancelAndWait() -proc drop*(server: HttpServerRef) {.async.} = +proc drop*(server: HttpServerRef) {.async: (raises: []).} = ## Drop all pending HTTP connections. var pending: seq[Future[void]] if server.state in {ServerStopped, ServerRunning}: @@ -1100,7 +1116,7 @@ proc drop*(server: HttpServerRef) {.async.} = await noCancel(allFutures(pending)) server.connections.clear() -proc closeWait*(server: HttpServerRef) {.async.} = +proc closeWait*(server: HttpServerRef) {.async: (raises: []).} = ## Stop HTTP server and drop all the pending connections. if server.state != ServerClosed: await server.stop() @@ -1108,7 +1124,8 @@ proc closeWait*(server: HttpServerRef) {.async.} = await server.instance.closeWait() server.lifetime.complete() -proc join*(server: HttpServerRef): Future[void] = +proc join*(server: HttpServerRef): Future[void] {. + async: (raw: true, raises: [CancelledError]).} = ## Wait until HTTP server will not be closed. var retFuture = newFuture[void]("http.server.join") @@ -1128,8 +1145,7 @@ proc join*(server: HttpServerRef): Future[void] = retFuture -proc getMultipartReader*(req: HttpRequestRef): HttpResult[MultiPartReaderRef] {. - raises: [].} = +proc getMultipartReader*(req: HttpRequestRef): HttpResult[MultiPartReaderRef] = ## Create new MultiPartReader interface for specific request. if req.meth in PostMethods: if MultipartForm in req.requestFlags: @@ -1144,7 +1160,8 @@ proc getMultipartReader*(req: HttpRequestRef): HttpResult[MultiPartReaderRef] {. else: err("Request's method do not supports multipart") -proc post*(req: HttpRequestRef): Future[HttpTable] {.async.} = +proc post*(req: HttpRequestRef): Future[HttpTable] {. + async: (raises: [CancelledError, HttpError]).} = ## Return POST parameters if req.postTable.isSome(): return req.postTable.get() @@ -1224,31 +1241,28 @@ proc post*(req: HttpRequestRef): Future[HttpTable] {.async.} = elif HttpRequestFlags.UnboundBody in req.requestFlags: raiseHttpCriticalError("Unsupported request body") -proc setHeader*(resp: HttpResponseRef, key, value: string) {. - raises: [].} = +proc setHeader*(resp: HttpResponseRef, key, value: string) = ## Sets value of header ``key`` to ``value``. doAssert(resp.state == HttpResponseState.Empty) resp.headersTable.set(key, value) -proc setHeaderDefault*(resp: HttpResponseRef, key, value: string) {. - raises: [].} = +proc setHeaderDefault*(resp: HttpResponseRef, key, value: string) = ## Sets value of header ``key`` to ``value``, only if header ``key`` is not ## present in the headers table. discard resp.headersTable.hasKeyOrPut(key, value) -proc addHeader*(resp: HttpResponseRef, key, value: string) {. - raises: [].} = +proc addHeader*(resp: HttpResponseRef, key, value: string) = ## Adds value ``value`` to header's ``key`` value. doAssert(resp.state == HttpResponseState.Empty) resp.headersTable.add(key, value) proc getHeader*(resp: HttpResponseRef, key: string, - default: string = ""): string {.raises: [].} = + default: string = ""): string = ## Returns value of header with name ``name`` or ``default``, if header is ## not present in the table. resp.headersTable.getString(key, default) -proc hasHeader*(resp: HttpResponseRef, key: string): bool {.raises: [].} = +proc hasHeader*(resp: HttpResponseRef, key: string): bool = ## Returns ``true`` if header with name ``key`` present in the headers table. key in resp.headersTable @@ -1267,8 +1281,7 @@ func createHeaders(resp: HttpResponseRef): string = answer.add("\r\n") answer -proc prepareLengthHeaders(resp: HttpResponseRef, length: int): string {. - raises: [].}= +proc prepareLengthHeaders(resp: HttpResponseRef, length: int): string = if not(resp.hasHeader(DateHeader)): resp.setHeader(DateHeader, httpDate()) if length > 0: @@ -1285,8 +1298,7 @@ proc prepareLengthHeaders(resp: HttpResponseRef, length: int): string {. resp.setHeader(ConnectionHeader, "close") resp.createHeaders() -proc prepareChunkedHeaders(resp: HttpResponseRef): string {. - raises: [].} = +proc prepareChunkedHeaders(resp: HttpResponseRef): string = if not(resp.hasHeader(DateHeader)): resp.setHeader(DateHeader, httpDate()) if not(resp.hasHeader(ContentTypeHeader)): @@ -1302,8 +1314,7 @@ proc prepareChunkedHeaders(resp: HttpResponseRef): string {. resp.setHeader(ConnectionHeader, "close") resp.createHeaders() -proc prepareServerSideEventHeaders(resp: HttpResponseRef): string {. - raises: [].} = +proc prepareServerSideEventHeaders(resp: HttpResponseRef): string = if not(resp.hasHeader(DateHeader)): resp.setHeader(DateHeader, httpDate()) if not(resp.hasHeader(ContentTypeHeader)): @@ -1315,8 +1326,7 @@ proc prepareServerSideEventHeaders(resp: HttpResponseRef): string {. resp.setHeader(ConnectionHeader, "close") resp.createHeaders() -proc preparePlainHeaders(resp: HttpResponseRef): string {. - raises: [].} = +proc preparePlainHeaders(resp: HttpResponseRef): string = if not(resp.hasHeader(DateHeader)): resp.setHeader(DateHeader, httpDate()) if not(resp.hasHeader(ServerHeader)): @@ -1326,7 +1336,8 @@ proc preparePlainHeaders(resp: HttpResponseRef): string {. resp.setHeader(ConnectionHeader, "close") resp.createHeaders() -proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {.async.} = +proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. + async: (raises: [CancelledError, HttpError]).} = ## Send HTTP response at once by using bytes pointer ``pbytes`` and length ## ``nbytes``. doAssert(not(isNil(pbytes)), "pbytes must not be nil") @@ -1343,11 +1354,12 @@ proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {.async.} = except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) -proc sendBody*(resp: HttpResponseRef, data: ByteChar) {.async.} = +proc sendBody*(resp: HttpResponseRef, data: ByteChar) {. + async: (raises: [CancelledError, HttpError]).} = ## Send HTTP response at once by using data ``data``. checkPending(resp) let responseHeaders = resp.prepareLengthHeaders(len(data)) @@ -1361,11 +1373,12 @@ proc sendBody*(resp: HttpResponseRef, data: ByteChar) {.async.} = except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) -proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {.async.} = +proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {. + async: (raises: [CancelledError, HttpError]).} = ## Send HTTP error status response. checkPending(resp) resp.status = code @@ -1380,12 +1393,13 @@ proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {.async.} = except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc prepare*(resp: HttpResponseRef, - streamType = HttpResponseStreamType.Chunked) {.async.} = + streamType = HttpResponseStreamType.Chunked) {. + async: (raises: [CancelledError, HttpError]).} = ## Prepare for HTTP stream response. ## ## Such responses will be sent chunk by chunk using ``chunked`` encoding. @@ -1412,27 +1426,31 @@ proc prepare*(resp: HttpResponseRef, except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) -proc prepareChunked*(resp: HttpResponseRef): Future[void] = +proc prepareChunked*(resp: HttpResponseRef): Future[void] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Prepare for HTTP chunked stream response. ## ## Such responses will be sent chunk by chunk using ``chunked`` encoding. resp.prepare(HttpResponseStreamType.Chunked) -proc preparePlain*(resp: HttpResponseRef): Future[void] = +proc preparePlain*(resp: HttpResponseRef): Future[void] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Prepare for HTTP plain stream response. ## ## Such responses will be sent without any encoding. resp.prepare(HttpResponseStreamType.Plain) -proc prepareSSE*(resp: HttpResponseRef): Future[void] = +proc prepareSSE*(resp: HttpResponseRef): Future[void] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Prepare for HTTP server-side event stream response. resp.prepare(HttpResponseStreamType.SSE) -proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {.async.} = +proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. + async: (raises: [CancelledError, HttpError]).} = ## Send single chunk of data pointed by ``pbytes`` and ``nbytes``. doAssert(not(isNil(pbytes)), "pbytes must not be nil") doAssert(nbytes >= 0, "nbytes should be bigger or equal to zero") @@ -1447,11 +1465,12 @@ proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {.async.} = except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) -proc send*(resp: HttpResponseRef, data: ByteChar) {.async.} = +proc send*(resp: HttpResponseRef, data: ByteChar) {. + async: (raises: [CancelledError, HttpError]).} = ## Send single chunk of data ``data``. if HttpResponseFlags.Stream notin resp.flags: raiseHttpCriticalError("Response was not prepared") @@ -1464,19 +1483,22 @@ proc send*(resp: HttpResponseRef, data: ByteChar) {.async.} = except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc sendChunk*(resp: HttpResponseRef, pbytes: pointer, - nbytes: int): Future[void] = + nbytes: int): Future[void] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = resp.send(pbytes, nbytes) -proc sendChunk*(resp: HttpResponseRef, data: ByteChar): Future[void] = +proc sendChunk*(resp: HttpResponseRef, data: ByteChar): Future[void] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = resp.send(data) proc sendEvent*(resp: HttpResponseRef, eventName: string, - data: string): Future[void] = + data: string): Future[void] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Send server-side event with name ``eventName`` and payload ``data`` to ## remote peer. let data = @@ -1492,7 +1514,8 @@ proc sendEvent*(resp: HttpResponseRef, eventName: string, res resp.send(data) -proc finish*(resp: HttpResponseRef) {.async.} = +proc finish*(resp: HttpResponseRef) {. + async: (raises: [CancelledError, HttpError]).} = ## Sending last chunk of data, so it will indicate end of HTTP response. if HttpResponseFlags.Stream notin resp.flags: raiseHttpCriticalError("Response was not prepared") @@ -1505,12 +1528,13 @@ proc finish*(resp: HttpResponseRef) {.async.} = except CancelledError as exc: resp.state = HttpResponseState.Cancelled raise exc - except AsyncStreamWriteError, AsyncStreamIncompleteError: + except AsyncStreamError as exc: resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response") + raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar, - headers: HttpTable): Future[HttpResponseRef] {.async.} = + headers: HttpTable): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpError]).} = ## Responds to the request with the specified ``HttpCode``, HTTP ``headers`` ## and ``content``. let response = req.getResponse() @@ -1518,19 +1542,22 @@ proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar, for k, v in headers.stringItems(): response.addHeader(k, v) await response.sendBody(content) - return response + response proc respond*(req: HttpRequestRef, code: HttpCode, - content: ByteChar): Future[HttpResponseRef] = + content: ByteChar): Future[HttpResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Responds to the request with specified ``HttpCode`` and ``content``. respond(req, code, content, HttpTable.init()) -proc respond*(req: HttpRequestRef, code: HttpCode): Future[HttpResponseRef] = +proc respond*(req: HttpRequestRef, code: HttpCode): Future[HttpResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Responds to the request with specified ``HttpCode`` only. respond(req, code, "", HttpTable.init()) proc redirect*(req: HttpRequestRef, code: HttpCode, - location: string, headers: HttpTable): Future[HttpResponseRef] = + location: string, headers: HttpTable): Future[HttpResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Responds to the request with redirection to location ``location`` and ## additional headers ``headers``. ## @@ -1541,7 +1568,8 @@ proc redirect*(req: HttpRequestRef, code: HttpCode, respond(req, code, "", mheaders) proc redirect*(req: HttpRequestRef, code: HttpCode, - location: Uri, headers: HttpTable): Future[HttpResponseRef] = + location: Uri, headers: HttpTable): Future[HttpResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Responds to the request with redirection to location ``location`` and ## additional headers ``headers``. ## @@ -1550,12 +1578,14 @@ proc redirect*(req: HttpRequestRef, code: HttpCode, redirect(req, code, $location, headers) proc redirect*(req: HttpRequestRef, code: HttpCode, - location: Uri): Future[HttpResponseRef] = + location: Uri): Future[HttpResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Responds to the request with redirection to location ``location``. redirect(req, code, location, HttpTable.init()) proc redirect*(req: HttpRequestRef, code: HttpCode, - location: string): Future[HttpResponseRef] = + location: string): Future[HttpResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]).} = ## Responds to the request with redirection to location ``location``. redirect(req, code, location, HttpTable.init()) @@ -1569,16 +1599,20 @@ proc responded*(req: HttpRequestRef): bool = else: false -proc remoteAddress*(conn: HttpConnectionRef): TransportAddress = +proc remoteAddress*(conn: HttpConnectionRef): TransportAddress {. + raises: [HttpAddressError].} = ## Returns address of the remote host that established connection ``conn``. - conn.transp.remoteAddress() + try: + conn.transp.remoteAddress() + except TransportOsError as exc: + raiseHttpAddressError($exc.msg) -proc remoteAddress*(request: HttpRequestRef): TransportAddress = +proc remoteAddress*(request: HttpRequestRef): TransportAddress {. + raises: [HttpAddressError].} = ## Returns address of the remote host that made request ``request``. request.connection.remoteAddress() -proc requestInfo*(req: HttpRequestRef, contentType = "text/text"): string {. - raises: [].} = +proc requestInfo*(req: HttpRequestRef, contentType = "text/text"): string = ## Returns comprehensive information about request for specific content ## type. ## diff --git a/chronos/apps/http/multipart.nim b/chronos/apps/http/multipart.nim index b936996..83a4b56 100644 --- a/chronos/apps/http/multipart.nim +++ b/chronos/apps/http/multipart.nim @@ -7,6 +7,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import std/[monotimes, strutils] import results, httputils import ../../asyncloop @@ -71,8 +74,7 @@ type BChar* = byte | char -proc startsWith(s, prefix: openArray[byte]): bool {. - raises: [].} = +proc startsWith(s, prefix: openArray[byte]): bool = # This procedure is copy of strutils.startsWith() procedure, however, # it is intended to work with arrays of bytes, but not with strings. var i = 0 @@ -81,8 +83,7 @@ proc startsWith(s, prefix: openArray[byte]): bool {. if i >= len(s) or s[i] != prefix[i]: return false inc(i) -proc parseUntil(s, until: openArray[byte]): int {. - raises: [].} = +proc parseUntil(s, until: openArray[byte]): int = # This procedure is copy of parseutils.parseUntil() procedure, however, # it is intended to work with arrays of bytes, but not with strings. var i = 0 @@ -95,8 +96,7 @@ proc parseUntil(s, until: openArray[byte]): int {. inc(i) -1 -func setPartNames(part: var MultiPart): HttpResult[void] {. - raises: [].} = +func setPartNames(part: var MultiPart): HttpResult[void] = if part.headers.count("content-disposition") != 1: return err("Content-Disposition header is incorrect") var header = part.headers.getString("content-disposition") @@ -120,8 +120,7 @@ func setPartNames(part: var MultiPart): HttpResult[void] {. proc init*[A: BChar, B: BChar](mpt: typedesc[MultiPartReader], buffer: openArray[A], - boundary: openArray[B]): MultiPartReader {. - raises: [].} = + boundary: openArray[B]): MultiPartReader = ## Create new MultiPartReader instance with `buffer` interface. ## ## ``buffer`` - is buffer which will be used to read data. @@ -145,8 +144,7 @@ proc init*[A: BChar, B: BChar](mpt: typedesc[MultiPartReader], proc new*[B: BChar](mpt: typedesc[MultiPartReaderRef], stream: HttpBodyReader, boundary: openArray[B], - partHeadersMaxSize = 4096): MultiPartReaderRef {. - raises: [].} = + partHeadersMaxSize = 4096): MultiPartReaderRef = ## Create new MultiPartReader instance with `stream` interface. ## ## ``stream`` is stream used to read data. @@ -173,7 +171,8 @@ proc new*[B: BChar](mpt: typedesc[MultiPartReaderRef], stream: stream, offset: 0, boundary: fboundary, buffer: newSeq[byte](partHeadersMaxSize)) -proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {.async.} = +proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {. + async: (raises: [CancelledError, HttpCriticalError]).} = doAssert(mpr.kind == MultiPartSource.Stream) if mpr.firstTime: try: @@ -240,7 +239,8 @@ proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {.async.} = else: raiseHttpCriticalError(UnableToReadMultipartBody) -proc getBody*(mp: MultiPart): Future[seq[byte]] {.async.} = +proc getBody*(mp: MultiPart): Future[seq[byte]] {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Get multipart's ``mp`` value as sequence of bytes. case mp.kind of MultiPartSource.Stream: @@ -255,7 +255,8 @@ proc getBody*(mp: MultiPart): Future[seq[byte]] {.async.} = of MultiPartSource.Buffer: return mp.buffer -proc consumeBody*(mp: MultiPart) {.async.} = +proc consumeBody*(mp: MultiPart) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Discard multipart's ``mp`` value. case mp.kind of MultiPartSource.Stream: @@ -269,8 +270,7 @@ proc consumeBody*(mp: MultiPart) {.async.} = of MultiPartSource.Buffer: discard -proc getBodyStream*(mp: MultiPart): HttpResult[AsyncStreamReader] {. - raises: [].} = +proc getBodyStream*(mp: MultiPart): HttpResult[AsyncStreamReader] = ## Get multipart's ``mp`` stream, which can be used to obtain value of the ## part. case mp.kind @@ -279,7 +279,7 @@ proc getBodyStream*(mp: MultiPart): HttpResult[AsyncStreamReader] {. else: err("Could not obtain stream from buffer-like part") -proc closeWait*(mp: MultiPart) {.async.} = +proc closeWait*(mp: MultiPart) {.async: (raises: []).} = ## Close and release MultiPart's ``mp`` stream and resources. case mp.kind of MultiPartSource.Stream: @@ -287,7 +287,7 @@ proc closeWait*(mp: MultiPart) {.async.} = else: discard -proc closeWait*(mpr: MultiPartReaderRef) {.async.} = +proc closeWait*(mpr: MultiPartReaderRef) {.async: (raises: []).} = ## Close and release MultiPartReader's ``mpr`` stream and resources. case mpr.kind of MultiPartSource.Stream: @@ -295,7 +295,7 @@ proc closeWait*(mpr: MultiPartReaderRef) {.async.} = else: discard -proc getBytes*(mp: MultiPart): seq[byte] {.raises: [].} = +proc getBytes*(mp: MultiPart): seq[byte] = ## Returns value for MultiPart ``mp`` as sequence of bytes. case mp.kind of MultiPartSource.Buffer: @@ -304,7 +304,7 @@ proc getBytes*(mp: MultiPart): seq[byte] {.raises: [].} = doAssert(not(mp.stream.atEof()), "Value is not obtained yet") mp.buffer -proc getString*(mp: MultiPart): string {.raises: [].} = +proc getString*(mp: MultiPart): string = ## Returns value for MultiPart ``mp`` as string. case mp.kind of MultiPartSource.Buffer: @@ -313,7 +313,7 @@ proc getString*(mp: MultiPart): string {.raises: [].} = doAssert(not(mp.stream.atEof()), "Value is not obtained yet") bytesToString(mp.buffer) -proc atEoM*(mpr: var MultiPartReader): bool {.raises: [].} = +proc atEoM*(mpr: var MultiPartReader): bool = ## Procedure returns ``true`` if MultiPartReader has reached the end of ## multipart message. case mpr.kind @@ -322,7 +322,7 @@ proc atEoM*(mpr: var MultiPartReader): bool {.raises: [].} = of MultiPartSource.Stream: mpr.stream.atEof() -proc atEoM*(mpr: MultiPartReaderRef): bool {.raises: [].} = +proc atEoM*(mpr: MultiPartReaderRef): bool = ## Procedure returns ``true`` if MultiPartReader has reached the end of ## multipart message. case mpr.kind @@ -331,8 +331,7 @@ proc atEoM*(mpr: MultiPartReaderRef): bool {.raises: [].} = of MultiPartSource.Stream: mpr.stream.atEof() -proc getPart*(mpr: var MultiPartReader): Result[MultiPart, string] {. - raises: [].} = +proc getPart*(mpr: var MultiPartReader): Result[MultiPart, string] = ## Get multipart part from MultiPartReader instance. ## ## This procedure will work only for MultiPartReader with buffer source. @@ -422,8 +421,7 @@ proc getPart*(mpr: var MultiPartReader): Result[MultiPart, string] {. else: err("Incorrect multipart form") -func isEmpty*(mp: MultiPart): bool {. - raises: [].} = +func isEmpty*(mp: MultiPart): bool = ## Returns ``true`` is multipart ``mp`` is not initialized/filled yet. mp.counter == 0 @@ -439,8 +437,7 @@ func validateBoundary[B: BChar](boundary: openArray[B]): HttpResult[void] = return err("Content-Type boundary alphabet incorrect") ok() -func getMultipartBoundary*(contentData: ContentTypeData): HttpResult[string] {. - raises: [].} = +func getMultipartBoundary*(contentData: ContentTypeData): HttpResult[string] = ## Returns ``multipart/form-data`` boundary value from ``Content-Type`` ## header. ## @@ -480,8 +477,7 @@ proc quoteCheck(name: string): HttpResult[string] = ok(name) proc init*[B: BChar](mpt: typedesc[MultiPartWriter], - boundary: openArray[B]): MultiPartWriter {. - raises: [].} = + boundary: openArray[B]): MultiPartWriter = ## Create new MultiPartWriter instance with `buffer` interface. ## ## ``boundary`` - is multipart boundary, this value must not be empty. @@ -510,8 +506,7 @@ proc init*[B: BChar](mpt: typedesc[MultiPartWriter], proc new*[B: BChar](mpt: typedesc[MultiPartWriterRef], stream: HttpBodyWriter, - boundary: openArray[B]): MultiPartWriterRef {. - raises: [].} = + boundary: openArray[B]): MultiPartWriterRef = doAssert(validateBoundary(boundary).isOk()) doAssert(not(isNil(stream))) @@ -576,7 +571,8 @@ proc prepareHeaders(partMark: openArray[byte], name: string, filename: string, buffer.add("\r\n") buffer -proc begin*(mpw: MultiPartWriterRef) {.async.} = +proc begin*(mpw: MultiPartWriterRef) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Starts multipart message form and write approprate markers to output ## stream. doAssert(mpw.kind == MultiPartSource.Stream) @@ -599,7 +595,8 @@ proc begin*(mpw: var MultiPartWriter) = mpw.state = MultiPartWriterState.MessageStarted proc beginPart*(mpw: MultiPartWriterRef, name: string, - filename: string, headers: HttpTable) {.async.} = + filename: string, headers: HttpTable) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Starts part of multipart message and write appropriate ``headers`` to the ## output stream. ## @@ -634,38 +631,44 @@ proc beginPart*(mpw: var MultiPartWriter, name: string, mpw.buffer.add(buffer.toOpenArrayByte(0, len(buffer) - 1)) mpw.state = MultiPartWriterState.PartStarted -proc write*(mpw: MultiPartWriterRef, pbytes: pointer, nbytes: int) {.async.} = +proc write*(mpw: MultiPartWriterRef, pbytes: pointer, nbytes: int) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Write part's data ``data`` to the output stream. doAssert(mpw.kind == MultiPartSource.Stream) doAssert(mpw.state == MultiPartWriterState.PartStarted) try: # write of data await mpw.stream.write(pbytes, nbytes) - except AsyncStreamError: + except AsyncStreamError as exc: mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to write multipart data") + raiseHttpCriticalError( + "Unable to write multipart data, reason: " & $exc.msg) -proc write*(mpw: MultiPartWriterRef, data: seq[byte]) {.async.} = +proc write*(mpw: MultiPartWriterRef, data: seq[byte]) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Write part's data ``data`` to the output stream. doAssert(mpw.kind == MultiPartSource.Stream) doAssert(mpw.state == MultiPartWriterState.PartStarted) try: # write of data await mpw.stream.write(data) - except AsyncStreamError: + except AsyncStreamError as exc: mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to write multipart data") + raiseHttpCriticalError( + "Unable to write multipart data, reason: " & $exc.msg) -proc write*(mpw: MultiPartWriterRef, data: string) {.async.} = +proc write*(mpw: MultiPartWriterRef, data: string) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Write part's data ``data`` to the output stream. doAssert(mpw.kind == MultiPartSource.Stream) doAssert(mpw.state == MultiPartWriterState.PartStarted) try: # write of data await mpw.stream.write(data) - except AsyncStreamError: + except AsyncStreamError as exc: mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to write multipart data") + raiseHttpCriticalError( + "Unable to write multipart data, reason: " & $exc.msg) proc write*(mpw: var MultiPartWriter, pbytes: pointer, nbytes: int) = ## Write part's data ``data`` to the output stream. @@ -688,16 +691,18 @@ proc write*(mpw: var MultiPartWriter, data: openArray[char]) = doAssert(mpw.state == MultiPartWriterState.PartStarted) mpw.buffer.add(data.toOpenArrayByte(0, len(data) - 1)) -proc finishPart*(mpw: MultiPartWriterRef) {.async.} = +proc finishPart*(mpw: MultiPartWriterRef) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Finish multipart's message part and send proper markers to output stream. doAssert(mpw.state == MultiPartWriterState.PartStarted) try: # write "--" await mpw.stream.write(mpw.finishPartMark) mpw.state = MultiPartWriterState.PartFinished - except AsyncStreamError: + except AsyncStreamError as exc: mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to finish multipart message part") + raiseHttpCriticalError( + "Unable to finish multipart message part, reason: " & $exc.msg) proc finishPart*(mpw: var MultiPartWriter) = ## Finish multipart's message part and send proper markers to output stream. @@ -707,7 +712,8 @@ proc finishPart*(mpw: var MultiPartWriter) = mpw.buffer.add(mpw.finishPartMark) mpw.state = MultiPartWriterState.PartFinished -proc finish*(mpw: MultiPartWriterRef) {.async.} = +proc finish*(mpw: MultiPartWriterRef) {. + async: (raises: [CancelledError, HttpCriticalError]).} = ## Finish multipart's message form and send finishing markers to the output ## stream. doAssert(mpw.kind == MultiPartSource.Stream) @@ -716,9 +722,10 @@ proc finish*(mpw: MultiPartWriterRef) {.async.} = # write "--" await mpw.stream.write(mpw.finishMark) mpw.state = MultiPartWriterState.MessageFinished - except AsyncStreamError: + except AsyncStreamError as exc: mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to finish multipart message") + raiseHttpCriticalError( + "Unable to finish multipart message, reason: " & $exc.msg) proc finish*(mpw: var MultiPartWriter): seq[byte] = ## Finish multipart's message form and send finishing markers to the output diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index 0300597..2373d95 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [].} + import httpserver import ../../asyncloop, ../../asyncsync import ../../streams/[asyncstream, tlsstream] @@ -24,7 +27,7 @@ type SecureHttpConnectionRef* = ref SecureHttpConnection -proc closeSecConnection(conn: HttpConnectionRef) {.async.} = +proc closeSecConnection(conn: HttpConnectionRef) {.async: (raises: []).} = if conn.state == HttpState.Alive: conn.state = HttpState.Closing var pending: seq[Future[void]] @@ -38,44 +41,44 @@ proc closeSecConnection(conn: HttpConnectionRef) {.async.} = untrackCounter(HttpServerSecureConnectionTrackerName) conn.state = HttpState.Closed -proc new*(ht: typedesc[SecureHttpConnectionRef], server: SecureHttpServerRef, - transp: StreamTransport): SecureHttpConnectionRef = +proc new(ht: typedesc[SecureHttpConnectionRef], server: SecureHttpServerRef, + transp: StreamTransport): Result[SecureHttpConnectionRef, string] = var res = SecureHttpConnectionRef() HttpConnection(res[]).init(HttpServerRef(server), transp) let tlsStream = - newTLSServerAsyncStream(res.mainReader, res.mainWriter, - server.tlsPrivateKey, - server.tlsCertificate, - minVersion = TLSVersion.TLS12, - flags = server.secureFlags) + try: + newTLSServerAsyncStream(res.mainReader, res.mainWriter, + server.tlsPrivateKey, + server.tlsCertificate, + minVersion = TLSVersion.TLS12, + flags = server.secureFlags) + except TLSStreamError as exc: + return err(exc.msg) res.tlsStream = tlsStream res.reader = AsyncStreamReader(tlsStream.reader) res.writer = AsyncStreamWriter(tlsStream.writer) res.closeCb = closeSecConnection trackCounter(HttpServerSecureConnectionTrackerName) - res + ok(res) proc createSecConnection(server: HttpServerRef, transp: StreamTransport): Future[HttpConnectionRef] {. - async.} = - let secureServ = cast[SecureHttpServerRef](server) - var sconn = SecureHttpConnectionRef.new(secureServ, transp) + async: (raises: [CancelledError, HttpConnectionError]).} = + let + secureServ = cast[SecureHttpServerRef](server) + sconn = SecureHttpConnectionRef.new(secureServ, transp).valueOr: + raiseHttpConnectionError(error) + try: await handshake(sconn.tlsStream) - return HttpConnectionRef(sconn) + HttpConnectionRef(sconn) except CancelledError as exc: await HttpConnectionRef(sconn).closeWait() raise exc - except TLSStreamError as exc: + except AsyncStreamError as exc: await HttpConnectionRef(sconn).closeWait() - let msg = "Unable to establish secure connection, reason [" & - $exc.msg & "]" - raiseHttpCriticalError(msg) - except CatchableError as exc: - await HttpConnectionRef(sconn).closeWait() - let msg = "Unexpected error while trying to establish secure connection, " & - "reason [" & $exc.msg & "]" - raiseHttpCriticalError(msg) + let msg = "Unable to establish secure connection, reason: " & $exc.msg + raiseHttpConnectionError(msg) proc new*(htype: typedesc[SecureHttpServerRef], address: TransportAddress, @@ -94,7 +97,7 @@ proc new*(htype: typedesc[SecureHttpServerRef], maxHeadersSize: int = 8192, maxRequestBodySize: int = 1_048_576, dualstack = DualStackType.Auto - ): HttpResult[SecureHttpServerRef] {.raises: [].} = + ): HttpResult[SecureHttpServerRef] = doAssert(not(isNil(tlsPrivateKey)), "TLS private key must not be nil!") doAssert(not(isNil(tlsCertificate)), "TLS certificate must not be nil!") @@ -114,8 +117,6 @@ proc new*(htype: typedesc[SecureHttpServerRef], backlog = backlogSize, dualstack = dualstack) except TransportOsError as exc: return err(exc.msg) - except CatchableError as exc: - return err(exc.msg) let res = SecureHttpServerRef( address: address, diff --git a/chronos/internal/asyncfutures.nim b/chronos/internal/asyncfutures.nim index ba7eaf0..a7fd961 100644 --- a/chronos/internal/asyncfutures.nim +++ b/chronos/internal/asyncfutures.nim @@ -983,7 +983,7 @@ template cancel*(future: FutureBase) {. cancelSoon(future, nil, nil, getSrcLocation()) proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] {. - async: (raw: true, raises: [CancelledError]).} = + async: (raw: true, raises: []).} = ## Perform cancellation ``future`` return Future which will be completed when ## ``future`` become finished (completed with value, failed or cancelled). ## @@ -1003,7 +1003,7 @@ proc cancelAndWait*(future: FutureBase, loc: ptr SrcLoc): Future[void] {. retFuture -template cancelAndWait*(future: FutureBase): Future[void].Raising([CancelledError]) = +template cancelAndWait*(future: FutureBase): Future[void].Raising([]) = ## Cancel ``future``. cancelAndWait(future, getSrcLocation()) diff --git a/chronos/internal/raisesfutures.nim b/chronos/internal/raisesfutures.nim index 79384d2..20fa6ed 100644 --- a/chronos/internal/raisesfutures.nim +++ b/chronos/internal/raisesfutures.nim @@ -57,8 +57,25 @@ template init*[T, E]( res, getSrcLocation(fromProc), FutureState.Pending, flags) res +proc dig(n: NimNode): NimNode {.compileTime.} = + # Dig through the layers of type to find the raises list + if n.eqIdent("void"): + n + elif n.kind == nnkBracketExpr: + if n[0].eqIdent("tuple"): + n + elif n[0].eqIdent("typeDesc"): + dig(getType(n[1])) + else: + echo astGenRepr(n) + raiseAssert "Unkown bracket" + elif n.kind == nnkTupleConstr: + n + else: + dig(getType(getTypeInst(n))) + proc isNoRaises*(n: NimNode): bool {.compileTime.} = - n.eqIdent("void") + dig(n).eqIdent("void") iterator members(tup: NimNode): NimNode = # Given a typedesc[tuple] = (A, B, C), yields the tuple members (A, B C) @@ -79,7 +96,7 @@ proc containsSignature(members: openArray[NimNode], typ: NimNode): bool {.compil false # Utilities for working with the E part of InternalRaisesFuture - unstable -macro prepend*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = +macro prepend*(tup: typedesc, typs: varargs[typed]): typedesc = result = nnkTupleConstr.newTree() for err in typs: if not tup.members().containsSignature(err): @@ -91,7 +108,7 @@ macro prepend*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = if result.len == 0: result = makeNoRaises() -macro remove*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = +macro remove*(tup: typedesc, typs: varargs[typed]): typedesc = result = nnkTupleConstr.newTree() for err in tup.members(): if not typs[0..^1].containsSignature(err): @@ -100,7 +117,7 @@ macro remove*(tup: typedesc[tuple], typs: varargs[typed]): typedesc = if result.len == 0: result = makeNoRaises() -macro union*(tup0: typedesc[tuple], tup1: typedesc[tuple]): typedesc = +macro union*(tup0: typedesc, tup1: typedesc): typedesc = ## Join the types of the two tuples deduplicating the entries result = nnkTupleConstr.newTree() diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 58aabc3..107bc6e 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -1832,10 +1832,21 @@ proc close*(server: StreamServer) = server.sock.closeSocket(continuation) proc closeWait*(server: StreamServer): Future[void] {. - async: (raw: true, raises: [CancelledError]).} = + async: (raw: true, raises: []).} = ## Close server ``server`` and release all resources. + let retFuture = newFuture[void]( + "stream.server.closeWait", {FutureFlag.OwnCancelSchedule}) + + proc continuation(udata: pointer) = + retFuture.complete() + server.close() - server.join() + + if not(server.loopFuture.finished()): + server.loopFuture.addCallback(continuation, cast[pointer](retFuture)) + else: + retFuture.complete() + retFuture proc getBacklogSize(backlog: int): cint = doAssert(backlog >= 0 and backlog <= high(int32)) From 28a100b1350d52668ce7b9c9ae8159fdc0acace2 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Tue, 28 Nov 2023 18:57:13 +0200 Subject: [PATCH 47/50] Fix processing callback missing asyncraises. (#479) --- chronos/apps/http/httpcommon.nim | 6 +-- chronos/apps/http/httpserver.nim | 72 ++++++++++++++++---------------- tests/testhttpclient.nim | 30 +++++++------ tests/testhttpserver.nim | 33 ++++++++------- tests/testshttpserver.nim | 4 +- 5 files changed, 77 insertions(+), 68 deletions(-) diff --git a/chronos/apps/http/httpcommon.nim b/chronos/apps/http/httpcommon.nim index d2148fb..3ebe3ca 100644 --- a/chronos/apps/http/httpcommon.nim +++ b/chronos/apps/http/httpcommon.nim @@ -53,10 +53,10 @@ type HttpDefect* = object of Defect HttpError* = object of AsyncError - HttpCriticalError* = object of HttpError - code*: HttpCode - HttpRecoverableError* = object of HttpError + HttpResponseError* = object of HttpError code*: HttpCode + HttpCriticalError* = object of HttpResponseError + HttpRecoverableError* = object of HttpResponseError HttpDisconnectError* = object of HttpError HttpConnectionError* = object of HttpError HttpInterruptError* = object of HttpError diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index e8326cc..7d1aea0 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -65,7 +65,7 @@ type HttpProcessCallback* = proc(req: RequestFence): Future[HttpResponseRef] {. - gcsafe, raises: [].} + async: (raises: [CancelledError, HttpResponseError]), gcsafe.} HttpConnectionCallback* = proc(server: HttpServerRef, @@ -448,7 +448,7 @@ proc getBodyReader*(request: HttpRequestRef): HttpResult[HttpBodyReader] = err("Request do not have body available") proc handleExpect*(request: HttpRequestRef) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Handle expectation for ``Expect`` header. ## https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect if HttpServerFlags.NoExpectHandler notin request.connection.server.flags: @@ -464,7 +464,7 @@ proc handleExpect*(request: HttpRequestRef) {. "Unable to send `100-continue` response, reason: " & $exc.msg) proc getBody*(request: HttpRequestRef): Future[seq[byte]] {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Obtain request's body as sequence of bytes. let bodyReader = request.getBodyReader() if bodyReader.isErr(): @@ -486,7 +486,7 @@ proc getBody*(request: HttpRequestRef): Future[seq[byte]] {. if not(isNil(reader)): await reader.closeWait() raise exc - except HttpError as exc: + except HttpCriticalError as exc: if not(isNil(reader)): await reader.closeWait() raise exc @@ -497,7 +497,7 @@ proc getBody*(request: HttpRequestRef): Future[seq[byte]] {. raiseHttpCriticalError(msg) proc consumeBody*(request: HttpRequestRef): Future[void] {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Consume/discard request's body. let bodyReader = request.getBodyReader() if bodyReader.isErr(): @@ -519,7 +519,7 @@ proc consumeBody*(request: HttpRequestRef): Future[void] {. if not(isNil(reader)): await reader.closeWait() raise exc - except HttpError as exc: + except HttpCriticalError as exc: if not(isNil(reader)): await reader.closeWait() raise exc @@ -905,10 +905,11 @@ proc getResponseFence*(connection: HttpConnectionRef, let address = connection.getRemoteAddress() ResponseFence.err(HttpProcessError.init( HttpServerError.RecoverableError, exc, address, exc.code)) - except CatchableError as exc: - let address = connection.getRemoteAddress() - ResponseFence.err(HttpProcessError.init( - HttpServerError.CatchableError, exc, address, Http503)) + except HttpResponseError as exc: + # There should be only 2 children of HttpResponseError, and all of them + # should be handled. + raiseAssert "Unexpected response error " & $exc.name & ", reason: " & + $exc.msg proc getResponseFence*(server: HttpServerRef, connFence: ConnectionFence): Future[ResponseFence] {. @@ -930,10 +931,11 @@ proc getResponseFence*(server: HttpServerRef, let address = Opt.none(TransportAddress) ResponseFence.err(HttpProcessError.init( HttpServerError.RecoverableError, exc, address, exc.code)) - except CatchableError as exc: - let address = Opt.none(TransportAddress) - ResponseFence.err(HttpProcessError.init( - HttpServerError.CatchableError, exc, address, Http503)) + except HttpResponseError as exc: + # There should be only 2 children of HttpResponseError, and all of them + # should be handled. + raiseAssert "Unexpected response error " & $exc.name & ", reason: " & + $exc.msg proc getRequestFence*(server: HttpServerRef, connection: HttpConnectionRef): Future[RequestFence] {. @@ -1161,7 +1163,7 @@ proc getMultipartReader*(req: HttpRequestRef): HttpResult[MultiPartReaderRef] = err("Request's method do not supports multipart") proc post*(req: HttpRequestRef): Future[HttpTable] {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Return POST parameters if req.postTable.isSome(): return req.postTable.get() @@ -1337,7 +1339,7 @@ proc preparePlainHeaders(resp: HttpResponseRef): string = resp.createHeaders() proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Send HTTP response at once by using bytes pointer ``pbytes`` and length ## ``nbytes``. doAssert(not(isNil(pbytes)), "pbytes must not be nil") @@ -1359,7 +1361,7 @@ proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc sendBody*(resp: HttpResponseRef, data: ByteChar) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Send HTTP response at once by using data ``data``. checkPending(resp) let responseHeaders = resp.prepareLengthHeaders(len(data)) @@ -1378,7 +1380,7 @@ proc sendBody*(resp: HttpResponseRef, data: ByteChar) {. raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Send HTTP error status response. checkPending(resp) resp.status = code @@ -1399,7 +1401,7 @@ proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {. proc prepare*(resp: HttpResponseRef, streamType = HttpResponseStreamType.Chunked) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Prepare for HTTP stream response. ## ## Such responses will be sent chunk by chunk using ``chunked`` encoding. @@ -1431,26 +1433,26 @@ proc prepare*(resp: HttpResponseRef, raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc prepareChunked*(resp: HttpResponseRef): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Prepare for HTTP chunked stream response. ## ## Such responses will be sent chunk by chunk using ``chunked`` encoding. resp.prepare(HttpResponseStreamType.Chunked) proc preparePlain*(resp: HttpResponseRef): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Prepare for HTTP plain stream response. ## ## Such responses will be sent without any encoding. resp.prepare(HttpResponseStreamType.Plain) proc prepareSSE*(resp: HttpResponseRef): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Prepare for HTTP server-side event stream response. resp.prepare(HttpResponseStreamType.SSE) proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Send single chunk of data pointed by ``pbytes`` and ``nbytes``. doAssert(not(isNil(pbytes)), "pbytes must not be nil") doAssert(nbytes >= 0, "nbytes should be bigger or equal to zero") @@ -1470,7 +1472,7 @@ proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) proc send*(resp: HttpResponseRef, data: ByteChar) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Send single chunk of data ``data``. if HttpResponseFlags.Stream notin resp.flags: raiseHttpCriticalError("Response was not prepared") @@ -1489,16 +1491,16 @@ proc send*(resp: HttpResponseRef, data: ByteChar) {. proc sendChunk*(resp: HttpResponseRef, pbytes: pointer, nbytes: int): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = resp.send(pbytes, nbytes) proc sendChunk*(resp: HttpResponseRef, data: ByteChar): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = resp.send(data) proc sendEvent*(resp: HttpResponseRef, eventName: string, data: string): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Send server-side event with name ``eventName`` and payload ``data`` to ## remote peer. let data = @@ -1515,7 +1517,7 @@ proc sendEvent*(resp: HttpResponseRef, eventName: string, resp.send(data) proc finish*(resp: HttpResponseRef) {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Sending last chunk of data, so it will indicate end of HTTP response. if HttpResponseFlags.Stream notin resp.flags: raiseHttpCriticalError("Response was not prepared") @@ -1534,7 +1536,7 @@ proc finish*(resp: HttpResponseRef) {. proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar, headers: HttpTable): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with the specified ``HttpCode``, HTTP ``headers`` ## and ``content``. let response = req.getResponse() @@ -1546,18 +1548,18 @@ proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar, proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with specified ``HttpCode`` and ``content``. respond(req, code, content, HttpTable.init()) proc respond*(req: HttpRequestRef, code: HttpCode): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with specified ``HttpCode`` only. respond(req, code, "", HttpTable.init()) proc redirect*(req: HttpRequestRef, code: HttpCode, location: string, headers: HttpTable): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with redirection to location ``location`` and ## additional headers ``headers``. ## @@ -1569,7 +1571,7 @@ proc redirect*(req: HttpRequestRef, code: HttpCode, proc redirect*(req: HttpRequestRef, code: HttpCode, location: Uri, headers: HttpTable): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with redirection to location ``location`` and ## additional headers ``headers``. ## @@ -1579,13 +1581,13 @@ proc redirect*(req: HttpRequestRef, code: HttpCode, proc redirect*(req: HttpRequestRef, code: HttpCode, location: Uri): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with redirection to location ``location``. redirect(req, code, location, HttpTable.init()) proc redirect*(req: HttpRequestRef, code: HttpCode, location: string): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]).} = + async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = ## Responds to the request with redirection to location ``location``. redirect(req, code, location, HttpTable.init()) diff --git a/tests/testhttpclient.nim b/tests/testhttpclient.nim index 1d9992f..eb1eaac 100644 --- a/tests/testhttpclient.nim +++ b/tests/testhttpclient.nim @@ -128,7 +128,7 @@ suite "HTTP client testing suite": (MethodPatch, "/test/patch") ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -195,7 +195,7 @@ suite "HTTP client testing suite": "LONGCHUNKRESPONSE") ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -311,7 +311,7 @@ suite "HTTP client testing suite": (MethodPost, "/test/big_request", 262400) ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -381,7 +381,7 @@ suite "HTTP client testing suite": (MethodPost, "/test/big_chunk_request", 262400) ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -455,7 +455,7 @@ suite "HTTP client testing suite": ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -554,7 +554,7 @@ suite "HTTP client testing suite": ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -649,7 +649,7 @@ suite "HTTP client testing suite": var lastAddress: Uri proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -706,7 +706,7 @@ suite "HTTP client testing suite": proc testSendCancelLeaksTest(secure: bool): Future[bool] {.async.} = proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = return defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) @@ -756,7 +756,7 @@ suite "HTTP client testing suite": proc testOpenCancelLeaksTest(secure: bool): Future[bool] {.async.} = proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = return defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) @@ -867,7 +867,8 @@ suite "HTTP client testing suite": return @[(data1.status, data1.data.bytesToString(), count), (data2.status, data2.data.bytesToString(), count)] - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -1002,7 +1003,8 @@ suite "HTTP client testing suite": await request.closeWait() return (data.status, data.data.bytesToString(), 0) - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -1061,7 +1063,8 @@ suite "HTTP client testing suite": await request.closeWait() return (data.status, data.data.bytesToString(), 0) - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -1176,7 +1179,8 @@ suite "HTTP client testing suite": return false true - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() if request.uri.path.startsWith("/test/single/"): diff --git a/tests/testhttpserver.nim b/tests/testhttpserver.nim index 85aeee5..33d5ea1 100644 --- a/tests/testhttpserver.nim +++ b/tests/testhttpserver.nim @@ -64,7 +64,7 @@ suite "HTTP server testing suite": proc testTooBigBodyChunked(operation: TooBigTest): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() try: @@ -128,7 +128,7 @@ suite "HTTP server testing suite": proc testTimeout(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() return await request.respond(Http200, "TEST_OK", HttpTable.init()) @@ -158,7 +158,7 @@ suite "HTTP server testing suite": proc testEmpty(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() return await request.respond(Http200, "TEST_OK", HttpTable.init()) @@ -188,7 +188,7 @@ suite "HTTP server testing suite": proc testTooBig(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() return await request.respond(Http200, "TEST_OK", HttpTable.init()) @@ -219,7 +219,7 @@ suite "HTTP server testing suite": proc testTooBigBody(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): discard else: @@ -266,7 +266,7 @@ suite "HTTP server testing suite": proc testQuery(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() var kres = newSeq[string]() @@ -307,7 +307,7 @@ suite "HTTP server testing suite": proc testHeaders(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() var kres = newSeq[string]() @@ -351,7 +351,7 @@ suite "HTTP server testing suite": proc testPostUrl(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() @@ -395,7 +395,7 @@ suite "HTTP server testing suite": proc testPostUrl2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() @@ -440,7 +440,7 @@ suite "HTTP server testing suite": proc testPostMultipart(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() @@ -496,7 +496,7 @@ suite "HTTP server testing suite": proc testPostMultipart2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() @@ -565,7 +565,8 @@ suite "HTTP server testing suite": var eventContinue = newAsyncEvent() var count = 0 - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() inc(count) @@ -1229,7 +1230,7 @@ suite "HTTP server testing suite": proc testPostMultipart2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() let response = request.getResponse() @@ -1304,7 +1305,8 @@ suite "HTTP server testing suite": {}, false, "close") ] - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() return await request.respond(Http200, "TEST_OK", HttpTable.init()) @@ -1357,7 +1359,8 @@ suite "HTTP server testing suite": TestsCount = 10 TestRequest = "GET /httpdebug HTTP/1.1\r\nConnection: keep-alive\r\n\r\n" - proc process(r: RequestFence): Future[HttpResponseRef] {.async.} = + proc process(r: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() return await request.respond(Http200, "TEST_OK", HttpTable.init()) diff --git a/tests/testshttpserver.nim b/tests/testshttpserver.nim index 8aacb8e..3ff2565 100644 --- a/tests/testshttpserver.nim +++ b/tests/testshttpserver.nim @@ -108,7 +108,7 @@ suite "Secure HTTP server testing suite": proc testHTTPS(address: TransportAddress): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() serverRes = true @@ -146,7 +146,7 @@ suite "Secure HTTP server testing suite": var serverRes = false var testFut = newFuture[void]() proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = + async: (raises: [CancelledError, HttpResponseError]).} = if r.isOk(): let request = r.get() serverRes = false From 48b2b08cfbe057242997a8aee615155ea76f3744 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 1 Dec 2023 12:33:28 +0100 Subject: [PATCH 48/50] Update docs (#480) * new mdbook version with built-in Nim highlighting support * describe examples in a dedicated page * fixes --- .github/workflows/doc.yml | 4 +-- docs/src/SUMMARY.md | 6 ++++- docs/src/async_procs.md | 13 ++++++---- docs/src/examples.md | 18 +++++++++++++ docs/src/introduction.md | 30 +++++++++++++++++----- docs/theme/highlight.js | 53 --------------------------------------- 6 files changed, 57 insertions(+), 67 deletions(-) create mode 100644 docs/src/examples.md delete mode 100644 docs/theme/highlight.js diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index dc718f8..5d4022c 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -22,7 +22,7 @@ jobs: with: crate: mdbook use-tool-cache: true - version: "0.4.35" + version: "0.4.36" - uses: actions-rs/install@v0.1 with: crate: mdbook-toc @@ -37,7 +37,7 @@ jobs: with: crate: mdbook-admonish use-tool-cache: true - version: "1.13.1" + version: "1.14.0" - uses: jiro4989/setup-nim-action@v1 with: diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 186fadd..4f2ee56 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -1,5 +1,5 @@ - [Introduction](./introduction.md) -- [Getting started](./getting_started.md) +- [Examples](./examples.md) # User guide @@ -8,3 +8,7 @@ - [Errors and exceptions](./error_handling.md) - [Tips, tricks and best practices](./tips.md) - [Porting code to `chronos`](./porting.md) + +# Developer guide + +- [Updating this book](./book.md) diff --git a/docs/src/async_procs.md b/docs/src/async_procs.md index 648f19b..c7ee9f3 100644 --- a/docs/src/async_procs.md +++ b/docs/src/async_procs.md @@ -2,8 +2,8 @@ Async procedures are those that interact with `chronos` to cooperatively suspend and resume their execution depending on the completion of other -async procedures which themselves may be waiting for I/O to complete, timers to -expire or tasks running on other threads to complete. +async procedures, timers, tasks on other threads or asynchronous I/O scheduled +with the operating system. Async procedures are marked with the `{.async.}` pragma and return a `Future` indicating the state of the operation. @@ -25,6 +25,9 @@ echo p().type # prints "Future[system.void]" ## `await` keyword +The `await` keyword operates on `Future` instances typically returned from an +`async` procedure. + Whenever `await` is encountered inside an async procedure, control is given back to the dispatcher for as many steps as it's necessary for the awaited future to complete, fail or be cancelled. `await` calls the @@ -53,13 +56,13 @@ waitFor p3() ```admonition warning Because `async` procedures are executed concurrently, they are subject to many -of the same risks that typically accompany multithreaded programming +of the same risks that typically accompany multithreaded programming. In particular, if two `async` procedures have access to the same mutable state, the value before and after `await` might not be the same as the order of execution is not guaranteed! ``` -## Raw procedures +## Raw async procedures Raw async procedures are those that interact with `chronos` via the `Future` type but whose body does not go through the async transformation. @@ -83,7 +86,7 @@ proc rawFailure(): Future[void] {.async: (raw: true).} = fut ``` -Raw functions can also use checked exceptions: +Raw procedures can also use checked exceptions: ```nim proc rawAsyncRaises(): Future[void] {.async: (raw: true, raises: [IOError]).} = diff --git a/docs/src/examples.md b/docs/src/examples.md new file mode 100644 index 0000000..c71247c --- /dev/null +++ b/docs/src/examples.md @@ -0,0 +1,18 @@ +# Examples + +Examples are available in the [`docs/examples/`](https://github.com/status-im/nim-chronos/tree/master/docs/examples/) folder. + +## Basic concepts + +* [cancellation](https://github.com/status-im/nim-chronos/tree/master/docs/examples/cancellation.nim) - Cancellation primer +* [timeoutsimple](https://github.com/status-im/nim-chronos/tree/master/docs/examples/timeoutsimple.nim) - Simple timeouts +* [timeoutcomposed](https://github.com/status-im/nim-chronos/tree/master/docs/examples/examples/timeoutcomposed.nim) - Shared timeout of multiple tasks + +## TCP + +* [tcpserver](https://github.com/status-im/nim-chronos/tree/master/docs/examples/tcpserver.nim) - Simple TCP/IP v4/v6 echo server + +## HTTP + +* [httpget](https://github.com/status-im/nim-chronos/tree/master/docs/examples/httpget.nim) - Downloading a web page using the http client +* [twogets](https://github.com/status-im/nim-chronos/tree/master/docs/examples/twogets.nim) - Download two pages concurrently diff --git a/docs/src/introduction.md b/docs/src/introduction.md index 9c2a308..bc43686 100644 --- a/docs/src/introduction.md +++ b/docs/src/introduction.md @@ -7,12 +7,34 @@ transformation features provided by Nim. Features include: * Asynchronous socket and process I/O -* HTTP server with SSL/TLS support out of the box (no OpenSSL needed) +* HTTP client / server with SSL/TLS support out of the box (no OpenSSL needed) * Synchronization primitivies like queues, events and locks -* Cancellation +* [Cancellation](./concepts.md#cancellation) * Efficient dispatch pipeline with excellent multi-platform support * Exception [effect support](./guide.md#error-handling) +## Installation + +Install `chronos` using `nimble`: + +```text +nimble install chronos +``` + +or add a dependency to your `.nimble` file: + +```text +requires "chronos" +``` + +and start using it: + +```nim +{{#include ../examples/httpget.nim}} +``` + +There are more [examples](./examples.md) throughout the manual! + ## Platform support Several platforms are supported, with different backend [options](./concepts.md#compile-time-configuration): @@ -22,10 +44,6 @@ Several platforms are supported, with different backend [options](./concepts.md# * OSX / BSD: [`kqueue`](https://en.wikipedia.org/wiki/Kqueue) / `poll` * Android / Emscripten / posix: `poll` -## Examples - -Examples are available in the [`docs/examples/`](https://github.com/status-im/nim-chronos/docs/examples) folder. - ## API documentation This guide covers basic usage of chronos - for details, see the diff --git a/docs/theme/highlight.js b/docs/theme/highlight.js deleted file mode 100644 index 3256c00..0000000 --- a/docs/theme/highlight.js +++ /dev/null @@ -1,53 +0,0 @@ -/* - Highlight.js 10.1.1 (93fd0d73) - License: BSD-3-Clause - Copyright (c) 2006-2020, Ivan Sagalaev -*/ -var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!Object.hasOwnProperty.call(n,r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}class n{constructor(e){void 0===e.data&&(e.data={}),this.data=e.data}ignoreMatch(){this.ignore=!0}}function t(e){return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'")}function r(e,...n){var t={};for(const n in e)t[n]=e[n];return n.forEach((function(e){for(const n in e)t[n]=e[n]})),t}function a(e){return e.nodeName.toLowerCase()}var i=Object.freeze({__proto__:null,escapeHTML:t,inherit:r,nodeStream:function(e){var n=[];return function e(t,r){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?r+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:r,node:i}),r=e(i,r),a(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:r,node:i}));return r}(e,0),n},mergeStreams:function(e,n,r){var i=0,s="",o=[];function l(){return e.length&&n.length?e[0].offset!==n[0].offset?e[0].offset"}function u(e){s+=""}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||n.length;){var g=l();if(s+=t(r.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+t(r.substr(i))}});const s="
",o=e=>!!e.kind;class l{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=t(e)}openNode(e){if(!o(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){o(e)&&(this.buffer+=s)}value(){return this.buffer}span(e){this.buffer+=``}}class c{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){const n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){"string"!=typeof e&&e.children&&(e.children.every(e=>"string"==typeof e)?e.children=[e.children.join("")]:e.children.forEach(e=>{c._collapse(e)}))}}class u extends c{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){const t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new l(this,this.options).value()}finalize(){return!0}}function d(e){return e?"string"==typeof e?e:e.source:null}const g="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",h={begin:"\\\\[\\s\\S]",relevance:0},f={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[h]},p={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[h]},b={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,t={}){var a=r({className:"comment",begin:e,end:n,contains:[]},t);return a.contains.push(b),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):",relevance:0}),a},v=m("//","$"),x=m("/\\*","\\*/"),E=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:g,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",SHEBANG:(e={})=>{const n=/^#![ ]*\//;return e.binary&&(e.begin=function(...e){return e.map(e=>d(e)).join("")}(n,/.*\b/,e.binary,/\b.*/)),r({className:"meta",begin:n,end:/$/,relevance:0,"on:begin":(e,n)=>{0!==e.index&&n.ignoreMatch()}},e)},BACKSLASH_ESCAPE:h,APOS_STRING_MODE:f,QUOTE_STRING_MODE:p,PHRASAL_WORDS_MODE:b,COMMENT:m,C_LINE_COMMENT_MODE:v,C_BLOCK_COMMENT_MODE:x,HASH_COMMENT_MODE:E,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:g,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[h,{begin:/\[/,end:/\]/,relevance:0,contains:[h]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0},END_SAME_AS_BEGIN:function(e){return Object.assign(e,{"on:begin":(e,n)=>{n.data._beginMatch=e[1]},"on:end":(e,n)=>{n.data._beginMatch!==e[1]&&n.ignoreMatch()}})}}),N="of and for in not or if then".split(" ");function w(e,n){return n?+n:function(e){return N.includes(e.toLowerCase())}(e)?0:1}const R=t,y=r,{nodeStream:k,mergeStreams:O}=i,M=Symbol("nomatch");return function(t){var a=[],i={},s={},o=[],l=!0,c=/(^(<[^>]+>|\t|)+|\n)/gm,g="Could not find the language '{}', did you forget to load/include a language module?";const h={disableAutodetect:!0,name:"Plain text",contains:[]};var f={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:null,__emitter:u};function p(e){return f.noHighlightRe.test(e)}function b(e,n,t,r){var a={code:n,language:e};S("before:highlight",a);var i=a.result?a.result:m(a.language,a.code,t,r);return i.code=a.code,S("after:highlight",i),i}function m(e,t,a,s){var o=t;function c(e,n){var t=E.case_insensitive?n[0].toLowerCase():n[0];return Object.prototype.hasOwnProperty.call(e.keywords,t)&&e.keywords[t]}function u(){null!=y.subLanguage?function(){if(""!==A){var e=null;if("string"==typeof y.subLanguage){if(!i[y.subLanguage])return void O.addText(A);e=m(y.subLanguage,A,!0,k[y.subLanguage]),k[y.subLanguage]=e.top}else e=v(A,y.subLanguage.length?y.subLanguage:null);y.relevance>0&&(I+=e.relevance),O.addSublanguage(e.emitter,e.language)}}():function(){if(!y.keywords)return void O.addText(A);let e=0;y.keywordPatternRe.lastIndex=0;let n=y.keywordPatternRe.exec(A),t="";for(;n;){t+=A.substring(e,n.index);const r=c(y,n);if(r){const[e,a]=r;O.addText(t),t="",I+=a,O.addKeyword(n[0],e)}else t+=n[0];e=y.keywordPatternRe.lastIndex,n=y.keywordPatternRe.exec(A)}t+=A.substr(e),O.addText(t)}(),A=""}function h(e){return e.className&&O.openNode(e.className),y=Object.create(e,{parent:{value:y}})}function p(e){return 0===y.matcher.regexIndex?(A+=e[0],1):(L=!0,0)}var b={};function x(t,r){var i=r&&r[0];if(A+=t,null==i)return u(),0;if("begin"===b.type&&"end"===r.type&&b.index===r.index&&""===i){if(A+=o.slice(r.index,r.index+1),!l){const n=Error("0 width match regex");throw n.languageName=e,n.badRule=b.rule,n}return 1}if(b=r,"begin"===r.type)return function(e){var t=e[0],r=e.rule;const a=new n(r),i=[r.__beforeBegin,r["on:begin"]];for(const n of i)if(n&&(n(e,a),a.ignore))return p(t);return r&&r.endSameAsBegin&&(r.endRe=RegExp(t.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),r.skip?A+=t:(r.excludeBegin&&(A+=t),u(),r.returnBegin||r.excludeBegin||(A=t)),h(r),r.returnBegin?0:t.length}(r);if("illegal"===r.type&&!a){const e=Error('Illegal lexeme "'+i+'" for mode "'+(y.className||"")+'"');throw e.mode=y,e}if("end"===r.type){var s=function(e){var t=e[0],r=o.substr(e.index),a=function e(t,r,a){let i=function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(t.endRe,a);if(i){if(t["on:end"]){const e=new n(t);t["on:end"](r,e),e.ignore&&(i=!1)}if(i){for(;t.endsParent&&t.parent;)t=t.parent;return t}}if(t.endsWithParent)return e(t.parent,r,a)}(y,e,r);if(!a)return M;var i=y;i.skip?A+=t:(i.returnEnd||i.excludeEnd||(A+=t),u(),i.excludeEnd&&(A=t));do{y.className&&O.closeNode(),y.skip||y.subLanguage||(I+=y.relevance),y=y.parent}while(y!==a.parent);return a.starts&&(a.endSameAsBegin&&(a.starts.endRe=a.endRe),h(a.starts)),i.returnEnd?0:t.length}(r);if(s!==M)return s}if("illegal"===r.type&&""===i)return 1;if(B>1e5&&B>3*r.index)throw Error("potential infinite loop, way more iterations than matches");return A+=i,i.length}var E=T(e);if(!E)throw console.error(g.replace("{}",e)),Error('Unknown language: "'+e+'"');var _=function(e){function n(n,t){return RegExp(d(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class t{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);const e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n="|"){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i0&&(a+=n),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"===l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("===l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;const n=this.matcherRe.exec(e);if(!n)return null;const t=n.findIndex((e,n)=>n>0&&void 0!==e),r=this.matchIndexes[t];return n.splice(0,t),Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];const n=new t;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){const n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;const t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e,n){const t=e.input[e.index-1],r=e.input[e.index+e[0].length];"."!==t&&"."!==r||n.ignoreMatch()}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");return function t(s,o){const l=s;if(s.compiled)return l;s.compiled=!0,s.__beforeBegin=null,s.keywords=s.keywords||s.beginKeywords;let c=null;if("object"==typeof s.keywords&&(c=s.keywords.$pattern,delete s.keywords.$pattern),s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,w(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemes&&c)throw Error("ERR: Prefer `keywords.$pattern` to `mode.lexemes`, BOTH are not allowed. (see mode reference) ");return l.keywordPatternRe=n(s.lexemes||c||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__beforeBegin=i),s.begin||(s.begin=/\B|\b/),l.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(l.endRe=n(s.end)),l.terminator_end=d(s.end)||"",s.endsWithParent&&o.terminator_end&&(l.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(l.illegalRe=n(s.illegal)),void 0===s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return r(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?r(e,{starts:e.starts?r(e.starts):null}):Object.isFrozen(e)?r(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){t(e,l)})),s.starts&&t(s.starts,o),l.matcher=function(e){const n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(l),l}(e)}(E),N="",y=s||_,k={},O=new f.__emitter(f);!function(){for(var e=[],n=y;n!==E;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>O.openNode(e))}();var A="",I=0,S=0,B=0,L=!1;try{for(y.matcher.considerAll();;){B++,L?L=!1:(y.matcher.lastIndex=S,y.matcher.considerAll());const e=y.matcher.exec(o);if(!e)break;const n=x(o.substring(S,e.index),e);S=e.index+n}return x(o.substr(S)),O.closeAllNodes(),O.finalize(),N=O.toHTML(),{relevance:I,value:N,language:e,illegal:!1,emitter:O,top:y}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:o.slice(S-100,S+100),mode:n.mode},sofar:N,relevance:0,value:R(o),emitter:O};if(l)return{illegal:!1,relevance:0,value:R(o),emitter:O,language:e,top:y,errorRaised:n};throw n}}function v(e,n){n=n||f.languages||Object.keys(i);var t=function(e){const n={relevance:0,emitter:new f.__emitter(f),value:R(e),illegal:!1,top:h};return n.emitter.addText(e),n}(e),r=t;return n.filter(T).filter(I).forEach((function(n){var a=m(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function x(e){return f.tabReplace||f.useBR?e.replace(c,e=>"\n"===e?f.useBR?"
":e:f.tabReplace?e.replace(/\t/g,f.tabReplace):e):e}function E(e){let n=null;const t=function(e){var n=e.className+" ";n+=e.parentNode?e.parentNode.className:"";const t=f.languageDetectRe.exec(n);if(t){var r=T(t[1]);return r||(console.warn(g.replace("{}",t[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?t[1]:"no-highlight"}return n.split(/\s+/).find(e=>p(e)||T(e))}(e);if(p(t))return;S("before:highlightBlock",{block:e,language:t}),f.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n"):n=e;const r=n.textContent,a=t?b(t,r,!0):v(r),i=k(n);if(i.length){const e=document.createElement("div");e.innerHTML=a.value,a.value=O(i,k(e),r)}a.value=x(a.value),S("after:highlightBlock",{block:e,result:a}),e.innerHTML=a.value,e.className=function(e,n,t){var r=n?s[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,t,a.language),e.result={language:a.language,re:a.relevance,relavance:a.relevance},a.second_best&&(e.second_best={language:a.second_best.language,re:a.second_best.relevance,relavance:a.second_best.relevance})}const N=()=>{if(!N.called){N.called=!0;var e=document.querySelectorAll("pre code");a.forEach.call(e,E)}};function T(e){return e=(e||"").toLowerCase(),i[e]||i[s[e]]}function A(e,{languageName:n}){"string"==typeof e&&(e=[e]),e.forEach(e=>{s[e]=n})}function I(e){var n=T(e);return n&&!n.disableAutodetect}function S(e,n){var t=e;o.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(t,{highlight:b,highlightAuto:v,fixMarkup:x,highlightBlock:E,configure:function(e){f=y(f,e)},initHighlighting:N,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",N,!1)},registerLanguage:function(e,n){var r=null;try{r=n(t)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!l)throw n;console.error(n),r=h}r.name||(r.name=e),i[e]=r,r.rawDefinition=n.bind(null,t),r.aliases&&A(r.aliases,{languageName:e})},listLanguages:function(){return Object.keys(i)},getLanguage:T,registerAliases:A,requireLanguage:function(e){var n=T(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:I,inherit:y,addPlugin:function(e){o.push(e)}}),t.debugMode=function(){l=!1},t.safeMode=function(){l=!0},t.versionString="10.1.1";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(t,_),t}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs); -hljs.registerLanguage("apache",function(){"use strict";return function(e){var n={className:"number",begin:"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?"};return{name:"Apache config",aliases:["apacheconf"],case_insensitive:!0,contains:[e.HASH_COMMENT_MODE,{className:"section",begin:"",contains:[n,{className:"number",begin:":\\d{1,5}"},e.inherit(e.QUOTE_STRING_MODE,{relevance:0})]},{className:"attribute",begin:/\w+/,relevance:0,keywords:{nomarkup:"order deny allow setenv rewriterule rewriteengine rewritecond documentroot sethandler errordocument loadmodule options header listen serverroot servername"},starts:{end:/$/,relevance:0,keywords:{literal:"on off all deny allow"},contains:[{className:"meta",begin:"\\s\\[",end:"\\]$"},{className:"variable",begin:"[\\$%]\\{",end:"\\}",contains:["self",{className:"number",begin:"[\\$%]\\d+"}]},n,{className:"number",begin:"\\d+"},e.QUOTE_STRING_MODE]}}],illegal:/\S/}}}()); -hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const t={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},n={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,t]};t.contains.push(n);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]},i=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10}),c={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b-?[a-z\._]+\b/,keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[i,e.SHEBANG(),c,a,e.HASH_COMMENT_MODE,n,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}()); -hljs.registerLanguage("c-like",function(){"use strict";return function(e){function t(e){return"(?:"+e+")?"}var n="(decltype\\(auto\\)|"+t("[a-zA-Z_]\\w*::")+"[a-zA-Z_]\\w*"+t("<.*?>")+")",r={className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},a={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},e.END_SAME_AS_BEGIN({begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},i={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},s={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(a,{className:"meta-string"}),{className:"meta-string",begin:/<.*?>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},o={className:"title",begin:t("[a-zA-Z_]\\w*::")+e.IDENT_RE,relevance:0},c=t("[a-zA-Z_]\\w*::")+e.IDENT_RE+"\\s*\\(",l={keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary",literal:"true false nullptr NULL"},d=[r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,i,a],_={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:l,contains:d.concat([{begin:/\(/,end:/\)/,keywords:l,contains:d.concat(["self"]),relevance:0}]),relevance:0},u={className:"function",begin:"("+n+"[\\*&\\s]+)+"+c,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:l,illegal:/[^\w\s\*&:<>]/,contains:[{begin:"decltype\\(auto\\)",keywords:l,relevance:0},{begin:c,returnBegin:!0,contains:[o],relevance:0},{className:"params",begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,i,r,{begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,i,r]}]},r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s]};return{aliases:["c","cc","h","c++","h++","hpp","hh","hxx","cxx"],keywords:l,disableAutodetect:!0,illegal:"",keywords:l,contains:["self",r]},{begin:e.IDENT_RE+"::",keywords:l},{className:"class",beginKeywords:"class struct",end:/[{;:]/,contains:[{begin://,contains:["self"]},e.TITLE_MODE]}]),exports:{preprocessor:s,strings:a,keywords:l}}}}()); -hljs.registerLanguage("c",function(){"use strict";return function(e){var n=e.getLanguage("c-like").rawDefinition();return n.name="C",n.aliases=["c","h"],n}}()); -hljs.registerLanguage("coffeescript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);return function(r){var t={keyword:e.concat(["then","unless","until","loop","by","when","and","or","is","isnt","not"]).filter((e=>n=>!e.includes(n))(["var","const","let","function","static"])).join(" "),literal:n.concat(["yes","no","on","off"]).join(" "),built_in:a.concat(["npm","print"]).join(" ")},i="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/}/,keywords:t},o=[r.BINARY_NUMBER_MODE,r.inherit(r.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[r.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[r.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[r.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[r.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,r.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+i},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=o;var c=r.inherit(r.TITLE_MODE,{begin:i}),l={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:t,contains:["self"].concat(o)}]};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:t,illegal:/\/\*/,contains:o.concat([r.COMMENT("###","###"),r.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+i+"\\s*=\\s*(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[c,l]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:"(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[l]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[c]},c]},{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}}()); -hljs.registerLanguage("cpp",function(){"use strict";return function(e){var t=e.getLanguage("c-like").rawDefinition();return t.disableAutodetect=!1,t.name="C++",t.aliases=["cc","c++","h++","hpp","hh","hxx","cxx"],t}}()); -hljs.registerLanguage("csharp",function(){"use strict";return function(e){var n={keyword:"abstract as base bool break byte case catch char checked const continue decimal default delegate do double enum event explicit extern finally fixed float for foreach goto if implicit in int interface internal is lock long object operator out override params private protected public readonly ref sbyte sealed short sizeof stackalloc static string struct switch this try typeof uint ulong unchecked unsafe ushort using virtual void volatile while add alias ascending async await by descending dynamic equals from get global group into join let nameof on orderby partial remove select set value var when where yield",literal:"null false true"},i=e.inherit(e.TITLE_MODE,{begin:"[a-zA-Z](\\.?\\w)*"}),a={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},s={className:"string",begin:'@"',end:'"',contains:[{begin:'""'}]},t=e.inherit(s,{illegal:/\n/}),l={className:"subst",begin:"{",end:"}",keywords:n},r=e.inherit(l,{illegal:/\n/}),c={className:"string",begin:/\$"/,end:'"',illegal:/\n/,contains:[{begin:"{{"},{begin:"}}"},e.BACKSLASH_ESCAPE,r]},o={className:"string",begin:/\$@"/,end:'"',contains:[{begin:"{{"},{begin:"}}"},{begin:'""'},l]},g=e.inherit(o,{illegal:/\n/,contains:[{begin:"{{"},{begin:"}}"},{begin:'""'},r]});l.contains=[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.C_BLOCK_COMMENT_MODE],r.contains=[g,c,t,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.inherit(e.C_BLOCK_COMMENT_MODE,{illegal:/\n/})];var d={variants:[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},E={begin:"<",end:">",contains:[{beginKeywords:"in out"},i]},_=e.IDENT_RE+"(<"+e.IDENT_RE+"(\\s*,\\s*"+e.IDENT_RE+")*>)?(\\[\\])?",b={begin:"@"+e.IDENT_RE,relevance:0};return{name:"C#",aliases:["cs","c#"],keywords:n,illegal:/::/,contains:[e.COMMENT("///","$",{returnBegin:!0,contains:[{className:"doctag",variants:[{begin:"///",relevance:0},{begin:"\x3c!--|--\x3e"},{begin:""}]}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"meta",begin:"#",end:"$",keywords:{"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum"}},d,a,{beginKeywords:"class interface",end:/[{;=]/,illegal:/[^\s:,]/,contains:[{beginKeywords:"where class"},i,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"namespace",end:/[{;=]/,illegal:/[^\s:]/,contains:[i,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"meta",begin:"^\\s*\\[",excludeBegin:!0,end:"\\]",excludeEnd:!0,contains:[{className:"meta-string",begin:/"/,end:/"/}]},{beginKeywords:"new return throw await else",relevance:0},{className:"function",begin:"("+_+"\\s+)+"+e.IDENT_RE+"\\s*(\\<.+\\>)?\\s*\\(",returnBegin:!0,end:/\s*[{;=]/,excludeEnd:!0,keywords:n,contains:[{begin:e.IDENT_RE+"\\s*(\\<.+\\>)?\\s*\\(",returnBegin:!0,contains:[e.TITLE_MODE,E],relevance:0},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,relevance:0,contains:[d,a,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},b]}}}()); -hljs.registerLanguage("css",function(){"use strict";return function(e){var n={begin:/(?:[A-Z\_\.\-]+|--[a-zA-Z0-9_-]+)\s*:/,returnBegin:!0,end:";",endsWithParent:!0,contains:[{className:"attribute",begin:/\S/,end:":",excludeEnd:!0,starts:{endsWithParent:!0,excludeEnd:!0,contains:[{begin:/[\w-]+\(/,returnBegin:!0,contains:[{className:"built_in",begin:/[\w-]+/},{begin:/\(/,end:/\)/,contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.CSS_NUMBER_MODE]}]},e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_BLOCK_COMMENT_MODE,{className:"number",begin:"#[0-9A-Fa-f]+"},{className:"meta",begin:"!important"}]}}]};return{name:"CSS",case_insensitive:!0,illegal:/[=\/|'\$]/,contains:[e.C_BLOCK_COMMENT_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/},{className:"selector-class",begin:/\.[A-Za-z0-9_-]+/},{className:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},{className:"selector-pseudo",begin:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{begin:"@(page|font-face)",lexemes:"@[a-z-]+",keywords:"@page @font-face"},{begin:"@",end:"[{;]",illegal:/:/,returnBegin:!0,contains:[{className:"keyword",begin:/@\-?\w[\w]*(\-\w+)*/},{begin:/\s/,endsWithParent:!0,excludeEnd:!0,relevance:0,keywords:"and or not only",contains:[{begin:/[a-z-]+:/,className:"attribute"},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.CSS_NUMBER_MODE]}]},{className:"selector-tag",begin:"[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0},{begin:"{",end:"}",illegal:/\S/,contains:[e.C_BLOCK_COMMENT_MODE,n]}]}}}()); -hljs.registerLanguage("diff",function(){"use strict";return function(e){return{name:"Diff",aliases:["patch"],contains:[{className:"meta",relevance:10,variants:[{begin:/^@@ +\-\d+,\d+ +\+\d+,\d+ +@@$/},{begin:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{begin:/^\-\-\- +\d+,\d+ +\-\-\-\-$/}]},{className:"comment",variants:[{begin:/Index: /,end:/$/},{begin:/={3,}/,end:/$/},{begin:/^\-{3}/,end:/$/},{begin:/^\*{3} /,end:/$/},{begin:/^\+{3}/,end:/$/},{begin:/^\*{15}$/}]},{className:"addition",begin:"^\\+",end:"$"},{className:"deletion",begin:"^\\-",end:"$"},{className:"addition",begin:"^\\!",end:"$"}]}}}()); -hljs.registerLanguage("go",function(){"use strict";return function(e){var n={keyword:"break default func interface select case map struct chan else goto package switch const fallthrough if range type continue for import return var go defer bool byte complex64 complex128 float32 float64 int8 int16 int32 int64 string uint8 uint16 uint32 uint64 int uint uintptr rune",literal:"true false iota nil",built_in:"append cap close complex copy imag len make new panic print println real recover delete"};return{name:"Go",aliases:["golang"],keywords:n,illegal:"e(n)).join("")}return function(a){var s={className:"number",relevance:0,variants:[{begin:/([\+\-]+)?[\d]+_[\d_]+/},{begin:a.NUMBER_RE}]},i=a.COMMENT();i.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];var t={className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)}/}]},r={className:"literal",begin:/\bon|off|true|false|yes|no\b/},l={className:"string",contains:[a.BACKSLASH_ESCAPE],variants:[{begin:"'''",end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"'},{begin:"'",end:"'"}]},c={begin:/\[/,end:/\]/,contains:[i,r,t,l,s,"self"],relevance:0},g="("+[/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/].map(n=>e(n)).join("|")+")";return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/,contains:[i,{className:"section",begin:/\[+/,end:/\]+/},{begin:n(g,"(\\s*\\.\\s*",g,")*",n("(?=",/\s*=\s*[^#\s]/,")")),className:"attr",starts:{end:/$/,contains:[i,c,r,t,l,s]}}]}}}()); -hljs.registerLanguage("java",function(){"use strict";function e(e){return e?"string"==typeof e?e:e.source:null}function n(e){return a("(",e,")?")}function a(...n){return n.map(n=>e(n)).join("")}function s(...n){return"("+n.map(n=>e(n)).join("|")+")"}return function(e){var t="false synchronized int abstract float private char boolean var static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",i={className:"meta",begin:"@[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",contains:[{begin:/\(/,end:/\)/,contains:["self"]}]},r=e=>a("[",e,"]+([",e,"_]*[",e,"]+)?"),c={className:"number",variants:[{begin:`\\b(0[bB]${r("01")})[lL]?`},{begin:`\\b(0${r("0-7")})[dDfFlL]?`},{begin:a(/\b0[xX]/,s(a(r("a-fA-F0-9"),/\./,r("a-fA-F0-9")),a(r("a-fA-F0-9"),/\.?/),a(/\./,r("a-fA-F0-9"))),/([pP][+-]?(\d+))?/,/[fFdDlL]?/)},{begin:a(/\b/,s(a(/\d*\./,r("\\d")),r("\\d")),/[eE][+-]?[\d]+[dDfF]?/)},{begin:a(/\b/,r(/\d/),n(/\.?/),n(r(/\d/)),/[dDfFlL]?/)}],relevance:0};return{name:"Java",aliases:["jsp"],keywords:t,illegal:/<\/|#/,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/,relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"class",beginKeywords:"class interface",end:/[{;=]/,excludeEnd:!0,keywords:"class interface",illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"new throw return else",relevance:0},{className:"function",begin:"([À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*(<[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*(\\s*,\\s*[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*)*>)?\\s+)+"+e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:t,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"params",begin:/\(/,end:/\)/,keywords:t,relevance:0,contains:[i,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},c,i]}}}()); -hljs.registerLanguage("javascript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);function s(e){return r("(?=",e,")")}function r(...e){return e.map(e=>(function(e){return e?"string"==typeof e?e:e.source:null})(e)).join("")}return function(t){var i="[A-Za-z$_][0-9A-Za-z$_]*",c={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/},o={$pattern:"[A-Za-z$_][0-9A-Za-z$_]*",keyword:e.join(" "),literal:n.join(" "),built_in:a.join(" ")},l={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:t.C_NUMBER_RE+"n?"}],relevance:0},E={className:"subst",begin:"\\$\\{",end:"\\}",keywords:o,contains:[]},d={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[t.BACKSLASH_ESCAPE,E],subLanguage:"xml"}},g={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[t.BACKSLASH_ESCAPE,E],subLanguage:"css"}},u={className:"string",begin:"`",end:"`",contains:[t.BACKSLASH_ESCAPE,E]};E.contains=[t.APOS_STRING_MODE,t.QUOTE_STRING_MODE,d,g,u,l,t.REGEXP_MODE];var b=E.contains.concat([{begin:/\(/,end:/\)/,contains:["self"].concat(E.contains,[t.C_BLOCK_COMMENT_MODE,t.C_LINE_COMMENT_MODE])},t.C_BLOCK_COMMENT_MODE,t.C_LINE_COMMENT_MODE]),_={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:b};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:o,contains:[t.SHEBANG({binary:"node",relevance:5}),{className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},t.APOS_STRING_MODE,t.QUOTE_STRING_MODE,d,g,u,t.C_LINE_COMMENT_MODE,t.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+",contains:[{className:"type",begin:"\\{",end:"\\}",relevance:0},{className:"variable",begin:i+"(?=\\s*(-)|$)",endsParent:!0,relevance:0},{begin:/(?=[^\n])\s/,relevance:0}]}]}),t.C_BLOCK_COMMENT_MODE,l,{begin:r(/[{,\n]\s*/,s(r(/(((\/\/.*)|(\/\*(.|\n)*\*\/))\s*)*/,i+"\\s*:"))),relevance:0,contains:[{className:"attr",begin:i+s("\\s*:"),relevance:0}]},{begin:"("+t.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[t.C_LINE_COMMENT_MODE,t.C_BLOCK_COMMENT_MODE,t.REGEXP_MODE,{className:"function",begin:"(\\([^(]*(\\([^(]*(\\([^(]*\\))?\\))?\\)|"+t.UNDERSCORE_IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:t.UNDERSCORE_IDENT_RE},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:o,contains:b}]}]},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{variants:[{begin:"<>",end:""},{begin:c.begin,end:c.end}],subLanguage:"xml",contains:[{begin:c.begin,end:c.end,skip:!0,contains:["self"]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[t.inherit(t.TITLE_MODE,{begin:i}),_],illegal:/\[|%/},{begin:/\$[(.]/},t.METHOD_GUARD,{className:"class",beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends"},t.UNDERSCORE_TITLE_MODE]},{beginKeywords:"constructor",end:/\{/,excludeEnd:!0},{begin:"(get|set)\\s+(?="+i+"\\()",end:/{/,keywords:"get set",contains:[t.inherit(t.TITLE_MODE,{begin:i}),{begin:/\(\)/},_]}],illegal:/#(?!!)/}}}()); -hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}()); -hljs.registerLanguage("kotlin",function(){"use strict";return function(e){var n={keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit init interface annotation data sealed internal infix operator out by constructor super tailrec where const inner suspend typealias external expect actual trait volatile transient native default",built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",literal:"true false null"},a={className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"@"},i={className:"subst",begin:"\\${",end:"}",contains:[e.C_NUMBER_MODE]},s={className:"variable",begin:"\\$"+e.UNDERSCORE_IDENT_RE},t={className:"string",variants:[{begin:'"""',end:'"""(?=[^"])',contains:[s,i]},{begin:"'",end:"'",illegal:/\n/,contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"',illegal:/\n/,contains:[e.BACKSLASH_ESCAPE,s,i]}]};i.contains.push(t);var r={className:"meta",begin:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UNDERSCORE_IDENT_RE+")?"},l={className:"meta",begin:"@"+e.UNDERSCORE_IDENT_RE,contains:[{begin:/\(/,end:/\)/,contains:[e.inherit(t,{className:"meta-string"})]}]},c=e.COMMENT("/\\*","\\*/",{contains:[e.C_BLOCK_COMMENT_MODE]}),o={variants:[{className:"type",begin:e.UNDERSCORE_IDENT_RE},{begin:/\(/,end:/\)/,contains:[]}]},d=o;return d.variants[1].contains=[o],o.variants[1].contains=[d],{name:"Kotlin",aliases:["kt"],keywords:n,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,c,{className:"keyword",begin:/\b(break|continue|return|this)\b/,starts:{contains:[{className:"symbol",begin:/@\w+/}]}},a,r,l,{className:"function",beginKeywords:"fun",end:"[(]|$",returnBegin:!0,excludeEnd:!0,keywords:n,illegal:/fun\s+(<.*>)?[^\s\(]+(\s+[^\s\(]+)\s*=/,relevance:5,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"type",begin://,keywords:"reified",relevance:0},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:n,relevance:0,contains:[{begin:/:/,end:/[=,\/]/,endsWithParent:!0,contains:[o,e.C_LINE_COMMENT_MODE,c],relevance:0},e.C_LINE_COMMENT_MODE,c,r,l,t,e.C_NUMBER_MODE]},c]},{className:"class",beginKeywords:"class interface trait",end:/[:\{(]|$/,excludeEnd:!0,illegal:"extends implements",contains:[{beginKeywords:"public protected internal private constructor"},e.UNDERSCORE_TITLE_MODE,{className:"type",begin://,excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:/[,:]\s*/,end:/[<\(,]|$/,excludeBegin:!0,returnEnd:!0},r,l]},t,{className:"meta",begin:"^#!/usr/bin/env",end:"$",illegal:"\n"},{className:"number",begin:"\\b(0[bB]([01]+[01_]+[01]+|[01]+)|0[xX]([a-fA-F0-9]+[a-fA-F0-9_]+[a-fA-F0-9]+|[a-fA-F0-9]+)|(([\\d]+[\\d_]+[\\d]+|[\\d]+)(\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))?|\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))([eE][-+]?\\d+)?)[lLfF]?",relevance:0}]}}}()); -hljs.registerLanguage("less",function(){"use strict";return function(e){var n="([\\w-]+|@{[\\w-]+})",a=[],s=[],t=function(e){return{className:"string",begin:"~?"+e+".*?"+e}},r=function(e,n,a){return{className:e,begin:n,relevance:a}},i={begin:"\\(",end:"\\)",contains:s,relevance:0};s.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,t("'"),t('"'),e.CSS_NUMBER_MODE,{begin:"(url|data-uri)\\(",starts:{className:"string",end:"[\\)\\n]",excludeEnd:!0}},r("number","#[0-9A-Fa-f]+\\b"),i,r("variable","@@?[\\w-]+",10),r("variable","@{[\\w-]+}"),r("built_in","~?`[^`]*?`"),{className:"attribute",begin:"[\\w-]+\\s*:",end:":",returnBegin:!0,excludeEnd:!0},{className:"meta",begin:"!important"});var c=s.concat({begin:"{",end:"}",contains:a}),l={beginKeywords:"when",endsWithParent:!0,contains:[{beginKeywords:"and not"}].concat(s)},o={begin:n+"\\s*:",returnBegin:!0,end:"[;}]",relevance:0,contains:[{className:"attribute",begin:n,end:":",excludeEnd:!0,starts:{endsWithParent:!0,illegal:"[<=$]",relevance:0,contains:s}}]},g={className:"keyword",begin:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b",starts:{end:"[;{}]",returnEnd:!0,contains:s,relevance:0}},d={className:"variable",variants:[{begin:"@[\\w-]+\\s*:",relevance:15},{begin:"@[\\w-]+"}],starts:{end:"[;}]",returnEnd:!0,contains:c}},b={variants:[{begin:"[\\.#:&\\[>]",end:"[;{}]"},{begin:n,end:"{"}],returnBegin:!0,returnEnd:!0,illegal:"[<='$\"]",relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,l,r("keyword","all\\b"),r("variable","@{[\\w-]+}"),r("selector-tag",n+"%?",0),r("selector-id","#"+n),r("selector-class","\\."+n,0),r("selector-tag","&",0),{className:"selector-attr",begin:"\\[",end:"\\]"},{className:"selector-pseudo",begin:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{begin:"\\(",end:"\\)",contains:c},{begin:"!important"}]};return a.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,g,d,o,b),{name:"Less",case_insensitive:!0,illegal:"[=>'/<($\"]",contains:a}}}()); -hljs.registerLanguage("lua",function(){"use strict";return function(e){var t={begin:"\\[=*\\[",end:"\\]=*\\]",contains:["self"]},a=[e.COMMENT("--(?!\\[=*\\[)","$"),e.COMMENT("--\\[=*\\[","\\]=*\\]",{contains:[t],relevance:10})];return{name:"Lua",keywords:{$pattern:e.UNDERSCORE_IDENT_RE,literal:"true false nil",keyword:"and break do else elseif end for goto if in local not or repeat return then until while",built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall arg self coroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"},contains:a.concat([{className:"function",beginKeywords:"function",end:"\\)",contains:[e.inherit(e.TITLE_MODE,{begin:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{className:"params",begin:"\\(",endsWithParent:!0,contains:a}].concat(a)},e.C_NUMBER_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string",begin:"\\[=*\\[",end:"\\]=*\\]",contains:[t],relevance:5}])}}}()); -hljs.registerLanguage("makefile",function(){"use strict";return function(e){var i={className:"variable",variants:[{begin:"\\$\\("+e.UNDERSCORE_IDENT_RE+"\\)",contains:[e.BACKSLASH_ESCAPE]},{begin:/\$[@%`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:")",end:">",keywords:{name:"style"},contains:[c],starts:{end:"",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:")",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}()); -hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}()); -hljs.registerLanguage("nginx",function(){"use strict";return function(e){var n={className:"variable",variants:[{begin:/\$\d+/},{begin:/\$\{/,end:/}/},{begin:"[\\$\\@]"+e.UNDERSCORE_IDENT_RE}]},a={endsWithParent:!0,keywords:{$pattern:"[a-z/_]+",literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll"},relevance:0,illegal:"=>",contains:[e.HASH_COMMENT_MODE,{className:"string",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:/"/,end:/"/},{begin:/'/,end:/'/}]},{begin:"([a-z]+):/",end:"\\s",endsWithParent:!0,excludeEnd:!0,contains:[n]},{className:"regexp",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:"\\s\\^",end:"\\s|{|;",returnEnd:!0},{begin:"~\\*?\\s+",end:"\\s|{|;",returnEnd:!0},{begin:"\\*(\\.[a-z\\-]+)+"},{begin:"([a-z\\-]+\\.)+\\*"}]},{className:"number",begin:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{className:"number",begin:"\\b\\d+[kKmMgGdshdwy]*\\b",relevance:0},n]};return{name:"Nginx config",aliases:["nginxconf"],contains:[e.HASH_COMMENT_MODE,{begin:e.UNDERSCORE_IDENT_RE+"\\s+{",returnBegin:!0,end:"{",contains:[{className:"section",begin:e.UNDERSCORE_IDENT_RE}],relevance:0},{begin:e.UNDERSCORE_IDENT_RE+"\\s",end:";|{",returnBegin:!0,contains:[{className:"attribute",begin:e.UNDERSCORE_IDENT_RE,starts:a}],relevance:0}],illegal:"[^\\s\\}]"}}}()); -hljs.registerLanguage("objectivec",function(){"use strict";return function(e){var n=/[a-zA-Z@][a-zA-Z0-9_]*/,_={$pattern:n,keyword:"@interface @class @protocol @implementation"};return{name:"Objective-C",aliases:["mm","objc","obj-c"],keywords:{$pattern:n,keyword:"int float while char export sizeof typedef const struct for union unsigned long volatile static bool mutable if do return goto void enum else break extern asm case short default double register explicit signed typename this switch continue wchar_t inline readonly assign readwrite self @synchronized id typeof nonatomic super unichar IBOutlet IBAction strong weak copy in out inout bycopy byref oneway __strong __weak __block __autoreleasing @private @protected @public @try @property @end @throw @catch @finally @autoreleasepool @synthesize @dynamic @selector @optional @required @encode @package @import @defs @compatibility_alias __bridge __bridge_transfer __bridge_retained __bridge_retain __covariant __contravariant __kindof _Nonnull _Nullable _Null_unspecified __FUNCTION__ __PRETTY_FUNCTION__ __attribute__ getter setter retain unsafe_unretained nonnull nullable null_unspecified null_resettable class instancetype NS_DESIGNATED_INITIALIZER NS_UNAVAILABLE NS_REQUIRES_SUPER NS_RETURNS_INNER_POINTER NS_INLINE NS_AVAILABLE NS_DEPRECATED NS_ENUM NS_OPTIONS NS_SWIFT_UNAVAILABLE NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_END NS_REFINED_FOR_SWIFT NS_SWIFT_NAME NS_SWIFT_NOTHROW NS_DURING NS_HANDLER NS_ENDHANDLER NS_VALUERETURN NS_VOIDRETURN",literal:"false true FALSE TRUE nil YES NO NULL",built_in:"BOOL dispatch_once_t dispatch_queue_t dispatch_sync dispatch_async dispatch_once"},illegal:"/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"class",begin:"("+_.keyword.split(" ").join("|")+")\\b",end:"({|$)",excludeEnd:!0,keywords:_,contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"\\."+e.UNDERSCORE_IDENT_RE,relevance:0}]}}}()); -hljs.registerLanguage("perl",function(){"use strict";return function(e){var n={$pattern:/[\w.]+/,keyword:"getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qq fileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent shutdown dump chomp connect getsockname die socketpair close flock exists index shmget sub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedir ioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when"},t={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:n},s={begin:"->{",end:"}"},r={variants:[{begin:/\$\d/},{begin:/[\$%@](\^\w\b|#\w+(::\w+)*|{\w+}|\w+(::\w*)*)/},{begin:/[\$%@][^\s\w{]/,relevance:0}]},i=[e.BACKSLASH_ESCAPE,t,r],a=[r,e.HASH_COMMENT_MODE,e.COMMENT("^\\=\\w","\\=cut",{endsWithParent:!0}),s,{className:"string",contains:i,variants:[{begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[",end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*\\<",end:"\\>",relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE]},{begin:"{\\w+}",contains:[],relevance:0},{begin:"-?\\w+\\s*\\=\\>",contains:[],relevance:0}]},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",keywords:"split return print reverse grep",relevance:0,contains:[e.HASH_COMMENT_MODE,{className:"regexp",begin:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",relevance:10},{className:"regexp",begin:"(m|qr)?/",end:"/[a-z]*",contains:[e.BACKSLASH_ESCAPE],relevance:0}]},{className:"function",beginKeywords:"sub",end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE]},{begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$",subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}]}];return t.contains=a,s.contains=a,{name:"Perl",aliases:["pl","pm"],keywords:n,contains:a}}}()); -hljs.registerLanguage("php",function(){"use strict";return function(e){var r={begin:"\\$+[a-zA-Z_-ÿ][a-zA-Z0-9_-ÿ]*"},t={className:"meta",variants:[{begin:/<\?php/,relevance:10},{begin:/<\?[=]?/},{begin:/\?>/}]},a={className:"string",contains:[e.BACKSLASH_ESCAPE,t],variants:[{begin:'b"',end:'"'},{begin:"b'",end:"'"},e.inherit(e.APOS_STRING_MODE,{illegal:null}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null})]},n={variants:[e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]},i={keyword:"__CLASS__ __DIR__ __FILE__ __FUNCTION__ __LINE__ __METHOD__ __NAMESPACE__ __TRAIT__ die echo exit include include_once print require require_once array abstract and as binary bool boolean break callable case catch class clone const continue declare default do double else elseif empty enddeclare endfor endforeach endif endswitch endwhile eval extends final finally float for foreach from global goto if implements instanceof insteadof int integer interface isset iterable list new object or private protected public real return string switch throw trait try unset use var void while xor yield",literal:"false null true",built_in:"Error|0 AppendIterator ArgumentCountError ArithmeticError ArrayIterator ArrayObject AssertionError BadFunctionCallException BadMethodCallException CachingIterator CallbackFilterIterator CompileError Countable DirectoryIterator DivisionByZeroError DomainException EmptyIterator ErrorException Exception FilesystemIterator FilterIterator GlobIterator InfiniteIterator InvalidArgumentException IteratorIterator LengthException LimitIterator LogicException MultipleIterator NoRewindIterator OutOfBoundsException OutOfRangeException OuterIterator OverflowException ParentIterator ParseError RangeException RecursiveArrayIterator RecursiveCachingIterator RecursiveCallbackFilterIterator RecursiveDirectoryIterator RecursiveFilterIterator RecursiveIterator RecursiveIteratorIterator RecursiveRegexIterator RecursiveTreeIterator RegexIterator RuntimeException SeekableIterator SplDoublyLinkedList SplFileInfo SplFileObject SplFixedArray SplHeap SplMaxHeap SplMinHeap SplObjectStorage SplObserver SplObserver SplPriorityQueue SplQueue SplStack SplSubject SplSubject SplTempFileObject TypeError UnderflowException UnexpectedValueException ArrayAccess Closure Generator Iterator IteratorAggregate Serializable Throwable Traversable WeakReference Directory __PHP_Incomplete_Class parent php_user_filter self static stdClass"};return{aliases:["php","php3","php4","php5","php6","php7"],case_insensitive:!0,keywords:i,contains:[e.HASH_COMMENT_MODE,e.COMMENT("//","$",{contains:[t]}),e.COMMENT("/\\*","\\*/",{contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.COMMENT("__halt_compiler.+?;",!1,{endsWithParent:!0,keywords:"__halt_compiler"}),{className:"string",begin:/<<<['"]?\w+['"]?$/,end:/^\w+;?$/,contains:[e.BACKSLASH_ESCAPE,{className:"subst",variants:[{begin:/\$\w+/},{begin:/\{\$/,end:/\}/}]}]},t,{className:"keyword",begin:/\$this\b/},r,{begin:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{className:"function",beginKeywords:"fn function",end:/[;{]/,excludeEnd:!0,illegal:"[$%\\[]",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",begin:"\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0,keywords:i,contains:["self",r,e.C_BLOCK_COMMENT_MODE,a,n]}]},{className:"class",beginKeywords:"class interface",end:"{",excludeEnd:!0,illegal:/[:\(\$"]/,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"namespace",end:";",illegal:/[\.']/,contains:[e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"use",end:";",contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"=>"},a,n]}}}()); -hljs.registerLanguage("php-template",function(){"use strict";return function(n){return{name:"PHP template",subLanguage:"xml",contains:[{begin:/<\?(php|=)?/,end:/\?>/,subLanguage:"php",contains:[{begin:"/\\*",end:"\\*/",skip:!0},{begin:'b"',end:'"',skip:!0},{begin:"b'",end:"'",skip:!0},n.inherit(n.APOS_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0}),n.inherit(n.QUOTE_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0})]}]}}}()); -hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}()); -hljs.registerLanguage("properties",function(){"use strict";return function(e){var n="[ \\t\\f]*",t="("+n+"[:=]"+n+"|[ \\t\\f]+)",a="([^\\\\:= \\t\\f\\n]|\\\\.)+",s={end:t,relevance:0,starts:{className:"string",end:/$/,relevance:0,contains:[{begin:"\\\\\\n"}]}};return{name:".properties",case_insensitive:!0,illegal:/\S/,contains:[e.COMMENT("^\\s*[!#]","$"),{begin:"([^\\\\\\W:= \\t\\f\\n]|\\\\.)+"+t,returnBegin:!0,contains:[{className:"attr",begin:"([^\\\\\\W:= \\t\\f\\n]|\\\\.)+",endsParent:!0,relevance:0}],starts:s},{begin:a+t,returnBegin:!0,relevance:0,contains:[{className:"meta",begin:a,endsParent:!0,relevance:0}],starts:s},{className:"attr",relevance:0,begin:a+n+"$"}]}}}()); -hljs.registerLanguage("python",function(){"use strict";return function(e){var n={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10",built_in:"Ellipsis NotImplemented",literal:"False None True"},a={className:"meta",begin:/^(>>>|\.\.\.) /},i={className:"subst",begin:/\{/,end:/\}/,keywords:n,illegal:/#/},s={begin:/\{\{/,relevance:0},r={className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/(u|b)?r?'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(u|b)?r?"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(fr|rf|f)'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(fr|rf|f)"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(u|r|ur)'/,end:/'/,relevance:10},{begin:/(u|r|ur)"/,end:/"/,relevance:10},{begin:/(b|br)'/,end:/'/},{begin:/(b|br)"/,end:/"/},{begin:/(fr|rf|f)'/,end:/'/,contains:[e.BACKSLASH_ESCAPE,s,i]},{begin:/(fr|rf|f)"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,i]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},l={className:"number",relevance:0,variants:[{begin:e.BINARY_NUMBER_RE+"[lLjJ]?"},{begin:"\\b(0o[0-7]+)[lLjJ]?"},{begin:e.C_NUMBER_RE+"[lLjJ]?"}]},t={className:"params",variants:[{begin:/\(\s*\)/,skip:!0,className:null},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:["self",a,l,r,e.HASH_COMMENT_MODE]}]};return i.contains=[r,l,a],{name:"Python",aliases:["py","gyp","ipython"],keywords:n,illegal:/(<\/|->|\?)|=>/,contains:[a,l,{beginKeywords:"if",relevance:0},r,e.HASH_COMMENT_MODE,{variants:[{className:"function",beginKeywords:"def"},{className:"class",beginKeywords:"class"}],end:/:/,illegal:/[${=;\n,]/,contains:[e.UNDERSCORE_TITLE_MODE,t,{begin:/->/,endsWithParent:!0,keywords:"None"}]},{className:"meta",begin:/^[\t ]*@/,end:/$/},{begin:/\b(print|exec)\(/}]}}}()); -hljs.registerLanguage("python-repl",function(){"use strict";return function(n){return{aliases:["pycon"],contains:[{className:"meta",starts:{end:/ |$/,starts:{end:"$",subLanguage:"python"}},variants:[{begin:/^>>>(?=[ ]|$)/},{begin:/^\.\.\.(?=[ ]|$)/}]}]}}}()); -hljs.registerLanguage("ruby",function(){"use strict";return function(e){var n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",a={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},s={className:"doctag",begin:"@[A-Za-z]+"},i={begin:"#<",end:">"},r=[e.COMMENT("#","$",{contains:[s]}),e.COMMENT("^\\=begin","^\\=end",{contains:[s],relevance:10}),e.COMMENT("^__END__","\\n$")],c={className:"subst",begin:"#\\{",end:"}",keywords:a},t={className:"string",contains:[e.BACKSLASH_ESCAPE,c],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:"%[qQwWx]?\\(",end:"\\)"},{begin:"%[qQwWx]?\\[",end:"\\]"},{begin:"%[qQwWx]?{",end:"}"},{begin:"%[qQwWx]?<",end:">"},{begin:"%[qQwWx]?/",end:"/"},{begin:"%[qQwWx]?%",end:"%"},{begin:"%[qQwWx]?-",end:"-"},{begin:"%[qQwWx]?\\|",end:"\\|"},{begin:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{begin:/<<[-~]?'?(\w+)(?:.|\n)*?\n\s*\1\b/,returnBegin:!0,contains:[{begin:/<<[-~]?'?/},e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,contains:[e.BACKSLASH_ESCAPE,c]})]}]},b={className:"params",begin:"\\(",end:"\\)",endsParent:!0,keywords:a},d=[t,i,{className:"class",beginKeywords:"class module",end:"$|;",illegal:/=/,contains:[e.inherit(e.TITLE_MODE,{begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{begin:"<\\s*",contains:[{begin:"("+e.IDENT_RE+"::)?"+e.IDENT_RE}]}].concat(r)},{className:"function",beginKeywords:"def",end:"$|;",contains:[e.inherit(e.TITLE_MODE,{begin:n}),b].concat(r)},{begin:e.IDENT_RE+"::"},{className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"(\\!|\\?)?:",relevance:0},{className:"symbol",begin:":(?!\\s)",contains:[t,{begin:n}],relevance:0},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{className:"params",begin:/\|/,end:/\|/,keywords:a},{begin:"("+e.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[i,{className:"regexp",contains:[e.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{begin:"/",end:"/[a-z]*"},{begin:"%r{",end:"}[a-z]*"},{begin:"%r\\(",end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}]}].concat(r),relevance:0}].concat(r);c.contains=d,b.contains=d;var g=[{begin:/^\s*=>/,starts:{end:"$",contains:d}},{className:"meta",begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+>|(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>)",starts:{end:"$",contains:d}}];return{name:"Ruby",aliases:["rb","gemspec","podspec","thor","irb"],keywords:a,illegal:/\/\*/,contains:r.concat(g).concat(d)}}}()); -hljs.registerLanguage("rust",function(){"use strict";return function(e){var n="([ui](8|16|32|64|128|size)|f(32|64))?",t="drop i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize f32 f64 str char bool Box Option Result String Vec Copy Send Sized Sync Drop Fn FnMut FnOnce ToOwned Clone Debug PartialEq PartialOrd Eq Ord AsRef AsMut Into From Default Iterator Extend IntoIterator DoubleEndedIterator ExactSizeIterator SliceConcatExt ToString assert! assert_eq! bitflags! bytes! cfg! col! concat! concat_idents! debug_assert! debug_assert_eq! env! panic! file! format! format_args! include_bin! include_str! line! local_data_key! module_path! option_env! print! println! select! stringify! try! unimplemented! unreachable! vec! write! writeln! macro_rules! assert_ne! debug_assert_ne!";return{name:"Rust",aliases:["rs"],keywords:{$pattern:e.IDENT_RE+"!?",keyword:"abstract as async await become box break const continue crate do dyn else enum extern false final fn for if impl in let loop macro match mod move mut override priv pub ref return self Self static struct super trait true try type typeof unsafe unsized use virtual where while yield",literal:"true false Some None Ok Err",built_in:t},illegal:""}]}}}()); -hljs.registerLanguage("scss",function(){"use strict";return function(e){var t={className:"variable",begin:"(\\$[a-zA-Z-][a-zA-Z0-9_-]*)\\b"},i={className:"number",begin:"#[0-9A-Fa-f]+"};return e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_BLOCK_COMMENT_MODE,{name:"SCSS",case_insensitive:!0,illegal:"[=/|']",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"selector-id",begin:"\\#[A-Za-z0-9_-]+",relevance:0},{className:"selector-class",begin:"\\.[A-Za-z0-9_-]+",relevance:0},{className:"selector-attr",begin:"\\[",end:"\\]",illegal:"$"},{className:"selector-tag",begin:"\\b(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|command|datalist|dd|del|details|dfn|div|dl|dt|em|embed|fieldset|figcaption|figure|footer|form|frame|frameset|(h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|keygen|label|legend|li|link|map|mark|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|rp|rt|ruby|samp|script|section|select|small|span|strike|strong|style|sub|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)\\b",relevance:0},{className:"selector-pseudo",begin:":(visited|valid|root|right|required|read-write|read-only|out-range|optional|only-of-type|only-child|nth-of-type|nth-last-of-type|nth-last-child|nth-child|not|link|left|last-of-type|last-child|lang|invalid|indeterminate|in-range|hover|focus|first-of-type|first-line|first-letter|first-child|first|enabled|empty|disabled|default|checked|before|after|active)"},{className:"selector-pseudo",begin:"::(after|before|choices|first-letter|first-line|repeat-index|repeat-item|selection|value)"},t,{className:"attribute",begin:"\\b(src|z-index|word-wrap|word-spacing|word-break|width|widows|white-space|visibility|vertical-align|unicode-bidi|transition-timing-function|transition-property|transition-duration|transition-delay|transition|transform-style|transform-origin|transform|top|text-underline-position|text-transform|text-shadow|text-rendering|text-overflow|text-indent|text-decoration-style|text-decoration-line|text-decoration-color|text-decoration|text-align-last|text-align|tab-size|table-layout|right|resize|quotes|position|pointer-events|perspective-origin|perspective|page-break-inside|page-break-before|page-break-after|padding-top|padding-right|padding-left|padding-bottom|padding|overflow-y|overflow-x|overflow-wrap|overflow|outline-width|outline-style|outline-offset|outline-color|outline|orphans|order|opacity|object-position|object-fit|normal|none|nav-up|nav-right|nav-left|nav-index|nav-down|min-width|min-height|max-width|max-height|mask|marks|margin-top|margin-right|margin-left|margin-bottom|margin|list-style-type|list-style-position|list-style-image|list-style|line-height|letter-spacing|left|justify-content|initial|inherit|ime-mode|image-orientation|image-resolution|image-rendering|icon|hyphens|height|font-weight|font-variant-ligatures|font-variant|font-style|font-stretch|font-size-adjust|font-size|font-language-override|font-kerning|font-feature-settings|font-family|font|float|flex-wrap|flex-shrink|flex-grow|flex-flow|flex-direction|flex-basis|flex|filter|empty-cells|display|direction|cursor|counter-reset|counter-increment|content|column-width|column-span|column-rule-width|column-rule-style|column-rule-color|column-rule|column-gap|column-fill|column-count|columns|color|clip-path|clip|clear|caption-side|break-inside|break-before|break-after|box-sizing|box-shadow|box-decoration-break|bottom|border-width|border-top-width|border-top-style|border-top-right-radius|border-top-left-radius|border-top-color|border-top|border-style|border-spacing|border-right-width|border-right-style|border-right-color|border-right|border-radius|border-left-width|border-left-style|border-left-color|border-left|border-image-width|border-image-source|border-image-slice|border-image-repeat|border-image-outset|border-image|border-color|border-collapse|border-bottom-width|border-bottom-style|border-bottom-right-radius|border-bottom-left-radius|border-bottom-color|border-bottom|border|background-size|background-repeat|background-position|background-origin|background-image|background-color|background-clip|background-attachment|background-blend-mode|background|backface-visibility|auto|animation-timing-function|animation-play-state|animation-name|animation-iteration-count|animation-fill-mode|animation-duration|animation-direction|animation-delay|animation|align-self|align-items|align-content)\\b",illegal:"[^\\s]"},{begin:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b"},{begin:":",end:";",contains:[t,i,e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,{className:"meta",begin:"!important"}]},{begin:"@(page|font-face)",lexemes:"@[a-z-]+",keywords:"@page @font-face"},{begin:"@",end:"[{;]",returnBegin:!0,keywords:"and or not only",contains:[{begin:"@[a-z-]+",className:"keyword"},t,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,i,e.CSS_NUMBER_MODE]}]}}}()); -hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}()); -hljs.registerLanguage("sql",function(){"use strict";return function(e){var t=e.COMMENT("--","$");return{name:"SQL",case_insensitive:!0,illegal:/[<>{}*]/,contains:[{beginKeywords:"begin end start commit rollback savepoint lock alter create drop rename call delete do handler insert load replace select truncate update set show pragma grant merge describe use explain help declare prepare execute deallocate release unlock purge reset change stop analyze cache flush optimize repair kill install uninstall checksum restore check backup revoke comment values with",end:/;/,endsWithParent:!0,keywords:{$pattern:/[\w\.]+/,keyword:"as abort abs absolute acc acce accep accept access accessed accessible account acos action activate add addtime admin administer advanced advise aes_decrypt aes_encrypt after agent aggregate ali alia alias all allocate allow alter always analyze ancillary and anti any anydata anydataset anyschema anytype apply archive archived archivelog are as asc ascii asin assembly assertion associate asynchronous at atan atn2 attr attri attrib attribu attribut attribute attributes audit authenticated authentication authid authors auto autoallocate autodblink autoextend automatic availability avg backup badfile basicfile before begin beginning benchmark between bfile bfile_base big bigfile bin binary_double binary_float binlog bit_and bit_count bit_length bit_or bit_xor bitmap blob_base block blocksize body both bound bucket buffer_cache buffer_pool build bulk by byte byteordermark bytes cache caching call calling cancel capacity cascade cascaded case cast catalog category ceil ceiling chain change changed char_base char_length character_length characters characterset charindex charset charsetform charsetid check checksum checksum_agg child choose chr chunk class cleanup clear client clob clob_base clone close cluster_id cluster_probability cluster_set clustering coalesce coercibility col collate collation collect colu colum column column_value columns columns_updated comment commit compact compatibility compiled complete composite_limit compound compress compute concat concat_ws concurrent confirm conn connec connect connect_by_iscycle connect_by_isleaf connect_by_root connect_time connection consider consistent constant constraint constraints constructor container content contents context contributors controlfile conv convert convert_tz corr corr_k corr_s corresponding corruption cos cost count count_big counted covar_pop covar_samp cpu_per_call cpu_per_session crc32 create creation critical cross cube cume_dist curdate current current_date current_time current_timestamp current_user cursor curtime customdatum cycle data database databases datafile datafiles datalength date_add date_cache date_format date_sub dateadd datediff datefromparts datename datepart datetime2fromparts day day_to_second dayname dayofmonth dayofweek dayofyear days db_role_change dbtimezone ddl deallocate declare decode decompose decrement decrypt deduplicate def defa defau defaul default defaults deferred defi defin define degrees delayed delegate delete delete_all delimited demand dense_rank depth dequeue des_decrypt des_encrypt des_key_file desc descr descri describ describe descriptor deterministic diagnostics difference dimension direct_load directory disable disable_all disallow disassociate discardfile disconnect diskgroup distinct distinctrow distribute distributed div do document domain dotnet double downgrade drop dumpfile duplicate duration each edition editionable editions element ellipsis else elsif elt empty enable enable_all enclosed encode encoding encrypt end end-exec endian enforced engine engines enqueue enterprise entityescaping eomonth error errors escaped evalname evaluate event eventdata events except exception exceptions exchange exclude excluding execu execut execute exempt exists exit exp expire explain explode export export_set extended extent external external_1 external_2 externally extract failed failed_login_attempts failover failure far fast feature_set feature_value fetch field fields file file_name_convert filesystem_like_logging final finish first first_value fixed flash_cache flashback floor flush following follows for forall force foreign form forma format found found_rows freelist freelists freepools fresh from from_base64 from_days ftp full function general generated get get_format get_lock getdate getutcdate global global_name globally go goto grant grants greatest group group_concat group_id grouping grouping_id groups gtid_subtract guarantee guard handler hash hashkeys having hea head headi headin heading heap help hex hierarchy high high_priority hosts hour hours http id ident_current ident_incr ident_seed identified identity idle_time if ifnull ignore iif ilike ilm immediate import in include including increment index indexes indexing indextype indicator indices inet6_aton inet6_ntoa inet_aton inet_ntoa infile initial initialized initially initrans inmemory inner innodb input insert install instance instantiable instr interface interleaved intersect into invalidate invisible is is_free_lock is_ipv4 is_ipv4_compat is_not is_not_null is_used_lock isdate isnull isolation iterate java join json json_exists keep keep_duplicates key keys kill language large last last_day last_insert_id last_value lateral lax lcase lead leading least leaves left len lenght length less level levels library like like2 like4 likec limit lines link list listagg little ln load load_file lob lobs local localtime localtimestamp locate locator lock locked log log10 log2 logfile logfiles logging logical logical_reads_per_call logoff logon logs long loop low low_priority lower lpad lrtrim ltrim main make_set makedate maketime managed management manual map mapping mask master master_pos_wait match matched materialized max maxextents maximize maxinstances maxlen maxlogfiles maxloghistory maxlogmembers maxsize maxtrans md5 measures median medium member memcompress memory merge microsecond mid migration min minextents minimum mining minus minute minutes minvalue missing mod mode model modification modify module monitoring month months mount move movement multiset mutex name name_const names nan national native natural nav nchar nclob nested never new newline next nextval no no_write_to_binlog noarchivelog noaudit nobadfile nocheck nocompress nocopy nocycle nodelay nodiscardfile noentityescaping noguarantee nokeep nologfile nomapping nomaxvalue nominimize nominvalue nomonitoring none noneditionable nonschema noorder nopr nopro noprom nopromp noprompt norely noresetlogs noreverse normal norowdependencies noschemacheck noswitch not nothing notice notnull notrim novalidate now nowait nth_value nullif nulls num numb numbe nvarchar nvarchar2 object ocicoll ocidate ocidatetime ociduration ociinterval ociloblocator ocinumber ociref ocirefcursor ocirowid ocistring ocitype oct octet_length of off offline offset oid oidindex old on online only opaque open operations operator optimal optimize option optionally or oracle oracle_date oradata ord ordaudio orddicom orddoc order ordimage ordinality ordvideo organization orlany orlvary out outer outfile outline output over overflow overriding package pad parallel parallel_enable parameters parent parse partial partition partitions pascal passing password password_grace_time password_lock_time password_reuse_max password_reuse_time password_verify_function patch path patindex pctincrease pctthreshold pctused pctversion percent percent_rank percentile_cont percentile_disc performance period period_add period_diff permanent physical pi pipe pipelined pivot pluggable plugin policy position post_transaction pow power pragma prebuilt precedes preceding precision prediction prediction_cost prediction_details prediction_probability prediction_set prepare present preserve prior priority private private_sga privileges procedural procedure procedure_analyze processlist profiles project prompt protection public publishingservername purge quarter query quick quiesce quota quotename radians raise rand range rank raw read reads readsize rebuild record records recover recovery recursive recycle redo reduced ref reference referenced references referencing refresh regexp_like register regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy reject rekey relational relative relaylog release release_lock relies_on relocate rely rem remainder rename repair repeat replace replicate replication required reset resetlogs resize resource respect restore restricted result result_cache resumable resume retention return returning returns reuse reverse revoke right rlike role roles rollback rolling rollup round row row_count rowdependencies rowid rownum rows rtrim rules safe salt sample save savepoint sb1 sb2 sb4 scan schema schemacheck scn scope scroll sdo_georaster sdo_topo_geometry search sec_to_time second seconds section securefile security seed segment select self semi sequence sequential serializable server servererror session session_user sessions_per_user set sets settings sha sha1 sha2 share shared shared_pool short show shrink shutdown si_averagecolor si_colorhistogram si_featurelist si_positionalcolor si_stillimage si_texture siblings sid sign sin size size_t sizes skip slave sleep smalldatetimefromparts smallfile snapshot some soname sort soundex source space sparse spfile split sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_small_result sql_variant_property sqlcode sqldata sqlerror sqlname sqlstate sqrt square standalone standby start starting startup statement static statistics stats_binomial_test stats_crosstab stats_ks_test stats_mode stats_mw_test stats_one_way_anova stats_t_test_ stats_t_test_indep stats_t_test_one stats_t_test_paired stats_wsr_test status std stddev stddev_pop stddev_samp stdev stop storage store stored str str_to_date straight_join strcmp strict string struct stuff style subdate subpartition subpartitions substitutable substr substring subtime subtring_index subtype success sum suspend switch switchoffset switchover sync synchronous synonym sys sys_xmlagg sysasm sysaux sysdate sysdatetimeoffset sysdba sysoper system system_user sysutcdatetime table tables tablespace tablesample tan tdo template temporary terminated tertiary_weights test than then thread through tier ties time time_format time_zone timediff timefromparts timeout timestamp timestampadd timestampdiff timezone_abbr timezone_minute timezone_region to to_base64 to_date to_days to_seconds todatetimeoffset trace tracking transaction transactional translate translation treat trigger trigger_nestlevel triggers trim truncate try_cast try_convert try_parse type ub1 ub2 ub4 ucase unarchived unbounded uncompress under undo unhex unicode uniform uninstall union unique unix_timestamp unknown unlimited unlock unnest unpivot unrecoverable unsafe unsigned until untrusted unusable unused update updated upgrade upped upper upsert url urowid usable usage use use_stored_outlines user user_data user_resources users using utc_date utc_timestamp uuid uuid_short validate validate_password_strength validation valist value values var var_samp varcharc vari varia variab variabl variable variables variance varp varraw varrawc varray verify version versions view virtual visible void wait wallet warning warnings week weekday weekofyear wellformed when whene whenev wheneve whenever where while whitespace window with within without work wrapped xdb xml xmlagg xmlattributes xmlcast xmlcolattval xmlelement xmlexists xmlforest xmlindex xmlnamespaces xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltype xor year year_to_month years yearweek",literal:"true false null unknown",built_in:"array bigint binary bit blob bool boolean char character date dec decimal float int int8 integer interval number numeric real record serial serial8 smallint text time timestamp tinyint varchar varchar2 varying void"},contains:[{className:"string",begin:"'",end:"'",contains:[{begin:"''"}]},{className:"string",begin:'"',end:'"',contains:[{begin:'""'}]},{className:"string",begin:"`",end:"`"},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,t,e.HASH_COMMENT_MODE]},e.C_BLOCK_COMMENT_MODE,t,e.HASH_COMMENT_MODE]}}}()); -hljs.registerLanguage("swift",function(){"use strict";return function(e){var i={keyword:"#available #colorLiteral #column #else #elseif #endif #file #fileLiteral #function #if #imageLiteral #line #selector #sourceLocation _ __COLUMN__ __FILE__ __FUNCTION__ __LINE__ Any as as! as? associatedtype associativity break case catch class continue convenience default defer deinit didSet do dynamic dynamicType else enum extension fallthrough false fileprivate final for func get guard if import in indirect infix init inout internal is lazy left let mutating nil none nonmutating open operator optional override postfix precedence prefix private protocol Protocol public repeat required rethrows return right self Self set static struct subscript super switch throw throws true try try! try? Type typealias unowned var weak where while willSet",literal:"true false nil",built_in:"abs advance alignof alignofValue anyGenerator assert assertionFailure bridgeFromObjectiveC bridgeFromObjectiveCUnconditional bridgeToObjectiveC bridgeToObjectiveCUnconditional c compactMap contains count countElements countLeadingZeros debugPrint debugPrintln distance dropFirst dropLast dump encodeBitsAsWords enumerate equal fatalError filter find getBridgedObjectiveCType getVaList indices insertionSort isBridgedToObjectiveC isBridgedVerbatimToObjectiveC isUniquelyReferenced isUniquelyReferencedNonObjC join lazy lexicographicalCompare map max maxElement min minElement numericCast overlaps partition posix precondition preconditionFailure print println quickSort readLine reduce reflect reinterpretCast reverse roundUpToAlignment sizeof sizeofValue sort split startsWith stride strideof strideofValue swap toString transcode underestimateCount unsafeAddressOf unsafeBitCast unsafeDowncast unsafeUnwrap unsafeReflect withExtendedLifetime withObjectAtPlusZero withUnsafePointer withUnsafePointerToObject withUnsafeMutablePointer withUnsafeMutablePointers withUnsafePointer withUnsafePointers withVaList zip"},n=e.COMMENT("/\\*","\\*/",{contains:["self"]}),t={className:"subst",begin:/\\\(/,end:"\\)",keywords:i,contains:[]},a={className:"string",contains:[e.BACKSLASH_ESCAPE,t],variants:[{begin:/"""/,end:/"""/},{begin:/"/,end:/"/}]},r={className:"number",begin:"\\b([\\d_]+(\\.[\\deE_]+)?|0x[a-fA-F0-9_]+(\\.[a-fA-F0-9p_]+)?|0b[01_]+|0o[0-7_]+)\\b",relevance:0};return t.contains=[r],{name:"Swift",keywords:i,contains:[a,e.C_LINE_COMMENT_MODE,n,{className:"type",begin:"\\b[A-Z][\\wÀ-ʸ']*[!?]"},{className:"type",begin:"\\b[A-Z][\\wÀ-ʸ']*",relevance:0},r,{className:"function",beginKeywords:"func",end:"{",excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][0-9A-Za-z$_]*/}),{begin://},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:i,contains:["self",r,a,e.C_BLOCK_COMMENT_MODE,{begin:":"}],illegal:/["']/}],illegal:/\[|%/},{className:"class",beginKeywords:"struct protocol class extension enum",keywords:i,end:"\\{",excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/})]},{className:"meta",begin:"(@discardableResult|@warn_unused_result|@exported|@lazy|@noescape|@NSCopying|@NSManaged|@objc|@objcMembers|@convention|@required|@noreturn|@IBAction|@IBDesignable|@IBInspectable|@IBOutlet|@infix|@prefix|@postfix|@autoclosure|@testable|@available|@nonobjc|@NSApplicationMain|@UIApplicationMain|@dynamicMemberLookup|@propertyWrapper)\\b"},{beginKeywords:"import",end:/$/,contains:[e.C_LINE_COMMENT_MODE,n]}]}}}()); -hljs.registerLanguage("typescript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);return function(r){var t={$pattern:"[A-Za-z$_][0-9A-Za-z$_]*",keyword:e.concat(["type","namespace","typedef","interface","public","private","protected","implements","declare","abstract","readonly"]).join(" "),literal:n.join(" "),built_in:a.concat(["any","void","number","boolean","string","object","never","enum"]).join(" ")},s={className:"meta",begin:"@[A-Za-z$_][0-9A-Za-z$_]*"},i={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:r.C_NUMBER_RE+"n?"}],relevance:0},o={className:"subst",begin:"\\$\\{",end:"\\}",keywords:t,contains:[]},c={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[r.BACKSLASH_ESCAPE,o],subLanguage:"xml"}},l={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[r.BACKSLASH_ESCAPE,o],subLanguage:"css"}},E={className:"string",begin:"`",end:"`",contains:[r.BACKSLASH_ESCAPE,o]};o.contains=[r.APOS_STRING_MODE,r.QUOTE_STRING_MODE,c,l,E,i,r.REGEXP_MODE];var d={begin:"\\(",end:/\)/,keywords:t,contains:["self",r.QUOTE_STRING_MODE,r.APOS_STRING_MODE,r.NUMBER_MODE]},u={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,contains:[r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,s,d]};return{name:"TypeScript",aliases:["ts"],keywords:t,contains:[r.SHEBANG(),{className:"meta",begin:/^\s*['"]use strict['"]/},r.APOS_STRING_MODE,r.QUOTE_STRING_MODE,c,l,E,r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,i,{begin:"("+r.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,r.REGEXP_MODE,{className:"function",begin:"(\\([^(]*(\\([^(]*(\\([^(]*\\))?\\))?\\)|"+r.UNDERSCORE_IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:r.UNDERSCORE_IDENT_RE},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,contains:d.contains}]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/[\{;]/,excludeEnd:!0,keywords:t,contains:["self",r.inherit(r.TITLE_MODE,{begin:"[A-Za-z$_][0-9A-Za-z$_]*"}),u],illegal:/%/,relevance:0},{beginKeywords:"constructor",end:/[\{;]/,excludeEnd:!0,contains:["self",u]},{begin:/module\./,keywords:{built_in:"module"},relevance:0},{beginKeywords:"module",end:/\{/,excludeEnd:!0},{beginKeywords:"interface",end:/\{/,excludeEnd:!0,keywords:"interface extends"},{begin:/\$[(.]/},{begin:"\\."+r.IDENT_RE,relevance:0},s,d]}}}()); -hljs.registerLanguage("yaml",function(){"use strict";return function(e){var n="true false yes no null",a="[\\w#;/?:@&=+$,.~*\\'()[\\]]+",s={className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable",variants:[{begin:"{{",end:"}}"},{begin:"%{",end:"}"}]}]},i=e.inherit(s,{variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),l={end:",",endsWithParent:!0,excludeEnd:!0,contains:[],keywords:n,relevance:0},t={begin:"{",end:"}",contains:[l],illegal:"\\n",relevance:0},g={begin:"\\[",end:"\\]",contains:[l],illegal:"\\n",relevance:0},b=[{className:"attr",variants:[{begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---s*$",relevance:10},{className:"string",begin:"[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*"},{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:"!\\w+!"+a},{className:"type",begin:"!<"+a+">"},{className:"type",begin:"!"+a},{className:"type",begin:"!!"+a},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta",begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"\\-(?=[ ]|$)",relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{className:"number",begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"},{className:"number",begin:e.C_NUMBER_RE+"\\b"},t,g,s],c=[...b];return c.pop(),c.push(i),l.contains=c,{name:"YAML",case_insensitive:!0,aliases:["yml","YAML"],contains:b}}}()); -hljs.registerLanguage("armasm",function(){"use strict";return function(s){const e={variants:[s.COMMENT("^[ \\t]*(?=#)","$",{relevance:0,excludeBegin:!0}),s.COMMENT("[;@]","$",{relevance:0}),s.C_LINE_COMMENT_MODE,s.C_BLOCK_COMMENT_MODE]};return{name:"ARM Assembly",case_insensitive:!0,aliases:["arm"],keywords:{$pattern:"\\.?"+s.IDENT_RE,meta:".2byte .4byte .align .ascii .asciz .balign .byte .code .data .else .end .endif .endm .endr .equ .err .exitm .extern .global .hword .if .ifdef .ifndef .include .irp .long .macro .rept .req .section .set .skip .space .text .word .arm .thumb .code16 .code32 .force_thumb .thumb_func .ltorg ALIAS ALIGN ARM AREA ASSERT ATTR CN CODE CODE16 CODE32 COMMON CP DATA DCB DCD DCDU DCDO DCFD DCFDU DCI DCQ DCQU DCW DCWU DN ELIF ELSE END ENDFUNC ENDIF ENDP ENTRY EQU EXPORT EXPORTAS EXTERN FIELD FILL FUNCTION GBLA GBLL GBLS GET GLOBAL IF IMPORT INCBIN INCLUDE INFO KEEP LCLA LCLL LCLS LTORG MACRO MAP MEND MEXIT NOFP OPT PRESERVE8 PROC QN READONLY RELOC REQUIRE REQUIRE8 RLIST FN ROUT SETA SETL SETS SN SPACE SUBT THUMB THUMBX TTL WHILE WEND ",built_in:"r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 pc lr sp ip sl sb fp a1 a2 a3 a4 v1 v2 v3 v4 v5 v6 v7 v8 f0 f1 f2 f3 f4 f5 f6 f7 p0 p1 p2 p3 p4 p5 p6 p7 p8 p9 p10 p11 p12 p13 p14 p15 c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 q0 q1 q2 q3 q4 q5 q6 q7 q8 q9 q10 q11 q12 q13 q14 q15 cpsr_c cpsr_x cpsr_s cpsr_f cpsr_cx cpsr_cxs cpsr_xs cpsr_xsf cpsr_sf cpsr_cxsf spsr_c spsr_x spsr_s spsr_f spsr_cx spsr_cxs spsr_xs spsr_xsf spsr_sf spsr_cxsf s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 s16 s17 s18 s19 s20 s21 s22 s23 s24 s25 s26 s27 s28 s29 s30 s31 d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15 d16 d17 d18 d19 d20 d21 d22 d23 d24 d25 d26 d27 d28 d29 d30 d31 {PC} {VAR} {TRUE} {FALSE} {OPT} {CONFIG} {ENDIAN} {CODESIZE} {CPU} {FPU} {ARCHITECTURE} {PCSTOREOFFSET} {ARMASM_VERSION} {INTER} {ROPI} {RWPI} {SWST} {NOSWST} . @"},contains:[{className:"keyword",begin:"\\b(adc|(qd?|sh?|u[qh]?)?add(8|16)?|usada?8|(q|sh?|u[qh]?)?(as|sa)x|and|adrl?|sbc|rs[bc]|asr|b[lx]?|blx|bxj|cbn?z|tb[bh]|bic|bfc|bfi|[su]bfx|bkpt|cdp2?|clz|clrex|cmp|cmn|cpsi[ed]|cps|setend|dbg|dmb|dsb|eor|isb|it[te]{0,3}|lsl|lsr|ror|rrx|ldm(([id][ab])|f[ds])?|ldr((s|ex)?[bhd])?|movt?|mvn|mra|mar|mul|[us]mull|smul[bwt][bt]|smu[as]d|smmul|smmla|mla|umlaal|smlal?([wbt][bt]|d)|mls|smlsl?[ds]|smc|svc|sev|mia([bt]{2}|ph)?|mrr?c2?|mcrr2?|mrs|msr|orr|orn|pkh(tb|bt)|rbit|rev(16|sh)?|sel|[su]sat(16)?|nop|pop|push|rfe([id][ab])?|stm([id][ab])?|str(ex)?[bhd]?|(qd?)?sub|(sh?|q|u[qh]?)?sub(8|16)|[su]xt(a?h|a?b(16)?)|srs([id][ab])?|swpb?|swi|smi|tst|teq|wfe|wfi|yield)(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al|hs|lo)?[sptrx]?(?=\\s)"},e,s.QUOTE_STRING_MODE,{className:"string",begin:"'",end:"[^\\\\]'",relevance:0},{className:"title",begin:"\\|",end:"\\|",illegal:"\\n",relevance:0},{className:"number",variants:[{begin:"[#$=]?0x[0-9a-f]+"},{begin:"[#$=]?0b[01]+"},{begin:"[#$=]\\d+"},{begin:"\\b\\d+"}],relevance:0},{className:"symbol",variants:[{begin:"^[ \\t]*[a-z_\\.\\$][a-z0-9_\\.\\$]+:"},{begin:"^[a-z_\\.\\$][a-z0-9_\\.\\$]+"},{begin:"[=#]\\w+"}],relevance:0}]}}}()); -hljs.registerLanguage("d",function(){"use strict";return function(e){var a={$pattern:e.UNDERSCORE_IDENT_RE,keyword:"abstract alias align asm assert auto body break byte case cast catch class const continue debug default delete deprecated do else enum export extern final finally for foreach foreach_reverse|10 goto if immutable import in inout int interface invariant is lazy macro mixin module new nothrow out override package pragma private protected public pure ref return scope shared static struct super switch synchronized template this throw try typedef typeid typeof union unittest version void volatile while with __FILE__ __LINE__ __gshared|10 __thread __traits __DATE__ __EOF__ __TIME__ __TIMESTAMP__ __VENDOR__ __VERSION__",built_in:"bool cdouble cent cfloat char creal dchar delegate double dstring float function idouble ifloat ireal long real short string ubyte ucent uint ulong ushort wchar wstring",literal:"false null true"},d="((0|[1-9][\\d_]*)|0[bB][01_]+|0[xX]([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*))",n="\\\\(['\"\\?\\\\abfnrtv]|u[\\dA-Fa-f]{4}|[0-7]{1,3}|x[\\dA-Fa-f]{2}|U[\\dA-Fa-f]{8})|&[a-zA-Z\\d]{2,};",t={className:"number",begin:"\\b"+d+"(L|u|U|Lu|LU|uL|UL)?",relevance:0},_={className:"number",begin:"\\b(((0[xX](([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*)\\.([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*)|\\.?([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*))[pP][+-]?(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d))|((0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)(\\.\\d*|([eE][+-]?(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)))|\\d+\\.(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)|\\.(0|[1-9][\\d_]*)([eE][+-]?(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d))?))([fF]|L|i|[fF]i|Li)?|"+d+"(i|[fF]i|Li))",relevance:0},r={className:"string",begin:"'("+n+"|.)",end:"'",illegal:"."},i={className:"string",begin:'"',contains:[{begin:n,relevance:0}],end:'"[cwd]?'},s=e.COMMENT("\\/\\+","\\+\\/",{contains:["self"],relevance:10});return{name:"D",keywords:a,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s,{className:"string",begin:'x"[\\da-fA-F\\s\\n\\r]*"[cwd]?',relevance:10},i,{className:"string",begin:'[rq]"',end:'"[cwd]?',relevance:5},{className:"string",begin:"`",end:"`[cwd]?"},{className:"string",begin:'q"\\{',end:'\\}"'},_,t,r,{className:"meta",begin:"^#!",end:"$",relevance:5},{className:"meta",begin:"#(line)",end:"$",relevance:5},{className:"keyword",begin:"@[a-zA-Z_][a-zA-Z_\\d]*"}]}}}()); -hljs.registerLanguage("handlebars",function(){"use strict";function e(...e){return e.map(e=>(function(e){return e?"string"==typeof e?e:e.source:null})(e)).join("")}return function(n){const a={"builtin-name":"action bindattr collection component concat debugger each each-in get hash if in input link-to loc log lookup mut outlet partial query-params render template textarea unbound unless view with yield"},t=/\[.*?\]/,s=/[^\s!"#%&'()*+,.\/;<=>@\[\\\]^`{|}~]+/,i=e("(",/'.*?'/,"|",/".*?"/,"|",t,"|",s,"|",/\.|\//,")+"),r=e("(",t,"|",s,")(?==)"),l={begin:i,lexemes:/[\w.\/]+/},c=n.inherit(l,{keywords:{literal:"true false undefined null"}}),o={begin:/\(/,end:/\)/},m={className:"attr",begin:r,relevance:0,starts:{begin:/=/,end:/=/,starts:{contains:[n.NUMBER_MODE,n.QUOTE_STRING_MODE,n.APOS_STRING_MODE,c,o]}}},d={contains:[n.NUMBER_MODE,n.QUOTE_STRING_MODE,n.APOS_STRING_MODE,{begin:/as\s+\|/,keywords:{keyword:"as"},end:/\|/,contains:[{begin:/\w+/}]},m,c,o],returnEnd:!0},g=n.inherit(l,{className:"name",keywords:a,starts:n.inherit(d,{end:/\)/})});o.contains=[g];const u=n.inherit(l,{keywords:a,className:"name",starts:n.inherit(d,{end:/}}/})}),b=n.inherit(l,{keywords:a,className:"name"}),h=n.inherit(l,{className:"name",keywords:a,starts:n.inherit(d,{end:/}}/})});return{name:"Handlebars",aliases:["hbs","html.hbs","html.handlebars","htmlbars"],case_insensitive:!0,subLanguage:"xml",contains:[{begin:/\\\{\{/,skip:!0},{begin:/\\\\(?=\{\{)/,skip:!0},n.COMMENT(/\{\{!--/,/--\}\}/),n.COMMENT(/\{\{!/,/\}\}/),{className:"template-tag",begin:/\{\{\{\{(?!\/)/,end:/\}\}\}\}/,contains:[u],starts:{end:/\{\{\{\{\//,returnEnd:!0,subLanguage:"xml"}},{className:"template-tag",begin:/\{\{\{\{\//,end:/\}\}\}\}/,contains:[b]},{className:"template-tag",begin:/\{\{#/,end:/\}\}/,contains:[u]},{className:"template-tag",begin:/\{\{(?=else\}\})/,end:/\}\}/,keywords:"else"},{className:"template-tag",begin:/\{\{\//,end:/\}\}/,contains:[b]},{className:"template-variable",begin:/\{\{\{/,end:/\}\}\}/,contains:[h]},{className:"template-variable",begin:/\{\{/,end:/\}\}/,contains:[h]}]}}}()); -hljs.registerLanguage("haskell",function(){"use strict";return function(e){var n={variants:[e.COMMENT("--","$"),e.COMMENT("{-","-}",{contains:["self"]})]},i={className:"meta",begin:"{-#",end:"#-}"},a={className:"meta",begin:"^#",end:"$"},s={className:"type",begin:"\\b[A-Z][\\w']*",relevance:0},l={begin:"\\(",end:"\\)",illegal:'"',contains:[i,a,{className:"type",begin:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},e.inherit(e.TITLE_MODE,{begin:"[_a-z][\\w']*"}),n]};return{name:"Haskell",aliases:["hs"],keywords:"let in if then else case of where do module import hiding qualified type data newtype deriving class instance as default infix infixl infixr foreign export ccall stdcall cplusplus jvm dotnet safe unsafe family forall mdo proc rec",contains:[{beginKeywords:"module",end:"where",keywords:"module where",contains:[l,n],illegal:"\\W\\.|;"},{begin:"\\bimport\\b",end:"$",keywords:"import qualified as hiding",contains:[l,n],illegal:"\\W\\.|;"},{className:"class",begin:"^(\\s*)?(class|instance)\\b",end:"where",keywords:"class family instance where",contains:[s,l,n]},{className:"class",begin:"\\b(data|(new)?type)\\b",end:"$",keywords:"data family type newtype deriving",contains:[i,s,l,{begin:"{",end:"}",contains:l.contains},n]},{beginKeywords:"default",end:"$",contains:[s,l,n]},{beginKeywords:"infix infixl infixr",end:"$",contains:[e.C_NUMBER_MODE,n]},{begin:"\\bforeign\\b",end:"$",keywords:"foreign import export ccall stdcall cplusplus jvm dotnet safe unsafe",contains:[s,e.QUOTE_STRING_MODE,n]},{className:"meta",begin:"#!\\/usr\\/bin\\/env runhaskell",end:"$"},i,a,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,s,e.inherit(e.TITLE_MODE,{begin:"^[_a-z][\\w']*"}),n,{begin:"->|<-"}]}}}()); -hljs.registerLanguage("julia",function(){"use strict";return function(e){var r="[A-Za-z_\\u00A1-\\uFFFF][A-Za-z_0-9\\u00A1-\\uFFFF]*",t={$pattern:r,keyword:"in isa where baremodule begin break catch ccall const continue do else elseif end export false finally for function global if import importall let local macro module quote return true try using while type immutable abstract bitstype typealias ",literal:"true false ARGS C_NULL DevNull ENDIAN_BOM ENV I Inf Inf16 Inf32 Inf64 InsertionSort JULIA_HOME LOAD_PATH MergeSort NaN NaN16 NaN32 NaN64 PROGRAM_FILE QuickSort RoundDown RoundFromZero RoundNearest RoundNearestTiesAway RoundNearestTiesUp RoundToZero RoundUp STDERR STDIN STDOUT VERSION catalan e|0 eu|0 eulergamma golden im nothing pi γ π φ ",built_in:"ANY AbstractArray AbstractChannel AbstractFloat AbstractMatrix AbstractRNG AbstractSerializer AbstractSet AbstractSparseArray AbstractSparseMatrix AbstractSparseVector AbstractString AbstractUnitRange AbstractVecOrMat AbstractVector Any ArgumentError Array AssertionError Associative Base64DecodePipe Base64EncodePipe Bidiagonal BigFloat BigInt BitArray BitMatrix BitVector Bool BoundsError BufferStream CachingPool CapturedException CartesianIndex CartesianRange Cchar Cdouble Cfloat Channel Char Cint Cintmax_t Clong Clonglong ClusterManager Cmd CodeInfo Colon Complex Complex128 Complex32 Complex64 CompositeException Condition ConjArray ConjMatrix ConjVector Cptrdiff_t Cshort Csize_t Cssize_t Cstring Cuchar Cuint Cuintmax_t Culong Culonglong Cushort Cwchar_t Cwstring DataType Date DateFormat DateTime DenseArray DenseMatrix DenseVecOrMat DenseVector Diagonal Dict DimensionMismatch Dims DirectIndexString Display DivideError DomainError EOFError EachLine Enum Enumerate ErrorException Exception ExponentialBackOff Expr Factorization FileMonitor Float16 Float32 Float64 Function Future GlobalRef GotoNode HTML Hermitian IO IOBuffer IOContext IOStream IPAddr IPv4 IPv6 IndexCartesian IndexLinear IndexStyle InexactError InitError Int Int128 Int16 Int32 Int64 Int8 IntSet Integer InterruptException InvalidStateException Irrational KeyError LabelNode LinSpace LineNumberNode LoadError LowerTriangular MIME Matrix MersenneTwister Method MethodError MethodTable Module NTuple NewvarNode NullException Nullable Number ObjectIdDict OrdinalRange OutOfMemoryError OverflowError Pair ParseError PartialQuickSort PermutedDimsArray Pipe PollingFileWatcher ProcessExitedException Ptr QuoteNode RandomDevice Range RangeIndex Rational RawFD ReadOnlyMemoryError Real ReentrantLock Ref Regex RegexMatch RemoteChannel RemoteException RevString RoundingMode RowVector SSAValue SegmentationFault SerializationState Set SharedArray SharedMatrix SharedVector Signed SimpleVector Slot SlotNumber SparseMatrixCSC SparseVector StackFrame StackOverflowError StackTrace StepRange StepRangeLen StridedArray StridedMatrix StridedVecOrMat StridedVector String SubArray SubString SymTridiagonal Symbol Symmetric SystemError TCPSocket Task Text TextDisplay Timer Tridiagonal Tuple Type TypeError TypeMapEntry TypeMapLevel TypeName TypeVar TypedSlot UDPSocket UInt UInt128 UInt16 UInt32 UInt64 UInt8 UndefRefError UndefVarError UnicodeError UniformScaling Union UnionAll UnitRange Unsigned UpperTriangular Val Vararg VecElement VecOrMat Vector VersionNumber Void WeakKeyDict WeakRef WorkerConfig WorkerPool "},a={keywords:t,illegal:/<\//},n={className:"subst",begin:/\$\(/,end:/\)/,keywords:t},o={className:"variable",begin:"\\$"+r},i={className:"string",contains:[e.BACKSLASH_ESCAPE,n,o],variants:[{begin:/\w*"""/,end:/"""\w*/,relevance:10},{begin:/\w*"/,end:/"\w*/}]},l={className:"string",contains:[e.BACKSLASH_ESCAPE,n,o],begin:"`",end:"`"},s={className:"meta",begin:"@"+r};return a.name="Julia",a.contains=[{className:"number",begin:/(\b0x[\d_]*(\.[\d_]*)?|0x\.\d[\d_]*)p[-+]?\d+|\b0[box][a-fA-F0-9][a-fA-F0-9_]*|(\b\d[\d_]*(\.[\d_]*)?|\.\d[\d_]*)([eEfF][-+]?\d+)?/,relevance:0},{className:"string",begin:/'(.|\\[xXuU][a-zA-Z0-9]+)'/},i,l,s,{className:"comment",variants:[{begin:"#=",end:"=#",relevance:10},{begin:"#",end:"$"}]},e.HASH_COMMENT_MODE,{className:"keyword",begin:"\\b(((abstract|primitive)\\s+)type|(mutable\\s+)?struct)\\b"},{begin:/<:/}],n.contains=a.contains,a}}()); -hljs.registerLanguage("nim",function(){"use strict";return function(e){return{name:"Nim",aliases:["nim"],keywords:{keyword:"addr and as asm bind block break case cast const continue converter discard distinct div do elif else end enum except export finally for from func generic if import in include interface is isnot iterator let macro method mixin mod nil not notin object of or out proc ptr raise ref return shl shr static template try tuple type using var when while with without xor yield",literal:"shared guarded stdin stdout stderr result true false",built_in:"int int8 int16 int32 int64 uint uint8 uint16 uint32 uint64 float float32 float64 bool char string cstring pointer expr stmt void auto any range array openarray varargs seq set clong culong cchar cschar cshort cint csize clonglong cfloat cdouble clongdouble cuchar cushort cuint culonglong cstringarray semistatic"},contains:[{className:"meta",begin:/{\./,end:/\.}/,relevance:10},{className:"string",begin:/[a-zA-Z]\w*"/,end:/"/,contains:[{begin:/""/}]},{className:"string",begin:/([a-zA-Z]\w*)?"""/,end:/"""/},e.QUOTE_STRING_MODE,{className:"type",begin:/\b[A-Z]\w+\b/,relevance:0},{className:"number",relevance:0,variants:[{begin:/\b(0[xX][0-9a-fA-F][_0-9a-fA-F]*)('?[iIuU](8|16|32|64))?/},{begin:/\b(0o[0-7][_0-7]*)('?[iIuUfF](8|16|32|64))?/},{begin:/\b(0(b|B)[01][_01]*)('?[iIuUfF](8|16|32|64))?/},{begin:/\b(\d[_\d]*)('?[iIuUfF](8|16|32|64))?/}]},e.HASH_COMMENT_MODE]}}}()); -hljs.registerLanguage("r",function(){"use strict";return function(e){var n="([a-zA-Z]|\\.[a-zA-Z.])[a-zA-Z0-9._]*";return{name:"R",contains:[e.HASH_COMMENT_MODE,{begin:n,keywords:{$pattern:n,keyword:"function if in break next repeat else for return switch while try tryCatch stop warning require library attach detach source setMethod setGeneric setGroupGeneric setClass ...",literal:"NULL NA TRUE FALSE T F Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10"},relevance:0},{className:"number",begin:"0[xX][0-9a-fA-F]+[Li]?\\b",relevance:0},{className:"number",begin:"\\d+(?:[eE][+\\-]?\\d*)?L\\b",relevance:0},{className:"number",begin:"\\d+\\.(?!\\d)(?:i\\b)?",relevance:0},{className:"number",begin:"\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",relevance:0},{className:"number",begin:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",relevance:0},{begin:"`",end:"`",relevance:0},{className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:'"',end:'"'},{begin:"'",end:"'"}]}]}}}()); -hljs.registerLanguage("scala",function(){"use strict";return function(e){var n={className:"subst",variants:[{begin:"\\$[A-Za-z0-9_]+"},{begin:"\\${",end:"}"}]},a={className:"string",variants:[{begin:'"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:'"""',end:'"""',relevance:10},{begin:'[a-z]+"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE,n]},{className:"string",begin:'[a-z]+"""',end:'"""',contains:[n],relevance:10}]},s={className:"type",begin:"\\b[A-Z][A-Za-z0-9_]*",relevance:0},t={className:"title",begin:/[^0-9\n\t "'(),.`{}\[\]:;][^\n\t "'(),.`{}\[\]:;]+|[^0-9\n\t "'(),.`{}\[\]:;=]/,relevance:0},i={className:"class",beginKeywords:"class object trait type",end:/[:={\[\n;]/,excludeEnd:!0,contains:[{beginKeywords:"extends with",relevance:10},{begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0,relevance:0,contains:[s]},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,relevance:0,contains:[s]},t]},l={className:"function",beginKeywords:"def",end:/[:={\[(\n;]/,excludeEnd:!0,contains:[t]};return{name:"Scala",keywords:{literal:"true false null",keyword:"type yield lazy override def with val var sealed abstract private trait object if forSome for while throw finally protected extends import final return else break new catch super class case package default try this match continue throws implicit"},contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,{className:"symbol",begin:"'\\w[\\w\\d_]*(?!')"},s,l,i,e.C_NUMBER_MODE,{className:"meta",begin:"@[A-Za-z]+"}]}}}()); -hljs.registerLanguage("x86asm",function(){"use strict";return function(s){return{name:"Intel x86 Assembly",case_insensitive:!0,keywords:{$pattern:"[.%]?"+s.IDENT_RE,keyword:"lock rep repe repz repne repnz xaquire xrelease bnd nobnd aaa aad aam aas adc add and arpl bb0_reset bb1_reset bound bsf bsr bswap bt btc btr bts call cbw cdq cdqe clc cld cli clts cmc cmp cmpsb cmpsd cmpsq cmpsw cmpxchg cmpxchg486 cmpxchg8b cmpxchg16b cpuid cpu_read cpu_write cqo cwd cwde daa das dec div dmint emms enter equ f2xm1 fabs fadd faddp fbld fbstp fchs fclex fcmovb fcmovbe fcmove fcmovnb fcmovnbe fcmovne fcmovnu fcmovu fcom fcomi fcomip fcomp fcompp fcos fdecstp fdisi fdiv fdivp fdivr fdivrp femms feni ffree ffreep fiadd ficom ficomp fidiv fidivr fild fimul fincstp finit fist fistp fisttp fisub fisubr fld fld1 fldcw fldenv fldl2e fldl2t fldlg2 fldln2 fldpi fldz fmul fmulp fnclex fndisi fneni fninit fnop fnsave fnstcw fnstenv fnstsw fpatan fprem fprem1 fptan frndint frstor fsave fscale fsetpm fsin fsincos fsqrt fst fstcw fstenv fstp fstsw fsub fsubp fsubr fsubrp ftst fucom fucomi fucomip fucomp fucompp fxam fxch fxtract fyl2x fyl2xp1 hlt ibts icebp idiv imul in inc incbin insb insd insw int int01 int1 int03 int3 into invd invpcid invlpg invlpga iret iretd iretq iretw jcxz jecxz jrcxz jmp jmpe lahf lar lds lea leave les lfence lfs lgdt lgs lidt lldt lmsw loadall loadall286 lodsb lodsd lodsq lodsw loop loope loopne loopnz loopz lsl lss ltr mfence monitor mov movd movq movsb movsd movsq movsw movsx movsxd movzx mul mwait neg nop not or out outsb outsd outsw packssdw packsswb packuswb paddb paddd paddsb paddsiw paddsw paddusb paddusw paddw pand pandn pause paveb pavgusb pcmpeqb pcmpeqd pcmpeqw pcmpgtb pcmpgtd pcmpgtw pdistib pf2id pfacc pfadd pfcmpeq pfcmpge pfcmpgt pfmax pfmin pfmul pfrcp pfrcpit1 pfrcpit2 pfrsqit1 pfrsqrt pfsub pfsubr pi2fd pmachriw pmaddwd pmagw pmulhriw pmulhrwa pmulhrwc pmulhw pmullw pmvgezb pmvlzb pmvnzb pmvzb pop popa popad popaw popf popfd popfq popfw por prefetch prefetchw pslld psllq psllw psrad psraw psrld psrlq psrlw psubb psubd psubsb psubsiw psubsw psubusb psubusw psubw punpckhbw punpckhdq punpckhwd punpcklbw punpckldq punpcklwd push pusha pushad pushaw pushf pushfd pushfq pushfw pxor rcl rcr rdshr rdmsr rdpmc rdtsc rdtscp ret retf retn rol ror rdm rsdc rsldt rsm rsts sahf sal salc sar sbb scasb scasd scasq scasw sfence sgdt shl shld shr shrd sidt sldt skinit smi smint smintold smsw stc std sti stosb stosd stosq stosw str sub svdc svldt svts swapgs syscall sysenter sysexit sysret test ud0 ud1 ud2b ud2 ud2a umov verr verw fwait wbinvd wrshr wrmsr xadd xbts xchg xlatb xlat xor cmove cmovz cmovne cmovnz cmova cmovnbe cmovae cmovnb cmovb cmovnae cmovbe cmovna cmovg cmovnle cmovge cmovnl cmovl cmovnge cmovle cmovng cmovc cmovnc cmovo cmovno cmovs cmovns cmovp cmovpe cmovnp cmovpo je jz jne jnz ja jnbe jae jnb jb jnae jbe jna jg jnle jge jnl jl jnge jle jng jc jnc jo jno js jns jpo jnp jpe jp sete setz setne setnz seta setnbe setae setnb setnc setb setnae setcset setbe setna setg setnle setge setnl setl setnge setle setng sets setns seto setno setpe setp setpo setnp addps addss andnps andps cmpeqps cmpeqss cmpleps cmpless cmpltps cmpltss cmpneqps cmpneqss cmpnleps cmpnless cmpnltps cmpnltss cmpordps cmpordss cmpunordps cmpunordss cmpps cmpss comiss cvtpi2ps cvtps2pi cvtsi2ss cvtss2si cvttps2pi cvttss2si divps divss ldmxcsr maxps maxss minps minss movaps movhps movlhps movlps movhlps movmskps movntps movss movups mulps mulss orps rcpps rcpss rsqrtps rsqrtss shufps sqrtps sqrtss stmxcsr subps subss ucomiss unpckhps unpcklps xorps fxrstor fxrstor64 fxsave fxsave64 xgetbv xsetbv xsave xsave64 xsaveopt xsaveopt64 xrstor xrstor64 prefetchnta prefetcht0 prefetcht1 prefetcht2 maskmovq movntq pavgb pavgw pextrw pinsrw pmaxsw pmaxub pminsw pminub pmovmskb pmulhuw psadbw pshufw pf2iw pfnacc pfpnacc pi2fw pswapd maskmovdqu clflush movntdq movnti movntpd movdqa movdqu movdq2q movq2dq paddq pmuludq pshufd pshufhw pshuflw pslldq psrldq psubq punpckhqdq punpcklqdq addpd addsd andnpd andpd cmpeqpd cmpeqsd cmplepd cmplesd cmpltpd cmpltsd cmpneqpd cmpneqsd cmpnlepd cmpnlesd cmpnltpd cmpnltsd cmpordpd cmpordsd cmpunordpd cmpunordsd cmppd comisd cvtdq2pd cvtdq2ps cvtpd2dq cvtpd2pi cvtpd2ps cvtpi2pd cvtps2dq cvtps2pd cvtsd2si cvtsd2ss cvtsi2sd cvtss2sd cvttpd2pi cvttpd2dq cvttps2dq cvttsd2si divpd divsd maxpd maxsd minpd minsd movapd movhpd movlpd movmskpd movupd mulpd mulsd orpd shufpd sqrtpd sqrtsd subpd subsd ucomisd unpckhpd unpcklpd xorpd addsubpd addsubps haddpd haddps hsubpd hsubps lddqu movddup movshdup movsldup clgi stgi vmcall vmclear vmfunc vmlaunch vmload vmmcall vmptrld vmptrst vmread vmresume vmrun vmsave vmwrite vmxoff vmxon invept invvpid pabsb pabsw pabsd palignr phaddw phaddd phaddsw phsubw phsubd phsubsw pmaddubsw pmulhrsw pshufb psignb psignw psignd extrq insertq movntsd movntss lzcnt blendpd blendps blendvpd blendvps dppd dpps extractps insertps movntdqa mpsadbw packusdw pblendvb pblendw pcmpeqq pextrb pextrd pextrq phminposuw pinsrb pinsrd pinsrq pmaxsb pmaxsd pmaxud pmaxuw pminsb pminsd pminud pminuw pmovsxbw pmovsxbd pmovsxbq pmovsxwd pmovsxwq pmovsxdq pmovzxbw pmovzxbd pmovzxbq pmovzxwd pmovzxwq pmovzxdq pmuldq pmulld ptest roundpd roundps roundsd roundss crc32 pcmpestri pcmpestrm pcmpistri pcmpistrm pcmpgtq popcnt getsec pfrcpv pfrsqrtv movbe aesenc aesenclast aesdec aesdeclast aesimc aeskeygenassist vaesenc vaesenclast vaesdec vaesdeclast vaesimc vaeskeygenassist vaddpd vaddps vaddsd vaddss vaddsubpd vaddsubps vandpd vandps vandnpd vandnps vblendpd vblendps vblendvpd vblendvps vbroadcastss vbroadcastsd vbroadcastf128 vcmpeq_ospd vcmpeqpd vcmplt_ospd vcmpltpd vcmple_ospd vcmplepd vcmpunord_qpd vcmpunordpd vcmpneq_uqpd vcmpneqpd vcmpnlt_uspd vcmpnltpd vcmpnle_uspd vcmpnlepd vcmpord_qpd vcmpordpd vcmpeq_uqpd vcmpnge_uspd vcmpngepd vcmpngt_uspd vcmpngtpd vcmpfalse_oqpd vcmpfalsepd vcmpneq_oqpd vcmpge_ospd vcmpgepd vcmpgt_ospd vcmpgtpd vcmptrue_uqpd vcmptruepd vcmplt_oqpd vcmple_oqpd vcmpunord_spd vcmpneq_uspd vcmpnlt_uqpd vcmpnle_uqpd vcmpord_spd vcmpeq_uspd vcmpnge_uqpd vcmpngt_uqpd vcmpfalse_ospd vcmpneq_ospd vcmpge_oqpd vcmpgt_oqpd vcmptrue_uspd vcmppd vcmpeq_osps vcmpeqps vcmplt_osps vcmpltps vcmple_osps vcmpleps vcmpunord_qps vcmpunordps vcmpneq_uqps vcmpneqps vcmpnlt_usps vcmpnltps vcmpnle_usps vcmpnleps vcmpord_qps vcmpordps vcmpeq_uqps vcmpnge_usps vcmpngeps vcmpngt_usps vcmpngtps vcmpfalse_oqps vcmpfalseps vcmpneq_oqps vcmpge_osps vcmpgeps vcmpgt_osps vcmpgtps vcmptrue_uqps vcmptrueps vcmplt_oqps vcmple_oqps vcmpunord_sps vcmpneq_usps vcmpnlt_uqps vcmpnle_uqps vcmpord_sps vcmpeq_usps vcmpnge_uqps vcmpngt_uqps vcmpfalse_osps vcmpneq_osps vcmpge_oqps vcmpgt_oqps vcmptrue_usps vcmpps vcmpeq_ossd vcmpeqsd vcmplt_ossd vcmpltsd vcmple_ossd vcmplesd vcmpunord_qsd vcmpunordsd vcmpneq_uqsd vcmpneqsd vcmpnlt_ussd vcmpnltsd vcmpnle_ussd vcmpnlesd vcmpord_qsd vcmpordsd vcmpeq_uqsd vcmpnge_ussd vcmpngesd vcmpngt_ussd vcmpngtsd vcmpfalse_oqsd vcmpfalsesd vcmpneq_oqsd vcmpge_ossd vcmpgesd vcmpgt_ossd vcmpgtsd vcmptrue_uqsd vcmptruesd vcmplt_oqsd vcmple_oqsd vcmpunord_ssd vcmpneq_ussd vcmpnlt_uqsd vcmpnle_uqsd vcmpord_ssd vcmpeq_ussd vcmpnge_uqsd vcmpngt_uqsd vcmpfalse_ossd vcmpneq_ossd vcmpge_oqsd vcmpgt_oqsd vcmptrue_ussd vcmpsd vcmpeq_osss vcmpeqss vcmplt_osss vcmpltss vcmple_osss vcmpless vcmpunord_qss vcmpunordss vcmpneq_uqss vcmpneqss vcmpnlt_usss vcmpnltss vcmpnle_usss vcmpnless vcmpord_qss vcmpordss vcmpeq_uqss vcmpnge_usss vcmpngess vcmpngt_usss vcmpngtss vcmpfalse_oqss vcmpfalsess vcmpneq_oqss vcmpge_osss vcmpgess vcmpgt_osss vcmpgtss vcmptrue_uqss vcmptruess vcmplt_oqss vcmple_oqss vcmpunord_sss vcmpneq_usss vcmpnlt_uqss vcmpnle_uqss vcmpord_sss vcmpeq_usss vcmpnge_uqss vcmpngt_uqss vcmpfalse_osss vcmpneq_osss vcmpge_oqss vcmpgt_oqss vcmptrue_usss vcmpss vcomisd vcomiss vcvtdq2pd vcvtdq2ps vcvtpd2dq vcvtpd2ps vcvtps2dq vcvtps2pd vcvtsd2si vcvtsd2ss vcvtsi2sd vcvtsi2ss vcvtss2sd vcvtss2si vcvttpd2dq vcvttps2dq vcvttsd2si vcvttss2si vdivpd vdivps vdivsd vdivss vdppd vdpps vextractf128 vextractps vhaddpd vhaddps vhsubpd vhsubps vinsertf128 vinsertps vlddqu vldqqu vldmxcsr vmaskmovdqu vmaskmovps vmaskmovpd vmaxpd vmaxps vmaxsd vmaxss vminpd vminps vminsd vminss vmovapd vmovaps vmovd vmovq vmovddup vmovdqa vmovqqa vmovdqu vmovqqu vmovhlps vmovhpd vmovhps vmovlhps vmovlpd vmovlps vmovmskpd vmovmskps vmovntdq vmovntqq vmovntdqa vmovntpd vmovntps vmovsd vmovshdup vmovsldup vmovss vmovupd vmovups vmpsadbw vmulpd vmulps vmulsd vmulss vorpd vorps vpabsb vpabsw vpabsd vpacksswb vpackssdw vpackuswb vpackusdw vpaddb vpaddw vpaddd vpaddq vpaddsb vpaddsw vpaddusb vpaddusw vpalignr vpand vpandn vpavgb vpavgw vpblendvb vpblendw vpcmpestri vpcmpestrm vpcmpistri vpcmpistrm vpcmpeqb vpcmpeqw vpcmpeqd vpcmpeqq vpcmpgtb vpcmpgtw vpcmpgtd vpcmpgtq vpermilpd vpermilps vperm2f128 vpextrb vpextrw vpextrd vpextrq vphaddw vphaddd vphaddsw vphminposuw vphsubw vphsubd vphsubsw vpinsrb vpinsrw vpinsrd vpinsrq vpmaddwd vpmaddubsw vpmaxsb vpmaxsw vpmaxsd vpmaxub vpmaxuw vpmaxud vpminsb vpminsw vpminsd vpminub vpminuw vpminud vpmovmskb vpmovsxbw vpmovsxbd vpmovsxbq vpmovsxwd vpmovsxwq vpmovsxdq vpmovzxbw vpmovzxbd vpmovzxbq vpmovzxwd vpmovzxwq vpmovzxdq vpmulhuw vpmulhrsw vpmulhw vpmullw vpmulld vpmuludq vpmuldq vpor vpsadbw vpshufb vpshufd vpshufhw vpshuflw vpsignb vpsignw vpsignd vpslldq vpsrldq vpsllw vpslld vpsllq vpsraw vpsrad vpsrlw vpsrld vpsrlq vptest vpsubb vpsubw vpsubd vpsubq vpsubsb vpsubsw vpsubusb vpsubusw vpunpckhbw vpunpckhwd vpunpckhdq vpunpckhqdq vpunpcklbw vpunpcklwd vpunpckldq vpunpcklqdq vpxor vrcpps vrcpss vrsqrtps vrsqrtss vroundpd vroundps vroundsd vroundss vshufpd vshufps vsqrtpd vsqrtps vsqrtsd vsqrtss vstmxcsr vsubpd vsubps vsubsd vsubss vtestps vtestpd vucomisd vucomiss vunpckhpd vunpckhps vunpcklpd vunpcklps vxorpd vxorps vzeroall vzeroupper pclmullqlqdq pclmulhqlqdq pclmullqhqdq pclmulhqhqdq pclmulqdq vpclmullqlqdq vpclmulhqlqdq vpclmullqhqdq vpclmulhqhqdq vpclmulqdq vfmadd132ps vfmadd132pd vfmadd312ps vfmadd312pd vfmadd213ps vfmadd213pd vfmadd123ps vfmadd123pd vfmadd231ps vfmadd231pd vfmadd321ps vfmadd321pd vfmaddsub132ps vfmaddsub132pd vfmaddsub312ps vfmaddsub312pd vfmaddsub213ps vfmaddsub213pd vfmaddsub123ps vfmaddsub123pd vfmaddsub231ps vfmaddsub231pd vfmaddsub321ps vfmaddsub321pd vfmsub132ps vfmsub132pd vfmsub312ps vfmsub312pd vfmsub213ps vfmsub213pd vfmsub123ps vfmsub123pd vfmsub231ps vfmsub231pd vfmsub321ps vfmsub321pd vfmsubadd132ps vfmsubadd132pd vfmsubadd312ps vfmsubadd312pd vfmsubadd213ps vfmsubadd213pd vfmsubadd123ps vfmsubadd123pd vfmsubadd231ps vfmsubadd231pd vfmsubadd321ps vfmsubadd321pd vfnmadd132ps vfnmadd132pd vfnmadd312ps vfnmadd312pd vfnmadd213ps vfnmadd213pd vfnmadd123ps vfnmadd123pd vfnmadd231ps vfnmadd231pd vfnmadd321ps vfnmadd321pd vfnmsub132ps vfnmsub132pd vfnmsub312ps vfnmsub312pd vfnmsub213ps vfnmsub213pd vfnmsub123ps vfnmsub123pd vfnmsub231ps vfnmsub231pd vfnmsub321ps vfnmsub321pd vfmadd132ss vfmadd132sd vfmadd312ss vfmadd312sd vfmadd213ss vfmadd213sd vfmadd123ss vfmadd123sd vfmadd231ss vfmadd231sd vfmadd321ss vfmadd321sd vfmsub132ss vfmsub132sd vfmsub312ss vfmsub312sd vfmsub213ss vfmsub213sd vfmsub123ss vfmsub123sd vfmsub231ss vfmsub231sd vfmsub321ss vfmsub321sd vfnmadd132ss vfnmadd132sd vfnmadd312ss vfnmadd312sd vfnmadd213ss vfnmadd213sd vfnmadd123ss vfnmadd123sd vfnmadd231ss vfnmadd231sd vfnmadd321ss vfnmadd321sd vfnmsub132ss vfnmsub132sd vfnmsub312ss vfnmsub312sd vfnmsub213ss vfnmsub213sd vfnmsub123ss vfnmsub123sd vfnmsub231ss vfnmsub231sd vfnmsub321ss vfnmsub321sd rdfsbase rdgsbase rdrand wrfsbase wrgsbase vcvtph2ps vcvtps2ph adcx adox rdseed clac stac xstore xcryptecb xcryptcbc xcryptctr xcryptcfb xcryptofb montmul xsha1 xsha256 llwpcb slwpcb lwpval lwpins vfmaddpd vfmaddps vfmaddsd vfmaddss vfmaddsubpd vfmaddsubps vfmsubaddpd vfmsubaddps vfmsubpd vfmsubps vfmsubsd vfmsubss vfnmaddpd vfnmaddps vfnmaddsd vfnmaddss vfnmsubpd vfnmsubps vfnmsubsd vfnmsubss vfrczpd vfrczps vfrczsd vfrczss vpcmov vpcomb vpcomd vpcomq vpcomub vpcomud vpcomuq vpcomuw vpcomw vphaddbd vphaddbq vphaddbw vphadddq vphaddubd vphaddubq vphaddubw vphaddudq vphadduwd vphadduwq vphaddwd vphaddwq vphsubbw vphsubdq vphsubwd vpmacsdd vpmacsdqh vpmacsdql vpmacssdd vpmacssdqh vpmacssdql vpmacsswd vpmacssww vpmacswd vpmacsww vpmadcsswd vpmadcswd vpperm vprotb vprotd vprotq vprotw vpshab vpshad vpshaq vpshaw vpshlb vpshld vpshlq vpshlw vbroadcasti128 vpblendd vpbroadcastb vpbroadcastw vpbroadcastd vpbroadcastq vpermd vpermpd vpermps vpermq vperm2i128 vextracti128 vinserti128 vpmaskmovd vpmaskmovq vpsllvd vpsllvq vpsravd vpsrlvd vpsrlvq vgatherdpd vgatherqpd vgatherdps vgatherqps vpgatherdd vpgatherqd vpgatherdq vpgatherqq xabort xbegin xend xtest andn bextr blci blcic blsi blsic blcfill blsfill blcmsk blsmsk blsr blcs bzhi mulx pdep pext rorx sarx shlx shrx tzcnt tzmsk t1mskc valignd valignq vblendmpd vblendmps vbroadcastf32x4 vbroadcastf64x4 vbroadcasti32x4 vbroadcasti64x4 vcompresspd vcompressps vcvtpd2udq vcvtps2udq vcvtsd2usi vcvtss2usi vcvttpd2udq vcvttps2udq vcvttsd2usi vcvttss2usi vcvtudq2pd vcvtudq2ps vcvtusi2sd vcvtusi2ss vexpandpd vexpandps vextractf32x4 vextractf64x4 vextracti32x4 vextracti64x4 vfixupimmpd vfixupimmps vfixupimmsd vfixupimmss vgetexppd vgetexpps vgetexpsd vgetexpss vgetmantpd vgetmantps vgetmantsd vgetmantss vinsertf32x4 vinsertf64x4 vinserti32x4 vinserti64x4 vmovdqa32 vmovdqa64 vmovdqu32 vmovdqu64 vpabsq vpandd vpandnd vpandnq vpandq vpblendmd vpblendmq vpcmpltd vpcmpled vpcmpneqd vpcmpnltd vpcmpnled vpcmpd vpcmpltq vpcmpleq vpcmpneqq vpcmpnltq vpcmpnleq vpcmpq vpcmpequd vpcmpltud vpcmpleud vpcmpnequd vpcmpnltud vpcmpnleud vpcmpud vpcmpequq vpcmpltuq vpcmpleuq vpcmpnequq vpcmpnltuq vpcmpnleuq vpcmpuq vpcompressd vpcompressq vpermi2d vpermi2pd vpermi2ps vpermi2q vpermt2d vpermt2pd vpermt2ps vpermt2q vpexpandd vpexpandq vpmaxsq vpmaxuq vpminsq vpminuq vpmovdb vpmovdw vpmovqb vpmovqd vpmovqw vpmovsdb vpmovsdw vpmovsqb vpmovsqd vpmovsqw vpmovusdb vpmovusdw vpmovusqb vpmovusqd vpmovusqw vpord vporq vprold vprolq vprolvd vprolvq vprord vprorq vprorvd vprorvq vpscatterdd vpscatterdq vpscatterqd vpscatterqq vpsraq vpsravq vpternlogd vpternlogq vptestmd vptestmq vptestnmd vptestnmq vpxord vpxorq vrcp14pd vrcp14ps vrcp14sd vrcp14ss vrndscalepd vrndscaleps vrndscalesd vrndscaless vrsqrt14pd vrsqrt14ps vrsqrt14sd vrsqrt14ss vscalefpd vscalefps vscalefsd vscalefss vscatterdpd vscatterdps vscatterqpd vscatterqps vshuff32x4 vshuff64x2 vshufi32x4 vshufi64x2 kandnw kandw kmovw knotw kortestw korw kshiftlw kshiftrw kunpckbw kxnorw kxorw vpbroadcastmb2q vpbroadcastmw2d vpconflictd vpconflictq vplzcntd vplzcntq vexp2pd vexp2ps vrcp28pd vrcp28ps vrcp28sd vrcp28ss vrsqrt28pd vrsqrt28ps vrsqrt28sd vrsqrt28ss vgatherpf0dpd vgatherpf0dps vgatherpf0qpd vgatherpf0qps vgatherpf1dpd vgatherpf1dps vgatherpf1qpd vgatherpf1qps vscatterpf0dpd vscatterpf0dps vscatterpf0qpd vscatterpf0qps vscatterpf1dpd vscatterpf1dps vscatterpf1qpd vscatterpf1qps prefetchwt1 bndmk bndcl bndcu bndcn bndmov bndldx bndstx sha1rnds4 sha1nexte sha1msg1 sha1msg2 sha256rnds2 sha256msg1 sha256msg2 hint_nop0 hint_nop1 hint_nop2 hint_nop3 hint_nop4 hint_nop5 hint_nop6 hint_nop7 hint_nop8 hint_nop9 hint_nop10 hint_nop11 hint_nop12 hint_nop13 hint_nop14 hint_nop15 hint_nop16 hint_nop17 hint_nop18 hint_nop19 hint_nop20 hint_nop21 hint_nop22 hint_nop23 hint_nop24 hint_nop25 hint_nop26 hint_nop27 hint_nop28 hint_nop29 hint_nop30 hint_nop31 hint_nop32 hint_nop33 hint_nop34 hint_nop35 hint_nop36 hint_nop37 hint_nop38 hint_nop39 hint_nop40 hint_nop41 hint_nop42 hint_nop43 hint_nop44 hint_nop45 hint_nop46 hint_nop47 hint_nop48 hint_nop49 hint_nop50 hint_nop51 hint_nop52 hint_nop53 hint_nop54 hint_nop55 hint_nop56 hint_nop57 hint_nop58 hint_nop59 hint_nop60 hint_nop61 hint_nop62 hint_nop63",built_in:"ip eip rip al ah bl bh cl ch dl dh sil dil bpl spl r8b r9b r10b r11b r12b r13b r14b r15b ax bx cx dx si di bp sp r8w r9w r10w r11w r12w r13w r14w r15w eax ebx ecx edx esi edi ebp esp eip r8d r9d r10d r11d r12d r13d r14d r15d rax rbx rcx rdx rsi rdi rbp rsp r8 r9 r10 r11 r12 r13 r14 r15 cs ds es fs gs ss st st0 st1 st2 st3 st4 st5 st6 st7 mm0 mm1 mm2 mm3 mm4 mm5 mm6 mm7 xmm0 xmm1 xmm2 xmm3 xmm4 xmm5 xmm6 xmm7 xmm8 xmm9 xmm10 xmm11 xmm12 xmm13 xmm14 xmm15 xmm16 xmm17 xmm18 xmm19 xmm20 xmm21 xmm22 xmm23 xmm24 xmm25 xmm26 xmm27 xmm28 xmm29 xmm30 xmm31 ymm0 ymm1 ymm2 ymm3 ymm4 ymm5 ymm6 ymm7 ymm8 ymm9 ymm10 ymm11 ymm12 ymm13 ymm14 ymm15 ymm16 ymm17 ymm18 ymm19 ymm20 ymm21 ymm22 ymm23 ymm24 ymm25 ymm26 ymm27 ymm28 ymm29 ymm30 ymm31 zmm0 zmm1 zmm2 zmm3 zmm4 zmm5 zmm6 zmm7 zmm8 zmm9 zmm10 zmm11 zmm12 zmm13 zmm14 zmm15 zmm16 zmm17 zmm18 zmm19 zmm20 zmm21 zmm22 zmm23 zmm24 zmm25 zmm26 zmm27 zmm28 zmm29 zmm30 zmm31 k0 k1 k2 k3 k4 k5 k6 k7 bnd0 bnd1 bnd2 bnd3 cr0 cr1 cr2 cr3 cr4 cr8 dr0 dr1 dr2 dr3 dr8 tr3 tr4 tr5 tr6 tr7 r0 r1 r2 r3 r4 r5 r6 r7 r0b r1b r2b r3b r4b r5b r6b r7b r0w r1w r2w r3w r4w r5w r6w r7w r0d r1d r2d r3d r4d r5d r6d r7d r0h r1h r2h r3h r0l r1l r2l r3l r4l r5l r6l r7l r8l r9l r10l r11l r12l r13l r14l r15l db dw dd dq dt ddq do dy dz resb resw resd resq rest resdq reso resy resz incbin equ times byte word dword qword nosplit rel abs seg wrt strict near far a32 ptr",meta:"%define %xdefine %+ %undef %defstr %deftok %assign %strcat %strlen %substr %rotate %elif %else %endif %if %ifmacro %ifctx %ifidn %ifidni %ifid %ifnum %ifstr %iftoken %ifempty %ifenv %error %warning %fatal %rep %endrep %include %push %pop %repl %pathsearch %depend %use %arg %stacksize %local %line %comment %endcomment .nolist __FILE__ __LINE__ __SECT__ __BITS__ __OUTPUT_FORMAT__ __DATE__ __TIME__ __DATE_NUM__ __TIME_NUM__ __UTC_DATE__ __UTC_TIME__ __UTC_DATE_NUM__ __UTC_TIME_NUM__ __PASS__ struc endstruc istruc at iend align alignb sectalign daz nodaz up down zero default option assume public bits use16 use32 use64 default section segment absolute extern global common cpu float __utf16__ __utf16le__ __utf16be__ __utf32__ __utf32le__ __utf32be__ __float8__ __float16__ __float32__ __float64__ __float80m__ __float80e__ __float128l__ __float128h__ __Infinity__ __QNaN__ __SNaN__ Inf NaN QNaN SNaN float8 float16 float32 float64 float80m float80e float128l float128h __FLOAT_DAZ__ __FLOAT_ROUND__ __FLOAT__"},contains:[s.COMMENT(";","$",{relevance:0}),{className:"number",variants:[{begin:"\\b(?:([0-9][0-9_]*)?\\.[0-9_]*(?:[eE][+-]?[0-9_]+)?|(0[Xx])?[0-9][0-9_]*\\.?[0-9_]*(?:[pP](?:[+-]?[0-9_]+)?)?)\\b",relevance:0},{begin:"\\$[0-9][0-9A-Fa-f]*",relevance:0},{begin:"\\b(?:[0-9A-Fa-f][0-9A-Fa-f_]*[Hh]|[0-9][0-9_]*[DdTt]?|[0-7][0-7_]*[QqOo]|[0-1][0-1_]*[BbYy])\\b"},{begin:"\\b(?:0[Xx][0-9A-Fa-f_]+|0[DdTt][0-9_]+|0[QqOo][0-7_]+|0[BbYy][0-1_]+)\\b"}]},s.QUOTE_STRING_MODE,{className:"string",variants:[{begin:"'",end:"[^\\\\]'"},{begin:"`",end:"[^\\\\]`"}],relevance:0},{className:"symbol",variants:[{begin:"^\\s*[A-Za-z._?][A-Za-z0-9_$#@~.?]*(:|\\s+label)"},{begin:"^\\s*%%[A-Za-z0-9_$#@~.?]*:"}],relevance:0},{className:"subst",begin:"%[0-9]+",relevance:0},{className:"subst",begin:"%!S+",relevance:0},{className:"meta",begin:/^\s*\.[\w_-]+/}]}}}()); \ No newline at end of file From e38ceb5378e7ce945eedbe1c6fb670095cfb9cc5 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 4 Dec 2023 14:19:29 +0100 Subject: [PATCH 49/50] fix v3 backwards compatibility for callbacks (#481) Because the callback types were used explicitly in some consumers of chronos, the change of type introduces a backwards incompatibility preventing a smooth transition to v4 for code that doesn't uses `raises`. This PR restores backwards compatibility at the expense of introducing a new type with a potentially ugly name - that said, there is already precedence for using numbered names to provide new error handling strategy in chronos. --- chronos/apps/http/httpserver.nim | 59 +++++++++++++++++++++++++++---- chronos/apps/http/shttpserver.nim | 52 ++++++++++++++++++++++++++- chronos/transports/stream.nim | 23 ++++++------ tests/testhttpclient.nim | 2 +- 4 files changed, 117 insertions(+), 19 deletions(-) diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index 7d1aea0..3bbee0f 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -63,18 +63,22 @@ type HttpResponseState* {.pure.} = enum Empty, Prepared, Sending, Finished, Failed, Cancelled, Default - HttpProcessCallback* = + # TODO Evaluate naming of raises-annotated callbacks + HttpProcessCallback2* = proc(req: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]), gcsafe.} + async: (raises: [CancelledError, HttpResponseError]).} + + HttpProcessCallback* {.deprecated.} = + proc(req: RequestFence): Future[HttpResponseRef] {.async.} HttpConnectionCallback* = proc(server: HttpServerRef, transp: StreamTransport): Future[HttpConnectionRef] {. - async: (raises: [CancelledError, HttpConnectionError]), gcsafe.} + async: (raises: [CancelledError, HttpConnectionError]).} HttpCloseConnectionCallback* = proc(connection: HttpConnectionRef): Future[void] {. - async: (raises: []), gcsafe.} + async: (raises: []).} HttpConnectionHolder* = object of RootObj connection*: HttpConnectionRef @@ -103,7 +107,7 @@ type bufferSize*: int maxHeadersSize*: int maxRequestBodySize*: int - processCallback*: HttpProcessCallback + processCallback*: HttpProcessCallback2 createConnCallback*: HttpConnectionCallback HttpServerRef* = ref HttpServer @@ -182,7 +186,7 @@ proc createConnection(server: HttpServerRef, proc new*(htype: typedesc[HttpServerRef], address: TransportAddress, - processCallback: HttpProcessCallback, + processCallback: HttpProcessCallback2, serverFlags: set[HttpServerFlags] = {}, socketFlags: set[ServerFlags] = {ReuseAddr}, serverUri = Uri(), @@ -236,6 +240,49 @@ proc new*(htype: typedesc[HttpServerRef], ) ok(res) +proc new*(htype: typedesc[HttpServerRef], + address: TransportAddress, + processCallback: HttpProcessCallback, + serverFlags: set[HttpServerFlags] = {}, + socketFlags: set[ServerFlags] = {ReuseAddr}, + serverUri = Uri(), + serverIdent = "", + maxConnections: int = -1, + bufferSize: int = 4096, + backlogSize: int = DefaultBacklogSize, + httpHeadersTimeout = 10.seconds, + maxHeadersSize: int = 8192, + maxRequestBodySize: int = 1_048_576, + dualstack = DualStackType.Auto): HttpResult[HttpServerRef] {. + deprecated: "raises missing from process callback".} = + + proc processCallback2(req: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = + try: + await processCallback(req) + except CancelledError as exc: + raise exc + except HttpResponseError as exc: + raise exc + except CatchableError as exc: + # Emulate 3.x behavior + raise (ref HttpCriticalError)(msg: exc.msg, code: Http503) + + HttpServerRef.new( + address = address, + processCallback = processCallback2, + serverFlags = serverFlags, + socketFlags = socketFlags, + serverUri = serverUri, + serverIdent = serverIdent, + maxConnections = maxConnections, + bufferSize = bufferSize, + backlogSize = backlogSize, + httpHeadersTimeout = httpHeadersTimeout, + maxHeadersSize = maxHeadersSize, + maxRequestBodySize = maxRequestBodySize, + dualstack = dualstack) + proc getServerFlags(req: HttpRequestRef): set[HttpServerFlags] = var defaultFlags: set[HttpServerFlags] = {} if isNil(req): return defaultFlags diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index 2373d95..f7e377f 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -82,7 +82,7 @@ proc createSecConnection(server: HttpServerRef, proc new*(htype: typedesc[SecureHttpServerRef], address: TransportAddress, - processCallback: HttpProcessCallback, + processCallback: HttpProcessCallback2, tlsPrivateKey: TLSPrivateKey, tlsCertificate: TLSCertificate, serverFlags: set[HttpServerFlags] = {}, @@ -145,3 +145,53 @@ proc new*(htype: typedesc[SecureHttpServerRef], secureFlags: secureFlags ) ok(res) + +proc new*(htype: typedesc[SecureHttpServerRef], + address: TransportAddress, + processCallback: HttpProcessCallback, + tlsPrivateKey: TLSPrivateKey, + tlsCertificate: TLSCertificate, + serverFlags: set[HttpServerFlags] = {}, + socketFlags: set[ServerFlags] = {ReuseAddr}, + serverUri = Uri(), + serverIdent = "", + secureFlags: set[TLSFlags] = {}, + maxConnections: int = -1, + bufferSize: int = 4096, + backlogSize: int = DefaultBacklogSize, + httpHeadersTimeout = 10.seconds, + maxHeadersSize: int = 8192, + maxRequestBodySize: int = 1_048_576, + dualstack = DualStackType.Auto + ): HttpResult[SecureHttpServerRef] {. + deprecated: "raises missing from process callback".} = + proc processCallback2(req: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError, HttpResponseError]).} = + try: + await processCallback(req) + except CancelledError as exc: + raise exc + except HttpResponseError as exc: + raise exc + except CatchableError as exc: + # Emulate 3.x behavior + raise (ref HttpCriticalError)(msg: exc.msg, code: Http503) + + SecureHttpServerRef.new( + address = address, + processCallback = processCallback2, + tlsPrivateKey = tlsPrivateKey, + tlsCertificate = tlsCertificate, + serverFlags = serverFlags, + socketFlags = socketFlags, + serverUri = serverUri, + serverIdent = serverIdent, + secureFlags = secureFlags, + maxConnections = maxConnections, + bufferSize = bufferSize, + backlogSize = backlogSize, + httpHeadersTimeout = httpHeadersTimeout, + maxHeadersSize = maxHeadersSize, + maxRequestBodySize = maxRequestBodySize, + dualstack = dualstack + ) \ No newline at end of file diff --git a/chronos/transports/stream.nim b/chronos/transports/stream.nim index 107bc6e..c0d1cfc 100644 --- a/chronos/transports/stream.nim +++ b/chronos/transports/stream.nim @@ -115,14 +115,15 @@ else: discard type - StreamCallback* = proc(server: StreamServer, - client: StreamTransport) {.async: (raises: []).} + # TODO evaluate naming of raises-annotated callbacks + StreamCallback2* = proc(server: StreamServer, + client: StreamTransport) {.async: (raises: []).} ## New remote client connection callback ## ``server`` - StreamServer object. ## ``client`` - accepted client transport. - UnsafeStreamCallback* = proc(server: StreamServer, - client: StreamTransport) {.async.} + StreamCallback* = proc(server: StreamServer, + client: StreamTransport) {.async.} ## Connection callback that doesn't check for exceptions at compile time ## ``server`` - StreamServer object. ## ``client`` - accepted client transport. @@ -135,7 +136,7 @@ type StreamServer* = ref object of SocketServer ## StreamServer object - function*: StreamCallback # callback which will be called after new + function*: StreamCallback2 # callback which will be called after new # client accepted init*: TransportInitCallback # callback which will be called before # transport for new client @@ -1870,7 +1871,7 @@ proc getBacklogSize(backlog: int): cint = cint(backlog) proc createStreamServer*(host: TransportAddress, - cbproc: StreamCallback, + cbproc: StreamCallback2, flags: set[ServerFlags] = {}, sock: AsyncFD = asyncInvalidSocket, backlog: int = DefaultBacklogSize, @@ -2092,7 +2093,7 @@ proc createStreamServer*(host: TransportAddress, sres proc createStreamServer*(host: TransportAddress, - cbproc: UnsafeStreamCallback, + cbproc: StreamCallback, flags: set[ServerFlags] = {}, sock: AsyncFD = asyncInvalidSocket, backlog: int = DefaultBacklogSize, @@ -2124,11 +2125,11 @@ proc createStreamServer*(host: TransportAddress, udata: pointer = nil, dualstack = DualStackType.Auto): StreamServer {. raises: [TransportOsError].} = - createStreamServer(host, StreamCallback(nil), flags, sock, backlog, bufferSize, + createStreamServer(host, StreamCallback2(nil), flags, sock, backlog, bufferSize, child, init, cast[pointer](udata), dualstack) proc createStreamServer*[T](host: TransportAddress, - cbproc: StreamCallback, + cbproc: StreamCallback2, flags: set[ServerFlags] = {}, udata: ref T, sock: AsyncFD = asyncInvalidSocket, @@ -2144,7 +2145,7 @@ proc createStreamServer*[T](host: TransportAddress, child, init, cast[pointer](udata), dualstack) proc createStreamServer*[T](host: TransportAddress, - cbproc: UnsafeStreamCallback, + cbproc: StreamCallback, flags: set[ServerFlags] = {}, udata: ref T, sock: AsyncFD = asyncInvalidSocket, @@ -2172,7 +2173,7 @@ proc createStreamServer*[T](host: TransportAddress, raises: [TransportOsError].} = var fflags = flags + {GCUserData} GC_ref(udata) - createStreamServer(host, StreamCallback(nil), fflags, sock, backlog, bufferSize, + createStreamServer(host, StreamCallback2(nil), fflags, sock, backlog, bufferSize, child, init, cast[pointer](udata), dualstack) proc getUserData*[T](server: StreamServer): T {.inline.} = diff --git a/tests/testhttpclient.nim b/tests/testhttpclient.nim index eb1eaac..d2a355d 100644 --- a/tests/testhttpclient.nim +++ b/tests/testhttpclient.nim @@ -85,7 +85,7 @@ suite "HTTP client testing suite": res proc createServer(address: TransportAddress, - process: HttpProcessCallback, secure: bool): HttpServerRef = + process: HttpProcessCallback2, secure: bool): HttpServerRef = let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} serverFlags = {HttpServerFlags.Http11Pipeline} From c41599a6d6d8b11c729032bf8913e06f4171e0fb Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Sat, 9 Dec 2023 06:50:35 +0200 Subject: [PATCH 50/50] Asyncraises HTTP layer V3 (#482) * No Critical and Recoverable errors anymore. * Recover raiseHttpCriticalError() * Post-rebase fixes. * Remove deprecated ResponseFence and getResponseFence(). * HttpProcessCallback and 2. * Fix callback holder. * Fix test issue. * Fix backwards compatibility of `HttpResponse.state` field. --- chronos/apps/http/httpcommon.nim | 79 ++- chronos/apps/http/httpserver.nim | 779 +++++++++++++----------------- chronos/apps/http/multipart.nim | 121 ++--- chronos/apps/http/shttpserver.nim | 27 +- tests/testhttpclient.nim | 395 ++++++++------- tests/testhttpserver.nim | 231 +++++---- tests/testshttpserver.nim | 25 +- 7 files changed, 840 insertions(+), 817 deletions(-) diff --git a/chronos/apps/http/httpcommon.nim b/chronos/apps/http/httpcommon.nim index 3ebe3ca..0f5370a 100644 --- a/chronos/apps/http/httpcommon.nim +++ b/chronos/apps/http/httpcommon.nim @@ -43,30 +43,48 @@ const ServerHeader* = "server" LocationHeader* = "location" AuthorizationHeader* = "authorization" + ContentDispositionHeader* = "content-disposition" UrlEncodedContentType* = MediaType.init("application/x-www-form-urlencoded") MultipartContentType* = MediaType.init("multipart/form-data") type + HttpMessage* = object + code*: HttpCode + contentType*: MediaType + message*: string + HttpResult*[T] = Result[T, string] HttpResultCode*[T] = Result[T, HttpCode] + HttpResultMessage*[T] = Result[T, HttpMessage] - HttpDefect* = object of Defect HttpError* = object of AsyncError - HttpResponseError* = object of HttpError - code*: HttpCode - HttpCriticalError* = object of HttpResponseError - HttpRecoverableError* = object of HttpResponseError - HttpDisconnectError* = object of HttpError - HttpConnectionError* = object of HttpError HttpInterruptError* = object of HttpError - HttpReadError* = object of HttpError - HttpWriteError* = object of HttpError - HttpProtocolError* = object of HttpError - HttpRedirectError* = object of HttpError - HttpAddressError* = object of HttpError - HttpUseClosedError* = object of HttpError + + HttpTransportError* = object of HttpError + HttpAddressError* = object of HttpTransportError + HttpRedirectError* = object of HttpTransportError + HttpConnectionError* = object of HttpTransportError + HttpReadError* = object of HttpTransportError HttpReadLimitError* = object of HttpReadError + HttpDisconnectError* = object of HttpReadError + HttpWriteError* = object of HttpTransportError + + HttpProtocolError* = object of HttpError + code*: HttpCode + + HttpCriticalError* = object of HttpProtocolError # deprecated + HttpRecoverableError* = object of HttpProtocolError # deprecated + + HttpRequestError* = object of HttpProtocolError + HttpRequestHeadersError* = object of HttpRequestError + HttpRequestBodyError* = object of HttpRequestError + HttpRequestHeadersTooLargeError* = object of HttpRequestHeadersError + HttpRequestBodyTooLargeError* = object of HttpRequestBodyError + HttpResponseError* = object of HttpProtocolError + + HttpInvalidUsageError* = object of HttpError + HttpUseClosedError* = object of HttpInvalidUsageError KeyValueTuple* = tuple key: string @@ -127,6 +145,11 @@ func toString*(error: HttpAddressErrorType): string = of HttpAddressErrorType.NoAddressResolved: "No address has been resolved" +proc raiseHttpRequestBodyTooLargeError*() {. + noinline, noreturn, raises: [HttpRequestBodyTooLargeError].} = + raise (ref HttpRequestBodyTooLargeError)( + code: Http413, msg: MaximumBodySizeError) + proc raiseHttpCriticalError*(msg: string, code = Http400) {. noinline, noreturn, raises: [HttpCriticalError].} = raise (ref HttpCriticalError)(code: code, msg: msg) @@ -135,9 +158,6 @@ proc raiseHttpDisconnectError*() {. noinline, noreturn, raises: [HttpDisconnectError].} = raise (ref HttpDisconnectError)(msg: "Remote peer disconnected") -proc raiseHttpDefect*(msg: string) {.noinline, noreturn.} = - raise (ref HttpDefect)(msg: msg) - proc raiseHttpConnectionError*(msg: string) {. noinline, noreturn, raises: [HttpConnectionError].} = raise (ref HttpConnectionError)(msg: msg) @@ -152,7 +172,15 @@ proc raiseHttpReadError*(msg: string) {. proc raiseHttpProtocolError*(msg: string) {. noinline, noreturn, raises: [HttpProtocolError].} = - raise (ref HttpProtocolError)(msg: msg) + raise (ref HttpProtocolError)(code: Http400, msg: msg) + +proc raiseHttpProtocolError*(code: HttpCode, msg: string) {. + noinline, noreturn, raises: [HttpProtocolError].} = + raise (ref HttpProtocolError)(code: code, msg: msg) + +proc raiseHttpProtocolError*(msg: HttpMessage) {. + noinline, noreturn, raises: [HttpProtocolError].} = + raise (ref HttpProtocolError)(code: msg.code, msg: msg.message) proc raiseHttpWriteError*(msg: string) {. noinline, noreturn, raises: [HttpWriteError].} = @@ -178,6 +206,23 @@ template newHttpWriteError*(message: string): ref HttpWriteError = template newHttpUseClosedError*(): ref HttpUseClosedError = newException(HttpUseClosedError, "Connection was already closed") +func init*(t: typedesc[HttpMessage], code: HttpCode, message: string, + contentType: MediaType): HttpMessage = + HttpMessage(code: code, message: message, contentType: contentType) + +func init*(t: typedesc[HttpMessage], code: HttpCode, message: string, + contentType: string): HttpMessage = + HttpMessage(code: code, message: message, + contentType: MediaType.init(contentType)) + +func init*(t: typedesc[HttpMessage], code: HttpCode, + message: string): HttpMessage = + HttpMessage(code: code, message: message, + contentType: MediaType.init("text/plain")) + +func init*(t: typedesc[HttpMessage], code: HttpCode): HttpMessage = + HttpMessage(code: code) + iterator queryParams*(query: string, flags: set[QueryParamsFlag] = {}): KeyValueTuple = ## Iterate over url-encoded query string. diff --git a/chronos/apps/http/httpserver.nim b/chronos/apps/http/httpserver.nim index 3bbee0f..9646956 100644 --- a/chronos/apps/http/httpserver.nim +++ b/chronos/apps/http/httpserver.nim @@ -32,8 +32,7 @@ type ## Enable HTTP/1.1 pipelining. HttpServerError* {.pure.} = enum - InterruptError, TimeoutError, CatchableError, RecoverableError, - CriticalError, DisconnectError + InterruptError, TimeoutError, ProtocolError, DisconnectError HttpServerState* {.pure.} = enum ServerRunning, ServerStopped, ServerClosed @@ -41,11 +40,10 @@ type HttpProcessError* = object kind*: HttpServerError code*: HttpCode - exc*: ref CatchableError + exc*: ref HttpError remote*: Opt[TransportAddress] ConnectionFence* = Result[HttpConnectionRef, HttpProcessError] - ResponseFence* = Result[HttpResponseRef, HttpProcessError] RequestFence* = Result[HttpRequestRef, HttpProcessError] HttpRequestFlags* {.pure.} = enum @@ -61,15 +59,15 @@ type KeepAlive, Graceful, Immediate HttpResponseState* {.pure.} = enum - Empty, Prepared, Sending, Finished, Failed, Cancelled, Default + Empty, Prepared, Sending, Finished, Failed, Cancelled, ErrorCode, Default + + HttpProcessCallback* = + proc(req: RequestFence): Future[HttpResponseRef] {. + gcsafe, raises: [].} - # TODO Evaluate naming of raises-annotated callbacks HttpProcessCallback2* = proc(req: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} - - HttpProcessCallback* {.deprecated.} = - proc(req: RequestFence): Future[HttpResponseRef] {.async.} + async: (raises: [CancelledError]).} HttpConnectionCallback* = proc(server: HttpServerRef, @@ -138,7 +136,7 @@ type headersTable: HttpTable body: seq[byte] flags: set[HttpResponseFlags] - state*: HttpResponseState + state*: HttpResponseState # TODO (cheatfate): Make this field private connection*: HttpConnectionRef streamType*: HttpResponseStreamType writer: AsyncStreamWriter @@ -163,14 +161,20 @@ type ByteChar* = string | seq[byte] proc init(htype: typedesc[HttpProcessError], error: HttpServerError, - exc: ref CatchableError, remote: Opt[TransportAddress], + exc: ref HttpError, remote: Opt[TransportAddress], code: HttpCode): HttpProcessError = HttpProcessError(kind: error, exc: exc, remote: remote, code: code) +proc init(htype: typedesc[HttpProcessError], error: HttpServerError, + remote: Opt[TransportAddress], code: HttpCode): HttpProcessError = + HttpProcessError(kind: error, remote: remote, code: code) + proc init(htype: typedesc[HttpProcessError], error: HttpServerError): HttpProcessError = HttpProcessError(kind: error) +proc defaultResponse*(exc: ref CatchableError): HttpResponseRef + proc new(htype: typedesc[HttpConnectionHolderRef], server: HttpServerRef, transp: StreamTransport, connectionId: string): HttpConnectionHolderRef = @@ -254,34 +258,22 @@ proc new*(htype: typedesc[HttpServerRef], maxHeadersSize: int = 8192, maxRequestBodySize: int = 1_048_576, dualstack = DualStackType.Auto): HttpResult[HttpServerRef] {. - deprecated: "raises missing from process callback".} = + deprecated: "Callback could raise only CancelledError, annotate with " & + "{.async: (raises: [CancelledError]).}".} = - proc processCallback2(req: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = - try: - await processCallback(req) - except CancelledError as exc: - raise exc - except HttpResponseError as exc: - raise exc - except CatchableError as exc: - # Emulate 3.x behavior - raise (ref HttpCriticalError)(msg: exc.msg, code: Http503) + proc wrap(req: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError]).} = + try: + await processCallback(req) + except CancelledError as exc: + raise exc + except CatchableError as exc: + defaultResponse(exc) - HttpServerRef.new( - address = address, - processCallback = processCallback2, - serverFlags = serverFlags, - socketFlags = socketFlags, - serverUri = serverUri, - serverIdent = serverIdent, - maxConnections = maxConnections, - bufferSize = bufferSize, - backlogSize = backlogSize, - httpHeadersTimeout = httpHeadersTimeout, - maxHeadersSize = maxHeadersSize, - maxRequestBodySize = maxRequestBodySize, - dualstack = dualstack) + HttpServerRef.new(address, wrap, serverFlags, socketFlags, serverUri, + serverIdent, maxConnections, bufferSize, backlogSize, + httpHeadersTimeout, maxHeadersSize, maxRequestBodySize, + dualstack) proc getServerFlags(req: HttpRequestRef): set[HttpServerFlags] = var defaultFlags: set[HttpServerFlags] = {} @@ -304,6 +296,12 @@ proc getResponseFlags(req: HttpRequestRef): set[HttpResponseFlags] = else: defaultFlags +proc getResponseState*(response: HttpResponseRef): HttpResponseState = + response.state + +proc setResponseState(response: HttpResponseRef, state: HttpResponseState) = + response.state = state + proc getResponseVersion(reqFence: RequestFence): HttpVersion = if reqFence.isErr(): HttpVersion11 @@ -335,6 +333,18 @@ proc defaultResponse*(): HttpResponseRef = ## Create an empty response to return when request processor got no request. HttpResponseRef(state: HttpResponseState.Default, version: HttpVersion11) +proc defaultResponse*(exc: ref CatchableError): HttpResponseRef = + ## Create response with error code based on exception type. + if exc of AsyncTimeoutError: + HttpResponseRef(state: HttpResponseState.ErrorCode, status: Http408) + elif exc of HttpTransportError: + HttpResponseRef(state: HttpResponseState.Failed) + elif exc of HttpProtocolError: + let code = cast[ref HttpProtocolError](exc).code + HttpResponseRef(state: HttpResponseState.ErrorCode, status: code) + else: + HttpResponseRef(state: HttpResponseState.ErrorCode, status: Http503) + proc dumbResponse*(): HttpResponseRef {. deprecated: "Please use defaultResponse() instead".} = ## Create an empty response to return when request processor got no request. @@ -353,11 +363,11 @@ proc hasBody*(request: HttpRequestRef): bool = HttpRequestFlags.UnboundBody} != {} proc prepareRequest(conn: HttpConnectionRef, - req: HttpRequestHeader): HttpResultCode[HttpRequestRef] = + req: HttpRequestHeader): HttpResultMessage[HttpRequestRef] = var request = HttpRequestRef(connection: conn, state: HttpState.Alive) if req.version notin {HttpVersion10, HttpVersion11}: - return err(Http505) + return err(HttpMessage.init(Http505, "Unsupported HTTP protocol version")) request.scheme = if HttpServerFlags.Secure in conn.server.flags: @@ -372,14 +382,14 @@ proc prepareRequest(conn: HttpConnectionRef, block: let res = req.uri() if len(res) == 0: - return err(Http400) + return err(HttpMessage.init(Http400, "Invalid request URI")) res request.uri = if request.rawPath != "*": let uri = parseUri(request.rawPath) if uri.scheme notin ["http", "https", ""]: - return err(Http400) + return err(HttpMessage.init(Http400, "Unsupported URI scheme")) uri else: var uri = initUri() @@ -407,59 +417,61 @@ proc prepareRequest(conn: HttpConnectionRef, # Validating HTTP request headers # Some of the headers must be present only once. if table.count(ContentTypeHeader) > 1: - return err(Http400) + return err(HttpMessage.init(Http400, "Multiple Content-Type headers")) if table.count(ContentLengthHeader) > 1: - return err(Http400) + return err(HttpMessage.init(Http400, "Multiple Content-Length headers")) if table.count(TransferEncodingHeader) > 1: - return err(Http400) + return err(HttpMessage.init(Http400, + "Multuple Transfer-Encoding headers")) table # Preprocessing "Content-Encoding" header. request.contentEncoding = - block: - let res = getContentEncoding( - request.headers.getList(ContentEncodingHeader)) - if res.isErr(): - return err(Http400) - else: - res.get() + getContentEncoding( + request.headers.getList(ContentEncodingHeader)).valueOr: + let msg = "Incorrect or unsupported Content-Encoding header value" + return err(HttpMessage.init(Http400, msg)) # Preprocessing "Transfer-Encoding" header. request.transferEncoding = - block: - let res = getTransferEncoding( - request.headers.getList(TransferEncodingHeader)) - if res.isErr(): - return err(Http400) - else: - res.get() + getTransferEncoding( + request.headers.getList(TransferEncodingHeader)).valueOr: + let msg = "Incorrect or unsupported Transfer-Encoding header value" + return err(HttpMessage.init(Http400, msg)) # Almost all HTTP requests could have body (except TRACE), we perform some # steps to reveal information about body. - if ContentLengthHeader in request.headers: - let length = request.headers.getInt(ContentLengthHeader) - if length >= 0: - if request.meth == MethodTrace: - return err(Http400) - # Because of coversion to `int` we should avoid unexpected OverflowError. - if length > uint64(high(int)): - return err(Http413) - if length > uint64(conn.server.maxRequestBodySize): - return err(Http413) - request.contentLength = int(length) - request.requestFlags.incl(HttpRequestFlags.BoundBody) - else: - if TransferEncodingFlags.Chunked in request.transferEncoding: - if request.meth == MethodTrace: - return err(Http400) - request.requestFlags.incl(HttpRequestFlags.UnboundBody) + request.contentLength = + if ContentLengthHeader in request.headers: + let length = request.headers.getInt(ContentLengthHeader) + if length != 0: + if request.meth == MethodTrace: + let msg = "TRACE requests could not have request body" + return err(HttpMessage.init(Http400, msg)) + # Because of coversion to `int` we should avoid unexpected OverflowError. + if length > uint64(high(int)): + return err(HttpMessage.init(Http413, "Unsupported content length")) + if length > uint64(conn.server.maxRequestBodySize): + return err(HttpMessage.init(Http413, "Content length exceeds limits")) + request.requestFlags.incl(HttpRequestFlags.BoundBody) + int(length) + else: + 0 + else: + if TransferEncodingFlags.Chunked in request.transferEncoding: + if request.meth == MethodTrace: + let msg = "TRACE requests could not have request body" + return err(HttpMessage.init(Http400, msg)) + request.requestFlags.incl(HttpRequestFlags.UnboundBody) + 0 if request.hasBody(): # If request has body, we going to understand how its encoded. if ContentTypeHeader in request.headers: let contentType = getContentType(request.headers.getList(ContentTypeHeader)).valueOr: - return err(Http415) + let msg = "Incorrect or missing Content-Type header" + return err(HttpMessage.init(Http415, msg)) if contentType == UrlEncodedContentType: request.requestFlags.incl(HttpRequestFlags.UrlencodedForm) elif contentType == MultipartContentType: @@ -486,16 +498,17 @@ proc getBodyReader*(request: HttpRequestRef): HttpResult[HttpBodyReader] = uint64(request.contentLength)) ok(newHttpBodyReader(bstream)) elif HttpRequestFlags.UnboundBody in request.requestFlags: - let maxBodySize = request.connection.server.maxRequestBodySize - let cstream = newChunkedStreamReader(request.connection.reader) - let bstream = newBoundedStreamReader(cstream, uint64(maxBodySize), - comparison = BoundCmp.LessOrEqual) + let + maxBodySize = request.connection.server.maxRequestBodySize + cstream = newChunkedStreamReader(request.connection.reader) + bstream = newBoundedStreamReader(cstream, uint64(maxBodySize), + comparison = BoundCmp.LessOrEqual) ok(newHttpBodyReader(bstream, cstream)) else: err("Request do not have body available") proc handleExpect*(request: HttpRequestRef) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Handle expectation for ``Expect`` header. ## https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect if HttpServerFlags.NoExpectHandler notin request.connection.server.flags: @@ -504,85 +517,50 @@ proc handleExpect*(request: HttpRequestRef) {. try: let message = $request.version & " " & $Http100 & "\r\n\r\n" await request.connection.writer.write(message) - except CancelledError as exc: - raise exc except AsyncStreamError as exc: - raiseHttpCriticalError( + raiseHttpWriteError( "Unable to send `100-continue` response, reason: " & $exc.msg) proc getBody*(request: HttpRequestRef): Future[seq[byte]] {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, + HttpTransportError, HttpProtocolError]).} = ## Obtain request's body as sequence of bytes. - let bodyReader = request.getBodyReader() - if bodyReader.isErr(): + let reader = request.getBodyReader().valueOr: return @[] - else: - var reader = bodyReader.get() - try: - await request.handleExpect() - let res = await reader.read() - if reader.hasOverflow(): - await reader.closeWait() - reader = nil - raiseHttpCriticalError(MaximumBodySizeError, Http413) - else: - await reader.closeWait() - reader = nil - return res - except CancelledError as exc: - if not(isNil(reader)): - await reader.closeWait() - raise exc - except HttpCriticalError as exc: - if not(isNil(reader)): - await reader.closeWait() - raise exc - except AsyncStreamError as exc: - let msg = "Unable to read request's body, reason: " & $exc.msg - if not(isNil(reader)): - await reader.closeWait() - raiseHttpCriticalError(msg) + try: + await request.handleExpect() + let res = await reader.read() + if reader.hasOverflow(): + raiseHttpRequestBodyTooLargeError() + res + except AsyncStreamError as exc: + let msg = "Unable to read request's body, reason: " & $exc.msg + raiseHttpReadError(msg) + finally: + await reader.closeWait() proc consumeBody*(request: HttpRequestRef): Future[void] {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpTransportError, + HttpProtocolError]).} = ## Consume/discard request's body. - let bodyReader = request.getBodyReader() - if bodyReader.isErr(): + let reader = request.getBodyReader().valueOr: return - else: - var reader = bodyReader.get() - try: - await request.handleExpect() - discard await reader.consume() - if reader.hasOverflow(): - await reader.closeWait() - reader = nil - raiseHttpCriticalError(MaximumBodySizeError, Http413) - else: - await reader.closeWait() - reader = nil - return - except CancelledError as exc: - if not(isNil(reader)): - await reader.closeWait() - raise exc - except HttpCriticalError as exc: - if not(isNil(reader)): - await reader.closeWait() - raise exc - except AsyncStreamError as exc: - let msg = "Unable to consume request's body, reason: " & $exc.msg - if not(isNil(reader)): - await reader.closeWait() - raiseHttpCriticalError(msg) + try: + await request.handleExpect() + discard await reader.consume() + if reader.hasOverflow(): raiseHttpRequestBodyTooLargeError() + except AsyncStreamError as exc: + let msg = "Unable to consume request's body, reason: " & $exc.msg + raiseHttpReadError(msg) + finally: + await reader.closeWait() proc getAcceptInfo*(request: HttpRequestRef): Result[AcceptInfo, cstring] = ## Returns value of `Accept` header as `AcceptInfo` object. ## ## If ``Accept`` header is missing in request headers, ``*/*`` content ## type will be returned. - let acceptHeader = request.headers.getString(AcceptHeaderName) - getAcceptInfo(acceptHeader) + getAcceptInfo(request.headers.getString(AcceptHeaderName)) proc preferredContentMediaType*(acceptHeader: string): MediaType = ## Returns preferred content-type using ``Accept`` header value specified by @@ -693,7 +671,7 @@ proc preferredContentType*(request: HttpRequestRef, proc sendErrorResponse(conn: HttpConnectionRef, version: HttpVersion, code: HttpCode, keepAlive = true, - datatype = "text/text", + datatype = "text/plain", databody = "") {. async: (raises: [CancelledError]).} = var answer = $version & " " & $code & "\r\n" @@ -721,39 +699,10 @@ proc sendErrorResponse(conn: HttpConnectionRef, version: HttpVersion, answer.add(databody) try: await conn.writer.write(answer) - except CancelledError as exc: - raise exc except AsyncStreamError: # We ignore errors here, because we indicating error already. discard -proc sendErrorResponse( - conn: HttpConnectionRef, - reqFence: RequestFence, - respError: HttpProcessError - ): Future[HttpProcessExitType] {.async: (raises: []).} = - let version = getResponseVersion(reqFence) - try: - if reqFence.isOk(): - case respError.kind - of HttpServerError.CriticalError: - await conn.sendErrorResponse(version, respError.code, false) - HttpProcessExitType.Graceful - of HttpServerError.RecoverableError: - await conn.sendErrorResponse(version, respError.code, true) - HttpProcessExitType.Graceful - of HttpServerError.CatchableError: - await conn.sendErrorResponse(version, respError.code, false) - HttpProcessExitType.Graceful - of HttpServerError.DisconnectError, - HttpServerError.InterruptError, - HttpServerError.TimeoutError: - raiseAssert("Unexpected response error: " & $respError.kind) - else: - HttpProcessExitType.Graceful - except CancelledError: - HttpProcessExitType.Immediate - proc sendDefaultResponse( conn: HttpConnectionRef, reqFence: RequestFence, @@ -794,6 +743,10 @@ proc sendDefaultResponse( await conn.sendErrorResponse(HttpVersion11, Http409, keepConnection.toBool()) keepConnection + of HttpResponseState.ErrorCode: + # Response with error code + await conn.sendErrorResponse(version, response.status, false) + HttpProcessExitType.Immediate of HttpResponseState.Sending, HttpResponseState.Failed, HttpResponseState.Cancelled: # Just drop connection, because we dont know at what stage we are @@ -810,27 +763,21 @@ proc sendDefaultResponse( of HttpServerError.TimeoutError: await conn.sendErrorResponse(version, reqFence.error.code, false) HttpProcessExitType.Graceful - of HttpServerError.CriticalError: - await conn.sendErrorResponse(version, reqFence.error.code, false) - HttpProcessExitType.Graceful - of HttpServerError.RecoverableError: - await conn.sendErrorResponse(version, reqFence.error.code, false) - HttpProcessExitType.Graceful - of HttpServerError.CatchableError: + of HttpServerError.ProtocolError: await conn.sendErrorResponse(version, reqFence.error.code, false) HttpProcessExitType.Graceful of HttpServerError.DisconnectError: # When `HttpServerFlags.NotifyDisconnect` is set. HttpProcessExitType.Immediate of HttpServerError.InterruptError: + # InterruptError should be handled earlier raiseAssert("Unexpected request error: " & $reqFence.error.kind) except CancelledError: HttpProcessExitType.Immediate - except CatchableError: - HttpProcessExitType.Immediate proc getRequest(conn: HttpConnectionRef): Future[HttpRequestRef] {. - async: (raises: [CancelledError, HttpError]).} = + async: (raises: [CancelledError, HttpDisconnectError, + HttpProtocolError]).} = try: conn.buffer.setLen(conn.server.maxHeadersSize) let res = await conn.reader.readUntil(addr conn.buffer[0], len(conn.buffer), @@ -838,15 +785,11 @@ proc getRequest(conn: HttpConnectionRef): Future[HttpRequestRef] {. conn.buffer.setLen(res) let header = parseRequest(conn.buffer) if header.failed(): - raiseHttpCriticalError("Malformed request recieved") - else: - let res = prepareRequest(conn, header) - if res.isErr(): - raiseHttpCriticalError("Invalid request received", res.error) - else: - return res.get() + raiseHttpProtocolError(Http400, "Malformed request recieved") + prepareRequest(conn, header).valueOr: + raiseHttpProtocolError(error) except AsyncStreamLimitError: - raiseHttpCriticalError("Maximum size of request headers reached", Http431) + raiseHttpProtocolError(Http431, "Maximum size of request headers reached") except AsyncStreamError: raiseHttpDisconnectError() @@ -915,7 +858,7 @@ proc createConnection(server: HttpServerRef, HttpConnectionRef.new(server, transp) proc `keepalive=`*(resp: HttpResponseRef, value: bool) = - doAssert(resp.state == HttpResponseState.Empty) + doAssert(resp.getResponseState() == HttpResponseState.Empty) if value: resp.flags.incl(HttpResponseFlags.KeepAlive) else: @@ -935,55 +878,6 @@ proc getRemoteAddress(connection: HttpConnectionRef): Opt[TransportAddress] = if isNil(connection): return Opt.none(TransportAddress) getRemoteAddress(connection.transp) -proc getResponseFence*(connection: HttpConnectionRef, - reqFence: RequestFence): Future[ResponseFence] {. - async: (raises: []).} = - try: - let res = await connection.server.processCallback(reqFence) - ResponseFence.ok(res) - except CancelledError: - ResponseFence.err(HttpProcessError.init( - HttpServerError.InterruptError)) - except HttpCriticalError as exc: - let address = connection.getRemoteAddress() - ResponseFence.err(HttpProcessError.init( - HttpServerError.CriticalError, exc, address, exc.code)) - except HttpRecoverableError as exc: - let address = connection.getRemoteAddress() - ResponseFence.err(HttpProcessError.init( - HttpServerError.RecoverableError, exc, address, exc.code)) - except HttpResponseError as exc: - # There should be only 2 children of HttpResponseError, and all of them - # should be handled. - raiseAssert "Unexpected response error " & $exc.name & ", reason: " & - $exc.msg - -proc getResponseFence*(server: HttpServerRef, - connFence: ConnectionFence): Future[ResponseFence] {. - async: (raises: []).} = - doAssert(connFence.isErr()) - try: - let - reqFence = RequestFence.err(connFence.error) - res = await server.processCallback(reqFence) - ResponseFence.ok(res) - except CancelledError: - ResponseFence.err(HttpProcessError.init( - HttpServerError.InterruptError)) - except HttpCriticalError as exc: - let address = Opt.none(TransportAddress) - ResponseFence.err(HttpProcessError.init( - HttpServerError.CriticalError, exc, address, exc.code)) - except HttpRecoverableError as exc: - let address = Opt.none(TransportAddress) - ResponseFence.err(HttpProcessError.init( - HttpServerError.RecoverableError, exc, address, exc.code)) - except HttpResponseError as exc: - # There should be only 2 children of HttpResponseError, and all of them - # should be handled. - raiseAssert "Unexpected response error " & $exc.name & ", reason: " & - $exc.msg - proc getRequestFence*(server: HttpServerRef, connection: HttpConnectionRef): Future[RequestFence] {. async: (raises: []).} = @@ -996,27 +890,21 @@ proc getRequestFence*(server: HttpServerRef, connection.currentRawQuery = Opt.some(res.rawPath) RequestFence.ok(res) except CancelledError: - RequestFence.err(HttpProcessError.init(HttpServerError.InterruptError)) - except AsyncTimeoutError as exc: + RequestFence.err( + HttpProcessError.init(HttpServerError.InterruptError)) + except AsyncTimeoutError: let address = connection.getRemoteAddress() - RequestFence.err(HttpProcessError.init( - HttpServerError.TimeoutError, exc, address, Http408)) - except HttpRecoverableError as exc: + RequestFence.err( + HttpProcessError.init(HttpServerError.TimeoutError, address, Http408)) + except HttpProtocolError as exc: let address = connection.getRemoteAddress() - RequestFence.err(HttpProcessError.init( - HttpServerError.RecoverableError, exc, address, exc.code)) - except HttpCriticalError as exc: + RequestFence.err( + HttpProcessError.init(HttpServerError.ProtocolError, exc, address, + exc.code)) + except HttpDisconnectError: let address = connection.getRemoteAddress() - RequestFence.err(HttpProcessError.init( - HttpServerError.CriticalError, exc, address, exc.code)) - except HttpDisconnectError as exc: - let address = connection.getRemoteAddress() - RequestFence.err(HttpProcessError.init( - HttpServerError.DisconnectError, exc, address, Http400)) - except CatchableError as exc: - let address = connection.getRemoteAddress() - RequestFence.err(HttpProcessError.init( - HttpServerError.CatchableError, exc, address, Http500)) + RequestFence.err( + HttpProcessError.init(HttpServerError.DisconnectError, address, Http400)) proc getConnectionFence*(server: HttpServerRef, transp: StreamTransport): Future[ConnectionFence] {. @@ -1026,16 +914,11 @@ proc getConnectionFence*(server: HttpServerRef, ConnectionFence.ok(res) except CancelledError: ConnectionFence.err(HttpProcessError.init(HttpServerError.InterruptError)) - except HttpCriticalError as exc: + except HttpConnectionError as exc: # On error `transp` will be closed by `createConnCallback()` call. let address = Opt.none(TransportAddress) ConnectionFence.err(HttpProcessError.init( - HttpServerError.CriticalError, exc, address, exc.code)) - except CatchableError as exc: - # On error `transp` will be closed by `createConnCallback()` call. - let address = Opt.none(TransportAddress) - ConnectionFence.err(HttpProcessError.init( - HttpServerError.CriticalError, exc, address, Http503)) + HttpServerError.DisconnectError, exc, address, Http400)) proc processRequest(server: HttpServerRef, connection: HttpConnectionRef, @@ -1045,30 +928,28 @@ proc processRequest(server: HttpServerRef, if requestFence.isErr(): case requestFence.error.kind of HttpServerError.InterruptError: + # Cancelled, exiting return HttpProcessExitType.Immediate of HttpServerError.DisconnectError: + # Remote peer disconnected if HttpServerFlags.NotifyDisconnect notin server.flags: return HttpProcessExitType.Immediate else: + # Request is incorrect or unsupported, sending notification discard - let responseFence = await getResponseFence(connection, requestFence) - if responseFence.isErr() and - (responseFence.error.kind == HttpServerError.InterruptError): + try: + let response = + try: + await connection.server.processCallback(requestFence) + except CancelledError: + # Cancelled, exiting + return HttpProcessExitType.Immediate + + await connection.sendDefaultResponse(requestFence, response) + finally: if requestFence.isOk(): await requestFence.get().closeWait() - return HttpProcessExitType.Immediate - - let res = - if responseFence.isErr(): - await connection.sendErrorResponse(requestFence, responseFence.error) - else: - await connection.sendDefaultResponse(requestFence, responseFence.get()) - - if requestFence.isOk(): - await requestFence.get().closeWait() - - res proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} = let @@ -1077,10 +958,11 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} = connectionId = holder.connectionId connection = block: - let res = await server.getConnectionFence(transp) + let res = await getConnectionFence(server, transp) if res.isErr(): if res.error.kind != HttpServerError.InterruptError: - discard await server.getResponseFence(res) + discard await noCancel( + server.processCallback(RequestFence.err(res.error))) server.connections.del(connectionId) return res.get() @@ -1089,13 +971,7 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} = var runLoop = HttpProcessExitType.KeepAlive while runLoop == HttpProcessExitType.KeepAlive: - runLoop = - try: - await server.processRequest(connection, connectionId) - except CancelledError: - HttpProcessExitType.Immediate - except CatchableError as exc: - raiseAssert "Unexpected error [" & $exc.name & "] happens: " & $exc.msg + runLoop = await server.processRequest(connection, connectionId) case runLoop of HttpProcessExitType.KeepAlive: @@ -1104,7 +980,6 @@ proc processLoop(holder: HttpConnectionHolderRef) {.async: (raises: []).} = await connection.closeWait() of HttpProcessExitType.Graceful: await connection.gracefulCloseWait() - server.connections.del(connectionId) proc acceptClientLoop(server: HttpServerRef) {.async: (raises: []).} = @@ -1210,89 +1085,84 @@ proc getMultipartReader*(req: HttpRequestRef): HttpResult[MultiPartReaderRef] = err("Request's method do not supports multipart") proc post*(req: HttpRequestRef): Future[HttpTable] {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpTransportError, + HttpProtocolError]).} = ## Return POST parameters if req.postTable.isSome(): return req.postTable.get() - else: - if req.meth notin PostMethods: - return HttpTable.init() - if UrlencodedForm in req.requestFlags: - let queryFlags = - if QueryCommaSeparatedArray in req.connection.server.flags: - {QueryParamsFlag.CommaSeparatedArray} - else: - {} - var table = HttpTable.init() - # getBody() will handle `Expect`. - var body = await req.getBody() - # TODO (cheatfate) double copy here, because of `byte` to `char` - # conversion. - var strbody = newString(len(body)) - if len(body) > 0: - copyMem(addr strbody[0], addr body[0], len(body)) - for key, value in queryParams(strbody, queryFlags): - table.add(key, value) - req.postTable = Opt.some(table) - return table - elif MultipartForm in req.requestFlags: - var table = HttpTable.init() - let res = getMultipartReader(req) - if res.isErr(): - raiseHttpCriticalError("Unable to retrieve multipart form data") - var mpreader = res.get() + if req.meth notin PostMethods: + return HttpTable.init() - # We must handle `Expect` first. + if UrlencodedForm in req.requestFlags: + let queryFlags = + if QueryCommaSeparatedArray in req.connection.server.flags: + {QueryParamsFlag.CommaSeparatedArray} + else: + {} + var table = HttpTable.init() + # getBody() will handle `Expect`. + var body = await req.getBody() + # TODO (cheatfate) double copy here, because of `byte` to `char` + # conversion. + var strbody = newString(len(body)) + if len(body) > 0: + copyMem(addr strbody[0], addr body[0], len(body)) + for key, value in queryParams(strbody, queryFlags): + table.add(key, value) + req.postTable = Opt.some(table) + return table + elif MultipartForm in req.requestFlags: + var table = HttpTable.init() + let mpreader = getMultipartReader(req).valueOr: + raiseHttpProtocolError(Http400, + "Unable to retrieve multipart form data, reason: " & $error) + # Reading multipart/form-data parts. + var runLoop = true + while runLoop: + var part: MultiPart try: - await req.handleExpect() - except CancelledError as exc: - await mpreader.closeWait() - raise exc - except HttpCriticalError as exc: - await mpreader.closeWait() - raise exc + part = await mpreader.readPart() + var value = await part.getBody() - # Reading multipart/form-data parts. - var runLoop = true - while runLoop: - var part: MultiPart - try: - part = await mpreader.readPart() - var value = await part.getBody() - # TODO (cheatfate) double copy here, because of `byte` to `char` - # conversion. - var strvalue = newString(len(value)) - if len(value) > 0: - copyMem(addr strvalue[0], addr value[0], len(value)) - table.add(part.name, strvalue) + # TODO (cheatfate) double copy here, because of `byte` to `char` + # conversion. + var strvalue = newString(len(value)) + if len(value) > 0: + copyMem(addr strvalue[0], addr value[0], len(value)) + table.add(part.name, strvalue) + await part.closeWait() + except MultipartEOMError: + runLoop = false + except HttpWriteError as exc: + if not(part.isEmpty()): await part.closeWait() - except MultipartEOMError: - runLoop = false - except HttpCriticalError as exc: - if not(part.isEmpty()): - await part.closeWait() - await mpreader.closeWait() - raise exc - except CancelledError as exc: - if not(part.isEmpty()): - await part.closeWait() - await mpreader.closeWait() - raise exc - await mpreader.closeWait() - req.postTable = Opt.some(table) - return table - else: - if HttpRequestFlags.BoundBody in req.requestFlags: - if req.contentLength != 0: - raiseHttpCriticalError("Unsupported request body") - return HttpTable.init() - elif HttpRequestFlags.UnboundBody in req.requestFlags: - raiseHttpCriticalError("Unsupported request body") + await mpreader.closeWait() + raise exc + except HttpProtocolError as exc: + if not(part.isEmpty()): + await part.closeWait() + await mpreader.closeWait() + raise exc + except CancelledError as exc: + if not(part.isEmpty()): + await part.closeWait() + await mpreader.closeWait() + raise exc + await mpreader.closeWait() + req.postTable = Opt.some(table) + return table + else: + if HttpRequestFlags.BoundBody in req.requestFlags: + if req.contentLength != 0: + raiseHttpProtocolError(Http400, "Unsupported request body") + return HttpTable.init() + elif HttpRequestFlags.UnboundBody in req.requestFlags: + raiseHttpProtocolError(Http400, "Unsupported request body") proc setHeader*(resp: HttpResponseRef, key, value: string) = ## Sets value of header ``key`` to ``value``. - doAssert(resp.state == HttpResponseState.Empty) + doAssert(resp.getResponseState() == HttpResponseState.Empty) resp.headersTable.set(key, value) proc setHeaderDefault*(resp: HttpResponseRef, key, value: string) = @@ -1302,7 +1172,7 @@ proc setHeaderDefault*(resp: HttpResponseRef, key, value: string) = proc addHeader*(resp: HttpResponseRef, key, value: string) = ## Adds value ``value`` to header's ``key`` value. - doAssert(resp.state == HttpResponseState.Empty) + doAssert(resp.getResponseState() == HttpResponseState.Empty) resp.headersTable.add(key, value) proc getHeader*(resp: HttpResponseRef, key: string, @@ -1316,8 +1186,22 @@ proc hasHeader*(resp: HttpResponseRef, key: string): bool = key in resp.headersTable template checkPending(t: untyped) = - if t.state != HttpResponseState.Empty: - raiseHttpCriticalError("Response body was already sent") + let currentState = t.getResponseState() + doAssert(currentState == HttpResponseState.Empty, + "Response body was already sent [" & $currentState & "]") + +template checkStreamResponse(t: untyped) = + doAssert(HttpResponseFlags.Stream in t.flags, + "Response was not prepared") + +template checkStreamResponseState(t: untyped) = + doAssert(t.getResponseState() in + {HttpResponseState.Prepared, HttpResponseState.Sending}, + "Response is in the wrong state") + +template checkPointerLength(t1, t2: untyped) = + doAssert(not(isNil(t1)), "pbytes must not be nil") + doAssert(t2 >= 0, "nbytes should be bigger or equal to zero") func createHeaders(resp: HttpResponseRef): string = var answer = $(resp.version) & " " & $(resp.status) & "\r\n" @@ -1386,69 +1270,68 @@ proc preparePlainHeaders(resp: HttpResponseRef): string = resp.createHeaders() proc sendBody*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Send HTTP response at once by using bytes pointer ``pbytes`` and length ## ``nbytes``. - doAssert(not(isNil(pbytes)), "pbytes must not be nil") - doAssert(nbytes >= 0, "nbytes should be bigger or equal to zero") + checkPointerLength(pbytes, nbytes) checkPending(resp) let responseHeaders = resp.prepareLengthHeaders(nbytes) - resp.state = HttpResponseState.Prepared + resp.setResponseState(HttpResponseState.Prepared) try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.connection.writer.write(responseHeaders) if nbytes > 0: await resp.connection.writer.write(pbytes, nbytes) - resp.state = HttpResponseState.Finished + resp.setResponseState(HttpResponseState.Finished) except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc sendBody*(resp: HttpResponseRef, data: ByteChar) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Send HTTP response at once by using data ``data``. checkPending(resp) let responseHeaders = resp.prepareLengthHeaders(len(data)) - resp.state = HttpResponseState.Prepared + resp.setResponseState(HttpResponseState.Prepared) try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.connection.writer.write(responseHeaders) if len(data) > 0: await resp.connection.writer.write(data) - resp.state = HttpResponseState.Finished + resp.setResponseState(HttpResponseState.Finished) except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc sendError*(resp: HttpResponseRef, code: HttpCode, body = "") {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Send HTTP error status response. checkPending(resp) resp.status = code let responseHeaders = resp.prepareLengthHeaders(len(body)) - resp.state = HttpResponseState.Prepared + resp.setResponseState(HttpResponseState.Prepared) try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.connection.writer.write(responseHeaders) if len(body) > 0: await resp.connection.writer.write(body) - resp.state = HttpResponseState.Finished + resp.setResponseState(HttpResponseState.Finished) except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc prepare*(resp: HttpResponseRef, streamType = HttpResponseStreamType.Chunked) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Prepare for HTTP stream response. ## ## Such responses will be sent chunk by chunk using ``chunked`` encoding. @@ -1462,9 +1345,9 @@ proc prepare*(resp: HttpResponseRef, of HttpResponseStreamType.Chunked: resp.prepareChunkedHeaders() resp.streamType = streamType - resp.state = HttpResponseState.Prepared + resp.setResponseState(HttpResponseState.Prepared) try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.connection.writer.write(responseHeaders) case streamType of HttpResponseStreamType.Plain, HttpResponseStreamType.SSE: @@ -1473,117 +1356,105 @@ proc prepare*(resp: HttpResponseRef, resp.writer = newChunkedStreamWriter(resp.connection.writer) resp.flags.incl(HttpResponseFlags.Stream) except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc prepareChunked*(resp: HttpResponseRef): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Prepare for HTTP chunked stream response. ## ## Such responses will be sent chunk by chunk using ``chunked`` encoding. resp.prepare(HttpResponseStreamType.Chunked) proc preparePlain*(resp: HttpResponseRef): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Prepare for HTTP plain stream response. ## ## Such responses will be sent without any encoding. resp.prepare(HttpResponseStreamType.Plain) proc prepareSSE*(resp: HttpResponseRef): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Prepare for HTTP server-side event stream response. resp.prepare(HttpResponseStreamType.SSE) proc send*(resp: HttpResponseRef, pbytes: pointer, nbytes: int) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Send single chunk of data pointed by ``pbytes`` and ``nbytes``. - doAssert(not(isNil(pbytes)), "pbytes must not be nil") - doAssert(nbytes >= 0, "nbytes should be bigger or equal to zero") - if HttpResponseFlags.Stream notin resp.flags: - raiseHttpCriticalError("Response was not prepared") - if resp.state notin {HttpResponseState.Prepared, HttpResponseState.Sending}: - raiseHttpCriticalError("Response in incorrect state") + checkPointerLength(pbytes, nbytes) + resp.checkStreamResponse() + resp.checkStreamResponseState() try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.writer.write(pbytes, nbytes) - resp.state = HttpResponseState.Sending except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc send*(resp: HttpResponseRef, data: ByteChar) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Send single chunk of data ``data``. - if HttpResponseFlags.Stream notin resp.flags: - raiseHttpCriticalError("Response was not prepared") - if resp.state notin {HttpResponseState.Prepared, HttpResponseState.Sending}: - raiseHttpCriticalError("Response in incorrect state") + resp.checkStreamResponse() + resp.checkStreamResponseState() try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.writer.write(data) - resp.state = HttpResponseState.Sending except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc sendChunk*(resp: HttpResponseRef, pbytes: pointer, nbytes: int): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = resp.send(pbytes, nbytes) proc sendChunk*(resp: HttpResponseRef, data: ByteChar): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = resp.send(data) proc sendEvent*(resp: HttpResponseRef, eventName: string, data: string): Future[void] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Send server-side event with name ``eventName`` and payload ``data`` to ## remote peer. - let data = - block: - var res = "" - if len(eventName) > 0: - res.add("event: ") - res.add(eventName) - res.add("\r\n") - res.add("data: ") - res.add(data) - res.add("\r\n\r\n") - res - resp.send(data) + var res = "" + if len(eventName) > 0: + res.add("event: ") + res.add(eventName) + res.add("\r\n") + res.add("data: ") + res.add(data) + res.add("\r\n\r\n") + resp.send(res) proc finish*(resp: HttpResponseRef) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Sending last chunk of data, so it will indicate end of HTTP response. - if HttpResponseFlags.Stream notin resp.flags: - raiseHttpCriticalError("Response was not prepared") - if resp.state notin {HttpResponseState.Prepared, HttpResponseState.Sending}: - raiseHttpCriticalError("Response in incorrect state") + resp.checkStreamResponse() + resp.checkStreamResponseState() try: - resp.state = HttpResponseState.Sending + resp.setResponseState(HttpResponseState.Sending) await resp.writer.finish() - resp.state = HttpResponseState.Finished + resp.setResponseState(HttpResponseState.Finished) except CancelledError as exc: - resp.state = HttpResponseState.Cancelled + resp.setResponseState(HttpResponseState.Cancelled) raise exc except AsyncStreamError as exc: - resp.state = HttpResponseState.Failed - raiseHttpCriticalError("Unable to send response, reason: " & $exc.msg) + resp.setResponseState(HttpResponseState.Failed) + raiseHttpWriteError("Unable to send response, reason: " & $exc.msg) proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar, headers: HttpTable): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with the specified ``HttpCode``, HTTP ``headers`` ## and ``content``. let response = req.getResponse() @@ -1595,18 +1466,18 @@ proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar, proc respond*(req: HttpRequestRef, code: HttpCode, content: ByteChar): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with specified ``HttpCode`` and ``content``. respond(req, code, content, HttpTable.init()) proc respond*(req: HttpRequestRef, code: HttpCode): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with specified ``HttpCode`` only. respond(req, code, "", HttpTable.init()) proc redirect*(req: HttpRequestRef, code: HttpCode, location: string, headers: HttpTable): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with redirection to location ``location`` and ## additional headers ``headers``. ## @@ -1618,7 +1489,7 @@ proc redirect*(req: HttpRequestRef, code: HttpCode, proc redirect*(req: HttpRequestRef, code: HttpCode, location: Uri, headers: HttpTable): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with redirection to location ``location`` and ## additional headers ``headers``. ## @@ -1628,13 +1499,13 @@ proc redirect*(req: HttpRequestRef, code: HttpCode, proc redirect*(req: HttpRequestRef, code: HttpCode, location: Uri): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with redirection to location ``location``. redirect(req, code, location, HttpTable.init()) proc redirect*(req: HttpRequestRef, code: HttpCode, location: string): Future[HttpResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpCriticalError]).} = + async: (raw: true, raises: [CancelledError, HttpWriteError]).} = ## Responds to the request with redirection to location ``location``. redirect(req, code, location, HttpTable.init()) diff --git a/chronos/apps/http/multipart.nim b/chronos/apps/http/multipart.nim index 83a4b56..302d6ef 100644 --- a/chronos/apps/http/multipart.nim +++ b/chronos/apps/http/multipart.nim @@ -18,7 +18,8 @@ import "."/[httptable, httpcommon, httpbodyrw] export asyncloop, httptable, httpcommon, httpbodyrw, asyncstream, httputils const - UnableToReadMultipartBody = "Unable to read multipart message body" + UnableToReadMultipartBody = "Unable to read multipart message body, reason: " + UnableToSendMultipartMessage = "Unable to send multipart message, reason: " type MultiPartSource* {.pure.} = enum @@ -69,7 +70,7 @@ type name*: string filename*: string - MultipartError* = object of HttpCriticalError + MultipartError* = object of HttpProtocolError MultipartEOMError* = object of MultipartError BChar* = byte | char @@ -105,7 +106,7 @@ func setPartNames(part: var MultiPart): HttpResult[void] = return err("Content-Disposition header value is incorrect") let dtype = disp.dispositionType(header.toOpenArrayByte(0, len(header) - 1)) if dtype.toLowerAscii() != "form-data": - return err("Content-Disposition type is incorrect") + return err("Content-Disposition header type is incorrect") for k, v in disp.fields(header.toOpenArrayByte(0, len(header) - 1)): case k.toLowerAscii() of "name": @@ -171,8 +172,17 @@ proc new*[B: BChar](mpt: typedesc[MultiPartReaderRef], stream: stream, offset: 0, boundary: fboundary, buffer: newSeq[byte](partHeadersMaxSize)) +template handleAsyncStreamReaderError(targ, excarg: untyped) = + if targ.hasOverflow(): + raiseHttpRequestBodyTooLargeError() + raiseHttpReadError(UnableToReadMultipartBody & $excarg.msg) + +template handleAsyncStreamWriterError(targ, excarg: untyped) = + targ.state = MultiPartWriterState.MessageFailure + raiseHttpWriteError(UnableToSendMultipartMessage & $excarg.msg) + proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpReadError, HttpProtocolError]).} = doAssert(mpr.kind == MultiPartSource.Stream) if mpr.firstTime: try: @@ -181,14 +191,11 @@ proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {. mpr.firstTime = false if not(startsWith(mpr.buffer.toOpenArray(0, len(mpr.boundary) - 3), mpr.boundary.toOpenArray(2, len(mpr.boundary) - 1))): - raiseHttpCriticalError("Unexpected boundary encountered") + raiseHttpProtocolError(Http400, "Unexpected boundary encountered") except CancelledError as exc: raise exc - except AsyncStreamError: - if mpr.stream.hasOverflow(): - raiseHttpCriticalError(MaximumBodySizeError, Http413) - else: - raiseHttpCriticalError(UnableToReadMultipartBody) + except AsyncStreamError as exc: + handleAsyncStreamReaderError(mpr.stream, exc) # Reading part's headers try: @@ -202,9 +209,9 @@ proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {. raise newException(MultipartEOMError, "End of multipart message") else: - raiseHttpCriticalError("Incorrect multipart header found") + raiseHttpProtocolError(Http400, "Incorrect multipart header found") if mpr.buffer[0] != 0x0D'u8 or mpr.buffer[1] != 0x0A'u8: - raiseHttpCriticalError("Incorrect multipart boundary found") + raiseHttpProtocolError(Http400, "Incorrect multipart boundary found") # If two bytes are CRLF we are at the part beginning. # Reading part's headers @@ -212,7 +219,7 @@ proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {. HeadersMark) var headersList = parseHeaders(mpr.buffer.toOpenArray(0, res - 1), false) if headersList.failed(): - raiseHttpCriticalError("Incorrect multipart's headers found") + raiseHttpProtocolError(Http400, "Incorrect multipart's headers found") inc(mpr.counter) var part = MultiPart( @@ -228,45 +235,35 @@ proc readPart*(mpr: MultiPartReaderRef): Future[MultiPart] {. let sres = part.setPartNames() if sres.isErr(): - raiseHttpCriticalError($sres.error) + raiseHttpProtocolError(Http400, $sres.error) return part except CancelledError as exc: raise exc - except AsyncStreamError: - if mpr.stream.hasOverflow(): - raiseHttpCriticalError(MaximumBodySizeError, Http413) - else: - raiseHttpCriticalError(UnableToReadMultipartBody) + except AsyncStreamError as exc: + handleAsyncStreamReaderError(mpr.stream, exc) proc getBody*(mp: MultiPart): Future[seq[byte]] {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpReadError, HttpProtocolError]).} = ## Get multipart's ``mp`` value as sequence of bytes. case mp.kind of MultiPartSource.Stream: try: - let res = await mp.stream.read() - return res - except AsyncStreamError: - if mp.breader.hasOverflow(): - raiseHttpCriticalError(MaximumBodySizeError, Http413) - else: - raiseHttpCriticalError(UnableToReadMultipartBody) + await mp.stream.read() + except AsyncStreamError as exc: + handleAsyncStreamReaderError(mp.breader, exc) of MultiPartSource.Buffer: - return mp.buffer + mp.buffer proc consumeBody*(mp: MultiPart) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpReadError, HttpProtocolError]).} = ## Discard multipart's ``mp`` value. case mp.kind of MultiPartSource.Stream: try: discard await mp.stream.consume() - except AsyncStreamError: - if mp.breader.hasOverflow(): - raiseHttpCriticalError(MaximumBodySizeError, Http413) - else: - raiseHttpCriticalError(UnableToReadMultipartBody) + except AsyncStreamError as exc: + handleAsyncStreamReaderError(mp.breader, exc) of MultiPartSource.Buffer: discard @@ -533,7 +530,7 @@ proc new*[B: BChar](mpt: typedesc[MultiPartWriterRef], proc prepareHeaders(partMark: openArray[byte], name: string, filename: string, headers: HttpTable): string = - const ContentDisposition = "Content-Disposition" + const ContentDispositionHeader = "Content-Disposition" let qname = block: let res = quoteCheck(name) @@ -546,10 +543,10 @@ proc prepareHeaders(partMark: openArray[byte], name: string, filename: string, res.get() var buffer = newString(len(partMark)) copyMem(addr buffer[0], unsafeAddr partMark[0], len(partMark)) - buffer.add(ContentDisposition) + buffer.add(ContentDispositionHeader) buffer.add(": ") - if ContentDisposition in headers: - buffer.add(headers.getString(ContentDisposition)) + if ContentDispositionHeader in headers: + buffer.add(headers.getString(ContentDispositionHeader)) buffer.add("\r\n") else: buffer.add("form-data; name=\"") @@ -562,7 +559,7 @@ proc prepareHeaders(partMark: openArray[byte], name: string, filename: string, buffer.add("\r\n") for k, v in headers.stringItems(): - if k != toLowerAscii(ContentDisposition): + if k != ContentDispositionHeader: if len(v) > 0: buffer.add(k) buffer.add(": ") @@ -572,7 +569,7 @@ proc prepareHeaders(partMark: openArray[byte], name: string, filename: string, buffer proc begin*(mpw: MultiPartWriterRef) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Starts multipart message form and write approprate markers to output ## stream. doAssert(mpw.kind == MultiPartSource.Stream) @@ -580,10 +577,9 @@ proc begin*(mpw: MultiPartWriterRef) {. # write "--" try: await mpw.stream.write(mpw.beginMark) - except AsyncStreamError: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to start multipart message") - mpw.state = MultiPartWriterState.MessageStarted + mpw.state = MultiPartWriterState.MessageStarted + except AsyncStreamError as exc: + handleAsyncStreamWriterError(mpw, exc) proc begin*(mpw: var MultiPartWriter) = ## Starts multipart message form and write approprate markers to output @@ -596,7 +592,7 @@ proc begin*(mpw: var MultiPartWriter) = proc beginPart*(mpw: MultiPartWriterRef, name: string, filename: string, headers: HttpTable) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Starts part of multipart message and write appropriate ``headers`` to the ## output stream. ## @@ -611,9 +607,8 @@ proc beginPart*(mpw: MultiPartWriterRef, name: string, try: await mpw.stream.write(buffer) mpw.state = MultiPartWriterState.PartStarted - except AsyncStreamError: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError("Unable to start multipart part") + except AsyncStreamError as exc: + handleAsyncStreamWriterError(mpw, exc) proc beginPart*(mpw: var MultiPartWriter, name: string, filename: string, headers: HttpTable) = @@ -632,7 +627,7 @@ proc beginPart*(mpw: var MultiPartWriter, name: string, mpw.state = MultiPartWriterState.PartStarted proc write*(mpw: MultiPartWriterRef, pbytes: pointer, nbytes: int) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Write part's data ``data`` to the output stream. doAssert(mpw.kind == MultiPartSource.Stream) doAssert(mpw.state == MultiPartWriterState.PartStarted) @@ -640,12 +635,10 @@ proc write*(mpw: MultiPartWriterRef, pbytes: pointer, nbytes: int) {. # write of data await mpw.stream.write(pbytes, nbytes) except AsyncStreamError as exc: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError( - "Unable to write multipart data, reason: " & $exc.msg) + handleAsyncStreamWriterError(mpw, exc) proc write*(mpw: MultiPartWriterRef, data: seq[byte]) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Write part's data ``data`` to the output stream. doAssert(mpw.kind == MultiPartSource.Stream) doAssert(mpw.state == MultiPartWriterState.PartStarted) @@ -653,12 +646,10 @@ proc write*(mpw: MultiPartWriterRef, data: seq[byte]) {. # write of data await mpw.stream.write(data) except AsyncStreamError as exc: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError( - "Unable to write multipart data, reason: " & $exc.msg) + handleAsyncStreamWriterError(mpw, exc) proc write*(mpw: MultiPartWriterRef, data: string) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Write part's data ``data`` to the output stream. doAssert(mpw.kind == MultiPartSource.Stream) doAssert(mpw.state == MultiPartWriterState.PartStarted) @@ -666,9 +657,7 @@ proc write*(mpw: MultiPartWriterRef, data: string) {. # write of data await mpw.stream.write(data) except AsyncStreamError as exc: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError( - "Unable to write multipart data, reason: " & $exc.msg) + handleAsyncStreamWriterError(mpw, exc) proc write*(mpw: var MultiPartWriter, pbytes: pointer, nbytes: int) = ## Write part's data ``data`` to the output stream. @@ -692,7 +681,7 @@ proc write*(mpw: var MultiPartWriter, data: openArray[char]) = mpw.buffer.add(data.toOpenArrayByte(0, len(data) - 1)) proc finishPart*(mpw: MultiPartWriterRef) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Finish multipart's message part and send proper markers to output stream. doAssert(mpw.state == MultiPartWriterState.PartStarted) try: @@ -700,9 +689,7 @@ proc finishPart*(mpw: MultiPartWriterRef) {. await mpw.stream.write(mpw.finishPartMark) mpw.state = MultiPartWriterState.PartFinished except AsyncStreamError as exc: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError( - "Unable to finish multipart message part, reason: " & $exc.msg) + handleAsyncStreamWriterError(mpw, exc) proc finishPart*(mpw: var MultiPartWriter) = ## Finish multipart's message part and send proper markers to output stream. @@ -713,7 +700,7 @@ proc finishPart*(mpw: var MultiPartWriter) = mpw.state = MultiPartWriterState.PartFinished proc finish*(mpw: MultiPartWriterRef) {. - async: (raises: [CancelledError, HttpCriticalError]).} = + async: (raises: [CancelledError, HttpWriteError]).} = ## Finish multipart's message form and send finishing markers to the output ## stream. doAssert(mpw.kind == MultiPartSource.Stream) @@ -723,9 +710,7 @@ proc finish*(mpw: MultiPartWriterRef) {. await mpw.stream.write(mpw.finishMark) mpw.state = MultiPartWriterState.MessageFinished except AsyncStreamError as exc: - mpw.state = MultiPartWriterState.MessageFailure - raiseHttpCriticalError( - "Unable to finish multipart message, reason: " & $exc.msg) + handleAsyncStreamWriterError(mpw, exc) proc finish*(mpw: var MultiPartWriter): seq[byte] = ## Finish multipart's message form and send finishing markers to the output diff --git a/chronos/apps/http/shttpserver.nim b/chronos/apps/http/shttpserver.nim index f7e377f..6272bb2 100644 --- a/chronos/apps/http/shttpserver.nim +++ b/chronos/apps/http/shttpserver.nim @@ -164,22 +164,21 @@ proc new*(htype: typedesc[SecureHttpServerRef], maxRequestBodySize: int = 1_048_576, dualstack = DualStackType.Auto ): HttpResult[SecureHttpServerRef] {. - deprecated: "raises missing from process callback".} = - proc processCallback2(req: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = - try: - await processCallback(req) - except CancelledError as exc: - raise exc - except HttpResponseError as exc: - raise exc - except CatchableError as exc: - # Emulate 3.x behavior - raise (ref HttpCriticalError)(msg: exc.msg, code: Http503) + deprecated: "Callback could raise only CancelledError, annotate with " & + "{.async: (raises: [CancelledError]).}".} = + + proc wrap(req: RequestFence): Future[HttpResponseRef] {. + async: (raises: [CancelledError]).} = + try: + await processCallback(req) + except CancelledError as exc: + raise exc + except CatchableError as exc: + defaultResponse(exc) SecureHttpServerRef.new( address = address, - processCallback = processCallback2, + processCallback = wrap, tlsPrivateKey = tlsPrivateKey, tlsCertificate = tlsCertificate, serverFlags = serverFlags, @@ -194,4 +193,4 @@ proc new*(htype: typedesc[SecureHttpServerRef], maxHeadersSize = maxHeadersSize, maxRequestBodySize = maxRequestBodySize, dualstack = dualstack - ) \ No newline at end of file + ) diff --git a/tests/testhttpclient.nim b/tests/testhttpclient.nim index d2a355d..967f896 100644 --- a/tests/testhttpclient.nim +++ b/tests/testhttpclient.nim @@ -85,7 +85,8 @@ suite "HTTP client testing suite": res proc createServer(address: TransportAddress, - process: HttpProcessCallback2, secure: bool): HttpServerRef = + process: HttpProcessCallback2, + secure: bool): HttpServerRef = let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} serverFlags = {HttpServerFlags.Http11Pipeline} @@ -128,18 +129,24 @@ suite "HTTP client testing suite": (MethodPatch, "/test/patch") ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() case request.uri.path of "/test/get", "/test/post", "/test/head", "/test/put", "/test/delete", "/test/trace", "/test/options", "/test/connect", "/test/patch", "/test/error": - return await request.respond(Http200, request.uri.path) + try: + await request.respond(Http200, request.uri.path) + except HttpWriteError as exc: + defaultResponse(exc) else: - return await request.respond(Http404, "Page not found") + try: + await request.respond(Http404, "Page not found") + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -195,7 +202,7 @@ suite "HTTP client testing suite": "LONGCHUNKRESPONSE") ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() case request.uri.path @@ -203,46 +210,58 @@ suite "HTTP client testing suite": var response = request.getResponse() var data = createBigMessage(ResponseTests[0][4], ResponseTests[0][2]) response.status = Http200 - await response.sendBody(data) - return response + try: + await response.sendBody(data) + except HttpWriteError as exc: + return defaultResponse(exc) + response of "/test/long_size_response": var response = request.getResponse() var data = createBigMessage(ResponseTests[1][4], ResponseTests[1][2]) response.status = Http200 - await response.sendBody(data) - return response + try: + await response.sendBody(data) + except HttpWriteError as exc: + return defaultResponse(exc) + response of "/test/short_chunked_response": var response = request.getResponse() var data = createBigMessage(ResponseTests[2][4], ResponseTests[2][2]) response.status = Http200 - await response.prepare() - var offset = 0 - while true: - if len(data) == offset: - break - let toWrite = min(1024, len(data) - offset) - await response.sendChunk(addr data[offset], toWrite) - offset = offset + toWrite - await response.finish() - return response + try: + await response.prepare() + var offset = 0 + while true: + if len(data) == offset: + break + let toWrite = min(1024, len(data) - offset) + await response.sendChunk(addr data[offset], toWrite) + offset = offset + toWrite + await response.finish() + except HttpWriteError as exc: + return defaultResponse(exc) + response of "/test/long_chunked_response": var response = request.getResponse() var data = createBigMessage(ResponseTests[3][4], ResponseTests[3][2]) response.status = Http200 - await response.prepare() - var offset = 0 - while true: - if len(data) == offset: - break - let toWrite = min(1024, len(data) - offset) - await response.sendChunk(addr data[offset], toWrite) - offset = offset + toWrite - await response.finish() - return response + try: + await response.prepare() + var offset = 0 + while true: + if len(data) == offset: + break + let toWrite = min(1024, len(data) - offset) + await response.sendChunk(addr data[offset], toWrite) + offset = offset + toWrite + await response.finish() + except HttpWriteError as exc: + return defaultResponse(exc) + response else: - return await request.respond(Http404, "Page not found") + defaultResponse() else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -311,21 +330,26 @@ suite "HTTP client testing suite": (MethodPost, "/test/big_request", 262400) ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() case request.uri.path of "/test/big_request": - if request.hasBody(): - let body = await request.getBody() - let digest = $secureHash(string.fromBytes(body)) - return await request.respond(Http200, digest) - else: - return await request.respond(Http400, "Missing content body") + try: + if request.hasBody(): + let body = await request.getBody() + let digest = $secureHash(string.fromBytes(body)) + await request.respond(Http200, digest) + else: + await request.respond(Http400, "Missing content body") + except HttpProtocolError as exc: + defaultResponse(exc) + except HttpTransportError as exc: + defaultResponse(exc) else: - return await request.respond(Http404, "Page not found") + defaultResponse() else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -381,21 +405,27 @@ suite "HTTP client testing suite": (MethodPost, "/test/big_chunk_request", 262400) ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() case request.uri.path of "/test/big_chunk_request": - if request.hasBody(): - let body = await request.getBody() - let digest = $secureHash(string.fromBytes(body)) - return await request.respond(Http200, digest) - else: - return await request.respond(Http400, "Missing content body") + try: + if request.hasBody(): + let + body = await request.getBody() + digest = $secureHash(string.fromBytes(body)) + await request.respond(Http200, digest) + else: + await request.respond(Http400, "Missing content body") + except HttpProtocolError as exc: + defaultResponse(exc) + except HttpTransportError as exc: + defaultResponse(exc) else: - return await request.respond(Http404, "Page not found") + defaultResponse() else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -455,23 +485,28 @@ suite "HTTP client testing suite": ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() case request.uri.path of "/test/post/urlencoded_size", "/test/post/urlencoded_chunked": - if request.hasBody(): - var postTable = await request.post() - let body = postTable.getString("field1") & ":" & - postTable.getString("field2") & ":" & - postTable.getString("field3") - return await request.respond(Http200, body) - else: - return await request.respond(Http400, "Missing content body") + try: + if request.hasBody(): + var postTable = await request.post() + let body = postTable.getString("field1") & ":" & + postTable.getString("field2") & ":" & + postTable.getString("field3") + await request.respond(Http200, body) + else: + await request.respond(Http400, "Missing content body") + except HttpTransportError as exc: + defaultResponse(exc) + except HttpProtocolError as exc: + defaultResponse(exc) else: - return await request.respond(Http404, "Page not found") + defaultResponse() else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -554,23 +589,28 @@ suite "HTTP client testing suite": ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() case request.uri.path of "/test/post/multipart_size", "/test/post/multipart_chunked": - if request.hasBody(): - var postTable = await request.post() - let body = postTable.getString("field1") & ":" & - postTable.getString("field2") & ":" & - postTable.getString("field3") - return await request.respond(Http200, body) - else: - return await request.respond(Http400, "Missing content body") + try: + if request.hasBody(): + var postTable = await request.post() + let body = postTable.getString("field1") & ":" & + postTable.getString("field2") & ":" & + postTable.getString("field3") + await request.respond(Http200, body) + else: + await request.respond(Http400, "Missing content body") + except HttpProtocolError as exc: + defaultResponse(exc) + except HttpTransportError as exc: + defaultResponse(exc) else: - return await request.respond(Http404, "Page not found") + defaultResponse() else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -649,26 +689,29 @@ suite "HTTP client testing suite": var lastAddress: Uri proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - case request.uri.path - of "/": - return await request.redirect(Http302, "/redirect/1") - of "/redirect/1": - return await request.redirect(Http302, "/next/redirect/2") - of "/next/redirect/2": - return await request.redirect(Http302, "redirect/3") - of "/next/redirect/redirect/3": - return await request.redirect(Http302, "next/redirect/4") - of "/next/redirect/redirect/next/redirect/4": - return await request.redirect(Http302, lastAddress) - of "/final/5": - return await request.respond(Http200, "ok-5") - else: - return await request.respond(Http404, "Page not found") + try: + case request.uri.path + of "/": + await request.redirect(Http302, "/redirect/1") + of "/redirect/1": + await request.redirect(Http302, "/next/redirect/2") + of "/next/redirect/2": + await request.redirect(Http302, "redirect/3") + of "/next/redirect/redirect/3": + await request.redirect(Http302, "next/redirect/4") + of "/next/redirect/redirect/next/redirect/4": + await request.redirect(Http302, lastAddress) + of "/final/5": + await request.respond(Http200, "ok-5") + else: + await request.respond(Http404, "Page not found") + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -706,8 +749,8 @@ suite "HTTP client testing suite": proc testSendCancelLeaksTest(secure: bool): Future[bool] {.async.} = proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = - return defaultResponse() + async: (raises: [CancelledError]).} = + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -756,8 +799,8 @@ suite "HTTP client testing suite": proc testOpenCancelLeaksTest(secure: bool): Future[bool] {.async.} = proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = - return defaultResponse() + async: (raises: [CancelledError]).} = + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() @@ -868,20 +911,23 @@ suite "HTTP client testing suite": (data2.status, data2.data.bytesToString(), count)] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - case request.uri.path - of "/keep": - let headers = HttpTable.init([("connection", "keep-alive")]) - return await request.respond(Http200, "ok", headers = headers) - of "/drop": - let headers = HttpTable.init([("connection", "close")]) - return await request.respond(Http200, "ok", headers = headers) - else: - return await request.respond(Http404, "Page not found") + try: + case request.uri.path + of "/keep": + let headers = HttpTable.init([("connection", "keep-alive")]) + await request.respond(Http200, "ok", headers = headers) + of "/drop": + let headers = HttpTable.init([("connection", "close")]) + await request.respond(Http200, "ok", headers = headers) + else: + await request.respond(Http404, "Page not found") + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, false) server.start() @@ -1004,16 +1050,19 @@ suite "HTTP client testing suite": return (data.status, data.data.bytesToString(), 0) proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - case request.uri.path - of "/test": - return await request.respond(Http200, "ok") - else: - return await request.respond(Http404, "Page not found") + try: + case request.uri.path + of "/test": + await request.respond(Http200, "ok") + else: + await request.respond(Http404, "Page not found") + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, false) server.start() @@ -1064,19 +1113,22 @@ suite "HTTP client testing suite": return (data.status, data.data.bytesToString(), 0) proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - case request.uri.path - of "/test": - return await request.respond(Http200, "ok") - of "/keep-test": - let headers = HttpTable.init([("Connection", "keep-alive")]) - return await request.respond(Http200, "not-alive", headers) - else: - return await request.respond(Http404, "Page not found") + try: + case request.uri.path + of "/test": + await request.respond(Http200, "ok") + of "/keep-test": + let headers = HttpTable.init([("Connection", "keep-alive")]) + await request.respond(Http200, "not-alive", headers) + else: + await request.respond(Http404, "Page not found") + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, false) server.start() @@ -1180,58 +1232,61 @@ suite "HTTP client testing suite": true proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - if request.uri.path.startsWith("/test/single/"): - let index = - block: - var res = -1 - for index, value in SingleGoodTests.pairs(): - if value[0] == request.uri.path: - res = index - break - res - if index < 0: - return await request.respond(Http404, "Page not found") - var response = request.getResponse() - response.status = Http200 - await response.sendBody(SingleGoodTests[index][1]) - return response - elif request.uri.path.startsWith("/test/multiple/"): - let index = - block: - var res = -1 - for index, value in MultipleGoodTests.pairs(): - if value[0] == request.uri.path: - res = index - break - res - if index < 0: - return await request.respond(Http404, "Page not found") - var response = request.getResponse() - response.status = Http200 - await response.sendBody(MultipleGoodTests[index][1]) - return response - elif request.uri.path.startsWith("/test/overflow/"): - let index = - block: - var res = -1 - for index, value in OverflowTests.pairs(): - if value[0] == request.uri.path: - res = index - break - res - if index < 0: - return await request.respond(Http404, "Page not found") - var response = request.getResponse() - response.status = Http200 - await response.sendBody(OverflowTests[index][1]) - return response - else: - return await request.respond(Http404, "Page not found") + try: + if request.uri.path.startsWith("/test/single/"): + let index = + block: + var res = -1 + for index, value in SingleGoodTests.pairs(): + if value[0] == request.uri.path: + res = index + break + res + if index < 0: + return await request.respond(Http404, "Page not found") + var response = request.getResponse() + response.status = Http200 + await response.sendBody(SingleGoodTests[index][1]) + response + elif request.uri.path.startsWith("/test/multiple/"): + let index = + block: + var res = -1 + for index, value in MultipleGoodTests.pairs(): + if value[0] == request.uri.path: + res = index + break + res + if index < 0: + return await request.respond(Http404, "Page not found") + var response = request.getResponse() + response.status = Http200 + await response.sendBody(MultipleGoodTests[index][1]) + response + elif request.uri.path.startsWith("/test/overflow/"): + let index = + block: + var res = -1 + for index, value in OverflowTests.pairs(): + if value[0] == request.uri.path: + res = index + break + res + if index < 0: + return await request.respond(Http404, "Page not found") + var response = request.getResponse() + response.status = Http200 + await response.sendBody(OverflowTests[index][1]) + response + else: + defaultResponse() + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() var server = createServer(initTAddress("127.0.0.1:0"), process, secure) server.start() diff --git a/tests/testhttpserver.nim b/tests/testhttpserver.nim index 33d5ea1..0183f1b 100644 --- a/tests/testhttpserver.nim +++ b/tests/testhttpserver.nim @@ -64,7 +64,7 @@ suite "HTTP server testing suite": proc testTooBigBodyChunked(operation: TooBigTest): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() try: @@ -77,13 +77,15 @@ suite "HTTP server testing suite": let ptable {.used.} = await request.post() of PostMultipartTest: let ptable {.used.} = await request.post() - except HttpCriticalError as exc: + defaultResponse() + except HttpTransportError as exc: + defaultResponse(exc) + except HttpProtocolError as exc: if exc.code == Http413: serverRes = true - # Reraising exception, because processor should properly handle it. - raise exc + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -128,14 +130,17 @@ suite "HTTP server testing suite": proc testTimeout(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - return await request.respond(Http200, "TEST_OK", HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK", HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: if r.error.kind == HttpServerError.TimeoutError: serverRes = true - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), @@ -158,14 +163,17 @@ suite "HTTP server testing suite": proc testEmpty(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - return await request.respond(Http200, "TEST_OK", HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK", HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: - if r.error.kind == HttpServerError.CriticalError: + if r.error.kind == HttpServerError.ProtocolError: serverRes = true - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), @@ -188,14 +196,17 @@ suite "HTTP server testing suite": proc testTooBig(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - return await request.respond(Http200, "TEST_OK", HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK", HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: - if r.error.error == HttpServerError.CriticalError: + if r.error.error == HttpServerError.ProtocolError: serverRes = true - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -219,13 +230,11 @@ suite "HTTP server testing suite": proc testTooBigBody(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = - if r.isOk(): - discard - else: - if r.error.error == HttpServerError.CriticalError: + async: (raises: [CancelledError]).} = + if r.isErr(): + if r.error.error == HttpServerError.ProtocolError: serverRes = true - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -266,7 +275,7 @@ suite "HTTP server testing suite": proc testQuery(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() var kres = newSeq[string]() @@ -274,11 +283,14 @@ suite "HTTP server testing suite": kres.add(k & ":" & v) sort(kres) serverRes = true - return await request.respond(Http200, "TEST_OK:" & kres.join(":"), - HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK:" & kres.join(":"), + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -296,10 +308,9 @@ suite "HTTP server testing suite": "GET /?a=%D0%9F&%D0%A4=%D0%91&b=%D0%A6&c=%D0%AE HTTP/1.0\r\n\r\n") await server.stop() await server.closeWait() - let r = serverRes and - (data1.find("TEST_OK:a:1:a:2:b:3:c:4") >= 0) and - (data2.find("TEST_OK:a:П:b:Ц:c:Ю:Ф:Б") >= 0) - return r + serverRes and + (data1.find("TEST_OK:a:1:a:2:b:3:c:4") >= 0) and + (data2.find("TEST_OK:a:П:b:Ц:c:Ю:Ф:Б") >= 0) check waitFor(testQuery()) == true @@ -307,7 +318,7 @@ suite "HTTP server testing suite": proc testHeaders(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() var kres = newSeq[string]() @@ -315,11 +326,14 @@ suite "HTTP server testing suite": kres.add(k & ":" & v) sort(kres) serverRes = true - return await request.respond(Http200, "TEST_OK:" & kres.join(":"), - HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK:" & kres.join(":"), + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -351,21 +365,30 @@ suite "HTTP server testing suite": proc testPostUrl(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() if request.meth in PostMethods: - let post = await request.post() + let post = + try: + await request.post() + except HttpProtocolError as exc: + return defaultResponse(exc) + except HttpTransportError as exc: + return defaultResponse(exc) for k, v in post.stringItems(): kres.add(k & ":" & v) sort(kres) - serverRes = true - return await request.respond(Http200, "TEST_OK:" & kres.join(":"), - HttpTable.init()) + serverRes = true + try: + await request.respond(Http200, "TEST_OK:" & kres.join(":"), + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -395,21 +418,30 @@ suite "HTTP server testing suite": proc testPostUrl2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() if request.meth in PostMethods: - let post = await request.post() + let post = + try: + await request.post() + except HttpProtocolError as exc: + return defaultResponse(exc) + except HttpTransportError as exc: + return defaultResponse(exc) for k, v in post.stringItems(): kres.add(k & ":" & v) sort(kres) - serverRes = true - return await request.respond(Http200, "TEST_OK:" & kres.join(":"), - HttpTable.init()) + serverRes = true + try: + await request.respond(Http200, "TEST_OK:" & kres.join(":"), + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -440,21 +472,30 @@ suite "HTTP server testing suite": proc testPostMultipart(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() if request.meth in PostMethods: - let post = await request.post() + let post = + try: + await request.post() + except HttpProtocolError as exc: + return defaultResponse(exc) + except HttpTransportError as exc: + return defaultResponse(exc) for k, v in post.stringItems(): kres.add(k & ":" & v) sort(kres) - serverRes = true - return await request.respond(Http200, "TEST_OK:" & kres.join(":"), - HttpTable.init()) + serverRes = true + try: + await request.respond(Http200, "TEST_OK:" & kres.join(":"), + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -496,21 +537,31 @@ suite "HTTP server testing suite": proc testPostMultipart2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): var kres = newSeq[string]() let request = r.get() if request.meth in PostMethods: - let post = await request.post() + let post = + try: + await request.post() + except HttpProtocolError as exc: + return defaultResponse(exc) + except HttpTransportError as exc: + return defaultResponse(exc) for k, v in post.stringItems(): kres.add(k & ":" & v) sort(kres) serverRes = true - return await request.respond(Http200, "TEST_OK:" & kres.join(":"), - HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK:" & kres.join(":"), + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -566,16 +617,19 @@ suite "HTTP server testing suite": var count = 0 proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() inc(count) if count == ClientsCount: eventWait.fire() await eventContinue.wait() - return await request.respond(Http404, "", HttpTable.init()) + try: + await request.respond(Http404, "", HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -1230,23 +1284,26 @@ suite "HTTP server testing suite": proc testPostMultipart2(): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() let response = request.getResponse() - await response.prepareSSE() - await response.send("event: event1\r\ndata: data1\r\n\r\n") - await response.send("event: event2\r\ndata: data2\r\n\r\n") - await response.sendEvent("event3", "data3") - await response.sendEvent("event4", "data4") - await response.send("data: data5\r\n\r\n") - await response.sendEvent("", "data6") - await response.finish() - serverRes = true - return response + try: + await response.prepareSSE() + await response.send("event: event1\r\ndata: data1\r\n\r\n") + await response.send("event: event2\r\ndata: data2\r\n\r\n") + await response.sendEvent("event3", "data3") + await response.sendEvent("event4", "data4") + await response.send("data: data5\r\n\r\n") + await response.sendEvent("", "data6") + await response.finish() + serverRes = true + response + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let res = HttpServerRef.new(initTAddress("127.0.0.1:0"), process, @@ -1306,12 +1363,15 @@ suite "HTTP server testing suite": ] proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - return await request.respond(Http200, "TEST_OK", HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK", HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() for test in TestMessages: let @@ -1360,12 +1420,15 @@ suite "HTTP server testing suite": TestRequest = "GET /httpdebug HTTP/1.1\r\nConnection: keep-alive\r\n\r\n" proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - return await request.respond(Http200, "TEST_OK", HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK", HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: - return defaultResponse() + defaultResponse() proc client(address: TransportAddress, data: string): Future[StreamTransport] {.async.} = diff --git a/tests/testshttpserver.nim b/tests/testshttpserver.nim index 3ff2565..18e84a9 100644 --- a/tests/testshttpserver.nim +++ b/tests/testshttpserver.nim @@ -108,15 +108,18 @@ suite "Secure HTTP server testing suite": proc testHTTPS(address: TransportAddress): Future[bool] {.async.} = var serverRes = false proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() serverRes = true - return await request.respond(Http200, "TEST_OK:" & $request.meth, - HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK:" & $request.meth, + HttpTable.init()) + except HttpWriteError as exc: + serverRes = false + defaultResponse(exc) else: - serverRes = false - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let serverFlags = {Secure} @@ -146,16 +149,18 @@ suite "Secure HTTP server testing suite": var serverRes = false var testFut = newFuture[void]() proc process(r: RequestFence): Future[HttpResponseRef] {. - async: (raises: [CancelledError, HttpResponseError]).} = + async: (raises: [CancelledError]).} = if r.isOk(): let request = r.get() - serverRes = false - return await request.respond(Http200, "TEST_OK:" & $request.meth, - HttpTable.init()) + try: + await request.respond(Http200, "TEST_OK:" & $request.meth, + HttpTable.init()) + except HttpWriteError as exc: + defaultResponse(exc) else: serverRes = true testFut.complete() - return defaultResponse() + defaultResponse() let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} let serverFlags = {Secure}