From 112da56662255d04bfc0ecc874379848d8f93089 Mon Sep 17 00:00:00 2001 From: kdeme Date: Fri, 26 Mar 2021 10:21:52 +0000 Subject: [PATCH] deploy: d1c1a0ca13e3aa2690a6550faca13210e1f46877 --- .gitmodules | 10 + .update.timestamp | 2 +- examples/v1/example.nim | 160 +- tests/test_helpers.nim | 14 +- tests/v1/test_waku_bridge.nim | 4 +- tests/v1/test_waku_connect.nim | 4 +- tests/v1/test_waku_mail.nim | 4 +- tests/v2/test_jsonrpc_waku.nim | 4 +- tests/v2/test_peer_manager.nim | 4 +- tests/v2/test_waku_bridge.nim | 3 +- tests/v2/test_waku_filter.nim | 4 +- tests/v2/test_waku_pagination.nim | 4 +- tests/v2/test_waku_payload.nim | 2 +- tests/v2/test_waku_rln_relay.nim | 9 +- tests/v2/test_waku_store.nim | 4 +- tests/v2/test_waku_swap.nim | 3 +- tests/v2/test_wakunode.nim | 2 +- vendor/nim-chronicles/chronicles.nim | 24 +- vendor/nim-chronicles/chronicles.nimble | 2 +- .../nim-chronicles/chronicles/log_output.nim | 4 +- .../nim-chronicles/tests/no_side_effect.nim | 6 + vendor/nim-chronos/README.md | 80 +- vendor/nim-chronos/chronos.nimble | 7 +- vendor/nim-chronos/chronos/apps.nim | 4 +- .../chronos/apps/http/httpserver.nim | 203 +- .../chronos/apps/http/shttpserver.nim | 105 + vendor/nim-chronos/chronos/asyncfutures2.nim | 192 +- vendor/nim-chronos/chronos/asyncloop.nim | 156 +- vendor/nim-chronos/chronos/asyncmacro2.nim | 146 +- vendor/nim-chronos/chronos/asyncsync.nim | 23 +- vendor/nim-chronos/chronos/debugutils.nim | 5 +- vendor/nim-chronos/chronos/handles.nim | 17 +- .../chronos/ioselects/ioselectors_epoll.nim | 524 +++++ .../chronos/ioselects/ioselectors_kqueue.nim | 625 ++++++ .../chronos/ioselects/ioselectors_poll.nim | 310 +++ .../chronos/ioselects/ioselectors_select.nim | 465 +++++ vendor/nim-chronos/chronos/selectors2.nim | 360 ++++ vendor/nim-chronos/chronos/sendfile.nim | 2 + vendor/nim-chronos/chronos/srcloc.nim | 2 + .../chronos/streams/asyncstream.nim | 25 +- vendor/nim-chronos/chronos/timer.nim | 2 + vendor/nim-chronos/chronos/transport.nim | 4 +- .../nim-chronos/chronos/transports/common.nim | 44 +- .../chronos/transports/datagram.nim | 71 +- .../nim-chronos/chronos/transports/ipnet.nim | 24 +- .../nim-chronos/chronos/transports/osnet.nim | 20 +- .../nim-chronos/chronos/transports/stream.nim | 318 ++- vendor/nim-chronos/tests/testaddress.nim | 2 +- vendor/nim-chronos/tests/testall.nim | 2 +- vendor/nim-chronos/tests/testasyncstream.nim | 2 +- vendor/nim-chronos/tests/testbugs.nim | 2 +- vendor/nim-chronos/tests/testdatagram.nim | 3 +- vendor/nim-chronos/tests/testfut.nim | 2 +- vendor/nim-chronos/tests/testhttpserver.nim | 169 +- vendor/nim-chronos/tests/testmacro.nim | 2 +- vendor/nim-chronos/tests/testnet.nim | 2 +- vendor/nim-chronos/tests/testserver.nim | 2 +- vendor/nim-chronos/tests/testshttpserver.nim | 180 ++ vendor/nim-chronos/tests/testsignal.nim | 12 +- vendor/nim-chronos/tests/testsoon.nim | 8 +- vendor/nim-chronos/tests/teststream.nim | 5 +- vendor/nim-chronos/tests/testsync.nim | 2 +- vendor/nim-chronos/tests/testtime.nim | 3 +- vendor/nim-chronos/tests/testutils.nim | 2 +- .../confutils/envvar/envvar_serialization.nim | 8 +- .../confutils/winreg/winreg_serialization.nim | 8 +- .../nim-eth/eth/p2p/discoveryv5/protocol.nim | 2 +- vendor/nim-eth/eth/p2p/private/p2p_types.nim | 7 +- vendor/nim-eth/eth/p2p/rlpx.nim | 20 +- .../rlpx_protocols/whisper/whisper_types.nim | 2 +- vendor/nim-http-utils/httputils.nim | 51 + .../json_serialization.nim | 14 +- .../json_serialization/format.nim | 10 + .../json_serialization/reader.nim | 20 +- .../json_serialization/writer.nim | 29 +- .../tests/test_json_flavor.nim | 41 + .../vendor/libbacktrace-upstream/libtool | 2 +- vendor/nim-libp2p/.github/workflows/ci.yml | 3 + .../nim-libp2p/libp2p/stream/connection.nim | 2 +- vendor/nim-libp2p/libp2p/switch.nim | 2 - .../libp2p/transports/tcptransport.nim | 2 +- vendor/nim-serialization/README.md | 6 +- vendor/nim-serialization/serialization.nim | 69 +- .../serialization/formats.nim | 40 + .../serialization/object_serialization.nim | 31 +- .../serialization/testing/generic_suite.nim | 10 +- vendor/nim-stew/stew/byteutils.nim | 5 + vendor/nim-testutils/.appveyor.yml | 39 + vendor/nim-testutils/.editorconfig | 5 + vendor/nim-testutils/.gitignore | 8 + vendor/nim-testutils/.travis.yml | 26 + vendor/nim-testutils/README.md | 12 + vendor/nim-testutils/nim.cfg | 1 + vendor/nim-testutils/ntu.nim | 470 +++++ vendor/nim-testutils/ntu.nim.cfg | 2 + .../scripts/install_honggfuzz.sh | 17 + vendor/nim-testutils/tests/hello.nim | 6 + vendor/nim-testutils/tests/hello/hello.test | 3 + .../tests/hello/hello_multiple.test | 15 + .../nim-testutils/tests/hello/hello_size.test | 5 + vendor/nim-testutils/tests/threads.nim | 4 + vendor/nim-testutils/tests/threads.test | 1 + vendor/nim-testutils/tests/tunits.nim | 11 + vendor/nim-testutils/tests/tunits.test | 1 + vendor/nim-testutils/testutils.nim | 6 + vendor/nim-testutils/testutils.nimble | 39 + vendor/nim-testutils/testutils/config.nim | 204 ++ vendor/nim-testutils/testutils/fuzzing.nim | 122 ++ .../testutils/fuzzing/fuzzing_on_windows.md | 148 ++ .../nim-testutils/testutils/fuzzing/readme.md | 207 ++ .../testutils/fuzzing_engines.nim | 153 ++ vendor/nim-testutils/testutils/helpers.nim | 103 + .../testutils/markdown_reports.nim | 44 + .../nim-testutils/testutils/moduletests.nim | 22 + .../nim-testutils/testutils/nimbletasks.nim | 5 + vendor/nim-testutils/testutils/readme.md | 189 ++ vendor/nim-testutils/testutils/spec.nim | 203 ++ vendor/nim-testutils/testutils/unittests.nim | 17 + vendor/nim-unittest2/.github/workflows/ci.yml | 148 ++ vendor/nim-unittest2/.gitignore | 2 + vendor/nim-unittest2/LICENSE.txt | 22 + vendor/nim-unittest2/README.md | 68 + vendor/nim-unittest2/config.nims | 8 + vendor/nim-unittest2/nim.cfg | 8 + vendor/nim-unittest2/tests/nim.cfg | 2 + vendor/nim-unittest2/tests/tunittest.nim | 183 ++ .../nim-unittest2/tests/tunittestparallel.nim | 134 ++ .../tests/tunittestparallel.nim.cfg | 2 + vendor/nim-unittest2/unittest2.html | 1830 +++++++++++++++++ vendor/nim-unittest2/unittest2.nim | 909 ++++++++ vendor/nim-unittest2/unittest2.nimble | 8 + waku/v1/node/wakunode1.nim | 8 +- waku/v2/node/wakunode2.nim | 9 +- 133 files changed, 9307 insertions(+), 913 deletions(-) create mode 100644 vendor/nim-chronos/chronos/apps/http/shttpserver.nim create mode 100644 vendor/nim-chronos/chronos/ioselects/ioselectors_epoll.nim create mode 100644 vendor/nim-chronos/chronos/ioselects/ioselectors_kqueue.nim create mode 100644 vendor/nim-chronos/chronos/ioselects/ioselectors_poll.nim create mode 100644 vendor/nim-chronos/chronos/ioselects/ioselectors_select.nim create mode 100644 vendor/nim-chronos/chronos/selectors2.nim create mode 100644 vendor/nim-chronos/tests/testshttpserver.nim create mode 100644 vendor/nim-json-serialization/json_serialization/format.nim create mode 100644 vendor/nim-json-serialization/tests/test_json_flavor.nim create mode 100644 vendor/nim-serialization/serialization/formats.nim create mode 100644 vendor/nim-testutils/.appveyor.yml create mode 100644 vendor/nim-testutils/.editorconfig create mode 100644 vendor/nim-testutils/.gitignore create mode 100644 vendor/nim-testutils/.travis.yml create mode 100644 vendor/nim-testutils/README.md create mode 100644 vendor/nim-testutils/nim.cfg create mode 100644 vendor/nim-testutils/ntu.nim create mode 100644 vendor/nim-testutils/ntu.nim.cfg create mode 100755 vendor/nim-testutils/scripts/install_honggfuzz.sh create mode 100644 vendor/nim-testutils/tests/hello.nim create mode 100644 vendor/nim-testutils/tests/hello/hello.test create mode 100644 vendor/nim-testutils/tests/hello/hello_multiple.test create mode 100644 vendor/nim-testutils/tests/hello/hello_size.test create mode 100644 vendor/nim-testutils/tests/threads.nim create mode 100644 vendor/nim-testutils/tests/threads.test create mode 100644 vendor/nim-testutils/tests/tunits.nim create mode 100644 vendor/nim-testutils/tests/tunits.test create mode 100644 vendor/nim-testutils/testutils.nim create mode 100644 vendor/nim-testutils/testutils.nimble create mode 100644 vendor/nim-testutils/testutils/config.nim create mode 100644 vendor/nim-testutils/testutils/fuzzing.nim create mode 100644 vendor/nim-testutils/testutils/fuzzing/fuzzing_on_windows.md create mode 100644 vendor/nim-testutils/testutils/fuzzing/readme.md create mode 100644 vendor/nim-testutils/testutils/fuzzing_engines.nim create mode 100644 vendor/nim-testutils/testutils/helpers.nim create mode 100644 vendor/nim-testutils/testutils/markdown_reports.nim create mode 100644 vendor/nim-testutils/testutils/moduletests.nim create mode 100644 vendor/nim-testutils/testutils/nimbletasks.nim create mode 100644 vendor/nim-testutils/testutils/readme.md create mode 100644 vendor/nim-testutils/testutils/spec.nim create mode 100644 vendor/nim-testutils/testutils/unittests.nim create mode 100644 vendor/nim-unittest2/.github/workflows/ci.yml create mode 100644 vendor/nim-unittest2/.gitignore create mode 100644 vendor/nim-unittest2/LICENSE.txt create mode 100644 vendor/nim-unittest2/README.md create mode 100644 vendor/nim-unittest2/config.nims create mode 100644 vendor/nim-unittest2/nim.cfg create mode 100644 vendor/nim-unittest2/tests/nim.cfg create mode 100644 vendor/nim-unittest2/tests/tunittest.nim create mode 100644 vendor/nim-unittest2/tests/tunittestparallel.nim create mode 100644 vendor/nim-unittest2/tests/tunittestparallel.nim.cfg create mode 100644 vendor/nim-unittest2/unittest2.html create mode 100644 vendor/nim-unittest2/unittest2.nim create mode 100644 vendor/nim-unittest2/unittest2.nimble diff --git a/.gitmodules b/.gitmodules index 1b005cb05..cdf466973 100644 --- a/.gitmodules +++ b/.gitmodules @@ -110,3 +110,13 @@ path = vendor/rln url = https://github.com/kilic/rln branch = full-node +[submodule "vendor/nim-testutils"] + path = vendor/nim-testutils + url = https://github.com/status-im/nim-testutils.git + ignore = untracked + branch = master +[submodule "vendor/nim-unittest2"] + path = vendor/nim-unittest2 + url = https://github.com/status-im/nim-unittest2.git + ignore = untracked + branch = master diff --git a/.update.timestamp b/.update.timestamp index fe717114b..16f737655 100644 --- a/.update.timestamp +++ b/.update.timestamp @@ -1 +1 @@ -1616748591 \ No newline at end of file +1616752324 \ No newline at end of file diff --git a/examples/v1/example.nim b/examples/v1/example.nim index 9121adab9..2fe4f150c 100644 --- a/examples/v1/example.nim +++ b/examples/v1/example.nim @@ -10,99 +10,103 @@ import const clientId = "Waku example v1" -let - # Load the cli configuration from `config_example.nim`. - config = WakuNodeConf.load() - # Seed the rng. - rng = keys.newRng() +proc run(config: WakuNodeConf, rng: ref BrHmacDrbgContext) = # Set up the address according to NAT information. - (ipExt, tcpPortExt, udpPortExt) = setupNat(config.nat, clientId, + let (ipExt, tcpPortExt, udpPortExt) = setupNat(config.nat, clientId, Port(config.tcpPort + config.portsShift), Port(config.udpPort + config.portsShift)) # TODO: EthereumNode should have a better split of binding address and # external address. Also, can't have different ports as it stands now. - address = if ipExt.isNone(): - Address(ip: parseIpAddress("0.0.0.0"), - tcpPort: Port(config.tcpPort + config.portsShift), - udpPort: Port(config.udpPort + config.portsShift)) - else: - Address(ip: ipExt.get(), - tcpPort: Port(config.tcpPort + config.portsShift), - udpPort: Port(config.udpPort + config.portsShift)) + let address = if ipExt.isNone(): + Address(ip: parseIpAddress("0.0.0.0"), + tcpPort: Port(config.tcpPort + config.portsShift), + udpPort: Port(config.udpPort + config.portsShift)) + else: + Address(ip: ipExt.get(), + tcpPort: Port(config.tcpPort + config.portsShift), + udpPort: Port(config.udpPort + config.portsShift)) -# Create Ethereum Node -var node = newEthereumNode(config.nodekey, # Node identifier - address, # Address reachable for incoming requests - NetworkId(1), # Network Id, only applicable for ETH protocol - nil, # Database, not required for Waku - clientId, # Client id string - addAllCapabilities = false, # Disable default all RLPx capabilities - rng = rng) + # Create Ethereum Node + var node = newEthereumNode(config.nodekey, # Node identifier + address, # Address reachable for incoming requests + NetworkId(1), # Network Id, only applicable for ETH protocol + nil, # Database, not required for Waku + clientId, # Client id string + addAllCapabilities = false, # Disable default all RLPx capabilities + rng = rng) -node.addCapability Waku # Enable only the Waku protocol. + node.addCapability Waku # Enable only the Waku protocol. -# Set up the Waku configuration. -let wakuConfig = WakuConfig(powRequirement: 0.002, - bloom: some(fullBloom()), # Full bloom filter - isLightNode: false, # Full node - maxMsgSize: waku_protocol.defaultMaxMsgSize, - topics: none(seq[waku_protocol.Topic]) # empty topic interest - ) -node.configureWaku(wakuConfig) + # Set up the Waku configuration. + let wakuConfig = WakuConfig(powRequirement: 0.002, + bloom: some(fullBloom()), # Full bloom filter + isLightNode: false, # Full node + maxMsgSize: waku_protocol.defaultMaxMsgSize, + topics: none(seq[waku_protocol.Topic]) # empty topic interest + ) + node.configureWaku(wakuConfig) -# Optionally direct connect to a set of nodes. -if config.staticnodes.len > 0: - connectToNodes(node, config.staticnodes) + # Optionally direct connect to a set of nodes. + if config.staticnodes.len > 0: + connectToNodes(node, config.staticnodes) -# Connect to the network, which will make the node start listening and/or -# connect to bootnodes, and/or start discovery. -# This will block until first connection is made, which in this case can only -# happen if we directly connect to nodes (step above) or if an incoming -# connection occurs, which is why we use a callback to exit on errors instead of -# using `await`. -# TODO: This looks a bit awkward and the API should perhaps be altered here. -let connectedFut = node.connectToNetwork(@[], - true, # Enable listening - false # Disable discovery (only discovery v4 is currently supported) - ) -connectedFut.callback = proc(data: pointer) {.gcsafe.} = - {.gcsafe.}: - if connectedFut.failed: - fatal "connectToNetwork failed", msg = connectedFut.readError.msg - quit(1) + # Connect to the network, which will make the node start listening and/or + # connect to bootnodes, and/or start discovery. + # This will block until first connection is made, which in this case can only + # happen if we directly connect to nodes (step above) or if an incoming + # connection occurs, which is why we use a callback to exit on errors instead of + # using `await`. + # TODO: This looks a bit awkward and the API should perhaps be altered here. + let connectedFut = node.connectToNetwork(@[], + true, # Enable listening + false # Disable discovery (only discovery v4 is currently supported) + ) + connectedFut.callback = proc(data: pointer) {.gcsafe.} = + {.gcsafe.}: + if connectedFut.failed: + fatal "connectToNetwork failed", msg = connectedFut.readError.msg + quit(1) -# Using a hardcoded symmetric key for encryption of the payload for the sake of -# simplicity. -var symKey: SymKey -symKey[31] = 1 -# Asymmetric keypair to sign the payload. -let signKeyPair = KeyPair.random(rng[]) + # Using a hardcoded symmetric key for encryption of the payload for the sake of + # simplicity. + var symKey: SymKey + symKey[31] = 1 + # Asymmetric keypair to sign the payload. + let signKeyPair = KeyPair.random(rng[]) -# Code to be executed on receival of a message on filter. -proc handler(msg: ReceivedMessage) = - if msg.decoded.src.isSome(): - echo "Received message from ", $msg.decoded.src.get(), ": ", - string.fromBytes(msg.decoded.payload) + # Code to be executed on receival of a message on filter. + proc handler(msg: ReceivedMessage) = + if msg.decoded.src.isSome(): + echo "Received message from ", $msg.decoded.src.get(), ": ", + string.fromBytes(msg.decoded.payload) -# Create and subscribe filter with above handler. -let - topic = [byte 0, 0, 0, 0] - filter = initFilter(symKey = some(symKey), topics = @[topic]) -discard node.subscribeFilter(filter, handler) + # Create and subscribe filter with above handler. + let + topic = [byte 0, 0, 0, 0] + filter = initFilter(symKey = some(symKey), topics = @[topic]) + discard node.subscribeFilter(filter, handler) -# Repeat the posting of a message every 5 seconds. -proc repeatMessage(udata: pointer) {.gcsafe.} = - {.gcsafe.}: - # Post a waku message on the network, encrypted with provided symmetric key, - # signed with asymmetric key, on topic and with ttl of 30 seconds. - let posted = node.postMessage( - symKey = some(symKey), src = some(signKeyPair.seckey), - ttl = 30, topic = topic, payload = @[byte 0x48, 0x65, 0x6C, 0x6C, 0x6F]) + # Repeat the posting of a message every 5 seconds. + # https://github.com/nim-lang/Nim/issues/17369 + var repeatMessage: proc(udata: pointer) {.gcsafe, raises: [Defect].} + repeatMessage = proc(udata: pointer) = + {.gcsafe.}: + # Post a waku message on the network, encrypted with provided symmetric key, + # signed with asymmetric key, on topic and with ttl of 30 seconds. + let posted = node.postMessage( + symKey = some(symKey), src = some(signKeyPair.seckey), + ttl = 30, topic = topic, payload = @[byte 0x48, 0x65, 0x6C, 0x6C, 0x6F]) - if posted: echo "Posted message as ", $signKeyPair.pubkey - else: echo "Posting message failed." + if posted: echo "Posted message as ", $signKeyPair.pubkey + else: echo "Posting message failed." + discard setTimer(Moment.fromNow(5.seconds), repeatMessage) discard setTimer(Moment.fromNow(5.seconds), repeatMessage) -discard setTimer(Moment.fromNow(5.seconds), repeatMessage) -runForever() + runForever() + +when isMainModule: + let + rng = keys.newRng() + conf = WakuNodeConf.load() + run(conf, rng) diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index ad745d19c..9acb2311f 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -1,5 +1,5 @@ import - unittest, chronos, bearssl, + chronos, bearssl, eth/[keys, p2p] import libp2p/crypto/crypto @@ -21,18 +21,6 @@ proc setupTestNode*( for capability in capabilities: result.addCapability capability -template asyncTest*(name, body: untyped) = - test name: - proc scenario {.async.} = body - waitFor scenario() - -template procSuite*(name, body: untyped) = - proc suitePayload = - suite name: - body - - suitePayload() - # Copied from here: https://github.com/status-im/nim-libp2p/blob/d522537b19a532bc4af94fcd146f779c1f23bad0/tests/helpers.nim#L28 type RngWrap = object rng: ref BrHmacDrbgContext diff --git a/tests/v1/test_waku_bridge.nim b/tests/v1/test_waku_bridge.nim index 0defd56db..aab6d7090 100644 --- a/tests/v1/test_waku_bridge.nim +++ b/tests/v1/test_waku_bridge.nim @@ -9,8 +9,8 @@ {.used.} import - std/[sequtils, unittest, tables], - chronos, eth/p2p, eth/p2p/peer_pool, + std/[sequtils, tables], + chronos, testutils/unittests, eth/p2p, eth/p2p/peer_pool, eth/p2p/rlpx_protocols/whisper_protocol as whisper, ../../waku/v1/protocol/waku_protocol as waku, ../../waku/v1/protocol/waku_bridge, diff --git a/tests/v1/test_waku_connect.nim b/tests/v1/test_waku_connect.nim index e045da5cc..06ef897d9 100644 --- a/tests/v1/test_waku_connect.nim +++ b/tests/v1/test_waku_connect.nim @@ -9,8 +9,8 @@ {.used.} import - std/[sequtils, tables, unittest], - chronos, eth/[keys, p2p], eth/p2p/peer_pool, + std/[sequtils, tables], + chronos, testutils/unittests, eth/[keys, p2p], eth/p2p/peer_pool, ../../waku/v1/protocol/waku_protocol, ../test_helpers diff --git a/tests/v1/test_waku_mail.nim b/tests/v1/test_waku_mail.nim index 3df1d23cc..e53c6b850 100644 --- a/tests/v1/test_waku_mail.nim +++ b/tests/v1/test_waku_mail.nim @@ -1,8 +1,8 @@ {.used.} import - std/[unittest, tables, sequtils, times], - chronos, eth/[p2p, async_utils], eth/p2p/peer_pool, + std/[tables, sequtils, times], + chronos, testutils/unittests, eth/[p2p, async_utils], eth/p2p/peer_pool, ../../waku/v1/protocol/[waku_protocol, waku_mail], ../test_helpers diff --git a/tests/v2/test_jsonrpc_waku.nim b/tests/v2/test_jsonrpc_waku.nim index 43e497595..a82ffe483 100644 --- a/tests/v2/test_jsonrpc_waku.nim +++ b/tests/v2/test_jsonrpc_waku.nim @@ -1,8 +1,8 @@ {.used.} import - std/[unittest, options, sets, tables, os, strutils, sequtils], - stew/shims/net as stewNet, + std/[options, sets, tables, os, strutils, sequtils], + testutils/unittests, stew/shims/net as stewNet, json_rpc/[rpcserver, rpcclient], eth/[keys, rlp], eth/common/eth_types, libp2p/[standard_setup, switch, multiaddress], diff --git a/tests/v2/test_peer_manager.nim b/tests/v2/test_peer_manager.nim index ad0516720..6007d5f9c 100644 --- a/tests/v2/test_peer_manager.nim +++ b/tests/v2/test_peer_manager.nim @@ -1,8 +1,8 @@ {.used.} import - std/[unittest, options, sets, tables, sequtils], - stew/shims/net as stewNet, + std/[options, sets, tables, sequtils], + testutils/unittests, stew/shims/net as stewNet, json_rpc/[rpcserver, rpcclient], eth/[keys, rlp], eth/common/eth_types, libp2p/[standard_setup, switch, multiaddress], diff --git a/tests/v2/test_waku_bridge.nim b/tests/v2/test_waku_bridge.nim index 0426e61d0..c0243154d 100644 --- a/tests/v2/test_waku_bridge.nim +++ b/tests/v2/test_waku_bridge.nim @@ -1,7 +1,8 @@ {.used.} import - std/[unittest, strutils], + std/strutils, + testutils/unittests, chronicles, chronos, stew/shims/net as stewNet, stew/byteutils, libp2p/crypto/crypto, libp2p/crypto/secp, diff --git a/tests/v2/test_waku_filter.nim b/tests/v2/test_waku_filter.nim index 96e2fdd89..2cc3fdc4c 100644 --- a/tests/v2/test_waku_filter.nim +++ b/tests/v2/test_waku_filter.nim @@ -1,8 +1,8 @@ {.used.} import - std/[unittest, options, tables, sets], - chronos, chronicles, + std/[options, tables, sets], + testutils/unittests, chronos, chronicles, libp2p/switch, libp2p/protobuf/minprotobuf, libp2p/stream/[bufferstream, connection], diff --git a/tests/v2/test_waku_pagination.nim b/tests/v2/test_waku_pagination.nim index a30eddf68..e2c88ca2f 100644 --- a/tests/v2/test_waku_pagination.nim +++ b/tests/v2/test_waku_pagination.nim @@ -1,7 +1,7 @@ {.used.} import - std/[unittest,algorithm,options], - nimcrypto/sha2, + std/[algorithm, options], + testutils/unittests, nimcrypto/sha2, ../../waku/v2/protocol/waku_store/waku_store, ../test_helpers diff --git a/tests/v2/test_waku_payload.nim b/tests/v2/test_waku_payload.nim index 5674ce10d..d57ddadaf 100644 --- a/tests/v2/test_waku_payload.nim +++ b/tests/v2/test_waku_payload.nim @@ -1,7 +1,7 @@ {.used.} import - std/unittest, + testutils/unittests, ../../waku/v2/protocol/waku_message, ../../waku/v2/node/waku_payload, ../test_helpers diff --git a/tests/v2/test_waku_rln_relay.nim b/tests/v2/test_waku_rln_relay.nim index 6b1c52ec1..b1a8085f5 100644 --- a/tests/v2/test_waku_rln_relay.nim +++ b/tests/v2/test_waku_rln_relay.nim @@ -1,17 +1,14 @@ {.used.} import - chronos, chronicles, options, stint, unittest, - web3, + std/options, + testutils/unittests, chronos, chronicles, stint, web3, stew/byteutils, stew/shims/net as stewNet, libp2p/crypto/crypto, ../../waku/v2/protocol/waku_rln_relay/[rln, waku_rln_relay_utils], ../../waku/v2/node/wakunode2, ../test_helpers, - test_utils - - - + ./test_utils # the address of Ethereum client (ganache-cli for now) # TODO this address in hardcoded in the code, we may need to take it as input from the user diff --git a/tests/v2/test_waku_store.nim b/tests/v2/test_waku_store.nim index 5f112a348..205951ed0 100644 --- a/tests/v2/test_waku_store.nim +++ b/tests/v2/test_waku_store.nim @@ -1,8 +1,8 @@ {.used.} import - std/[unittest, options, tables, sets], - chronos, chronicles, + std/[options, tables, sets], + testutils/unittests, chronos, chronicles, libp2p/switch, libp2p/protobuf/minprotobuf, libp2p/stream/[bufferstream, connection], diff --git a/tests/v2/test_waku_swap.nim b/tests/v2/test_waku_swap.nim index 427f380c2..f2fcfed75 100644 --- a/tests/v2/test_waku_swap.nim +++ b/tests/v2/test_waku_swap.nim @@ -1,7 +1,8 @@ {.used.} import - std/[unittest, options, tables, sets], + std/[options, tables, sets], + testutils/unittests, chronos, chronicles, stew/shims/net as stewNet, stew/byteutils, libp2p/switch, libp2p/protobuf/minprotobuf, diff --git a/tests/v2/test_wakunode.nim b/tests/v2/test_wakunode.nim index 5b89e59dd..1f3f90a26 100644 --- a/tests/v2/test_wakunode.nim +++ b/tests/v2/test_wakunode.nim @@ -1,7 +1,7 @@ {.used.} import - std/unittest, + testutils/unittests, chronicles, chronos, stew/shims/net as stewNet, stew/byteutils, libp2p/crypto/crypto, libp2p/crypto/secp, diff --git a/vendor/nim-chronicles/chronicles.nim b/vendor/nim-chronicles/chronicles.nim index 5dc136f0a..fcd780b39 100644 --- a/vendor/nim-chronicles/chronicles.nim +++ b/vendor/nim-chronicles/chronicles.nim @@ -366,13 +366,23 @@ template logFn(name: untyped, severity: typed, debug=false) {.dirty.} = wrapSideEffects(debug): log(instantiationInfo(), stream, severity, eventName, props) -logFn trace , LogLevel.TRACE, debug=true -logFn debug , LogLevel.DEBUG -logFn info , LogLevel.INFO -logFn notice, LogLevel.NOTICE -logFn warn , LogLevel.WARN -logFn error , LogLevel.ERROR -logFn fatal , LogLevel.FATAL +# workaround for https://github.com/status-im/nim-chronicles/issues/92 +when defined(windows) and (NimMajor, NimMinor, NimPatch) < (1, 4, 4): + logFn trace , LogLevel.TRACE, debug=true + logFn debug , LogLevel.DEBUG, debug=true + logFn info , LogLevel.INFO, debug=true + logFn notice, LogLevel.NOTICE, debug=true + logFn warn , LogLevel.WARN, debug=true + logFn error , LogLevel.ERROR, debug=true + logFn fatal , LogLevel.FATAL, debug=true +else: + logFn trace , LogLevel.TRACE, debug=true + logFn debug , LogLevel.DEBUG + logFn info , LogLevel.INFO + logFn notice, LogLevel.NOTICE + logFn warn , LogLevel.WARN + logFn error , LogLevel.ERROR + logFn fatal , LogLevel.FATAL # TODO: # diff --git a/vendor/nim-chronicles/chronicles.nimble b/vendor/nim-chronicles/chronicles.nimble index 5e1b4a198..0c7c582ad 100644 --- a/vendor/nim-chronicles/chronicles.nimble +++ b/vendor/nim-chronicles/chronicles.nimble @@ -1,7 +1,7 @@ mode = ScriptMode.Verbose packageName = "chronicles" -version = "0.10.0" +version = "0.10.1" author = "Status Research & Development GmbH" description = "A crafty implementation of structured logging for Nim" license = "Apache License 2.0" diff --git a/vendor/nim-chronicles/chronicles/log_output.nim b/vendor/nim-chronicles/chronicles/log_output.nim index 2a58b3d2a..69571c7cc 100644 --- a/vendor/nim-chronicles/chronicles/log_output.nim +++ b/vendor/nim-chronicles/chronicles/log_output.nim @@ -86,7 +86,7 @@ else: JsonRecord*[Output; timestamps: static[TimestampsScheme]] = object output*: Output outStream: OutputStream - jsonWriter: JsonWriter + jsonWriter: Json.Writer export JsonString @@ -659,7 +659,7 @@ proc initLogRecord*(r: var JsonRecord, r.record = newJsObject() else: r.outStream = memoryOutput() - r.jsonWriter = JsonWriter.init(r.outStream, pretty = false) + r.jsonWriter = Json.Writer.init(r.outStream, pretty = false) r.jsonWriter.beginRecord() if level != NONE: diff --git a/vendor/nim-chronicles/tests/no_side_effect.nim b/vendor/nim-chronicles/tests/no_side_effect.nim index 411c3ba88..7071a9cfa 100644 --- a/vendor/nim-chronicles/tests/no_side_effect.nim +++ b/vendor/nim-chronicles/tests/no_side_effect.nim @@ -4,3 +4,9 @@ func main = trace "effect-free" main() + +# issue #92 +proc test() {.raises: [Defect].} = + error "should not raises exception" + +test() diff --git a/vendor/nim-chronos/README.md b/vendor/nim-chronos/README.md index 25bb57a16..6f33d17bc 100644 --- a/vendor/nim-chronos/README.md +++ b/vendor/nim-chronos/README.md @@ -1,6 +1,6 @@ # Chronos - An efficient library for asynchronous programming -![Github action](https://github.com/status-im/nim-chronos/workflows/nim-chronos%20CI/badge.svg) +[![Github action](https://github.com/status-im/nim-chronos/workflows/nim-chronos%20CI/badge.svg)](https://github.com/status-im/nim-chronos/actions/workflows/ci.yml) [![Windows build status (AppVeyor)](https://img.shields.io/appveyor/ci/nimbus/nim-asyncdispatch2/master.svg?label=Windows "Windows build status (Appveyor)")](https://ci.appveyor.com/project/nimbus/nim-asyncdispatch2) [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) @@ -8,16 +8,39 @@ ## Introduction -Chronos is an [asyncdispatch](https://nim-lang.org/docs/asyncdispatch.html) -fork with a unified callback type, FIFO processing order for Future callbacks and [many other changes](https://github.com/status-im/nim-chronos/wiki/AsyncDispatch-comparison) that diverged from upstream's philosophy. +Chronos is an efficient [async/await](https://en.wikipedia.org/wiki/Async/await) framework for Nim. Features include: + +* Efficient dispatch pipeline for asynchronous execution +* HTTP server with SSL/TLS support out of the box (no OpenSSL needed) +* Cancellation support +* Synchronization primitivies like queues, events and locks +* FIFO processing order of dispatch queue +* Minimal exception effect support (see [exception effects](#exception-effects)) ## Installation + You can use Nim's official package manager Nimble to install Chronos: +```text +nimble install https://github.com/status-im/nim-chronos.git ``` -$ nimble install https://github.com/status-im/nim-chronos.git + +or add a dependency to your `.nimble` file: + +```text +requires "chronos" ``` +## Projects using `chronos` + +* [libp2p](https://github.com/status-im/nim-libp2p) - Peer-to-Peer networking stack implemented in many languages +* [Looper](https://github.com/bung87/Looper) - Web framework +* [2DeFi](https://github.com/gogolxdong/2DeFi) - Decentralised file system + +`chronos` is available in the [Nim Playground](https://play.nim-lang.org/#ix=2TpS) + +Submit a PR to add yours! + ## Documentation ### Concepts @@ -176,15 +199,49 @@ proc p3() {.async.} = fut2 = p2() try: await fut1 - except: + except CachableError: echo "p1() failed: ", fut1.error.name, ": ", fut1.error.msg echo "reachable code here" await fut2 ``` -Exceptions inheriting from `Defect` are treated differently, being raised -directly. Don't try to catch them coming out of `poll()`, because this would -leave behind some zombie futures. +Chronos does not allow that future continuations and other callbacks raise +`CatchableError` - as such, calls to `poll` will never raise exceptions caused +originating from tasks on the dispatcher queue. It is however possible that +`Defect` that happen in tasks bubble up through `poll` as these are not caught +by the transformation. + +### Platform independence + +Several functions in `chronos` are backed by the operating system, such as +waiting for network events, creating files and sockets etc. The specific +exceptions that are raised by the OS is platform-dependent, thus such functions +are declared as raising `CatchableError` but will in general raise something +more specific. In particular, it's possible that some functions that are +annotated as raising `CatchableError` only raise on _some_ platforms - in order +to work on all platforms, calling code must assume that they will raise even +when they don't seem to do so on one platform. + +### Exception effects + +`chronos` currently offers minimal support for exception effects and `raises` +annotations. In general, during the `async` transformation, a generic +`except CatchableError` handler is added around the entire function being +transformed, in order to catch any exceptions and transfer them to the `Future`. +Because of this, the effect system thinks no exceptions are "leaking" because in +fact, exception _handling_ is deferred to when the future is being read. + +Effectively, this means that while code can be compiled with +`{.push raises: [Defect]}`, the intended effect propagation and checking is +**disabled** for `async` functions. + +To enable checking exception effects in `async` code, enable strict mode with +`-d:chronosStrictException`. + +In the strict mode, `async` functions are checked such that they only raise +`CatchableError` and thus must make sure to explicitly specify exception +effects on forward declarations, callbacks and methods using +`{.raises: [CatchableError].}` (or more strict) annotations. ## TODO * Pipe/Subprocess Transports. @@ -194,6 +251,12 @@ leave behind some zombie futures. When submitting pull requests, please add test cases for any new features or fixes and make sure `nimble test` is still able to execute the entire test suite successfully. +`chronos` follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/). + +## Other resources + +* [Historical differences with asyncdispatch](https://github.com/status-im/nim-chronos/wiki/AsyncDispatch-comparison) + ## License Licensed and distributed under either of @@ -205,4 +268,3 @@ or * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) at your option. These files may not be copied, modified, or distributed except according to those terms. - diff --git a/vendor/nim-chronos/chronos.nimble b/vendor/nim-chronos/chronos.nimble index 89e93af16..8d376347e 100644 --- a/vendor/nim-chronos/chronos.nimble +++ b/vendor/nim-chronos/chronos.nimble @@ -1,5 +1,5 @@ packageName = "chronos" -version = "2.6.1" +version = "3.0.0" author = "Status Research & Development GmbH" description = "Chronos" license = "Apache License 2.0 or MIT" @@ -10,12 +10,13 @@ skipDirs = @["tests"] requires "nim > 1.2.0", "stew", "bearssl", - "httputils" + "httputils", + "https://github.com/status-im/nim-unittest2.git#head" task test, "Run all tests": var commands = @[ "nim c -r -d:useSysAssert -d:useGcAssert tests/", - "nim c -r -d:chronosStackTrace tests/", + "nim c -r -d:chronosStackTrace -d:chronosStrictException tests/", "nim c -r -d:release tests/", "nim c -r -d:release -d:chronosFutureTracking tests/" ] diff --git a/vendor/nim-chronos/chronos/apps.nim b/vendor/nim-chronos/chronos/apps.nim index 53b3fac7b..a044ca3c0 100644 --- a/vendor/nim-chronos/chronos/apps.nim +++ b/vendor/nim-chronos/chronos/apps.nim @@ -6,5 +6,5 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import apps/http/httpserver -export httpserver +import ./apps/http/[httpserver, shttpserver] +export httpserver, shttpserver diff --git a/vendor/nim-chronos/chronos/apps/http/httpserver.nim b/vendor/nim-chronos/chronos/apps/http/httpserver.nim index 20f8ca0fa..00f60319f 100644 --- a/vendor/nim-chronos/chronos/apps/http/httpserver.nim +++ b/vendor/nim-chronos/chronos/apps/http/httpserver.nim @@ -9,9 +9,9 @@ import std/[tables, options, uri, strutils] import stew/[results, base10], httputils import ../../asyncloop, ../../asyncsync -import ../../streams/[asyncstream, boundstream, chunkstream, tlsstream] +import ../../streams/[asyncstream, boundstream, chunkstream] import httptable, httpcommon, multipart -export httptable, httpcommon, httputils, multipart, tlsstream, asyncstream, +export httptable, httpcommon, httputils, multipart, asyncstream, uri, tables, options, results type @@ -44,7 +44,12 @@ type Empty, Prepared, Sending, Finished, Failed, Cancelled, Dumb HttpProcessCallback* = - proc(req: RequestFence): Future[HttpResponseRef] {.gcsafe.} + proc(req: RequestFence): Future[HttpResponseRef] {. + gcsafe, raises: [Defect, CatchableError].} + + HttpConnectionCallback* = + proc(server: HttpServerRef, + transp: StreamTransport): Future[HttpConnectionRef] {.gcsafe, raises: [Defect].} HttpServer* = object of RootObj instance*: StreamServer @@ -56,16 +61,15 @@ type serverIdent*: string flags*: set[HttpServerFlags] socketFlags*: set[ServerFlags] - secureFlags*: set[TLSFlags] connections*: Table[string, Future[void]] acceptLoop*: Future[void] lifetime*: Future[void] headersTimeout: Duration + bufferSize: int maxHeadersSize: int maxRequestBodySize: int processCallback: HttpProcessCallback - tlsPrivateKey: TLSPrivateKey - tlsCertificate: TLSCertificate + createConnCallback: HttpConnectionCallback HttpServerRef* = ref HttpServer @@ -103,9 +107,8 @@ type HttpConnection* = object of RootObj server*: HttpServerRef transp: StreamTransport - mainReader: AsyncStreamReader - mainWriter: AsyncStreamWriter - tlsStream: TLSAsyncStream + mainReader*: AsyncStreamReader + mainWriter*: AsyncStreamWriter reader*: AsyncStreamReader writer*: AsyncStreamWriter buffer: seq[byte] @@ -119,6 +122,50 @@ proc init(htype: typedesc[HttpProcessError], error: HTTPServerError, code: HttpCode): HttpProcessError {.raises: [Defect].} = HttpProcessError(error: error, exc: exc, remote: remote, code: code) +proc init*(value: var HttpServer, + address: TransportAddress, + server: StreamServer, + processCallback: HttpProcessCallback, + createConnCallback: HttpConnectionCallback, + serverUri: Uri, + serverFlags: set[HttpServerFlags] = {}, + socketFlags: set[ServerFlags] = {ReuseAddr}, + serverIdent = "", + maxConnections: int = -1, + bufferSize: int = 4096, + backlogSize: int = 100, + httpHeadersTimeout = 10.seconds, + maxHeadersSize: int = 8192, + maxRequestBodySize: int = 1_048_576) = + + value = HttpServer( + address: address, + instance: server, + processCallback: processCallback, + createConnCallback: createConnCallback, + baseUri: serverUri, + serverIdent: serverIdent, + flags: serverFlags, + socketFlags: socketFlags, + maxConnections: maxConnections, + bufferSize: bufferSize, + backlogSize: backlogSize, + headersTimeout: httpHeadersTimeout, + maxHeadersSize: maxHeadersSize, + maxRequestBodySize: maxRequestBodySize, + # semaphore: + # if maxConnections > 0: + # newAsyncSemaphore(maxConnections) + # else: + # nil + lifetime: newFuture[void]("http.server.lifetime"), + connections: initTable[string, Future[void]]() + ) + +proc createConnection(server: HttpServerRef, + transp: StreamTransport): Future[HttpConnectionRef] {. + gcsafe.} + proc new*(htype: typedesc[HttpServerRef], address: TransportAddress, processCallback: HttpProcessCallback, @@ -126,9 +173,6 @@ proc new*(htype: typedesc[HttpServerRef], socketFlags: set[ServerFlags] = {ReuseAddr}, serverUri = Uri(), serverIdent = "", - tlsPrivateKey: TLSPrivateKey = nil, - tlsCertificate: TLSCertificate = nil, - secureFlags: set[TLSFlags] = {}, maxConnections: int = -1, bufferSize: int = 4096, backlogSize: int = 100, @@ -136,50 +180,30 @@ proc new*(htype: typedesc[HttpServerRef], maxHeadersSize: int = 8192, maxRequestBodySize: int = 1_048_576): HttpResult[HttpServerRef] = - if HttpServerFlags.Secure in serverFlags: - if isNil(tlsPrivateKey) or isNil(tlsCertificate): - return err("PrivateKey or Certificate is missing") + let serverUri = + if len(serverUri.hostname) > 0: + serverUri + else: + try: + parseUri("http://" & $address & "/") + except TransportAddressError as exc: + return err(exc.msg) - var res = HttpServerRef( - address: address, - serverIdent: serverIdent, - maxConnections: maxConnections, - headersTimeout: httpHeadersTimeout, - maxHeadersSize: maxHeadersSize, - maxRequestBodySize: maxRequestBodySize, - processCallback: processCallback, - backLogSize: backLogSize, - flags: serverFlags, - socketFlags: socketFlags, - tlsPrivateKey: tlsPrivateKey, - tlsCertificate: tlsCertificate - ) - - res.baseUri = + let serverInstance = try: - if len(serverUri.hostname) > 0 and isAbsolute(serverUri): - serverUri - else: - if HttpServerFlags.Secure in serverFlags: - parseUri("https://" & $address & "/") - else: - parseUri("http://" & $address & "/") - except TransportAddressError as exc: + createStreamServer(address, flags = socketFlags, bufferSize = bufferSize, + backlog = backlogSize) + except TransportOsError as exc: + return err(exc.msg) + except CatchableError as exc: return err(exc.msg) - try: - res.instance = createStreamServer(address, flags = socketFlags, - bufferSize = bufferSize, - backlog = backlogSize) - # if maxConnections > 0: - # res.semaphore = newAsyncSemaphore(maxConnections) - res.lifetime = newFuture[void]("http.server.lifetime") - res.connections = initTable[string, Future[void]]() - return ok(res) - except TransportOsError as exc: - return err(exc.msg) - except CatchableError as exc: - return err(exc.msg) + var res = HttpServerRef() + res[].init(address, serverInstance, processCallback, createConnection, + serverUri, serverFlags, socketFlags, serverIdent, maxConnections, + bufferSize, backLogSize, httpHeadersTimeout, maxHeadersSize, + maxRequestBodySize) + ok(res) proc getResponse*(req: HttpRequestRef): HttpResponseRef {.raises: [Defect].} = if req.response.isNone(): @@ -450,47 +474,32 @@ proc getRequest(conn: HttpConnectionRef): Future[HttpRequestRef] {.async.} = except AsyncStreamLimitError: raiseHttpCriticalError("Maximum size of request headers reached", Http431) -proc new(ht: typedesc[HttpConnectionRef], server: HttpServerRef, - transp: StreamTransport): HttpConnectionRef = - let mainReader = newAsyncStreamReader(transp) - let mainWriter = newAsyncStreamWriter(transp) - let tlsStream = - if HttpServerFlags.Secure in server.flags: - newTLSServerAsyncStream(mainReader, mainWriter, server.tlsPrivateKey, - server.tlsCertificate, - minVersion = TLSVersion.TLS12, - flags = server.secureFlags) - else: - nil - - let reader = - if isNil(tlsStream): - mainReader - else: - cast[AsyncStreamReader](tlsStream.reader) - - let writer = - if isNil(tlsStream): - mainWriter - else: - cast[AsyncStreamWriter](tlsStream.writer) - - HttpConnectionRef( - transp: transp, +proc init*(value: var HttpConnection, server: HttpServerRef, + transp: StreamTransport) = + value = HttpConnection( server: server, + transp: transp, buffer: newSeq[byte](server.maxHeadersSize), - mainReader: mainReader, - mainWriter: mainWriter, - tlsStream: tlsStream, - reader: reader, - writer: writer + mainReader: newAsyncStreamReader(transp), + mainWriter: newAsyncStreamWriter(transp) ) -proc closeWait(conn: HttpConnectionRef) {.async.} = - if HttpServerFlags.Secure in conn.server.flags: - # First we will close TLS streams. - await allFutures(conn.reader.closeWait(), conn.writer.closeWait()) +proc new(ht: typedesc[HttpConnectionRef], server: HttpServerRef, + transp: StreamTransport): HttpConnectionRef = + var res = HttpConnectionRef() + res[].init(server, transp) + res.reader = res.mainReader + res.writer = res.mainWriter + res +proc closeWait*(conn: HttpConnectionRef) {.async.} = + var pending: seq[Future[void]] + if conn.reader != conn.mainReader: + pending.add(conn.reader.closeWait()) + if conn.writer != conn.mainWriter: + pending.add(conn.writer.closeWait()) + if len(pending) > 0: + await allFutures(pending) # After we going to close everything else. await allFutures(conn.mainReader.closeWait(), conn.mainWriter.closeWait(), conn.transp.closeWait()) @@ -505,20 +514,7 @@ proc closeWait(req: HttpRequestRef) {.async.} = proc createConnection(server: HttpServerRef, transp: StreamTransport): Future[HttpConnectionRef] {. async.} = - var conn = HttpConnectionRef.new(server, transp) - if HttpServerFlags.Secure notin server.flags: - # Non secure connection - return conn - - try: - await handshake(conn.tlsStream) - return conn - except CancelledError as exc: - await conn.closeWait() - raise exc - except TLSStreamError: - await conn.closeWait() - raiseHttpCriticalError("Unable to establish secure connection") + return HttpConnectionRef.new(server, transp) proc processLoop(server: HttpServerRef, transp: StreamTransport) {.async.} = var @@ -527,10 +523,9 @@ proc processLoop(server: HttpServerRef, transp: StreamTransport) {.async.} = runLoop = false try: - conn = await createConnection(server, transp) + conn = await server.createConnCallback(server, transp) runLoop = true except CancelledError: - # We could be cancelled only when we perform TLS handshake, connection server.connections.del(transp.getId()) await transp.closeWait() return diff --git a/vendor/nim-chronos/chronos/apps/http/shttpserver.nim b/vendor/nim-chronos/chronos/apps/http/shttpserver.nim new file mode 100644 index 000000000..ca22ccc4c --- /dev/null +++ b/vendor/nim-chronos/chronos/apps/http/shttpserver.nim @@ -0,0 +1,105 @@ +# +# Chronos HTTP/S server implementation +# (c) Copyright 2021-Present +# Status Research & Development GmbH +# +# Licensed under either of +# Apache License, version 2.0, (LICENSE-APACHEv2) +# MIT license (LICENSE-MIT) +import httpserver +import ../../asyncloop, ../../asyncsync +import ../../streams/[asyncstream, tlsstream] +export httpserver, asyncstream, tlsstream + +type + SecureHttpServer* = object of HttpServer + secureFlags*: set[TLSFlags] + tlsPrivateKey: TLSPrivateKey + tlsCertificate: TLSCertificate + + SecureHttpServerRef* = ref SecureHttpServer + + SecureHttpConnection* = object of HttpConnection + tlsStream*: TLSAsyncStream + + SecureHttpConnectionRef* = ref SecureHttpConnection + +proc new*(ht: typedesc[SecureHttpConnectionRef], server: SecureHttpServerRef, + transp: StreamTransport): SecureHttpConnectionRef = + var res = SecureHttpConnectionRef() + HttpConnection(res[]).init(HttpServerRef(server), transp) + let tlsStream = + newTLSServerAsyncStream(res.mainReader, res.mainWriter, + server.tlsPrivateKey, + server.tlsCertificate, + minVersion = TLSVersion.TLS12, + flags = server.secureFlags) + res.tlsStream = tlsStream + res.reader = AsyncStreamReader(tlsStream.reader) + res.writer = AsyncStreamWriter(tlsStream.writer) + res + +proc createSecConnection(server: HttpServerRef, + transp: StreamTransport): Future[HttpConnectionRef] {. + async.} = + let secureServ = cast[SecureHttpServerRef](server) + var sconn = SecureHttpConnectionRef.new(secureServ, transp) + try: + await handshake(sconn.tlsStream) + return HttpConnectionRef(sconn) + except CancelledError as exc: + await HttpConnectionRef(sconn).closeWait() + raise exc + except TLSStreamError: + await HttpConnectionRef(sconn).closeWait() + raiseHttpCriticalError("Unable to establish secure connection") + +proc new*(htype: typedesc[SecureHttpServerRef], + address: TransportAddress, + processCallback: HttpProcessCallback, + tlsPrivateKey: TLSPrivateKey, + tlsCertificate: TLSCertificate, + serverFlags: set[HttpServerFlags] = {}, + socketFlags: set[ServerFlags] = {ReuseAddr}, + serverUri = Uri(), + serverIdent = "", + secureFlags: set[TLSFlags] = {}, + maxConnections: int = -1, + bufferSize: int = 4096, + backlogSize: int = 100, + httpHeadersTimeout = 10.seconds, + maxHeadersSize: int = 8192, + maxRequestBodySize: int = 1_048_576 + ): HttpResult[SecureHttpServerRef] = + + doAssert(not(isNil(tlsPrivateKey)), "TLS private key must not be nil!") + doAssert(not(isNil(tlsCertificate)), "TLS certificate must not be nil!") + + let serverUri = + if len(serverUri.hostname) > 0: + serverUri + else: + try: + parseUri("https://" & $address & "/") + except TransportAddressError as exc: + return err(exc.msg) + + let serverInstance = + try: + createStreamServer(address, flags = socketFlags, bufferSize = bufferSize, + backlog = backlogSize) + except TransportOsError as exc: + return err(exc.msg) + except CatchableError as exc: + return err(exc.msg) + + var res = SecureHttpServerRef() + HttpServer(res[]).init(address, serverInstance, processCallback, + createSecConnection, serverUri, serverFlags, + socketFlags, serverIdent, maxConnections, + bufferSize, backLogSize, httpHeadersTimeout, + maxHeadersSize, maxRequestBodySize) + res.tlsCertificate = tlsCertificate + res.tlsPrivateKey = tlsPrivateKey + res.secureFlags = secureFlags + ok(res) diff --git a/vendor/nim-chronos/chronos/asyncfutures2.nim b/vendor/nim-chronos/chronos/asyncfutures2.nim index 4d87bf28f..15355b233 100644 --- a/vendor/nim-chronos/chronos/asyncfutures2.nim +++ b/vendor/nim-chronos/chronos/asyncfutures2.nim @@ -9,7 +9,7 @@ # MIT license (LICENSE-MIT) import std/[os, tables, strutils, heapqueue, options, deques, cstrutils, sequtils] -import srcloc +import ./srcloc export srcloc const @@ -29,7 +29,7 @@ type cancelcb*: CallbackFunc child*: FutureBase state*: FutureState - error*: ref Exception ## Stored exception + error*: ref CatchableError ## Stored exception mustCancel*: bool id*: int @@ -171,7 +171,7 @@ proc done*(future: FutureBase): bool {.inline.} = completed(future) when defined(chronosFutureTracking): - proc futureDestructor(udata: pointer) {.gcsafe.} = + proc futureDestructor(udata: pointer) = ## This procedure will be called when Future[T] got finished, cancelled or ## failed and all Future[T].callbacks are already scheduled and processed. let future = cast[FutureBase](udata) @@ -271,7 +271,7 @@ template complete*[T](futvar: FutureVar[T], val: T) = ## Any previously stored value will be overwritten. complete(futvar, val, getSrcLocation()) -proc fail[T](future: Future[T], error: ref Exception, loc: ptr SrcLoc) = +proc fail[T](future: Future[T], error: ref CatchableError, loc: ptr SrcLoc) = if not(future.cancelled()): checkFinished(FutureBase(future), loc) future.error = error @@ -282,7 +282,7 @@ proc fail[T](future: Future[T], error: ref Exception, loc: ptr SrcLoc) = getStackTrace(error) future.finish(FutureState.Failed) -template fail*[T](future: Future[T], error: ref Exception) = +template fail*[T](future: Future[T], error: ref CatchableError) = ## Completes ``future`` with ``error``. fail(future, error, getSrcLocation()) @@ -406,37 +406,39 @@ proc getHint(entry: StackTraceEntry): string = return "Resumes an async procedure" proc `$`*(entries: seq[StackTraceEntry]): string = - result = "" - # Find longest filename & line number combo for alignment purposes. - var longestLeft = 0 - for entry in entries: - if isNil(entry.procName): continue + try: + # Find longest filename & line number combo for alignment purposes. + var longestLeft = 0 + for entry in entries: + if isNil(entry.procName): continue - let left = $entry.filename & $entry.line - if left.len > longestLeft: - longestLeft = left.len + let left = $entry.filename & $entry.line + if left.len > longestLeft: + longestLeft = left.len - var indent = 2 - # Format the entries. - for entry in entries: - if isNil(entry.procName): - if entry.line == -10: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - else: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue + var indent = 2 + # Format the entries. + for entry in entries: + if isNil(entry.procName): + if entry.line == -10: + result.add(spaces(indent) & "#[\n") + indent.inc(2) + else: + indent.dec(2) + result.add(spaces(indent) & "]#\n") + continue - let left = "$#($#)" % [$entry.filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ - left, - spaces(longestLeft - left.len + 2), - $entry.procName - ]) - let hint = getHint(entry) - if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + let left = "$#($#)" % [$entry.filename, $entry.line] + result.add((spaces(indent) & "$#$# $#\n") % [ + left, + spaces(longestLeft - left.len + 2), + $entry.procName + ]) + let hint = getHint(entry) + if hint.len > 0: + result.add(spaces(indent+2) & "## " & hint & "\n") + except ValueError as exc: + return exc.msg # Shouldn't actually happen since we set the formatting string when defined(chronosStackTrace): proc injectStacktrace(future: FutureBase) = @@ -462,7 +464,7 @@ when defined(chronosStackTrace): # newMsg.add "\n" & $entry future.error.msg = newMsg -proc internalCheckComplete*(fut: FutureBase) = +proc internalCheckComplete*(fut: FutureBase) {.raises: [Defect, CatchableError].} = # For internal use only. Used in asyncmacro if not(isNil(fut.error)): when defined(chronosStackTrace): @@ -474,22 +476,19 @@ proc internalRead*[T](fut: Future[T] | FutureVar[T]): T {.inline.} = when T isnot void: return fut.value -proc read*[T](future: Future[T] | FutureVar[T]): T = +proc read*[T](future: Future[T] | FutureVar[T]): T {.raises: [Defect, CatchableError].} = ## Retrieves the value of ``future``. Future must be finished otherwise ## this function will fail with a ``ValueError`` exception. ## ## If the result of the future is an error then that error will be raised. - {.push hint[ConvFromXtoItselfNotNeeded]: off.} - let fut = Future[T](future) - {.pop.} - if fut.finished(): + if future.finished(): internalCheckComplete(future) internalRead(future) else: # TODO: Make a custom exception type for this? raise newException(ValueError, "Future still in progress.") -proc readError*[T](future: Future[T]): ref Exception = +proc readError*[T](future: Future[T]): ref CatchableError {.raises: [Defect, ValueError].} = ## Retrieves the exception stored in ``future``. ## ## An ``ValueError`` exception will be thrown if no exception exists @@ -507,18 +506,18 @@ proc mget*[T](future: FutureVar[T]): var T = ## Future has not been finished. result = Future[T](future).value -proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future - ## finished with an error. - ## - ## This should be used instead of ``discard`` to discard void futures. - doAssert(not isNil(future), "Future is nil") - proc cb(data: pointer) = - if future.failed() or future.cancelled(): - when defined(chronosStackTrace): - injectStacktrace(future) - raise future.error - future.callback = cb +template taskFutureLocation(future: FutureBase): string = + let loc = future.location[0] + "[" & ( + if len(loc.procedure) == 0: "[unspecified]" else: $loc.procedure & "()" + ) & " at " & $loc.file & ":" & $(loc.line) & "]" + +template taskErrorMessage(future: FutureBase): string = + "Asynchronous task " & taskFutureLocation(future) & + " finished with an exception \"" & $future.error.name & "\"!\nStack trace: " & + future.error.getStackTrace() +template taskCancelMessage(future: FutureBase): string = + "Asynchronous task " & taskFutureLocation(future) & " was cancelled!" proc asyncSpawn*(future: Future[void]) = ## Spawns a new concurrent async task. @@ -534,35 +533,45 @@ proc asyncSpawn*(future: Future[void]) = ## and processed immediately. doAssert(not isNil(future), "Future is nil") - template getFutureLocation(): string = - let loc = future.location[0] - "[" & ( - if len(loc.procedure) == 0: "[unspecified]" else: $loc.procedure & "()" - ) & " at " & $loc.file & ":" & $(loc.line) & "]" - - template getErrorMessage(): string = - "Asynchronous task " & getFutureLocation() & - " finished with an exception \"" & $future.error.name & "\"!" - template getCancelMessage(): string = - "Asynchronous task " & getFutureLocation() & " was cancelled!" - proc cb(data: pointer) = if future.failed(): - raise newException(FutureDefect, getErrorMessage()) + raise newException(FutureDefect, taskErrorMessage(future)) elif future.cancelled(): - raise newException(FutureDefect, getCancelMessage()) + raise newException(FutureDefect, taskCancelMessage(future)) if not(future.finished()): # We adding completion callback only if ``future`` is not finished yet. future.addCallback(cb) else: - if future.failed(): - raise newException(FutureDefect, getErrorMessage()) - elif future.cancelled(): - raise newException(FutureDefect, getCancelMessage()) + cb(nil) -proc asyncDiscard*[T](future: Future[T]) {.deprecated.} = discard - ## This is async workaround for discard ``Future[T]``. +proc asyncCheck*[T](future: Future[T]) {. + deprecated: "Raises Defect on future failure, fix your code and use asyncSpawn!".} = + ## This function used to raise an exception through the `poll` call if + ## the given future failed - there's no way to handle such exceptions so this + ## function is now an alias for `asyncSpawn` + ## + when T is void: + asyncSpawn(future) + else: + proc cb(data: pointer) = + if future.failed(): + raise newException(FutureDefect, taskErrorMessage(future)) + elif future.cancelled(): + raise newException(FutureDefect, taskCancelMessage(future)) + + if not(future.finished()): + # We adding completion callback only if ``future`` is not finished yet. + future.addCallback(cb) + else: + cb(nil) + +proc asyncDiscard*[T](future: Future[T]) {. + deprecated: "Use asyncSpawn or `discard await`".} = discard + ## `asyncDiscard` will discard the outcome of the operation - unlike `discard` + ## it also throws away exceptions! Use `asyncSpawn` if you're sure your + ## code doesn't raise exceptions, or `discard await` to ignore successful + ## outcomes proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] {. deprecated: "Use allFutures[T](varargs[Future[T]])".} = @@ -587,7 +596,7 @@ proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] {. fut1.callback = cb fut2.callback = cb - proc cancellation(udata: pointer) {.gcsafe.} = + proc cancellation(udata: pointer) = # On cancel we remove all our callbacks only. if not(fut1.finished()): fut1.removeCallback(cb) @@ -611,7 +620,8 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = ## ## If cancelled, ``fut1`` and ``fut2`` futures WILL NOT BE cancelled. var retFuture = newFuture[void]("chronos.or") - proc cb(udata: pointer) {.gcsafe.} = + var cb: proc(udata: pointer) {.gcsafe, raises: [Defect].} + cb = proc(udata: pointer) = if not(retFuture.finished()): var fut = cast[FutureBase](udata) if cast[pointer](fut1) == udata: @@ -623,7 +633,7 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = else: retFuture.complete() - proc cancellation(udata: pointer) {.gcsafe.} = + proc cancellation(udata: pointer) = # On cancel we remove all our callbacks only. if not(fut1.finished()): fut1.removeCallback(cb) @@ -676,7 +686,7 @@ proc all*[T](futs: varargs[Future[T]]): auto {. when T is void: var retFuture = newFuture[void]("chronos.all(void)") - proc cb(udata: pointer) {.gcsafe.} = + proc cb(udata: pointer) = if not(retFuture.finished()): inc(completedFutures) if completedFutures == totalFutures: @@ -698,7 +708,7 @@ proc all*[T](futs: varargs[Future[T]]): auto {. var retFuture = newFuture[seq[T]]("chronos.all(T)") var retValues = newSeq[T](totalFutures) - proc cb(udata: pointer) {.gcsafe.} = + proc cb(udata: pointer) = if not(retFuture.finished()): inc(completedFutures) if completedFutures == totalFutures: @@ -707,7 +717,7 @@ proc all*[T](futs: varargs[Future[T]]): auto {. retFuture.fail(nfut.error) break else: - retValues[k] = nfut.read() + retValues[k] = nfut.value if not(retFuture.failed()): retFuture.complete(retValues) @@ -731,7 +741,7 @@ proc oneIndex*[T](futs: varargs[Future[T]]): Future[int] {. var nfuts = @futs var retFuture = newFuture[int]("chronos.oneIndex(T)") - proc cb(udata: pointer) {.gcsafe.} = + proc cb(udata: pointer) = var res = -1 if not(retFuture.finished()): var rfut = cast[FutureBase](udata) @@ -762,7 +772,7 @@ proc oneValue*[T](futs: varargs[Future[T]]): Future[T] {. var nfuts = @futs var retFuture = newFuture[T]("chronos.oneValue(T)") - proc cb(udata: pointer) {.gcsafe.} = + proc cb(udata: pointer) = var resFut: Future[T] if not(retFuture.finished()): var rfut = cast[FutureBase](udata) @@ -794,10 +804,10 @@ proc cancelAndWait*[T](fut: Future[T]): Future[void] = ## If ``fut`` is already finished (completed, failed or cancelled) result ## Future[void] object will be returned complete. var retFuture = newFuture[void]("chronos.cancelAndWait(T)") - proc continuation(udata: pointer) {.gcsafe.} = + proc continuation(udata: pointer) = if not(retFuture.finished()): retFuture.complete() - proc cancellation(udata: pointer) {.gcsafe.} = + proc cancellation(udata: pointer) = if not(fut.finished()): fut.removeCallback(continuation) if fut.finished(): @@ -823,13 +833,13 @@ proc allFutures*[T](futs: varargs[Future[T]]): Future[void] = # Because we can't capture varargs[T] in closures we need to create copy. var nfuts = @futs - proc cb(udata: pointer) {.gcsafe.} = + proc cb(udata: pointer) = if not(retFuture.finished()): inc(completedFutures) if completedFutures == totalFutures: retFuture.complete() - proc cancellation(udata: pointer) {.gcsafe.} = + proc cancellation(udata: pointer) = # On cancel we remove all our callbacks only. for i in 0.. createCb(retFuture) # NOTE: The "_continue" suffix is checked for in asyncfutures.nim to produce # friendlier stack traces: - var cbName = genSym(nskProc, prcName & "_continue") + var cbName = genSym(nskVar, prcName & "_continue") var procCb = getAst createCb(retFutureSym, iteratorNameSym, newStrLitNode(prcName), cbName, @@ -281,7 +342,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = #if prcName == "recvLineInto": # echo(toStrLit(result)) -template await*[T](f: Future[T]): auto = +template await*[T](f: Future[T]): untyped = when declared(chronosInternalRetFuture): when not declaredInScope(chronosInternalTmpFuture): var chronosInternalTmpFuture {.inject.}: FutureBase @@ -304,7 +365,8 @@ template await*[T](f: Future[T]): auto = if chronosInternalRetFuture.mustCancel: raise newCancelledError() chronosInternalTmpFuture.internalCheckComplete() - cast[type(f)](chronosInternalTmpFuture).internalRead() + when T isnot void: + cast[type(f)](chronosInternalTmpFuture).internalRead() else: unsupported "await is only available within {.async.}" diff --git a/vendor/nim-chronos/chronos/asyncsync.nim b/vendor/nim-chronos/chronos/asyncsync.nim index b352a8958..44f09ae96 100644 --- a/vendor/nim-chronos/chronos/asyncsync.nim +++ b/vendor/nim-chronos/chronos/asyncsync.nim @@ -9,6 +9,9 @@ # MIT license (LICENSE-MIT) ## This module implements some core synchronization primitives + +{.push raises: [Defect].} + import std/[sequtils, deques] import ./asyncloop @@ -115,7 +118,7 @@ proc locked*(lock: AsyncLock): bool = ## Return `true` if the lock ``lock`` is acquired, `false` otherwise. lock.locked -proc release*(lock: AsyncLock) = +proc release*(lock: AsyncLock) {.raises: [Defect, AsyncLockError].} = ## Release a lock ``lock``. ## ## When the ``lock`` is locked, reset it to unlocked, and return. If any @@ -220,7 +223,8 @@ proc empty*[T](aq: AsyncQueue[T]): bool {.inline.} = ## Return ``true`` if the queue is empty, ``false`` otherwise. (len(aq.queue) == 0) -proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) = +proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) {. + raises: [Defect, AsyncQueueFullError].}= ## Put an item ``item`` to the beginning of the queue ``aq`` immediately. ## ## If queue ``aq`` is full, then ``AsyncQueueFullError`` exception raised. @@ -229,7 +233,8 @@ proc addFirstNoWait*[T](aq: AsyncQueue[T], item: T) = aq.queue.addFirst(item) aq.getters.wakeupNext() -proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) = +proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) {. + raises: [Defect, AsyncQueueFullError].}= ## Put an item ``item`` at the end of the queue ``aq`` immediately. ## ## If queue ``aq`` is full, then ``AsyncQueueFullError`` exception raised. @@ -238,7 +243,8 @@ proc addLastNoWait*[T](aq: AsyncQueue[T], item: T) = aq.queue.addLast(item) aq.getters.wakeupNext() -proc popFirstNoWait*[T](aq: AsyncQueue[T]): T = +proc popFirstNoWait*[T](aq: AsyncQueue[T]): T {. + raises: [Defect, AsyncQueueEmptyError].} = ## Get an item from the beginning of the queue ``aq`` immediately. ## ## If queue ``aq`` is empty, then ``AsyncQueueEmptyError`` exception raised. @@ -248,7 +254,8 @@ proc popFirstNoWait*[T](aq: AsyncQueue[T]): T = aq.putters.wakeupNext() res -proc popLastNoWait*[T](aq: AsyncQueue[T]): T = +proc popLastNoWait*[T](aq: AsyncQueue[T]): T {. + raises: [Defect, AsyncQueueEmptyError].} = ## Get an item from the end of the queue ``aq`` immediately. ## ## If queue ``aq`` is empty, then ``AsyncQueueEmptyError`` exception raised. @@ -314,11 +321,13 @@ proc popLast*[T](aq: AsyncQueue[T]): Future[T] {.async.} = raise exc return aq.popLastNoWait() -proc putNoWait*[T](aq: AsyncQueue[T], item: T) {.inline.} = +proc putNoWait*[T](aq: AsyncQueue[T], item: T) {. + raises: [Defect, AsyncQueueFullError].} = ## Alias of ``addLastNoWait()``. aq.addLastNoWait(item) -proc getNoWait*[T](aq: AsyncQueue[T]): T {.inline.} = +proc getNoWait*[T](aq: AsyncQueue[T]): T {. + raises: [Defect, AsyncQueueEmptyError].} = ## Alias of ``popFirstNoWait()``. aq.popFirstNoWait() diff --git a/vendor/nim-chronos/chronos/debugutils.nim b/vendor/nim-chronos/chronos/debugutils.nim index f00d80532..87a7225d8 100644 --- a/vendor/nim-chronos/chronos/debugutils.nim +++ b/vendor/nim-chronos/chronos/debugutils.nim @@ -6,7 +6,10 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import asyncloop + +{.push raises: [Defect].} + +import ./asyncloop const AllFutureStates* = {FutureState.Pending, FutureState.Cancelled, diff --git a/vendor/nim-chronos/chronos/handles.nim b/vendor/nim-chronos/chronos/handles.nim index eef66cb39..03b7f00f8 100644 --- a/vendor/nim-chronos/chronos/handles.nim +++ b/vendor/nim-chronos/chronos/handles.nim @@ -7,7 +7,12 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import net, nativesockets, asyncloop +{.push raises: [Defect].} + +import + std/[net, nativesockets], + ./selectors2, + ./asyncloop when defined(windows): import os, winlean @@ -88,7 +93,8 @@ proc getSocketError*(socket: AsyncFD, err: var int): bool = result = getSockOpt(socket, cint(SOL_SOCKET), cint(SO_ERROR), err) proc createAsyncSocket*(domain: Domain, sockType: SockType, - protocol: Protocol): AsyncFD = + protocol: Protocol): AsyncFD {. + raises: [Defect, CatchableError].} = ## Creates new asynchronous socket. ## Returns ``asyncInvalidSocket`` on error. let handle = createNativeSocket(domain, sockType, protocol) @@ -104,7 +110,8 @@ proc createAsyncSocket*(domain: Domain, sockType: SockType, result = AsyncFD(handle) register(result) -proc wrapAsyncSocket*(sock: SocketHandle): AsyncFD = +proc wrapAsyncSocket*(sock: SocketHandle): AsyncFD {. + raises: [Defect, CatchableError].} = ## Wraps socket to asynchronous socket handle. ## Return ``asyncInvalidSocket`` on error. if not setSocketBlocking(sock, false): @@ -117,7 +124,7 @@ proc wrapAsyncSocket*(sock: SocketHandle): AsyncFD = result = AsyncFD(sock) register(result) -proc getMaxOpenFiles*(): int = +proc getMaxOpenFiles*(): int {.raises: [Defect, OSError].} = ## Returns maximum file descriptor number that can be opened by this process. ## ## Note: On Windows its impossible to obtain such number, so getMaxOpenFiles() @@ -131,7 +138,7 @@ proc getMaxOpenFiles*(): int = raiseOSError(osLastError()) result = int(limits.rlim_cur) -proc setMaxOpenFiles*(count: int) = +proc setMaxOpenFiles*(count: int) {.raises: [Defect, OSError].} = ## Set maximum file descriptor number that can be opened by this process. ## ## Note: On Windows its impossible to set this value, so it just a nop call. diff --git a/vendor/nim-chronos/chronos/ioselects/ioselectors_epoll.nim b/vendor/nim-chronos/chronos/ioselects/ioselectors_epoll.nim new file mode 100644 index 000000000..16daa2dc0 --- /dev/null +++ b/vendor/nim-chronos/chronos/ioselects/ioselectors_epoll.nim @@ -0,0 +1,524 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Linux epoll(). + +{.push raises: [Defect].} + +import posix, times, epoll + +# Maximum number of events that can be returned +const MAX_EPOLL_EVENTS = 64 + +when not defined(android): + type + SignalFdInfo* {.importc: "struct signalfd_siginfo", + header: "", pure, final.} = object + ssi_signo*: uint32 + ssi_errno*: int32 + ssi_code*: int32 + ssi_pid*: uint32 + ssi_uid*: uint32 + ssi_fd*: int32 + ssi_tid*: uint32 + ssi_band*: uint32 + ssi_overrun*: uint32 + ssi_trapno*: uint32 + ssi_status*: int32 + ssi_int*: int32 + ssi_ptr*: uint64 + ssi_utime*: uint64 + ssi_stime*: uint64 + ssi_addr*: uint64 + pad* {.importc: "__pad".}: array[0..47, uint8] + +proc timerfd_create(clock_id: ClockId, flags: cint): cint + {.cdecl, importc: "timerfd_create", header: "".} +proc timerfd_settime(ufd: cint, flags: cint, + utmr: var Itimerspec, otmr: var Itimerspec): cint + {.cdecl, importc: "timerfd_settime", header: "".} +proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "".} + +when not defined(android): + proc signalfd(fd: cint, mask: var Sigset, flags: cint): cint + {.cdecl, importc: "signalfd", header: "".} + +when hasThreadSupport: + type + SelectorImpl[T] = object + epollFD: cint + numFD: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + epollFD: cint + numFD: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] +type + SelectEventImpl = object + efd: cint + SelectEvent* = ptr SelectEventImpl + +proc newSelector*[T](): Selector[T] {.raises: [Defect, OSError].} = + # Retrieve the maximum fd count (for current OS) via getrlimit() + var a = RLimit() + # Start with a reasonable size, checkFd() will grow this on demand + const numFD = 1024 + + var epollFD = epoll_create(MAX_EPOLL_EVENTS) + if epollFD < 0: + raiseOSError(osLastError()) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.epollFD = epollFD + result.numFD = numFD + result.fds = allocSharedArray[SelectorKey[T]](numFD) + else: + result = Selector[T]() + result.epollFD = epollFD + result.numFD = numFD + result.fds = newSeq[SelectorKey[T]](numFD) + + for i in 0 ..< numFD: + result.fds[i].ident = InvalidIdent + +proc close*[T](s: Selector[T]) = + let res = posix.close(s.epollFD) + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + if res != 0: + raiseIOSelectorsError(osLastError()) + +proc newSelectEvent*(): SelectEvent {.raises: [Defect, OSError, IOSelectorsException].} = + let fdci = eventfd(0, 0) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fdci) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.efd = fdci + +proc trigger*(ev: SelectEvent) {.raises: [Defect, IOSelectorsException].} = + var data: uint64 = 1 + if posix.write(ev.efd, addr data, sizeof(uint64)) == -1: + raiseIOSelectorsError(osLastError()) + +proc close*(ev: SelectEvent) {.raises: [Defect, IOSelectorsException].} = + let res = posix.close(ev.efd) + deallocShared(cast[pointer](ev)) + if res != 0: + raiseIOSelectorsError(osLastError()) + +template checkFd(s, f) = + if f >= s.numFD: + var numFD = s.numFD + while numFD <= f: numFD *= 2 + when hasThreadSupport: + s.fds = reallocSharedArray(s.fds, numFD) + else: + s.fds.setLen(numFD) + for i in s.numFD ..< numFD: + s.fds[i].ident = InvalidIdent + s.numFD = numFD + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) {. + raises: [Defect, IOSelectorsException].} = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent, "Descriptor " & $fdi & " already registered") + s.setKey(fdi, events, 0, data) + if events != {}: + var epv = EpollEvent(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, events: set[Event]) {. + raises: [Defect, IOSelectorsException].} = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor " & $fdi & " is not registered in the selector!") + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + var epv = EpollEvent(events: EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if Event.Read in events: epv.events = epv.events or EPOLLIN + if Event.Write in events: epv.events = epv.events or EPOLLOUT + + if pkey.events == {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + inc(s.count) + else: + if events != {}: + if epoll_ctl(s.epollFD, EPOLL_CTL_MOD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + else: + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) {.raises: [Defect, IOSelectorsException].} = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor " & $fdi & " is not registered in the selector!") + if pkey.events != {}: + when not defined(android): + if Event.Read in pkey.events or Event.Write in pkey.events or Event.User in pkey.events: + var epv = EpollEvent() + # TODO: Refactor all these EPOLL_CTL_DEL + dec(s.count) into a proc. + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + if Event.Finished notin pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Signal in pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(s.fds[fdi].param)) + unblockSignals(nmask, omask) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Process in pkey.events: + if Event.Finished notin pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, SIGCHLD) + unblockSignals(nmask, omask) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + else: + if Event.Read in pkey.events or Event.Write in pkey.events or Event.User in pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + elif Event.Timer in pkey.events: + if Event.Finished notin pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + if posix.close(cint(fdi)) != 0: + raiseIOSelectorsError(osLastError()) + clearKey(pkey) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) {. + raises: [Defect, IOSelectorsException].} = + let fdi = int(ev.efd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") + doAssert(Event.User in pkey.events) + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + dec(s.count) + clearKey(pkey) + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {. + discardable, raises: [Defect, IOSelectorsException].} = + var + newTs: Itimerspec + oldTs: Itimerspec + let fdi = timerfd_create(CLOCK_MONOTONIC, 0).int + if fdi == -1: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var events = {Event.Timer} + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + + if oneshot: + newTs.it_interval.tv_sec = posix.Time(0) + newTs.it_interval.tv_nsec = 0 + newTs.it_value.tv_sec = posix.Time(timeout div 1_000) + newTs.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000 + incl(events, Event.Oneshot) + epv.events = epv.events or EPOLLONESHOT + else: + newTs.it_interval.tv_sec = posix.Time(timeout div 1000) + newTs.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000 + newTs.it_value.tv_sec = newTs.it_interval.tv_sec + newTs.it_value.tv_nsec = newTs.it_interval.tv_nsec + + if timerfd_settime(fdi.cint, cint(0), newTs, oldTs) != 0: + raiseIOSelectorsError(osLastError()) + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + s.setKey(fdi, events, 0, data) + inc(s.count) + result = fdi + +when not defined(android): + proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {. + discardable, raises: [Defect, OSError, IOSelectorsException].} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + s.setKey(fdi, {Event.Signal}, signal, data) + inc(s.count) + result = fdi + + proc registerProcess*[T](s: Selector, pid: int, + data: T): int {. + discardable, raises: [Defect, IOSelectorsException].} = + var + nmask: Sigset + omask: Sigset + + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, posix.SIGCHLD) + blockSignals(nmask, omask) + + let fdi = signalfd(-1, nmask, 0).int + if fdi == -1: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fdi.cint) + + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = fdi.uint + epv.events = EPOLLIN or EPOLLRDHUP + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, fdi.cint, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + s.setKey(fdi, {Event.Process, Event.Oneshot}, pid, data) + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = int(ev.efd) + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") + s.setKey(fdi, {Event.User}, 0, data) + var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP) + epv.data.u64 = ev.efd.uint + if epoll_ctl(s.epollFD, EPOLL_CTL_ADD, ev.efd, addr epv) != 0: + raiseIOSelectorsError(osLastError()) + inc(s.count) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int {.raises: [Defect, IOSelectorsException].} = + var + resTable: array[MAX_EPOLL_EVENTS, EpollEvent] + maxres = MAX_EPOLL_EVENTS + i, k: int + + if maxres > len(results): + maxres = len(results) + + verifySelectParams(timeout) + + let count = epoll_wait(s.epollFD, addr(resTable[0]), maxres.cint, + timeout.cint) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + i = 0 + k = 0 + while i < count: + let fdi = int(resTable[i].data.u64) + let pevents = resTable[i].events + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent) + var rkey = ReadyKey(fd: fdi, events: {}) + + if (pevents and EPOLLERR) != 0 or (pevents and EPOLLHUP) != 0: + if (pevents and EPOLLHUP) != 0: + rkey.errorCode = OSErrorCode ECONNRESET + else: + # Try reading SO_ERROR from fd. + var error: cint + var size = SockLen sizeof(error) + if getsockopt(SocketHandle fdi, SOL_SOCKET, SO_ERROR, addr(error), + addr(size)) == 0'i32: + rkey.errorCode = OSErrorCode error + + rkey.events.incl(Event.Error) + if (pevents and EPOLLOUT) != 0: + rkey.events.incl(Event.Write) + when not defined(android): + if (pevents and EPOLLIN) != 0: + if Event.Read in pkey.events: + rkey.events.incl(Event.Read) + elif Event.Timer in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + rkey.events.incl(Event.Timer) + elif Event.Signal in pkey.events: + var data = SignalFdInfo() + if posix.read(cint(fdi), addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseIOSelectorsError(osLastError()) + rkey.events.incl(Event.Signal) + elif Event.Process in pkey.events: + var data = SignalFdInfo() + if posix.read(cint(fdi), addr data, + sizeof(SignalFdInfo)) != sizeof(SignalFdInfo): + raiseIOSelectorsError(osLastError()) + if cast[int](data.ssi_pid) == pkey.param: + rkey.events.incl(Event.Process) + else: + inc(i) + continue + elif Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseIOSelectorsError(err) + rkey.events.incl(Event.User) + else: + if (pevents and EPOLLIN) != 0: + if Event.Read in pkey.events: + rkey.events.incl(Event.Read) + elif Event.Timer in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + rkey.events.incl(Event.Timer) + elif Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(cint(fdi), addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + inc(i) + continue + else: + raiseIOSelectorsError(err) + rkey.events.incl(Event.User) + + if Event.Oneshot in pkey.events: + var epv = EpollEvent() + if epoll_ctl(s.epollFD, EPOLL_CTL_DEL, cint(fdi), addr epv) != 0: + raiseIOSelectorsError(osLastError()) + # we will not clear key until it will be unregistered, so + # application can obtain data, but we will decrease counter, + # because epoll is empty. + dec(s.count) + # we are marking key with `Finished` event, to avoid double decrease. + pkey.events.incl(Event.Finished) + + results[k] = rkey + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](MAX_EPOLL_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + let fdi = int(fd) + fdi < s.numFD and s.fds[fdi].ident != InvalidIdent + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + s.fds[fdi].data = data + result = true + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + if fdi in s: + var value = addr(s.fds[fdi].data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, + body2: untyped) = + let fdi = int(fd) + if fdi in s: + var value = addr(s.fds[fdi].data) + body1 + else: + body2 + +proc getFd*[T](s: Selector[T]): int = + return s.epollFd.int diff --git a/vendor/nim-chronos/chronos/ioselects/ioselectors_kqueue.nim b/vendor/nim-chronos/chronos/ioselects/ioselectors_kqueue.nim new file mode 100644 index 000000000..e346f8220 --- /dev/null +++ b/vendor/nim-chronos/chronos/ioselects/ioselectors_kqueue.nim @@ -0,0 +1,625 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements BSD kqueue(). + +import posix, times, kqueue + +const + # Maximum number of events that can be returned. + MAX_KQUEUE_EVENTS = 64 + # SIG_IGN and SIG_DFL declared in posix.nim as variables, but we need them + # to be constants and GC-safe. + SIG_DFL = cast[proc(x: cint) {.raises: [],noconv,gcsafe.}](0) + SIG_IGN = cast[proc(x: cint) {.raises: [],noconv,gcsafe.}](1) + +when defined(kqcache): + const CACHE_EVENTS = true + +when defined(macosx) or defined(freebsd) or defined(dragonfly): + when defined(macosx): + const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) + else: + const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, + newp: pointer, newplen: csize_t): cint + {.importc: "sysctl",header: """#include + #include """} +elif defined(netbsd) or defined(openbsd): + # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using + # KERN_MAXFILES, because KERN_MAXFILES is always bigger, + # than KERN_MAXFILESPERPROC. + const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, + newp: pointer, newplen: csize_t): cint + {.importc: "sysctl",header: """#include + #include """} + +when hasThreadSupport: + type + SelectorImpl[T] = object + kqFD: cint + maxFD: int + changes: ptr SharedArray[KEvent] + fds: ptr SharedArray[SelectorKey[T]] + count: int + changesLock: Lock + changesSize: int + changesLength: int + sock: cint + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + kqFD: cint + maxFD: int + changes: seq[KEvent] + fds: seq[SelectorKey[T]] + count: int + sock: cint + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + + SelectEvent* = ptr SelectEventImpl + # SelectEvent is declared as `ptr` to be placed in `shared memory`, + # so you can share one SelectEvent handle between threads. + +proc getUnique[T](s: Selector[T]): int {.inline.} = + # we create duplicated handles to get unique indexes for our `fds` array. + result = posix.fcntl(s.sock, F_DUPFD, s.sock) + if result == -1: + raiseIOSelectorsError(osLastError()) + +proc newSelector*[T](): owned(Selector[T]) = + var maxFD = 0.cint + var size = csize_t(sizeof(cint)) + var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] + # Obtain maximum number of opened file descriptors for process + if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, + nil, 0) != 0: + raiseIOSelectorsError(osLastError()) + + var kqFD = kqueue() + if kqFD < 0: + raiseIOSelectorsError(osLastError()) + + # we allocating empty socket to duplicate it handle in future, to get unique + # indexes for `fds` array. This is needed to properly identify + # {Event.Timer, Event.Signal, Event.Process} events. + let usock = posix.socket(posix.AF_INET, posix.SOCK_STREAM, + posix.IPPROTO_TCP).cint + if usock == -1: + let err = osLastError() + discard posix.close(kqFD) + raiseIOSelectorsError(err) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.changes = allocSharedArray[KEvent](MAX_KQUEUE_EVENTS) + result.changesSize = MAX_KQUEUE_EVENTS + initLock(result.changesLock) + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](maxFD) + result.changes = newSeqOfCap[KEvent](MAX_KQUEUE_EVENTS) + + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + + result.sock = usock + result.kqFD = kqFD + result.maxFD = maxFD.int + +proc close*[T](s: Selector[T]) = + let res1 = posix.close(s.kqFD) + let res2 = posix.close(s.sock) + when hasThreadSupport: + deinitLock(s.changesLock) + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + +proc close*(ev: SelectEvent) = + let res1 = posix.close(ev.rfd) + let res2 = posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +template checkFd(s, f) = + if f >= s.maxFD: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + +when hasThreadSupport: + template withChangeLock[T](s: Selector[T], body: untyped) = + acquire(s.changesLock) + {.locks: [s.changesLock].}: + try: + body + finally: + release(s.changesLock) +else: + template withChangeLock(s, body: untyped) = + body + +when hasThreadSupport: + template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + mixin withChangeLock + s.withChangeLock(): + if s.changesLength == s.changesSize: + # if cache array is full, we allocating new with size * 2 + let newSize = s.changesSize shl 1 + let rdata = allocSharedArray[KEvent](newSize) + copyMem(rdata, s.changes, s.changesSize * sizeof(KEvent)) + s.changesSize = newSize + s.changes[s.changesLength] = KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata) + inc(s.changesLength) + + when not declared(CACHE_EVENTS): + template flushKQueue[T](s: Selector[T]) = + mixin withChangeLock + s.withChangeLock(): + if s.changesLength > 0: + if kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), + nil, 0, nil) == -1: + raiseIOSelectorsError(osLastError()) + s.changesLength = 0 +else: + template modifyKQueue[T](s: Selector[T], nident: uint, nfilter: cshort, + nflags: cushort, nfflags: cuint, ndata: int, + nudata: pointer) = + s.changes.add(KEvent(ident: nident, + filter: nfilter, flags: nflags, + fflags: nfflags, data: ndata, + udata: nudata)) + + when not declared(CACHE_EVENTS): + template flushKQueue[T](s: Selector[T]) = + let length = cint(len(s.changes)) + if length > 0: + if kevent(s.kqFD, addr(s.changes[0]), length, + nil, 0, nil) == -1: + raiseIOSelectorsError(osLastError()) + s.changes.setLen(0) + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + let fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + s.setKey(fdi, events, 0, data) + + if events != {}: + if Event.Read in events: + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if Event.Write in events: + modifyKQueue(s, uint(fdi), EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor $# is not registered in the queue!" % $fdi) + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + modifyKQueue(s, fdi.uint, EVFILT_WRITE, EV_ADD, 0, 0, nil) + inc(s.count) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + pkey.events = events + +proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool, + data: T): int {.discardable.} = + let fdi = getUnique(s) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + let events = if oneshot: {Event.Timer, Event.Oneshot} else: {Event.Timer} + let flags: cushort = if oneshot: EV_ONESHOT or EV_ADD else: EV_ADD + + s.setKey(fdi, events, 0, data) + + # EVFILT_TIMER on Open/Net(BSD) has granularity of only milliseconds, + # but MacOS and FreeBSD allow use `0` as `fflags` to use milliseconds + # too + modifyKQueue(s, fdi.uint, EVFILT_TIMER, flags, 0, cint(timeout), nil) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + result = fdi + +proc registerSignal*[T](s: Selector[T], signal: int, + data: T): int {.discardable.} = + let fdi = getUnique(s) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + s.setKey(fdi, {Event.Signal}, signal, data) + var nmask, omask: Sigset + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, cint(signal)) + blockSignals(nmask, omask) + # to be compatible with linux semantic we need to "eat" signals + posix.signal(cint(signal), SIG_IGN) + + modifyKQueue(s, signal.uint, EVFILT_SIGNAL, EV_ADD, 0, 0, + cast[pointer](fdi)) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + result = fdi + +proc registerProcess*[T](s: Selector[T], pid: int, + data: T): int {.discardable.} = + let fdi = getUnique(s) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + + var kflags: cushort = EV_ONESHOT or EV_ADD + setKey(s, fdi, {Event.Process, Event.Oneshot}, pid, data) + + modifyKQueue(s, pid.uint, EVFILT_PROC, kflags, NOTE_EXIT, 0, + cast[pointer](fdi)) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + result = fdi + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + let fdi = ev.rfd.int + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") + setKey(s, fdi, {Event.User}, 0, data) + + modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + +template processVnodeEvents(events: set[Event]): cuint = + var rfflags = 0.cuint + if events == {Event.VnodeWrite, Event.VnodeDelete, Event.VnodeExtend, + Event.VnodeAttrib, Event.VnodeLink, Event.VnodeRename, + Event.VnodeRevoke}: + rfflags = NOTE_DELETE or NOTE_WRITE or NOTE_EXTEND or NOTE_ATTRIB or + NOTE_LINK or NOTE_RENAME or NOTE_REVOKE + else: + if Event.VnodeDelete in events: rfflags = rfflags or NOTE_DELETE + if Event.VnodeWrite in events: rfflags = rfflags or NOTE_WRITE + if Event.VnodeExtend in events: rfflags = rfflags or NOTE_EXTEND + if Event.VnodeAttrib in events: rfflags = rfflags or NOTE_ATTRIB + if Event.VnodeLink in events: rfflags = rfflags or NOTE_LINK + if Event.VnodeRename in events: rfflags = rfflags or NOTE_RENAME + if Event.VnodeRevoke in events: rfflags = rfflags or NOTE_REVOKE + rfflags + +proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], data: T) = + let fdi = fd.int + setKey(s, fdi, {Event.Vnode} + events, 0, data) + var fflags = processVnodeEvents(events) + + modifyKQueue(s, fdi.uint, EVFILT_VNODE, EV_ADD or EV_CLEAR, fflags, 0, nil) + + when not declared(CACHE_EVENTS): + flushKQueue(s) + + inc(s.count) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor [" & $fdi & "] is not registered in the queue!") + + if pkey.events != {}: + if pkey.events * {Event.Read, Event.Write} != {}: + if Event.Read in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) + dec(s.count) + if Event.Write in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_WRITE, EV_DELETE, 0, 0, nil) + dec(s.count) + when not declared(CACHE_EVENTS): + flushKQueue(s) + elif Event.Timer in pkey.events: + if Event.Finished notin pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_TIMER, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + if posix.close(cint(pkey.ident)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Signal in pkey.events: + var nmask, omask: Sigset + let signal = cint(pkey.param) + discard sigemptyset(nmask) + discard sigemptyset(omask) + discard sigaddset(nmask, signal) + unblockSignals(nmask, omask) + posix.signal(signal, SIG_DFL) + modifyKQueue(s, uint(pkey.param), EVFILT_SIGNAL, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + if posix.close(cint(pkey.ident)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Process in pkey.events: + if Event.Finished notin pkey.events: + modifyKQueue(s, uint(pkey.param), EVFILT_PROC, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + if posix.close(cint(pkey.ident)) != 0: + raiseIOSelectorsError(osLastError()) + elif Event.Vnode in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_VNODE, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + elif Event.User in pkey.events: + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + dec(s.count) + + clearKey(pkey) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") + doAssert(Event.User in pkey.events) + modifyKQueue(s, uint(fdi), EVFILT_READ, EV_DELETE, 0, 0, nil) + when not declared(CACHE_EVENTS): + flushKQueue(s) + clearKey(pkey) + dec(s.count) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openArray[ReadyKey]): int = + var + tv: Timespec + resTable: array[MAX_KQUEUE_EVENTS, KEvent] + ptv = addr tv + maxres = MAX_KQUEUE_EVENTS + + verifySelectParams(timeout) + + if timeout != -1: + if timeout >= 1000: + tv.tv_sec = posix.Time(timeout div 1_000) + tv.tv_nsec = (timeout %% 1_000) * 1_000_000 + else: + tv.tv_sec = posix.Time(0) + tv.tv_nsec = timeout * 1_000_000 + else: + ptv = nil + + if maxres > len(results): + maxres = len(results) + + var count = 0 + when not declared(CACHE_EVENTS): + count = kevent(s.kqFD, nil, cint(0), addr(resTable[0]), cint(maxres), ptv) + else: + when hasThreadSupport: + s.withChangeLock(): + if s.changesLength > 0: + count = kevent(s.kqFD, addr(s.changes[0]), cint(s.changesLength), + addr(resTable[0]), cint(maxres), ptv) + s.changesLength = 0 + else: + count = kevent(s.kqFD, nil, cint(0), addr(resTable[0]), cint(maxres), + ptv) + else: + let length = cint(len(s.changes)) + if length > 0: + count = kevent(s.kqFD, addr(s.changes[0]), length, + addr(resTable[0]), cint(maxres), ptv) + s.changes.setLen(0) + else: + count = kevent(s.kqFD, nil, cint(0), addr(resTable[0]), cint(maxres), + ptv) + + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 # do not delete this, because `continue` used in cycle. + var pkey: ptr SelectorKey[T] + while i < count: + let kevent = addr(resTable[i]) + var rkey = ReadyKey(fd: int(kevent.ident), events: {}) + + if (kevent.flags and EV_ERROR) != 0: + rkey.events = {Event.Error} + rkey.errorCode = OSErrorCode(kevent.data) + + case kevent.filter: + of EVFILT_READ: + pkey = addr(s.fds[int(kevent.ident)]) + rkey.events.incl(Event.Read) + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(cint(kevent.ident), addr data, + sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err == OSErrorCode(EAGAIN): + # someone already consumed event data + inc(i) + continue + else: + raiseIOSelectorsError(err) + rkey.events = {Event.User} + of EVFILT_WRITE: + pkey = addr(s.fds[int(kevent.ident)]) + rkey.events.incl(Event.Write) + rkey.events = {Event.Write} + of EVFILT_TIMER: + pkey = addr(s.fds[int(kevent.ident)]) + if Event.Oneshot in pkey.events: + # we will not clear key until it will be unregistered, so + # application can obtain data, but we will decrease counter, + # because kqueue is empty. + dec(s.count) + # we are marking key with `Finished` event, to avoid double decrease. + pkey.events.incl(Event.Finished) + rkey.events.incl(Event.Timer) + of EVFILT_VNODE: + pkey = addr(s.fds[int(kevent.ident)]) + rkey.events.incl(Event.Vnode) + if (kevent.fflags and NOTE_DELETE) != 0: + rkey.events.incl(Event.VnodeDelete) + if (kevent.fflags and NOTE_WRITE) != 0: + rkey.events.incl(Event.VnodeWrite) + if (kevent.fflags and NOTE_EXTEND) != 0: + rkey.events.incl(Event.VnodeExtend) + if (kevent.fflags and NOTE_ATTRIB) != 0: + rkey.events.incl(Event.VnodeAttrib) + if (kevent.fflags and NOTE_LINK) != 0: + rkey.events.incl(Event.VnodeLink) + if (kevent.fflags and NOTE_RENAME) != 0: + rkey.events.incl(Event.VnodeRename) + if (kevent.fflags and NOTE_REVOKE) != 0: + rkey.events.incl(Event.VnodeRevoke) + of EVFILT_SIGNAL: + pkey = addr(s.fds[cast[int](kevent.udata)]) + rkey.fd = cast[int](kevent.udata) + rkey.events.incl(Event.Signal) + of EVFILT_PROC: + rkey.fd = cast[int](kevent.udata) + pkey = addr(s.fds[cast[int](kevent.udata)]) + # we will not clear key, until it will be unregistered, so + # application can obtain data, but we will decrease counter, + # because kqueue is empty. + dec(s.count) + # we are marking key with `Finished` event, to avoid double decrease. + pkey.events.incl(Event.Finished) + rkey.events.incl(Event.Process) + else: + doAssert(true, "Unsupported kqueue filter in the queue!") + + if (kevent.flags and EV_EOF) != 0: + # TODO this error handling needs to be rethought. + # `fflags` can sometimes be `0x80000000` and thus we use 'cast' + # here: + if kevent.fflags != 0: + rkey.errorCode = cast[OSErrorCode](kevent.fflags) + else: + # This assumes we are dealing with sockets. + # TODO: For future-proofing it might be a good idea to give the + # user access to the raw `kevent`. + rkey.errorCode = OSErrorCode(ECONNRESET) + rkey.events.incl(Event.Error) + + results[k] = rkey + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](MAX_KQUEUE_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + let fdi = fd.int + fdi < s.maxFD and s.fds[fd.int].ident != InvalidIdent + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + let fdi = int(fd) + if fdi in s: + s.fds[fdi].data = data + result = true + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + let fdi = int(fd) + if fdi in s: + var value = addr(s.fds[fdi].data) + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, + body2: untyped) = + let fdi = int(fd) + if fdi in s: + var value = addr(s.fds[fdi].data) + body1 + else: + body2 + + +proc getFd*[T](s: Selector[T]): int = + return s.kqFD.int diff --git a/vendor/nim-chronos/chronos/ioselects/ioselectors_poll.nim b/vendor/nim-chronos/chronos/ioselects/ioselectors_poll.nim new file mode 100644 index 000000000..1af2a46db --- /dev/null +++ b/vendor/nim-chronos/chronos/ioselects/ioselectors_poll.nim @@ -0,0 +1,310 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix poll(). + +import posix, times + +# Maximum number of events that can be returned +const MAX_POLL_EVENTS = 64 + +when hasThreadSupport: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: ptr SharedArray[SelectorKey[T]] + pollfds: ptr SharedArray[TPollFd] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + maxFD : int + pollcnt: int + fds: seq[SelectorKey[T]] + pollfds: seq[TPollFd] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rfd: cint + wfd: cint + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withPollLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withPollLock(s, body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + var a = RLimit() + if getrlimit(posix.RLIMIT_NOFILE, a) != 0: + raiseIOSelectorsError(osLastError()) + var maxFD = int(a.rlim_max) + + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.maxFD = maxFD + result.fds = allocSharedArray[SelectorKey[T]](maxFD) + result.pollfds = allocSharedArray[TPollFd](maxFD) + initLock(result.lock) + else: + result = Selector[T]() + result.maxFD = maxFD + result.fds = newSeq[SelectorKey[T]](maxFD) + result.pollfds = newSeq[TPollFd](maxFD) + + for i in 0 ..< maxFD: + result.fds[i].ident = InvalidIdent + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deinitLock(s.lock) + deallocSharedArray(s.fds) + deallocSharedArray(s.pollfds) + deallocShared(cast[pointer](s)) + +template pollAdd[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + s.pollfds[s.pollcnt].fd = cint(sock) + s.pollfds[s.pollcnt].events = pollev + inc(s.count) + inc(s.pollcnt) + +template pollUpdate[T](s: Selector[T], sock: cint, events: set[Event]) = + withPollLock(s): + var i = 0 + var pollev: cshort = 0 + if Event.Read in events: pollev = pollev or POLLIN + if Event.Write in events: pollev = pollev or POLLOUT + + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + s.pollfds[i].events = pollev + break + inc(i) + doAssert(i < s.pollcnt, + "Descriptor [" & $sock & "] is not registered in the queue!") + +template pollRemove[T](s: Selector[T], sock: cint) = + withPollLock(s): + var i = 0 + while i < s.pollcnt: + if s.pollfds[i].fd == sock: + if i == s.pollcnt - 1: + s.pollfds[i].fd = 0 + s.pollfds[i].events = 0 + s.pollfds[i].revents = 0 + else: + while i < (s.pollcnt - 1): + s.pollfds[i].fd = s.pollfds[i + 1].fd + s.pollfds[i].events = s.pollfds[i + 1].events + inc(i) + break + inc(i) + dec(s.pollcnt) + dec(s.count) + +template checkFd(s, f) = + if f >= s.maxFD: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + var fdi = int(fd) + s.checkFd(fdi) + doAssert(s.fds[fdi].ident == InvalidIdent) + setKey(s, fdi, events, 0, data) + if events != {}: s.pollAdd(fdi.cint, events) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor [" & $fdi & "] is not registered in the queue!") + doAssert(pkey.events * maskEvents == {}) + + if pkey.events != events: + if pkey.events == {}: + s.pollAdd(fd.cint, events) + else: + if events != {}: + s.pollUpdate(fd.cint, events) + else: + s.pollRemove(fd.cint) + pkey.events = events + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + var fdi = int(ev.rfd) + doAssert(s.fds[fdi].ident == InvalidIdent, "Event is already registered in the queue!") + var events = {Event.User} + setKey(s, fdi, events, 0, data) + events.incl(Event.Read) + s.pollAdd(fdi.cint, events) + +proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = + let fdi = int(fd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, + "Descriptor [" & $fdi & "] is not registered in the queue!") + pkey.ident = InvalidIdent + if pkey.events != {}: + pkey.events = {} + s.pollRemove(fdi.cint) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fdi = int(ev.rfd) + s.checkFd(fdi) + var pkey = addr(s.fds[fdi]) + doAssert(pkey.ident != InvalidIdent, "Event is not registered in the queue!") + doAssert(Event.User in pkey.events) + pkey.ident = InvalidIdent + pkey.events = {} + s.pollRemove(fdi.cint) + +proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + +proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(ev.wfd, addr data, sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + +proc close*(ev: SelectEvent) = + let res1 = posix.close(ev.rfd) + let res2 = posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey]): int = + var maxres = MAX_POLL_EVENTS + if maxres > len(results): + maxres = len(results) + + verifySelectParams(timeout) + + s.withPollLock(): + let count = posix.poll(addr(s.pollfds[0]), Tnfds(s.pollcnt), timeout) + if count < 0: + result = 0 + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + var i = 0 + var k = 0 + var rindex = 0 + while (i < s.pollcnt) and (k < count) and (rindex < maxres): + let revents = s.pollfds[i].revents + if revents != 0: + let fd = s.pollfds[i].fd + var pkey = addr(s.fds[fd]) + var rkey = ReadyKey(fd: int(fd), events: {}) + + if (revents and POLLIN) != 0: + rkey.events.incl(Event.Read) + if Event.User in pkey.events: + var data: uint64 = 0 + if posix.read(fd, addr data, sizeof(uint64)) != sizeof(uint64): + let err = osLastError() + if err != OSErrorCode(EAGAIN): + raiseIOSelectorsError(err) + else: + # someone already consumed event data + inc(i) + continue + rkey.events = {Event.User} + if (revents and POLLOUT) != 0: + rkey.events.incl(Event.Write) + if (revents and POLLERR) != 0 or (revents and POLLHUP) != 0 or + (revents and POLLNVAL) != 0: + rkey.events.incl(Event.Error) + results[rindex] = rkey + s.pollfds[i].revents = 0 + inc(rindex) + inc(k) + inc(i) + result = k + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](MAX_POLL_EVENTS) + let count = selectInto(s, timeout, result) + result.setLen(count) + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + return s.fds[fd.int].ident != InvalidIdent + +proc getData*[T](s: Selector[T], fd: SocketHandle|int): var T = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + result = s.fds[fdi].data + +proc setData*[T](s: Selector[T], fd: SocketHandle|int, data: T): bool = + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + s.fds[fdi].data = data + result = true + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, + body: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.getData(fdi)) + body + +template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, + body2: untyped) = + mixin checkFd + let fdi = int(fd) + s.checkFd(fdi) + if fdi in s: + var value = addr(s.getData(fdi)) + body1 + else: + body2 + + +proc getFd*[T](s: Selector[T]): int = + return -1 diff --git a/vendor/nim-chronos/chronos/ioselects/ioselectors_select.nim b/vendor/nim-chronos/chronos/ioselects/ioselectors_select.nim new file mode 100644 index 000000000..02a853b42 --- /dev/null +++ b/vendor/nim-chronos/chronos/ioselects/ioselectors_select.nim @@ -0,0 +1,465 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Eugene Kabanov +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# This module implements Posix and Windows select(). + +import times, nativesockets + +when defined(windows): + import winlean + when defined(gcc): + {.passl: "-lws2_32".} + elif defined(vcc): + {.passl: "ws2_32.lib".} + const platformHeaders = """#include + #include """ + const EAGAIN = WSAEWOULDBLOCK +else: + const platformHeaders = """#include + #include + #include + #include """ +type + Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object +var + FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint + +proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} +proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) + {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} +proc IOFD_ZERO(fdset: ptr Fdset) + {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} + +when defined(windows): + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.stdcall, importc: "select", header: platformHeaders.} +else: + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + timeout: ptr Timeval): cint + {.cdecl, importc: "select", header: platformHeaders.} + +when hasThreadSupport: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: ptr SharedArray[SelectorKey[T]] + count: int + lock: Lock + Selector*[T] = ptr SelectorImpl[T] +else: + type + SelectorImpl[T] = object + rSet: FdSet + wSet: FdSet + eSet: FdSet + maxFD: int + fds: seq[SelectorKey[T]] + count: int + Selector*[T] = ref SelectorImpl[T] + +type + SelectEventImpl = object + rsock: SocketHandle + wsock: SocketHandle + SelectEvent* = ptr SelectEventImpl + +when hasThreadSupport: + template withSelectLock[T](s: Selector[T], body: untyped) = + acquire(s.lock) + {.locks: [s.lock].}: + try: + body + finally: + release(s.lock) +else: + template withSelectLock[T](s: Selector[T], body: untyped) = + body + +proc newSelector*[T](): Selector[T] = + when hasThreadSupport: + result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) + result.fds = allocSharedArray[SelectorKey[T]](FD_SETSIZE) + initLock result.lock + else: + result = Selector[T]() + result.fds = newSeq[SelectorKey[T]](FD_SETSIZE) + + for i in 0 ..< FD_SETSIZE: + result.fds[i].ident = InvalidIdent + + IOFD_ZERO(addr result.rSet) + IOFD_ZERO(addr result.wSet) + IOFD_ZERO(addr result.eSet) + +proc close*[T](s: Selector[T]) = + when hasThreadSupport: + deallocSharedArray(s.fds) + deallocShared(cast[pointer](s)) + +when defined(windows): + proc newSelectEvent*(): SelectEvent = + var ssock = createNativeSocket() + var wsock = createNativeSocket() + var rsock: SocketHandle = INVALID_SOCKET + var saddr = Sockaddr_in() + + saddr.sin_family = winlean.AF_INET + saddr.sin_port = 0 + saddr.sin_addr.s_addr = INADDR_ANY + if bindAddr(ssock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) < 0'i32: + raiseIOSelectorsError(osLastError()) + + if winlean.listen(ssock, 1) != 0: + raiseIOSelectorsError(osLastError()) + + var namelen = sizeof(saddr).SockLen + if getsockname(ssock, cast[ptr SockAddr](addr(saddr)), + addr(namelen)) != 0'i32: + raiseIOSelectorsError(osLastError()) + + saddr.sin_addr.s_addr = 0x0100007F + if winlean.connect(wsock, cast[ptr SockAddr](addr(saddr)), + sizeof(saddr).SockLen) != 0: + raiseIOSelectorsError(osLastError()) + namelen = sizeof(saddr).SockLen + rsock = winlean.accept(ssock, cast[ptr SockAddr](addr(saddr)), + cast[ptr SockLen](addr(namelen))) + if rsock == SocketHandle(-1): + raiseIOSelectorsError(osLastError()) + + if winlean.closesocket(ssock) != 0: + raiseIOSelectorsError(osLastError()) + + var mode = clong(1) + if ioctlsocket(rsock, FIONBIO, addr(mode)) != 0: + raiseIOSelectorsError(osLastError()) + mode = clong(1) + if ioctlsocket(wsock, FIONBIO, addr(mode)) != 0: + raiseIOSelectorsError(osLastError()) + + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = rsock + result.wsock = wsock + + proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if winlean.send(ev.wsock, cast[pointer](addr data), + cint(sizeof(uint64)), 0) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + + proc close*(ev: SelectEvent) = + let res1 = winlean.closesocket(ev.rsock) + let res2 = winlean.closesocket(ev.wsock) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +else: + proc newSelectEvent*(): SelectEvent = + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rsock = SocketHandle(fds[0]) + result.wsock = SocketHandle(fds[1]) + + proc trigger*(ev: SelectEvent) = + var data: uint64 = 1 + if posix.write(cint(ev.wsock), addr data, sizeof(uint64)) != sizeof(uint64): + raiseIOSelectorsError(osLastError()) + + proc close*(ev: SelectEvent) = + let res1 = posix.close(cint(ev.rsock)) + let res2 = posix.close(cint(ev.wsock)) + deallocShared(cast[pointer](ev)) + if res1 != 0 or res2 != 0: + raiseIOSelectorsError(osLastError()) + +proc setSelectKey[T](s: Selector[T], fd: SocketHandle, events: set[Event], + data: T) = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == InvalidIdent: + var pkey = addr(s.fds[i]) + pkey.ident = fdi + pkey.events = events + pkey.data = data + break + inc(i) + if i >= FD_SETSIZE: + raiseIOSelectorsError("Maximum number of descriptors is exhausted!") + +proc getKey[T](s: Selector[T], fd: SocketHandle): ptr SelectorKey[T] = + var i = 0 + let fdi = int(fd) + while i < FD_SETSIZE: + if s.fds[i].ident == fdi: + result = addr(s.fds[i]) + break + inc(i) + doAssert(i < FD_SETSIZE, + "Descriptor [" & $int(fd) & "] is not registered in the queue!") + +proc delKey[T](s: Selector[T], fd: SocketHandle) = + var empty: T + var i = 0 + while i < FD_SETSIZE: + if s.fds[i].ident == fd.int: + s.fds[i].ident = InvalidIdent + s.fds[i].events = {} + s.fds[i].data = empty + break + inc(i) + doAssert(i < FD_SETSIZE, + "Descriptor [" & $int(fd) & "] is not registered in the queue!") + +proc registerHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) = + when not defined(windows): + let fdi = int(fd) + s.withSelectLock(): + s.setSelectKey(fd, events, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + if Event.Read in events: + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if Event.Write in events: + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + +proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = + when not defined(windows): + let fdi = int(ev.rsock) + s.withSelectLock(): + s.setSelectKey(ev.rsock, {Event.User}, data) + when not defined(windows): + if fdi > s.maxFD: s.maxFD = fdi + IOFD_SET(ev.rsock, addr s.rSet) + inc(s.count) + +proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) = + let maskEvents = {Event.Timer, Event.Signal, Event.Process, Event.Vnode, + Event.User, Event.Oneshot, Event.Error} + s.withSelectLock(): + var pkey = s.getKey(fd) + doAssert(pkey.events * maskEvents == {}) + if pkey.events != events: + if (Event.Read in pkey.events) and (Event.Read notin events): + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if (Event.Write in pkey.events) and (Event.Write notin events): + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + if (Event.Read notin pkey.events) and (Event.Read in events): + IOFD_SET(fd, addr s.rSet) + inc(s.count) + if (Event.Write notin pkey.events) and (Event.Write in events): + IOFD_SET(fd, addr s.wSet) + IOFD_SET(fd, addr s.eSet) + inc(s.count) + pkey.events = events + +proc unregister*[T](s: Selector[T], fd: SocketHandle|int) = + s.withSelectLock(): + let fd = fd.SocketHandle + var pkey = s.getKey(fd) + if Event.Read in pkey.events or Event.User in pkey.events: + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + if Event.Write in pkey.events: + IOFD_CLR(fd, addr s.wSet) + IOFD_CLR(fd, addr s.eSet) + dec(s.count) + s.delKey(fd) + +proc unregister*[T](s: Selector[T], ev: SelectEvent) = + let fd = ev.rsock + s.withSelectLock(): + var pkey = s.getKey(fd) + IOFD_CLR(fd, addr s.rSet) + dec(s.count) + s.delKey(fd) + +proc selectInto*[T](s: Selector[T], timeout: int, + results: var openarray[ReadyKey]): int = + var tv = Timeval() + var ptv = addr tv + var rset, wset, eset: FdSet + + verifySelectParams(timeout) + + if timeout != -1: + when defined(genode): + tv.tv_sec = Time(timeout div 1_000) + else: + tv.tv_sec = timeout.int32 div 1_000 + tv.tv_usec = (timeout.int32 %% 1_000) * 1_000 + else: + ptv = nil + + s.withSelectLock(): + rset = s.rSet + wset = s.wSet + eset = s.eSet + + var count = ioselect(cint(s.maxFD) + 1, addr(rset), addr(wset), + addr(eset), ptv) + if count < 0: + result = 0 + when defined(windows): + raiseIOSelectorsError(osLastError()) + else: + let err = osLastError() + if cint(err) != EINTR: + raiseIOSelectorsError(err) + elif count == 0: + result = 0 + else: + var rindex = 0 + var i = 0 + var k = 0 + + while (i < FD_SETSIZE) and (k < count): + if s.fds[i].ident != InvalidIdent: + var flag = false + var pkey = addr(s.fds[i]) + var rkey = ReadyKey(fd: int(pkey.ident), events: {}) + let fd = SocketHandle(pkey.ident) + if IOFD_ISSET(fd, addr rset) != 0: + if Event.User in pkey.events: + var data: uint64 = 0 + if recv(fd, cast[pointer](addr(data)), + sizeof(uint64).cint, 0) != sizeof(uint64): + let err = osLastError() + if cint(err) != EAGAIN: + raiseIOSelectorsError(err) + else: + inc(i) + inc(k) + continue + else: + flag = true + rkey.events = {Event.User} + else: + flag = true + rkey.events = {Event.Read} + if IOFD_ISSET(fd, addr wset) != 0: + rkey.events.incl(Event.Write) + if IOFD_ISSET(fd, addr eset) != 0: + rkey.events.incl(Event.Error) + flag = true + if flag: + results[rindex] = rkey + inc(rindex) + inc(k) + inc(i) + result = rindex + +proc select*[T](s: Selector[T], timeout: int): seq[ReadyKey] = + result = newSeq[ReadyKey](FD_SETSIZE) + var count = selectInto(s, timeout, result) + result.setLen(count) + +proc flush*[T](s: Selector[T]) = discard + +template isEmpty*[T](s: Selector[T]): bool = + (s.count == 0) + +proc contains*[T](s: Selector[T], fd: SocketHandle|int): bool {.inline.} = + s.withSelectLock(): + result = false + + let fdi = int(fd) + for i in 0..= -1, "Cannot select with a negative value, got " & $timeout) + + when defined(linux): + include ./ioselects/ioselectors_epoll + elif bsdPlatform: + include ./ioselects/ioselectors_kqueue + elif defined(windows): + include ./ioselects/ioselectors_select + elif defined(solaris): + include ./ioselects/ioselectors_poll # need to replace it with event ports + elif defined(genode): + include ./ioselects/ioselectors_select # TODO: use the native VFS layer + elif defined(nintendoswitch): + include ./ioselects/ioselectors_select + else: + include ./ioselects/ioselectors_poll + +proc register*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event], data: T) {.deprecated: "use registerHandle instead".} = + ## **Deprecated since v0.18.0:** Use ``registerHandle`` instead. + s.registerHandle(fd, events, data) + +proc setEvent*(ev: SelectEvent) {.deprecated: "use trigger instead", + raises: [Defect, IOSelectorsException].} = + ## Trigger event ``ev``. + ## + ## **Deprecated since v0.18.0:** Use ``trigger`` instead. + ev.trigger() + +proc update*[T](s: Selector[T], fd: int | SocketHandle, + events: set[Event]) {.deprecated: "use updateHandle instead".} = + ## Update file/socket descriptor ``fd``, registered in selector + ## ``s`` with new events set ``event``. + ## + ## **Deprecated since v0.18.0:** Use ``updateHandle`` instead. + s.updateHandle() diff --git a/vendor/nim-chronos/chronos/sendfile.nim b/vendor/nim-chronos/chronos/sendfile.nim index 358643f9a..ca3829c90 100644 --- a/vendor/nim-chronos/chronos/sendfile.nim +++ b/vendor/nim-chronos/chronos/sendfile.nim @@ -9,6 +9,8 @@ ## This module provides cross-platform wrapper for ``sendfile()`` syscall. +{.push raises: [Defect].} + when defined(nimdoc): proc sendfile*(outfd, infd: int, offset: int, count: var int): int = ## Copies data between file descriptor ``infd`` and ``outfd``. Because this diff --git a/vendor/nim-chronos/chronos/srcloc.nim b/vendor/nim-chronos/chronos/srcloc.nim index 9d7f942b7..576fc58a0 100644 --- a/vendor/nim-chronos/chronos/srcloc.nim +++ b/vendor/nim-chronos/chronos/srcloc.nim @@ -1,3 +1,5 @@ +{.push raises: [].} + type SrcLoc* = object procedure*: cstring diff --git a/vendor/nim-chronos/chronos/streams/asyncstream.nim b/vendor/nim-chronos/chronos/streams/asyncstream.nim index ce7b1f7e4..05b48ceec 100644 --- a/vendor/nim-chronos/chronos/streams/asyncstream.nim +++ b/vendor/nim-chronos/chronos/streams/asyncstream.nim @@ -6,6 +6,9 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) + +{.push raises: [Defect].} + import ../asyncloop, ../asyncsync import ../transports/common, ../transports/stream export asyncsync, stream, common @@ -58,9 +61,9 @@ type Finished, ## Stream was properly finished Closed ## Stream was closed - StreamReaderLoop* = proc (stream: AsyncStreamReader): Future[void] {.gcsafe.} + StreamReaderLoop* = proc (stream: AsyncStreamReader): Future[void] {.gcsafe, raises: [Defect].} ## Main read loop for read streams. - StreamWriterLoop* = proc (stream: AsyncStreamWriter): Future[void] {.gcsafe.} + StreamWriterLoop* = proc (stream: AsyncStreamWriter): Future[void] {.gcsafe, raises: [Defect].} ## Main write loop for write streams. AsyncStreamReader* = ref object of RootRef @@ -202,16 +205,20 @@ proc newAsyncStreamUseClosedError*(): ref AsyncStreamUseClosedError {. noinline.} = newException(AsyncStreamUseClosedError, "Stream is already closed") -proc raiseAsyncStreamUseClosedError*() {.noinline, noreturn.} = +proc raiseAsyncStreamUseClosedError*() {. + noinline, noreturn, raises: [Defect, AsyncStreamUseClosedError].} = raise newAsyncStreamUseClosedError() -proc raiseAsyncStreamLimitError*() {.noinline, noreturn.} = +proc raiseAsyncStreamLimitError*() {. + noinline, noreturn, raises: [Defect, AsyncStreamLimitError].} = raise newAsyncStreamLimitError() -proc raiseAsyncStreamIncompleteError*() {.noinline, noreturn.} = +proc raiseAsyncStreamIncompleteError*() {. + noinline, noreturn, raises: [Defect, AsyncStreamIncompleteError].} = raise newAsyncStreamIncompleteError() -proc raiseAsyncStreamIncorrectDefect*(m: string) {.noinline, noreturn.} = +proc raiseAsyncStreamIncorrectDefect*(m: string) {. + noinline, noreturn, raises: [Defect].} = raise newException(AsyncStreamIncorrectDefect, m) proc raiseEmptyMessageDefect*() {.noinline, noreturn.} = @@ -248,8 +255,8 @@ proc running*(rw: AsyncStreamRW): bool {.inline.} = ## Returns ``true`` is reading/writing stream is still pending. (rw.state == AsyncStreamState.Running) -proc setupAsyncStreamReaderTracker(): AsyncStreamTracker {.gcsafe.} -proc setupAsyncStreamWriterTracker(): AsyncStreamTracker {.gcsafe.} +proc setupAsyncStreamReaderTracker(): AsyncStreamTracker {.gcsafe, raises: [Defect].} +proc setupAsyncStreamWriterTracker(): AsyncStreamTracker {.gcsafe, raises: [Defect].} proc getAsyncStreamReaderTracker(): AsyncStreamTracker {.inline.} = var res = cast[AsyncStreamTracker](getTracker(AsyncStreamReaderTrackerName)) @@ -873,7 +880,7 @@ proc close*(rw: AsyncStreamRW) = rw.state = AsyncStreamState.Closed - proc continuation(udata: pointer) = + proc continuation(udata: pointer) {.raises: [Defect].} = if not isNil(rw.udata): GC_unref(cast[ref int](rw.udata)) if not(rw.future.finished()): diff --git a/vendor/nim-chronos/chronos/timer.nim b/vendor/nim-chronos/chronos/timer.nim index c61ff6d48..c4584233d 100644 --- a/vendor/nim-chronos/chronos/timer.nim +++ b/vendor/nim-chronos/chronos/timer.nim @@ -24,6 +24,8 @@ ## You can specify which timer you want to use ``-d:asyncTimer=``. const asyncTimer* {.strdefine.} = "mono" +{.push raises: [Defect].} + when defined(windows): when asyncTimer == "system": from winlean import getSystemTimeAsFileTime, FILETIME diff --git a/vendor/nim-chronos/chronos/transport.nim b/vendor/nim-chronos/chronos/transport.nim index 48538582a..61905d9b6 100644 --- a/vendor/nim-chronos/chronos/transport.nim +++ b/vendor/nim-chronos/chronos/transport.nim @@ -6,8 +6,8 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import transports/[datagram, stream, common, ipnet, osnet] -import streams/[asyncstream, chunkstream] +import ./transports/[datagram, stream, common, ipnet, osnet] +import ./streams/[asyncstream, chunkstream] export datagram, common, stream, ipnet, osnet export asyncstream, chunkstream diff --git a/vendor/nim-chronos/chronos/transports/common.nim b/vendor/nim-chronos/chronos/transports/common.nim index 5de8a4d68..f79ca4493 100644 --- a/vendor/nim-chronos/chronos/transports/common.nim +++ b/vendor/nim-chronos/chronos/transports/common.nim @@ -6,7 +6,10 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import os, strutils, nativesockets, net + +{.push raises: [Defect].} + +import std/[os, strutils, nativesockets, net] import ../asyncloop export net @@ -183,9 +186,10 @@ proc `$`*(address: TransportAddress): string = else: result = "" else: - raise newException(TransportAddressError, "Unknown address family!") + result = "Unknown address family: " & $address.family -proc initTAddress*(address: string): TransportAddress = +proc initTAddress*(address: string): TransportAddress {. + raises: [Defect, TransportAddressError].} = ## Parses string representation of ``address``. ``address`` can be IPv4, IPv6 ## or Unix domain address. ## @@ -230,7 +234,8 @@ proc initTAddress*(address: string): TransportAddress = else: result = TransportAddress(family: AddressFamily.Unix) -proc initTAddress*(address: string, port: Port): TransportAddress = +proc initTAddress*(address: string, port: Port): TransportAddress {. + raises: [Defect, TransportAddressError].} = ## Initialize ``TransportAddress`` with IP (IPv4 or IPv6) address ``address`` ## and port number ``port``. try: @@ -246,7 +251,8 @@ proc initTAddress*(address: string, port: Port): TransportAddress = except CatchableError as exc: raise newException(TransportAddressError, exc.msg) -proc initTAddress*(address: string, port: int): TransportAddress {.inline.} = +proc initTAddress*(address: string, port: int): TransportAddress {. + raises: [Defect, TransportAddressError].} = ## Initialize ``TransportAddress`` with IP (IPv4 or IPv6) address ``address`` ## and port number ``port``. if port < 0 or port >= 65536: @@ -267,7 +273,8 @@ proc initTAddress*(address: IpAddress, port: Port): TransportAddress = proc getAddrInfo(address: string, port: Port, domain: Domain, sockType: SockType = SockType.SOCK_STREAM, - protocol: Protocol = Protocol.IPPROTO_TCP): ptr AddrInfo = + protocol: Protocol = Protocol.IPPROTO_TCP): ptr AddrInfo {. + raises: [Defect, TransportAddressError].} = ## We have this one copy of ``getAddrInfo()`` because of AI_V4MAPPED in ## ``net.nim:getAddrInfo()``, which is not cross-platform. var hints: AddrInfo @@ -346,7 +353,7 @@ proc toSAddr*(address: TransportAddress, sa: var Sockaddr_storage, else: discard -proc address*(ta: TransportAddress): IpAddress = +proc address*(ta: TransportAddress): IpAddress {.raises: [Defect, ValueError].} = ## Converts ``TransportAddress`` to ``net.IpAddress`` object. ## ## Note its impossible to convert ``TransportAddress`` of ``Unix`` family, @@ -361,7 +368,8 @@ proc address*(ta: TransportAddress): IpAddress = raise newException(ValueError, "IpAddress supports only IPv4/IPv6!") proc resolveTAddress*(address: string, - family = AddressFamily.IPv4): seq[TransportAddress] = + family = AddressFamily.IPv4): seq[TransportAddress] {. + raises: [Defect, TransportAddressError].} = ## Resolve string representation of ``address``. ## ## Supported formats are: @@ -412,7 +420,8 @@ proc resolveTAddress*(address: string, freeAddrInfo(aiList) proc resolveTAddress*(address: string, port: Port, - family = AddressFamily.IPv4): seq[TransportAddress] = + family = AddressFamily.IPv4): seq[TransportAddress] {. + raises: [Defect, TransportAddressError].} = ## Resolve string representation of ``address``. ## ## ``address`` could be dot IPv4/IPv6 address or hostname. @@ -439,7 +448,7 @@ proc resolveTAddress*(address: string, port: Port, proc resolveTAddress*(address: string, family: IpAddressFamily): seq[TransportAddress] {. - deprecated.} = + deprecated, raises: [Defect, TransportAddressError].} = if family == IpAddressFamily.IPv4: result = resolveTAddress(address, AddressFamily.IPv4) elif family == IpAddressFamily.IPv6: @@ -447,22 +456,24 @@ proc resolveTAddress*(address: string, proc resolveTAddress*(address: string, port: Port, family: IpAddressFamily): seq[TransportAddress] {. - deprecated.} = + deprecated, raises: [Defect, TransportAddressError].} = if family == IpAddressFamily.IPv4: result = resolveTAddress(address, port, AddressFamily.IPv4) elif family == IpAddressFamily.IPv6: result = resolveTAddress(address, port, AddressFamily.IPv6) -proc windowsAnyAddressFix*(a: TransportAddress): TransportAddress {.inline.} = +proc windowsAnyAddressFix*(a: TransportAddress): TransportAddress = ## BSD Sockets on *nix systems are able to perform connections to ## `0.0.0.0` or `::0` which are equal to `127.0.0.1` or `::1`. when defined(windows): if (a.family == AddressFamily.IPv4 and a.address_v4 == AnyAddress.address_v4): - result = initTAddress("127.0.0.1", a.port) + result = try: initTAddress("127.0.0.1", a.port) + except TransportAddressError as exc: raiseAssert exc.msg elif (a.family == AddressFamily.IPv6 and a.address_v6 == AnyAddress6.address_v6): - result = initTAddress("::1", a.port) + result = try: initTAddress("::1", a.port) + except TransportAddressError as exc: raiseAssert exc.msg else: result = a else: @@ -484,7 +495,7 @@ template checkWriteEof*(t: untyped, future: untyped) = "Transport connection is already dropped!")) return future -template getError*(t: untyped): ref Exception = +template getError*(t: untyped): ref CatchableError = var err = (t).error (t).error = nil err @@ -507,7 +518,8 @@ template getTransportOsError*(err: OSErrorCode): ref TransportOsError = template getTransportOsError*(err: cint): ref TransportOsError = getTransportOsError(OSErrorCode(err)) -proc raiseTransportOsError*(err: OSErrorCode) = +proc raiseTransportOsError*(err: OSErrorCode) {. + raises: [Defect, TransportOsError].} = ## Raises transport specific OS error. raise getTransportOsError(err) diff --git a/vendor/nim-chronos/chronos/transports/datagram.nim b/vendor/nim-chronos/chronos/transports/datagram.nim index f3f2eb46c..7624e91c9 100644 --- a/vendor/nim-chronos/chronos/transports/datagram.nim +++ b/vendor/nim-chronos/chronos/transports/datagram.nim @@ -7,9 +7,11 @@ # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import net, nativesockets, os, deques -import ../asyncloop, ../handles -import common +{.push raises: [Defect].} + +import std/[net, nativesockets, os, deques] +import ".."/[selectors2, asyncloop, handles] +import ./common when defined(windows): import winlean @@ -33,7 +35,7 @@ type writer: Future[void] # Writer vector completion Future DatagramCallback* = proc(transp: DatagramTransport, - remote: TransportAddress): Future[void] {.gcsafe.} + remote: TransportAddress): Future[void] {.gcsafe, raises: [Defect].} DatagramTransport* = ref object of RootRef fd*: AsyncFD # File descriptor @@ -41,7 +43,7 @@ type flags: set[ServerFlags] # Flags buffer: seq[byte] # Reading buffer buflen: int # Reading buffer effective size - error: ref Exception # Current error + error: ref CatchableError # Current error queue: Deque[GramVector] # Writer queue local: TransportAddress # Local address remote: TransportAddress # Remote address @@ -66,7 +68,8 @@ type const DgramTransportTrackerName = "datagram.transport" -proc remoteAddress*(transp: DatagramTransport): TransportAddress = +proc remoteAddress*(transp: DatagramTransport): TransportAddress {. + raises: [Defect, TransportOsError].} = ## Returns ``transp`` remote socket address. if transp.remote.family == AddressFamily.None: var saddr: Sockaddr_storage @@ -77,7 +80,8 @@ proc remoteAddress*(transp: DatagramTransport): TransportAddress = fromSAddr(addr saddr, slen, transp.remote) result = transp.remote -proc localAddress*(transp: DatagramTransport): TransportAddress = +proc localAddress*(transp: DatagramTransport): TransportAddress {. + raises: [Defect, TransportOsError].} = ## Returns ``transp`` local socket address. if transp.local.family == AddressFamily.None: var saddr: Sockaddr_storage @@ -92,7 +96,7 @@ template setReadError(t, e: untyped) = (t).state.incl(ReadError) (t).error = getTransportOsError(e) -proc setupDgramTransportTracker(): DgramTransportTracker {.gcsafe.} +proc setupDgramTransportTracker(): DgramTransportTracker {.gcsafe, raises: [Defect].} proc getDgramTransportTracker(): DgramTransportTracker {.inline.} = result = cast[DgramTransportTracker](getTracker(DgramTransportTrackerName)) @@ -286,7 +290,8 @@ when defined(windows): udata: pointer, child: DatagramTransport, bufferSize: int, - ttl: int): DatagramTransport = + ttl: int): DatagramTransport {. + raises: [Defect, CatchableError].} = var localSock: AsyncFD doAssert(remote.family == local.family) doAssert(not isNil(cbproc)) @@ -300,6 +305,7 @@ when defined(windows): if sock == asyncInvalidSocket: localSock = createAsyncSocket(local.getDomain(), SockType.SOCK_DGRAM, Protocol.IPPROTO_UDP) + if localSock == asyncInvalidSocket: raiseTransportOsError(osLastError()) else: @@ -397,7 +403,7 @@ when defined(windows): else: # Linux/BSD/MacOS part - proc readDatagramLoop(udata: pointer) = + proc readDatagramLoop(udata: pointer) {.raises: Defect.}= var raddr: TransportAddress doAssert(not isNil(udata)) var cdata = cast[ptr CompletionData](udata) @@ -466,15 +472,30 @@ else: break else: transp.state.incl(WritePaused) - transp.fd.removeWriter() + try: + transp.fd.removeWriter() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeWriter" + except ValueError as exc: + raiseAsDefect exc, "removeWriter" proc resumeWrite(transp: DatagramTransport) {.inline.} = transp.state.excl(WritePaused) - addWriter(transp.fd, writeDatagramLoop, cast[pointer](transp)) + try: + addWriter(transp.fd, writeDatagramLoop, cast[pointer](transp)) + except IOSelectorsException as exc: + raiseAsDefect exc, "addWriter" + except ValueError as exc: + raiseAsDefect exc, "addWriter" proc resumeRead(transp: DatagramTransport) {.inline.} = transp.state.excl(ReadPaused) - addReader(transp.fd, readDatagramLoop, cast[pointer](transp)) + try: + addReader(transp.fd, readDatagramLoop, cast[pointer](transp)) + except IOSelectorsException as exc: + raiseAsDefect exc, "addReader" + except ValueError as exc: + raiseAsDefect exc, "addReader" proc newDatagramTransportCommon(cbproc: DatagramCallback, remote: TransportAddress, @@ -482,9 +503,10 @@ else: sock: AsyncFD, flags: set[ServerFlags], udata: pointer, - child: DatagramTransport = nil, + child: DatagramTransport, bufferSize: int, - ttl: int): DatagramTransport = + ttl: int): DatagramTransport {. + raises: [Defect, CatchableError].} = var localSock: AsyncFD doAssert(remote.family == local.family) doAssert(not isNil(cbproc)) @@ -580,7 +602,7 @@ else: proc close*(transp: DatagramTransport) = ## Closes and frees resources of transport ``transp``. - proc continuation(udata: pointer) = + proc continuation(udata: pointer) {.raises: Defect.} = if not(transp.future.finished()): # Stop tracking transport untrackDgram(transp) @@ -612,7 +634,8 @@ proc newDatagramTransport*(cbproc: DatagramCallback, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, ttl: int = 0 - ): DatagramTransport = + ): DatagramTransport {. + raises: [Defect, CatchableError].} = ## Create new UDP datagram transport (IPv4). ## ## ``cbproc`` - callback which will be called, when new datagram received. @@ -637,7 +660,8 @@ proc newDatagramTransport*[T](cbproc: DatagramCallback, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, ttl: int = 0 - ): DatagramTransport = + ): DatagramTransport {. + raises: [Defect, CatchableError].} = var fflags = flags + {GCUserData} GC_ref(udata) result = newDatagramTransportCommon(cbproc, remote, local, sock, @@ -653,7 +677,8 @@ proc newDatagramTransport6*(cbproc: DatagramCallback, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, ttl: int = 0 - ): DatagramTransport = + ): DatagramTransport {. + raises: [Defect, CatchableError].} = ## Create new UDP datagram transport (IPv6). ## ## ``cbproc`` - callback which will be called, when new datagram received. @@ -678,7 +703,8 @@ proc newDatagramTransport6*[T](cbproc: DatagramCallback, child: DatagramTransport = nil, bufSize: int = DefaultDatagramBufferSize, ttl: int = 0 - ): DatagramTransport = + ): DatagramTransport {. + raises: [Defect, CatchableError].} = var fflags = flags + {GCUserData} GC_ref(udata) result = newDatagramTransportCommon(cbproc, remote, local, sock, @@ -815,7 +841,7 @@ proc sendTo*[T](transp: DatagramTransport, remote: TransportAddress, return retFuture proc peekMessage*(transp: DatagramTransport, msg: var seq[byte], - msglen: var int) = + msglen: var int) {.raises: [Defect, CatchableError].} = ## Get access to internal message buffer and length of incoming datagram. if ReadError in transp.state: transp.state.excl(ReadError) @@ -823,7 +849,8 @@ proc peekMessage*(transp: DatagramTransport, msg: var seq[byte], shallowCopy(msg, transp.buffer) msglen = transp.buflen -proc getMessage*(transp: DatagramTransport): seq[byte] = +proc getMessage*(transp: DatagramTransport): seq[byte] {. + raises: [Defect, CatchableError].} = ## Copy data from internal message buffer and return result. if ReadError in transp.state: transp.state.excl(ReadError) diff --git a/vendor/nim-chronos/chronos/transports/ipnet.nim b/vendor/nim-chronos/chronos/transports/ipnet.nim index 615e42b96..cd56123e0 100644 --- a/vendor/nim-chronos/chronos/transports/ipnet.nim +++ b/vendor/nim-chronos/chronos/transports/ipnet.nim @@ -8,8 +8,11 @@ # MIT license (LICENSE-MIT) ## This module implements various IP network utility procedures. -import stew/endians2, strutils -import common + +{.push raises: [Defect].} + +import stew/endians2, std/strutils +import ./common export common type @@ -325,9 +328,9 @@ proc `$`*(mask: IpMask, include0x = false): string = else: result.add(chr(ord('A') + (c - 10))) else: - raise newException(ValueError, "Invalid mask") + return "Unknown mask family: " & $host.family -proc ip*(mask: IpMask): string = +proc ip*(mask: IpMask): string {.raises: [Defect, ValueError].} = ## Returns IP address text representation of IP mask ``mask``. if mask.family == AddressFamily.IPv4: var ip = IpAddress(family: IpAddressFamily.IPv4) @@ -363,7 +366,8 @@ proc init*(t: typedesc[IpNet], host: TransportAddress, result.mask = mask result.host = host -proc init*(t: typedesc[IpNet], network: string): IpNet = +proc init*(t: typedesc[IpNet], network: string): IpNet {. + raises: [Defect, TransportAddressError].} = ## Initialize IP Network from string representation in format ##
/ or
/. var parts = network.rsplit("/", maxsplit = 1) @@ -549,7 +553,10 @@ proc `$`*(net: IpNet): string = result.add("/") let prefix = net.mask.prefix() if prefix == -1: - result.add(net.mask.ip()) + try: + result.add(net.mask.ip()) + except ValueError as exc: + result.add(exc.msg) else: result.add($prefix) elif net.host.family == AddressFamily.IPv6: @@ -559,7 +566,10 @@ proc `$`*(net: IpNet): string = result.add("/") let prefix = net.mask.prefix() if prefix == -1: - result.add(net.mask.ip()) + try: + result.add(net.mask.ip()) + except ValueError as exc: + result.add(exc.msg) else: result.add($prefix) diff --git a/vendor/nim-chronos/chronos/transports/osnet.nim b/vendor/nim-chronos/chronos/transports/osnet.nim index 3a7cecc57..7f8bdfd4d 100644 --- a/vendor/nim-chronos/chronos/transports/osnet.nim +++ b/vendor/nim-chronos/chronos/transports/osnet.nim @@ -9,9 +9,12 @@ ## This module implements cross-platform network interfaces list. ## Currently supported OSes are Windows, Linux, MacOS, BSD(not tested). -import algorithm + +{.push raises: [Defect].} + +import std/algorithm from strutils import toHex -import ipnet +import ./ipnet export ipnet const @@ -19,7 +22,7 @@ const type InterfaceType* = enum - IfError = 0, # This is workaround element for ProoveInit warnings. + IfError = 0, # This is workaround element for ProveInit warnings. IfOther = 1, IfRegular1822 = 2, IfHdh1822 = 3, @@ -316,21 +319,22 @@ proc `$`*(iface: NetworkInterface): string = res.add("inet6 ") res.add($item) res.add(" netmask ") - res.add($(item.netmask().address())) + res.add(try: $(item.netmask().address()) except ValueError as exc: exc.msg) res.add(" brd ") - res.add($(item.broadcast().address())) + res.add( + try: $(item.broadcast().address()) except ValueError as exc: exc.msg) res proc `$`*(route: Route): string = - var res = $route.dest.address() + var res = try: $route.dest.address() except ValueError as exc: exc.msg res.add(" via ") if route.gateway.family != AddressFamily.None: res.add("gateway ") - res.add($route.gateway.address()) + res.add(try: $route.gateway.address() except ValueError as exc: exc.msg) else: res.add("link") res.add(" src ") - res.add($route.source.address()) + res.add(try: $route.source.address() except ValueError as exc: exc.msg) res proc cmp*(a, b: NetworkInterface): int = diff --git a/vendor/nim-chronos/chronos/transports/stream.nim b/vendor/nim-chronos/chronos/transports/stream.nim index aae041d72..a97e6b0f2 100644 --- a/vendor/nim-chronos/chronos/transports/stream.nim +++ b/vendor/nim-chronos/chronos/transports/stream.nim @@ -6,11 +6,12 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import net, nativesockets, os, deques -import ../asyncloop, ../handles -import common -{.deadCodeElim: on.} +{.push raises: [Defect].} + +import std/[net, nativesockets, os, deques] +import ".."/[asyncloop, handles, selectors2] +import common when defined(windows): import winlean @@ -62,7 +63,7 @@ type ReadMessagePredicate* = proc (data: openarray[byte]): tuple[consumed: int, done: bool] {. - gcsafe, raises: [].} + gcsafe, raises: [Defect].} const StreamTransportTrackerName = "stream.transport" @@ -78,7 +79,7 @@ when defined(windows): reader: Future[void] # Current reader Future buffer: seq[byte] # Reading buffer offset: int # Reading buffer offset - error: ref Exception # Current error + error: ref CatchableError # Current error queue: Deque[StreamVector] # Writer queue future: Future[void] # Stream life future # Windows specific part @@ -105,7 +106,7 @@ else: reader: Future[void] # Current reader Future buffer: seq[byte] # Reading buffer offset: int # Reading buffer offset - error: ref Exception # Current error + error: ref CatchableError # Current error queue: Deque[StreamVector] # Writer queue future: Future[void] # Stream life future case kind*: TransportKind @@ -120,13 +121,13 @@ else: type StreamCallback* = proc(server: StreamServer, - client: StreamTransport): Future[void] {.gcsafe.} + client: StreamTransport): Future[void] {.gcsafe, raises: [Defect].} ## New remote client connection callback ## ``server`` - StreamServer object. ## ``client`` - accepted client transport. TransportInitCallback* = proc(server: StreamServer, - fd: AsyncFD): StreamTransport {.gcsafe.} + fd: AsyncFD): StreamTransport {.gcsafe, raises: [Defect].} ## Custom transport initialization procedure, which can allocate inherited ## StreamTransport object. @@ -137,7 +138,8 @@ type init*: TransportInitCallback # callback which will be called before # transport for new client -proc remoteAddress*(transp: StreamTransport): TransportAddress = +proc remoteAddress*(transp: StreamTransport): TransportAddress {. + raises: [Defect, TransportError].} = ## Returns ``transp`` remote socket address. if transp.kind != TransportKind.Socket: raise newException(TransportError, "Socket required!") @@ -150,7 +152,8 @@ proc remoteAddress*(transp: StreamTransport): TransportAddress = fromSAddr(addr saddr, slen, transp.remote) result = transp.remote -proc localAddress*(transp: StreamTransport): TransportAddress = +proc localAddress*(transp: StreamTransport): TransportAddress {. + raises: [Defect, TransportError].} = ## Returns ``transp`` local socket address. if transp.kind != TransportKind.Socket: raise newException(TransportError, "Socket required!") @@ -196,8 +199,8 @@ template shiftVectorFile(v, o: untyped) = (v).buf = cast[pointer](cast[uint]((v).buf) - cast[uint](o)) (v).offset += cast[uint]((o)) -proc setupStreamTransportTracker(): StreamTransportTracker {.gcsafe.} -proc setupStreamServerTracker(): StreamServerTracker {.gcsafe.} +proc setupStreamTransportTracker(): StreamTransportTracker {.gcsafe, raises: [Defect].} +proc setupStreamServerTracker(): StreamServerTracker {.gcsafe, raises: [Defect].} proc getStreamTransportTracker(): StreamTransportTracker {.inline.} = result = cast[StreamTransportTracker](getTracker(StreamTransportTrackerName)) @@ -267,7 +270,7 @@ proc completePendingWriteQueue(queue: var Deque[StreamVector], vector.writer.complete(v) proc failPendingWriteQueue(queue: var Deque[StreamVector], - error: ref Exception) {.inline.} = + error: ref CatchableError) {.inline.} = while len(queue) > 0: var vector = queue.popFirst() if not(vector.writer.finished()): @@ -704,8 +707,11 @@ when defined(windows): toSAddr(raddress, saddr, slen) proto = Protocol.IPPROTO_TCP - sock = createAsyncSocket(raddress.getDomain(), SockType.SOCK_STREAM, + sock = try: createAsyncSocket(raddress.getDomain(), SockType.SOCK_STREAM, proto) + except CatchableError as exc: + retFuture.fail(exc) + return retFuture if sock == asyncInvalidSocket: retFuture.fail(getTransportOsError(osLastError())) return retFuture @@ -760,7 +766,8 @@ when defined(windows): elif address.family == AddressFamily.Unix: ## Unix domain socket emulation with Windows Named Pipes. var pipeHandle = INVALID_HANDLE_VALUE - proc pipeContinuation(udata: pointer) {.gcsafe.} = + var pipeContinuation: proc (udata: pointer) {.gcsafe, raises: [Defect].} + pipeContinuation = proc (udata: pointer) = # Continue only if `retFuture` is not cancelled. if not(retFuture.finished()): var pipeSuffix = $cast[cstring](unsafeAddr address.address_un[0]) @@ -777,9 +784,17 @@ when defined(windows): else: retFuture.fail(getTransportOsError(err)) else: - register(AsyncFD(pipeHandle)) - let transp = newStreamPipeTransport(AsyncFD(pipeHandle), + try: + register(AsyncFD(pipeHandle)) + except CatchableError as exc: + retFuture.fail(exc) + return + + let transp = try: newStreamPipeTransport(AsyncFD(pipeHandle), bufferSize, child) + except CatchableError as exc: + retFuture.fail(exc) + return # Start tracking transport trackStream(transp) retFuture.complete(transp) @@ -787,7 +802,8 @@ when defined(windows): return retFuture - proc createAcceptPipe(server: StreamServer) = + proc createAcceptPipe(server: StreamServer) {. + raises: [Defect, CatchableError].} = let pipeSuffix = $cast[cstring](addr server.local.address_un) let pipeName = newWideCString(r"\\.\pipe\" & pipeSuffix[1 .. ^1]) var openMode = PIPE_ACCESS_DUPLEX or FILE_FLAG_OVERLAPPED @@ -840,7 +856,7 @@ when defined(windows): # We should not raise defects in this loop. discard disconnectNamedPipe(Handle(server.sock)) discard closeHandle(HANDLE(server.sock)) - raiseTransportOsError(osLastError()) + raiseAssert osErrorMsg(osLastError()) else: # Server close happens in callback, and we are not started new # connectNamedPipe session. @@ -864,10 +880,12 @@ when defined(windows): DWORD(server.bufferSize), DWORD(0), nil) if pipeHandle == INVALID_HANDLE_VALUE: - raiseTransportOsError(osLastError()) + raiseAssert osErrorMsg(osLastError()) server.sock = AsyncFD(pipeHandle) server.aovl.data.fd = AsyncFD(pipeHandle) - register(server.sock) + try: register(server.sock) + except CatchableError as exc: + raiseAsDefect exc, "register" let res = connectNamedPipe(pipeHandle, cast[POVERLAPPED](addr server.aovl)) if res == 0: @@ -880,7 +898,7 @@ when defined(windows): elif int32(err) == ERROR_PIPE_CONNECTED: discard else: - raiseTransportOsError(err) + raiseAssert osErrorMsg(err) break else: # Server close happens in callback, and we are not started new @@ -905,7 +923,7 @@ when defined(windows): SockLen(sizeof(SocketHandle))) != 0'i32: let err = OSErrorCode(wsaGetLastError()) server.asock.closeSocket() - raiseTransportOsError(err) + raiseAssert osErrorMsg(err) else: var ntransp: StreamTransport if not isNil(server.init): @@ -930,7 +948,7 @@ when defined(windows): break else: server.asock.closeSocket() - raiseTransportOsError(ovl.data.errCode) + raiseAssert $(ovl.data.errCode) else: # Server close happens in callback, and we are not started new # AcceptEx session. @@ -941,10 +959,12 @@ when defined(windows): ## Initiation if server.status notin {ServerStatus.Stopped, ServerStatus.Closed}: server.apending = true - server.asock = createAsyncSocket(server.domain, SockType.SOCK_STREAM, + # TODO No way to report back errors! + server.asock = try: createAsyncSocket(server.domain, SockType.SOCK_STREAM, Protocol.IPPROTO_TCP) + except CatchableError as exc: raiseAsDefect exc, "createAsyncSocket" if server.asock == asyncInvalidSocket: - raiseTransportOsError(OSErrorCode(wsaGetLastError())) + raiseAssert osErrorMsg(OSErrorCode(wsaGetLastError())) var dwBytesReceived = DWORD(0) let dwReceiveDataLength = DWORD(0) @@ -965,7 +985,7 @@ when defined(windows): elif int32(err) == ERROR_IO_PENDING: discard else: - raiseTransportOsError(err) + raiseAssert osErrorMsg(err) break else: # Server close happens in callback, and we are not started new @@ -1071,8 +1091,13 @@ when defined(windows): ntransp = newStreamPipeTransport(server.sock, server.bufferSize, nil, flags) # Start tracking transport + try: + server.createAcceptPipe() + except CatchableError as exc: + closeHandle(server.sock) + retFuture.fail(exc) + return trackStream(ntransp) - server.createAcceptPipe() retFuture.complete(ntransp) elif int32(ovl.data.errCode) in {ERROR_OPERATION_ABORTED, @@ -1082,8 +1107,14 @@ when defined(windows): server.clean() else: let sock = server.sock - server.createAcceptPipe() + try: + server.createAcceptPipe() + except CatchableError as exc: + closeHandle(sock) + retFuture.fail(exc) + return closeHandle(sock) + retFuture.fail(getTransportOsError(ovl.data.errCode)) proc cancellationPipe(udata: pointer) {.gcsafe.} = @@ -1092,8 +1123,12 @@ when defined(windows): if server.local.family in {AddressFamily.IPv4, AddressFamily.IPv6}: # TCP Sockets part var loop = getThreadDispatcher() - server.asock = createAsyncSocket(server.domain, SockType.SOCK_STREAM, + server.asock = try: createAsyncSocket(server.domain, SockType.SOCK_STREAM, Protocol.IPPROTO_TCP) + except CatchableError as exc: + retFuture.fail(exc) + return retFuture + if server.asock == asyncInvalidSocket: let err = osLastError() if int32(err) == ERROR_TOO_MANY_OPEN_FILES: @@ -1173,7 +1208,8 @@ else: result = (err == OSErrorCode(ECONNRESET)) or (err == OSErrorCode(EPIPE)) - proc writeStreamLoop(udata: pointer) {.gcsafe.} = + proc writeStreamLoop(udata: pointer) = + # TODO fix Defect raises - they "shouldn't" happen var cdata = cast[ptr CompletionData](udata) var transp = cast[StreamTransport](cdata.udata) let fd = SocketHandle(cdata.fd) @@ -1206,7 +1242,13 @@ else: if int(err) == EINTR: continue else: - transp.fd.removeWriter() + try: + transp.fd.removeWriter() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeWriter" + except ValueError as exc: + raiseAsDefect exc, "removeWriter" + if isConnResetError(err): # Soft error happens which indicates that remote peer got # disconnected, complete all pending writes in queue with 0. @@ -1239,7 +1281,13 @@ else: if int(err) == EINTR: continue else: - transp.fd.removeWriter() + try: + transp.fd.removeWriter() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeWriter" + except ValueError as exc: + raiseAsDefect exc, "removeWriter" + if isConnResetError(err): # Soft error happens which indicates that remote peer got # disconnected, complete all pending writes in queue with 0. @@ -1270,7 +1318,13 @@ else: if int(err) == EINTR: continue else: - transp.fd.removeWriter() + try: + transp.fd.removeWriter() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeWriter" + except ValueError as exc: + raiseAsDefect exc, "removeWriter" + if isConnResetError(err): # Soft error happens which indicates that remote peer got # disconnected, complete all pending writes in queue with 0. @@ -1303,7 +1357,12 @@ else: if int(err) == EINTR: continue else: - transp.fd.removeWriter() + try: + transp.fd.removeWriter() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeWriter" + except ValueError as exc: + raiseAsDefect exc, "removeWriter" if isConnResetError(err): # Soft error happens which indicates that remote peer got # disconnected, complete all pending writes in queue with 0. @@ -1320,9 +1379,15 @@ else: break else: transp.state.incl(WritePaused) - transp.fd.removeWriter() + try: + transp.fd.removeWriter() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeWriter" + except ValueError as exc: + raiseAsDefect exc, "removeWriter" - proc readStreamLoop(udata: pointer) {.gcsafe.} = + proc readStreamLoop(udata: pointer) = + # TODO fix Defect raises - they "shouldn't" happen var cdata = cast[ptr CompletionData](udata) var transp = cast[StreamTransport](cdata.udata) let fd = SocketHandle(cdata.fd) @@ -1345,19 +1410,39 @@ else: continue elif int(err) in {ECONNRESET}: transp.state.incl({ReadEof, ReadPaused}) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" else: transp.state.incl(ReadPaused) transp.setReadError(err) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" elif res == 0: transp.state.incl({ReadEof, ReadPaused}) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" else: transp.offset += res if transp.offset == len(transp.buffer): transp.state.incl(ReadPaused) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" transp.completeReader() break elif transp.kind == TransportKind.Pipe: @@ -1371,15 +1456,30 @@ else: else: transp.state.incl(ReadPaused) transp.setReadError(err) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" elif res == 0: transp.state.incl({ReadEof, ReadPaused}) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" else: transp.offset += res if transp.offset == len(transp.buffer): transp.state.incl(ReadPaused) - cdata.fd.removeReader() + try: + cdata.fd.removeReader() + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" transp.completeReader() break @@ -1424,7 +1524,6 @@ else: var saddr: Sockaddr_storage slen: SockLen - sock: AsyncFD proto: Protocol var retFuture = newFuture[StreamTransport]("stream.transport.connect") address.toSAddr(saddr, slen) @@ -1433,8 +1532,13 @@ else: # `Protocol` enum is missing `0` value, so we making here cast, until # `Protocol` enum will not support IPPROTO_IP == 0. proto = cast[Protocol](0) - sock = createAsyncSocket(address.getDomain(), SockType.SOCK_STREAM, - proto) + + let sock = try: createAsyncSocket(address.getDomain(), SockType.SOCK_STREAM, + proto) + except CatchableError as exc: + retFuture.fail(exc) + return retFuture + if sock == asyncInvalidSocket: let err = osLastError() if int(err) == EMFILE: @@ -1443,12 +1547,20 @@ else: retFuture.fail(getTransportOsError(err)) return retFuture - proc continuation(udata: pointer) {.gcsafe.} = + proc continuation(udata: pointer) = if not(retFuture.finished()): var data = cast[ptr CompletionData](udata) var err = 0 let fd = data.fd - fd.removeWriter() + try: + fd.removeWriter() + except IOSelectorsException as exc: + retFuture.fail(exc) + return + except ValueError as exc: + retFuture.fail(exc) + return + if not fd.getSocketError(err): closeSocket(fd) retFuture.fail(getTransportOsError(osLastError())) @@ -1462,7 +1574,7 @@ else: trackStream(transp) retFuture.complete(transp) - proc cancel(udata: pointer) {.gcsafe.} = + proc cancel(udata: pointer) = closeSocket(sock) while true: @@ -1483,11 +1595,18 @@ else: # # http://www.madore.org/~david/computers/connect-intr.html if int(err) == EINPROGRESS or int(err) == EINTR: - sock.addWriter(continuation) + try: + sock.addWriter(continuation) + except CatchableError as exc: + closeSocket(sock) + retFuture.fail(exc) + return retFuture + retFuture.cancelCallback = cancel break else: sock.closeSocket() + retFuture.fail(getTransportOsError(err)) break return retFuture @@ -1504,7 +1623,9 @@ else: let res = posix.accept(SocketHandle(server.sock), cast[ptr SockAddr](addr saddr), addr slen) if int(res) > 0: - let sock = wrapAsyncSocket(res) + let sock = try: wrapAsyncSocket(res) + except CatchableError as exc: + raiseAsDefect exc, "wrapAsyncSocket" if sock != asyncInvalidSocket: var ntransp: StreamTransport if not isNil(server.init): @@ -1526,23 +1647,37 @@ else: break else: ## Critical unrecoverable error - raiseTransportOsError(err) + raiseAssert $err - proc resumeAccept(server: StreamServer) = + proc resumeAccept(server: StreamServer) {. + raises: [Defect, IOSelectorsException, ValueError].} = addReader(server.sock, acceptLoop, cast[pointer](server)) - proc pauseAccept(server: StreamServer) = + proc pauseAccept(server: StreamServer) {. + raises: [Defect, IOSelectorsException, ValueError].} = removeReader(server.sock) proc resumeRead(transp: StreamTransport) {.inline.} = if ReadPaused in transp.state: transp.state.excl(ReadPaused) - addReader(transp.fd, readStreamLoop, cast[pointer](transp)) + # TODO reset flag on exception?? + try: + addReader(transp.fd, readStreamLoop, cast[pointer](transp)) + except IOSelectorsException as exc: + raiseAsDefect exc, "addReader" + except ValueError as exc: + raiseAsDefect exc, "addReader" proc resumeWrite(transp: StreamTransport) {.inline.} = if WritePaused in transp.state: transp.state.excl(WritePaused) - addWriter(transp.fd, writeStreamLoop, cast[pointer](transp)) + # TODO reset flag on exception?? + try: + addWriter(transp.fd, writeStreamLoop, cast[pointer](transp)) + except IOSelectorsException as exc: + raiseAsDefect exc, "addWriter" + except ValueError as exc: + raiseAsDefect exc, "addWriter" proc accept*(server: StreamServer): Future[StreamTransport] = var retFuture = newFuture[StreamTransport]("stream.server.accept") @@ -1565,7 +1700,12 @@ else: let res = posix.accept(SocketHandle(server.sock), cast[ptr SockAddr](addr saddr), addr slen) if int(res) > 0: - let sock = wrapAsyncSocket(res) + let sock = try: wrapAsyncSocket(res) + except CatchableError as exc: + close(res) + retFuture.fail(exc) + return + if sock != asyncInvalidSocket: var ntransp: StreamTransport if not isNil(server.init): @@ -1592,23 +1732,41 @@ else: else: retFuture.fail(getTransportOsError(err)) break - removeReader(server.sock) + try: + removeReader(server.sock) + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" - proc cancellation(udata: pointer) {.gcsafe.} = - removeReader(server.sock) + proc cancellation(udata: pointer) = + try: + removeReader(server.sock) + except IOSelectorsException as exc: + raiseAsDefect exc, "removeReader" + except ValueError as exc: + raiseAsDefect exc, "removeReader" + + try: + addReader(server.sock, continuation, nil) + except IOSelectorsException as exc: + raiseAsDefect exc, "addReader" + except ValueError as exc: + raiseAsDefect exc, "addReader" - addReader(server.sock, continuation, nil) retFuture.cancelCallback = cancellation return retFuture -proc start*(server: StreamServer) = +proc start*(server: StreamServer) {. + raises: [Defect, IOSelectorsException, ValueError].} = ## Starts ``server``. doAssert(not(isNil(server.function))) if server.status == ServerStatus.Starting: server.resumeAccept() server.status = ServerStatus.Running -proc stop*(server: StreamServer) = +proc stop*(server: StreamServer) {. + raises: [Defect, IOSelectorsException, ValueError].} = ## Stops ``server``. if server.status == ServerStatus.Running: server.pauseAccept() @@ -1620,10 +1778,10 @@ proc join*(server: StreamServer): Future[void] = ## Waits until ``server`` is not closed. var retFuture = newFuture[void]("stream.transport.server.join") - proc continuation(udata: pointer) {.gcsafe.} = + proc continuation(udata: pointer) = retFuture.complete() - proc cancel(udata: pointer) {.gcsafe.} = + proc cancel(udata: pointer) = server.loopFuture.removeCallback(continuation, cast[pointer](retFuture)) if not(server.loopFuture.finished()): @@ -1638,7 +1796,7 @@ proc close*(server: StreamServer) = ## ## Please note that release of resources is not completed immediately, to be ## sure all resources got released please use ``await server.join()``. - proc continuation(udata: pointer) {.gcsafe.} = + proc continuation(udata: pointer) = # Stop tracking server if not(server.loopFuture.finished()): server.clean() @@ -1680,7 +1838,8 @@ proc createStreamServer*(host: TransportAddress, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil, - udata: pointer = nil): StreamServer = + udata: pointer = nil): StreamServer {. + raises: [Defect, CatchableError].} = ## Create new TCP stream server. ## ## ``host`` - address to which server will be bound. @@ -1707,6 +1866,7 @@ proc createStreamServer*(host: TransportAddress, serverSocket = createAsyncSocket(host.getDomain(), SockType.SOCK_STREAM, Protocol.IPPROTO_TCP) + if serverSocket == asyncInvalidSocket: raiseTransportOsError(osLastError()) else: @@ -1770,6 +1930,7 @@ proc createStreamServer*(host: TransportAddress, if not setSocketBlocking(SocketHandle(sock), false): raiseTransportOsError(osLastError()) register(sock) + serverSocket = sock if host.family in {AddressFamily.IPv4, AddressFamily.IPv6}: @@ -1869,7 +2030,8 @@ proc createStreamServer*(host: TransportAddress, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, init: TransportInitCallback = nil, - udata: pointer = nil): StreamServer = + udata: pointer = nil): StreamServer {. + raises: [Defect, CatchableError].} = result = createStreamServer(host, nil, flags, sock, backlog, bufferSize, child, init, cast[pointer](udata)) @@ -1881,7 +2043,8 @@ proc createStreamServer*[T](host: TransportAddress, backlog: int = 100, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, - init: TransportInitCallback = nil): StreamServer = + init: TransportInitCallback = nil): StreamServer {. + raises: [Defect, CatchableError].} = var fflags = flags + {GCUserData} GC_ref(udata) result = createStreamServer(host, cbproc, fflags, sock, backlog, bufferSize, @@ -1894,7 +2057,8 @@ proc createStreamServer*[T](host: TransportAddress, backlog: int = 100, bufferSize: int = DefaultStreamBufferSize, child: StreamServer = nil, - init: TransportInitCallback = nil): StreamServer = + init: TransportInitCallback = nil): StreamServer {. + raises: [Defect, CatchableError].} = var fflags = flags + {GCUserData} GC_ref(udata) result = createStreamServer(host, nil, fflags, sock, backlog, bufferSize, @@ -1959,10 +2123,12 @@ proc writeFile*(transp: StreamTransport, handle: int, ## ## You can specify starting ``offset`` in opened file and number of bytes ## to transfer from file to transport via ``size``. + var retFuture = newFuture[int]("stream.transport.writeFile") when defined(windows): if transp.kind != TransportKind.Socket: - raise newException(TransportNoSupport, "writeFile() is not supported!") - var retFuture = newFuture[int]("stream.transport.writeFile") + retFuture.fail(newException( + TransportNoSupport, "writeFile() is not supported!")) + return retFuture transp.checkClosed(retFuture) transp.checkWriteEof(retFuture) var vector = StreamVector(kind: DataFile, writer: retFuture, @@ -2295,11 +2461,13 @@ proc closed*(transp: StreamTransport): bool {.inline.} = result = ({ReadClosed, WriteClosed} * transp.state != {}) proc fromPipe*(fd: AsyncFD, child: StreamTransport = nil, - bufferSize = DefaultStreamBufferSize): StreamTransport = + bufferSize = DefaultStreamBufferSize): StreamTransport {. + raises: [Defect, CatchableError].} = ## Create new transport object using pipe's file descriptor. ## ## ``bufferSize`` is size of internal buffer for transport. register(fd) + result = newStreamPipeTransport(fd, bufferSize, child) # Start tracking transport trackStream(result) diff --git a/vendor/nim-chronos/tests/testaddress.nim b/vendor/nim-chronos/tests/testaddress.nim index 5348e7272..a1a588ece 100644 --- a/vendor/nim-chronos/tests/testaddress.nim +++ b/vendor/nim-chronos/tests/testaddress.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testall.nim b/vendor/nim-chronos/tests/testall.nim index 59a66c099..decec74c4 100644 --- a/vendor/nim-chronos/tests/testall.nim +++ b/vendor/nim-chronos/tests/testall.nim @@ -7,5 +7,5 @@ # MIT license (LICENSE-MIT) import testmacro, testsync, testsoon, testtime, testfut, testsignal, testaddress, testdatagram, teststream, testserver, testbugs, testnet, - testasyncstream, testhttpserver + testasyncstream, testhttpserver, testshttpserver import testutils diff --git a/vendor/nim-chronos/tests/testasyncstream.nim b/vendor/nim-chronos/tests/testasyncstream.nim index 12b445f08..7a4f2112d 100644 --- a/vendor/nim-chronos/tests/testasyncstream.nim +++ b/vendor/nim-chronos/tests/testasyncstream.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos import ../chronos/streams/[tlsstream, chunkstream, boundstream] diff --git a/vendor/nim-chronos/tests/testbugs.nim b/vendor/nim-chronos/tests/testbugs.nim index 7fc9a9c2b..0d9f74744 100644 --- a/vendor/nim-chronos/tests/testbugs.nim +++ b/vendor/nim-chronos/tests/testbugs.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testdatagram.nim b/vendor/nim-chronos/tests/testdatagram.nim index d6557f2e3..c353cce08 100644 --- a/vendor/nim-chronos/tests/testdatagram.nim +++ b/vendor/nim-chronos/tests/testdatagram.nim @@ -5,7 +5,8 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import strutils, net, unittest +import std/[strutils, net] +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testfut.nim b/vendor/nim-chronos/tests/testfut.nim index 2d903d795..9e48432ba 100644 --- a/vendor/nim-chronos/tests/testfut.nim +++ b/vendor/nim-chronos/tests/testfut.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testhttpserver.nim b/vendor/nim-chronos/tests/testhttpserver.nim index a57b174c9..86fd4f124 100644 --- a/vendor/nim-chronos/tests/testhttpserver.nim +++ b/vendor/nim-chronos/tests/testhttpserver.nim @@ -5,69 +5,12 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import std/[strutils, unittest, algorithm, strutils] -import ../chronos, ../chronos/apps +import std/[strutils, algorithm, strutils] +import unittest2 +import ../chronos, ../chronos/apps/http/httpserver import stew/base10 -# To create self-signed certificate and key you can use openssl -# openssl req -new -x509 -sha256 -newkey rsa:2048 -nodes \ -# -keyout example-com.key.pem -days 3650 -out example-com.cert.pem -const HttpsSelfSignedRsaKey = """ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCn7tXGLKMIMzOG -tVzUixax1/ftlSLcpEAkZMORuiCCnYjtIJhGZdzRFZC8fBlfAJZpLIAOfX2L2f1J -ZuwpwDkOIvNqKMBrl5Mvkl5azPT0rtnjuwrcqN5NFtbmZPKFYvbjex2aXGqjl5MW -nQIs/ZA++DVEXmaN9oDxcZsvRMDKfrGQf9iLeoVL47Gx9KpqNqD/JLIn4LpieumV -yYidm6ukTOqHRvrWm36y6VvKW4TE97THacULmkeahtTf8zDJbbh4EO+gifgwgJ2W -BUS0+5hMcWu8111mXmanlOVlcoW8fH8RmPjL1eK1Z3j3SVHEf7oWZtIVW5gGA0jQ -nfA4K51RAgMBAAECggEANZ7/R13tWKrwouy6DWuz/WlWUtgx333atUQvZhKmWs5u -cDjeJmxUC7b1FhoSB9GqNT7uTLIpKkSaqZthgRtNnIPwcU890Zz+dEwqMJgNByvl -it+oYjjRco/+YmaNQaYN6yjelPE5Y678WlYb4b29Fz4t0/zIhj/VgEKkKH2tiXpS -TIicoM7pSOscEUfaW3yp5bS5QwNU6/AaF1wws0feBACd19ZkcdPvr52jopbhxlXw -h3XTV/vXIJd5zWGp0h/Jbd4xcD4MVo2GjfkeORKY6SjDaNzt8OGtePcKnnbUVu8b -2XlDxukhDQXqJ3g0sHz47mhvo4JeIM+FgymRm+3QmQKBgQDTawrEA3Zy9WvucaC7 -Zah02oE9nuvpF12lZ7WJh7+tZ/1ss+Fm7YspEKaUiEk7nn1CAVFtem4X4YCXTBiC -Oqq/o+ipv1yTur0ae6m4pwLm5wcMWBh3H5zjfQTfrClNN8yjWv8u3/sq8KesHPnT -R92/sMAptAChPgTzQphWbxFiYwKBgQDLWFaBqXfZYVnTyUvKX8GorS6jGWc6Eh4l -lAFA+2EBWDICrUxsDPoZjEXrWCixdqLhyehaI3KEFIx2bcPv6X2c7yx3IG5lA/Gx -TZiKlY74c6jOTstkdLW9RJbg1VUHUVZMf/Owt802YmEfUI5S5v7jFmKW6VG+io+K -+5KYeHD1uwKBgQDMf53KPA82422jFwYCPjLT1QduM2q97HwIomhWv5gIg63+l4BP -rzYMYq6+vZUYthUy41OAMgyLzPQ1ZMXQMi83b7R9fTxvKRIBq9xfYCzObGnE5vHD -SDDZWvR75muM5Yxr9nkfPkgVIPMO6Hg+hiVYZf96V0LEtNjU9HWmJYkLQQKBgQCQ -ULGUdGHKtXy7AjH3/t3CiKaAupa4cANVSCVbqQy/l4hmvfdu+AbH+vXkgTzgNgKD -nHh7AI1Vj//gTSayLlQn/Nbh9PJkXtg5rYiFUn+VdQBo6yMOuIYDPZqXFtCx0Nge -kvCwisHpxwiG4PUhgS+Em259DDonsM8PJFx2OYRx4QKBgEQpGhg71Oi9MhPJshN7 -dYTowaMS5eLTk2264ARaY+hAIV7fgvUa+5bgTVaWL+Cfs33hi4sMRqlEwsmfds2T -cnQiJ4cU20Euldfwa5FLnk6LaWdOyzYt/ICBJnKFRwfCUbS4Bu5rtMEM+3t0wxnJ -IgaD04WhoL9EX0Qo3DC1+0kG ------END PRIVATE KEY----- -""" - -# This SSL certificate will expire 13 October 2030. -const HttpsSelfSignedRsaCert = """ ------BEGIN CERTIFICATE----- -MIIDnzCCAoegAwIBAgIUUdcusjDd3XQi3FPM8urdFG3qI+8wDQYJKoZIhvcNAQEL -BQAwXzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEYMBYGA1UEAwwPMTI3LjAuMC4xOjQz -ODA4MB4XDTIwMTAxMjIxNDUwMVoXDTMwMTAxMDIxNDUwMVowXzELMAkGA1UEBhMC -QVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdp -dHMgUHR5IEx0ZDEYMBYGA1UEAwwPMTI3LjAuMC4xOjQzODA4MIIBIjANBgkqhkiG -9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp+7VxiyjCDMzhrVc1IsWsdf37ZUi3KRAJGTD -kboggp2I7SCYRmXc0RWQvHwZXwCWaSyADn19i9n9SWbsKcA5DiLzaijAa5eTL5Je -Wsz09K7Z47sK3KjeTRbW5mTyhWL243sdmlxqo5eTFp0CLP2QPvg1RF5mjfaA8XGb -L0TAyn6xkH/Yi3qFS+OxsfSqajag/ySyJ+C6YnrplcmInZurpEzqh0b61pt+sulb -yluExPe0x2nFC5pHmobU3/MwyW24eBDvoIn4MICdlgVEtPuYTHFrvNddZl5mp5Tl -ZXKFvHx/EZj4y9XitWd490lRxH+6FmbSFVuYBgNI0J3wOCudUQIDAQABo1MwUTAd -BgNVHQ4EFgQUBKha84woY5WkFxKw7qx1cONg1H8wHwYDVR0jBBgwFoAUBKha84wo -Y5WkFxKw7qx1cONg1H8wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC -AQEAHZMYt9Ry+Xj3vTbzpGFQzYQVTJlfJWSN6eWNOivRFQE5io9kOBEe5noa8aLo -dLkw6ztxRP2QRJmlhGCO9/HwS17ckrkgZp3EC2LFnzxcBmoZu+owfxOT1KqpO52O -IKOl8eVohi1pEicE4dtTJVcpI7VCMovnXUhzx1Ci4Vibns4a6H+BQa19a1JSpifN -tO8U5jkjJ8Jprs/VPFhJj2O3di53oDHaYSE5eOrm2ZO14KFHSk9cGcOGmcYkUv8B -nV5vnGadH5Lvfxb/BCpuONabeRdOxMt9u9yQ89vNpxFtRdZDCpGKZBCfmUP+5m3m -N8r5CwGcIX/XPC3lKazzbZ8baA== ------END CERTIFICATE----- -""" +when defined(nimHasUsed): {.used.} suite "HTTP server testing suite": type @@ -89,34 +32,6 @@ suite "HTTP server testing suite": if not(isNil(transp)): await closeWait(transp) - proc httpsClient(address: TransportAddress, - data: string, flags = {NoVerifyHost, NoVerifyServerName} - ): Future[string] {.async.} = - var - transp: StreamTransport - tlsstream: TlsAsyncStream - reader: AsyncStreamReader - writer: AsyncStreamWriter - - try: - transp = await connect(address) - reader = newAsyncStreamReader(transp) - writer = newAsyncStreamWriter(transp) - tlsstream = newTLSClientAsyncStream(reader, writer, "", flags = flags) - if len(data) > 0: - await tlsstream.writer.write(data) - var rres = await tlsstream.reader.read() - return bytesToString(rres) - except CatchableError: - return "EXCEPTION" - finally: - if not(isNil(tlsstream)): - await allFutures(tlsstream.reader.closeWait(), - tlsstream.writer.closeWait()) - if not(isNil(reader)): - await allFutures(reader.closeWait(), writer.closeWait(), - transp.closeWait()) - proc testTooBigBodyChunked(address: TransportAddress, operation: TooBigTest): Future[bool] {.async.} = var serverRes = false @@ -606,82 +521,6 @@ suite "HTTP server testing suite": check waitFor(testPostMultipart2(initTAddress("127.0.0.1:30080"))) == true - test "HTTPS server (successful handshake) test": - proc testHTTPS(address: TransportAddress): Future[bool] {.async.} = - var serverRes = false - proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = - if r.isOk(): - let request = r.get() - serverRes = true - return await request.respond(Http200, "TEST_OK:" & $request.meth, - HttpTable.init()) - else: - serverRes = false - return dumbResponse() - - let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let serverFlags = {Secure} - let secureKey = TLSPrivateKey.init(HttpsSelfSignedRsaKey) - let secureCert = TLSCertificate.init(HttpsSelfSignedRsaCert) - let res = HttpServerRef.new(address, process, - socketFlags = socketFlags, - serverFlags = serverFlags, - tlsPrivateKey = secureKey, - tlsCertificate = secureCert) - if res.isErr(): - return false - - let server = res.get() - server.start() - let message = "GET / HTTP/1.0\r\nHost: https://127.0.0.1:80\r\n\r\n" - let data = await httpsClient(address, message) - - await server.stop() - await server.closeWait() - return serverRes and (data.find("TEST_OK:GET") >= 0) - - check waitFor(testHTTPS(initTAddress("127.0.0.1:30080"))) == true - - test "HTTPS server (failed handshake) test": - proc testHTTPS2(address: TransportAddress): Future[bool] {.async.} = - var serverRes = false - var testFut = newFuture[void]() - proc process(r: RequestFence): Future[HttpResponseRef] {. - async.} = - if r.isOk(): - let request = r.get() - serverRes = false - return await request.respond(Http200, "TEST_OK:" & $request.meth, - HttpTable.init()) - else: - serverRes = true - testFut.complete() - return dumbResponse() - - let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} - let serverFlags = {Secure} - let secureKey = TLSPrivateKey.init(HttpsSelfSignedRsaKey) - let secureCert = TLSCertificate.init(HttpsSelfSignedRsaCert) - let res = HttpServerRef.new(address, process, - socketFlags = socketFlags, - serverFlags = serverFlags, - tlsPrivateKey = secureKey, - tlsCertificate = secureCert) - if res.isErr(): - return false - - let server = res.get() - server.start() - let message = "GET / HTTP/1.0\r\nHost: https://127.0.0.1:80\r\n\r\n" - let data = await httpsClient(address, message, {NoVerifyServerName}) - await testFut - await server.stop() - await server.closeWait() - return serverRes and data == "EXCEPTION" - - check waitFor(testHTTPS2(initTAddress("127.0.0.1:30080"))) == true - test "drop() connections test": const ClientsCount = 10 diff --git a/vendor/nim-chronos/tests/testmacro.nim b/vendor/nim-chronos/tests/testmacro.nim index fc9ca1969..c18b37ee9 100644 --- a/vendor/nim-chronos/tests/testmacro.nim +++ b/vendor/nim-chronos/tests/testmacro.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testnet.nim b/vendor/nim-chronos/tests/testnet.nim index b0bf5fa17..31c3d66c0 100644 --- a/vendor/nim-chronos/tests/testnet.nim +++ b/vendor/nim-chronos/tests/testnet.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos/transports/[osnet, ipnet] when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testserver.nim b/vendor/nim-chronos/tests/testserver.nim index d079648ba..8828bb227 100644 --- a/vendor/nim-chronos/tests/testserver.nim +++ b/vendor/nim-chronos/tests/testserver.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testshttpserver.nim b/vendor/nim-chronos/tests/testshttpserver.nim new file mode 100644 index 000000000..9e563b5ce --- /dev/null +++ b/vendor/nim-chronos/tests/testshttpserver.nim @@ -0,0 +1,180 @@ +# Chronos Test Suite +# (c) Copyright 2021-Present +# Status Research & Development GmbH +# +# Licensed under either of +# Apache License, version 2.0, (LICENSE-APACHEv2) +# MIT license (LICENSE-MIT) +import std/[strutils, strutils] +import unittest2 +import ../chronos, ../chronos/apps/http/shttpserver +import stew/base10 + +when defined(nimHasUsed): {.used.} + +# To create self-signed certificate and key you can use openssl +# openssl req -new -x509 -sha256 -newkey rsa:2048 -nodes \ +# -keyout example-com.key.pem -days 3650 -out example-com.cert.pem +const HttpsSelfSignedRsaKey = """ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCn7tXGLKMIMzOG +tVzUixax1/ftlSLcpEAkZMORuiCCnYjtIJhGZdzRFZC8fBlfAJZpLIAOfX2L2f1J +ZuwpwDkOIvNqKMBrl5Mvkl5azPT0rtnjuwrcqN5NFtbmZPKFYvbjex2aXGqjl5MW +nQIs/ZA++DVEXmaN9oDxcZsvRMDKfrGQf9iLeoVL47Gx9KpqNqD/JLIn4LpieumV +yYidm6ukTOqHRvrWm36y6VvKW4TE97THacULmkeahtTf8zDJbbh4EO+gifgwgJ2W +BUS0+5hMcWu8111mXmanlOVlcoW8fH8RmPjL1eK1Z3j3SVHEf7oWZtIVW5gGA0jQ +nfA4K51RAgMBAAECggEANZ7/R13tWKrwouy6DWuz/WlWUtgx333atUQvZhKmWs5u +cDjeJmxUC7b1FhoSB9GqNT7uTLIpKkSaqZthgRtNnIPwcU890Zz+dEwqMJgNByvl +it+oYjjRco/+YmaNQaYN6yjelPE5Y678WlYb4b29Fz4t0/zIhj/VgEKkKH2tiXpS +TIicoM7pSOscEUfaW3yp5bS5QwNU6/AaF1wws0feBACd19ZkcdPvr52jopbhxlXw +h3XTV/vXIJd5zWGp0h/Jbd4xcD4MVo2GjfkeORKY6SjDaNzt8OGtePcKnnbUVu8b +2XlDxukhDQXqJ3g0sHz47mhvo4JeIM+FgymRm+3QmQKBgQDTawrEA3Zy9WvucaC7 +Zah02oE9nuvpF12lZ7WJh7+tZ/1ss+Fm7YspEKaUiEk7nn1CAVFtem4X4YCXTBiC +Oqq/o+ipv1yTur0ae6m4pwLm5wcMWBh3H5zjfQTfrClNN8yjWv8u3/sq8KesHPnT +R92/sMAptAChPgTzQphWbxFiYwKBgQDLWFaBqXfZYVnTyUvKX8GorS6jGWc6Eh4l +lAFA+2EBWDICrUxsDPoZjEXrWCixdqLhyehaI3KEFIx2bcPv6X2c7yx3IG5lA/Gx +TZiKlY74c6jOTstkdLW9RJbg1VUHUVZMf/Owt802YmEfUI5S5v7jFmKW6VG+io+K ++5KYeHD1uwKBgQDMf53KPA82422jFwYCPjLT1QduM2q97HwIomhWv5gIg63+l4BP +rzYMYq6+vZUYthUy41OAMgyLzPQ1ZMXQMi83b7R9fTxvKRIBq9xfYCzObGnE5vHD +SDDZWvR75muM5Yxr9nkfPkgVIPMO6Hg+hiVYZf96V0LEtNjU9HWmJYkLQQKBgQCQ +ULGUdGHKtXy7AjH3/t3CiKaAupa4cANVSCVbqQy/l4hmvfdu+AbH+vXkgTzgNgKD +nHh7AI1Vj//gTSayLlQn/Nbh9PJkXtg5rYiFUn+VdQBo6yMOuIYDPZqXFtCx0Nge +kvCwisHpxwiG4PUhgS+Em259DDonsM8PJFx2OYRx4QKBgEQpGhg71Oi9MhPJshN7 +dYTowaMS5eLTk2264ARaY+hAIV7fgvUa+5bgTVaWL+Cfs33hi4sMRqlEwsmfds2T +cnQiJ4cU20Euldfwa5FLnk6LaWdOyzYt/ICBJnKFRwfCUbS4Bu5rtMEM+3t0wxnJ +IgaD04WhoL9EX0Qo3DC1+0kG +-----END PRIVATE KEY----- +""" + +# This SSL certificate will expire 13 October 2030. +const HttpsSelfSignedRsaCert = """ +-----BEGIN CERTIFICATE----- +MIIDnzCCAoegAwIBAgIUUdcusjDd3XQi3FPM8urdFG3qI+8wDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEYMBYGA1UEAwwPMTI3LjAuMC4xOjQz +ODA4MB4XDTIwMTAxMjIxNDUwMVoXDTMwMTAxMDIxNDUwMVowXzELMAkGA1UEBhMC +QVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdp +dHMgUHR5IEx0ZDEYMBYGA1UEAwwPMTI3LjAuMC4xOjQzODA4MIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp+7VxiyjCDMzhrVc1IsWsdf37ZUi3KRAJGTD +kboggp2I7SCYRmXc0RWQvHwZXwCWaSyADn19i9n9SWbsKcA5DiLzaijAa5eTL5Je +Wsz09K7Z47sK3KjeTRbW5mTyhWL243sdmlxqo5eTFp0CLP2QPvg1RF5mjfaA8XGb +L0TAyn6xkH/Yi3qFS+OxsfSqajag/ySyJ+C6YnrplcmInZurpEzqh0b61pt+sulb +yluExPe0x2nFC5pHmobU3/MwyW24eBDvoIn4MICdlgVEtPuYTHFrvNddZl5mp5Tl +ZXKFvHx/EZj4y9XitWd490lRxH+6FmbSFVuYBgNI0J3wOCudUQIDAQABo1MwUTAd +BgNVHQ4EFgQUBKha84woY5WkFxKw7qx1cONg1H8wHwYDVR0jBBgwFoAUBKha84wo +Y5WkFxKw7qx1cONg1H8wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC +AQEAHZMYt9Ry+Xj3vTbzpGFQzYQVTJlfJWSN6eWNOivRFQE5io9kOBEe5noa8aLo +dLkw6ztxRP2QRJmlhGCO9/HwS17ckrkgZp3EC2LFnzxcBmoZu+owfxOT1KqpO52O +IKOl8eVohi1pEicE4dtTJVcpI7VCMovnXUhzx1Ci4Vibns4a6H+BQa19a1JSpifN +tO8U5jkjJ8Jprs/VPFhJj2O3di53oDHaYSE5eOrm2ZO14KFHSk9cGcOGmcYkUv8B +nV5vnGadH5Lvfxb/BCpuONabeRdOxMt9u9yQ89vNpxFtRdZDCpGKZBCfmUP+5m3m +N8r5CwGcIX/XPC3lKazzbZ8baA== +-----END CERTIFICATE----- +""" + + +suite "Secure HTTP server testing suite": + + proc httpsClient(address: TransportAddress, + data: string, flags = {NoVerifyHost, NoVerifyServerName} + ): Future[string] {.async.} = + var + transp: StreamTransport + tlsstream: TlsAsyncStream + reader: AsyncStreamReader + writer: AsyncStreamWriter + + try: + transp = await connect(address) + reader = newAsyncStreamReader(transp) + writer = newAsyncStreamWriter(transp) + tlsstream = newTLSClientAsyncStream(reader, writer, "", flags = flags) + if len(data) > 0: + await tlsstream.writer.write(data) + var rres = await tlsstream.reader.read() + return bytesToString(rres) + except CatchableError: + return "EXCEPTION" + finally: + if not(isNil(tlsstream)): + await allFutures(tlsstream.reader.closeWait(), + tlsstream.writer.closeWait()) + if not(isNil(reader)): + await allFutures(reader.closeWait(), writer.closeWait(), + transp.closeWait()) + + test "HTTPS server (successful handshake) test": + proc testHTTPS(address: TransportAddress): Future[bool] {.async.} = + var serverRes = false + proc process(r: RequestFence): Future[HttpResponseRef] {. + async.} = + if r.isOk(): + let request = r.get() + serverRes = true + return await request.respond(Http200, "TEST_OK:" & $request.meth, + HttpTable.init()) + else: + serverRes = false + return dumbResponse() + + let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} + let serverFlags = {Secure} + let secureKey = TLSPrivateKey.init(HttpsSelfSignedRsaKey) + let secureCert = TLSCertificate.init(HttpsSelfSignedRsaCert) + let res = SecureHttpServerRef.new(address, process, + socketFlags = socketFlags, + serverFlags = serverFlags, + tlsPrivateKey = secureKey, + tlsCertificate = secureCert) + if res.isErr(): + return false + + let server = res.get() + server.start() + let message = "GET / HTTP/1.0\r\nHost: https://127.0.0.1:80\r\n\r\n" + let data = await httpsClient(address, message) + + await server.stop() + await server.closeWait() + return serverRes and (data.find("TEST_OK:GET") >= 0) + + check waitFor(testHTTPS(initTAddress("127.0.0.1:30080"))) == true + + test "HTTPS server (failed handshake) test": + proc testHTTPS2(address: TransportAddress): Future[bool] {.async.} = + var serverRes = false + var testFut = newFuture[void]() + proc process(r: RequestFence): Future[HttpResponseRef] {. + async.} = + if r.isOk(): + let request = r.get() + serverRes = false + return await request.respond(Http200, "TEST_OK:" & $request.meth, + HttpTable.init()) + else: + serverRes = true + testFut.complete() + return dumbResponse() + + let socketFlags = {ServerFlags.TcpNoDelay, ServerFlags.ReuseAddr} + let serverFlags = {Secure} + let secureKey = TLSPrivateKey.init(HttpsSelfSignedRsaKey) + let secureCert = TLSCertificate.init(HttpsSelfSignedRsaCert) + let res = SecureHttpServerRef.new(address, process, + socketFlags = socketFlags, + serverFlags = serverFlags, + tlsPrivateKey = secureKey, + tlsCertificate = secureCert) + if res.isErr(): + return false + + let server = res.get() + server.start() + let message = "GET / HTTP/1.0\r\nHost: https://127.0.0.1:80\r\n\r\n" + let data = await httpsClient(address, message, {NoVerifyServerName}) + await testFut + await server.stop() + await server.closeWait() + return serverRes and data == "EXCEPTION" + + check waitFor(testHTTPS2(initTAddress("127.0.0.1:30080"))) == true diff --git a/vendor/nim-chronos/tests/testsignal.nim b/vendor/nim-chronos/tests/testsignal.nim index 027f61931..ee32f8ac5 100644 --- a/vendor/nim-chronos/tests/testsignal.nim +++ b/vendor/nim-chronos/tests/testsignal.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} @@ -20,13 +20,19 @@ suite "Signal handling test suite": proc signalProc(udata: pointer) = var cdata = cast[ptr CompletionData](udata) signalCounter = cast[int](cdata.udata) - removeSignal(int(cdata.fd)) + try: + removeSignal(int(cdata.fd)) + except Exception as exc: + raiseAssert exc.msg proc asyncProc() {.async.} = await sleepAsync(500.milliseconds) proc test(signal, value: int): bool = - discard addSignal(signal, signalProc, cast[pointer](value)) + try: + discard addSignal(signal, signalProc, cast[pointer](value)) + except Exception as exc: + raiseAssert exc.msg var fut = asyncProc() discard posix.kill(posix.getpid(), cint(signal)) waitFor(fut) diff --git a/vendor/nim-chronos/tests/testsoon.nim b/vendor/nim-chronos/tests/testsoon.nim index dfa712b34..b2ea84a03 100644 --- a/vendor/nim-chronos/tests/testsoon.nim +++ b/vendor/nim-chronos/tests/testsoon.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} @@ -46,9 +46,11 @@ suite "callSoon() tests suite": await sleepAsync(100.milliseconds) timeoutsTest1 += 1 - proc callbackProc(udata: pointer) {.gcsafe.} = + var callbackproc: proc(udata: pointer) {.gcsafe, raises: [Defect].} + callbackproc = proc (udata: pointer) {.gcsafe, raises: [Defect].} = timeoutsTest2 += 1 - callSoon(callbackProc) + {.gcsafe.}: + callSoon(callbackProc) proc test2(timers, callbacks: var int) = callSoon(callbackProc) diff --git a/vendor/nim-chronos/tests/teststream.nim b/vendor/nim-chronos/tests/teststream.nim index 7d20a53b9..00ffb37f2 100644 --- a/vendor/nim-chronos/tests/teststream.nim +++ b/vendor/nim-chronos/tests/teststream.nim @@ -5,7 +5,8 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import strutils, unittest, os +import std/[strutils, os] +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} @@ -866,7 +867,7 @@ suite "Stream Transport test suite": var valueLen = 0'u32 res: seq[byte] - error: ref Exception + error: ref CatchableError proc predicate(data: openarray[byte]): tuple[consumed: int, done: bool] = if len(data) == 0: diff --git a/vendor/nim-chronos/tests/testsync.nim b/vendor/nim-chronos/tests/testsync.nim index 5bb7acc2e..75ccd120f 100644 --- a/vendor/nim-chronos/tests/testsync.nim +++ b/vendor/nim-chronos/tests/testsync.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testtime.nim b/vendor/nim-chronos/tests/testtime.nim index 3a659e52b..8a1f84d77 100644 --- a/vendor/nim-chronos/tests/testtime.nim +++ b/vendor/nim-chronos/tests/testtime.nim @@ -5,7 +5,8 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import os, unittest +import std/os +import unittest2 import ../chronos, ../chronos/timer when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-chronos/tests/testutils.nim b/vendor/nim-chronos/tests/testutils.nim index 6d8ff8af6..79e55ec7e 100644 --- a/vendor/nim-chronos/tests/testutils.nim +++ b/vendor/nim-chronos/tests/testutils.nim @@ -5,7 +5,7 @@ # Licensed under either of # Apache License, version 2.0, (LICENSE-APACHEv2) # MIT license (LICENSE-MIT) -import unittest +import unittest2 import ../chronos when defined(nimHasUsed): {.used.} diff --git a/vendor/nim-confutils/confutils/envvar/envvar_serialization.nim b/vendor/nim-confutils/confutils/envvar/envvar_serialization.nim index bf03ee0df..18f37e028 100644 --- a/vendor/nim-confutils/confutils/envvar/envvar_serialization.nim +++ b/vendor/nim-confutils/confutils/envvar/envvar_serialization.nim @@ -5,10 +5,10 @@ import export serialization, reader, writer -serializationFormat Envvar, - Reader = EnvvarReader, - Writer = EnvvarWriter, - PreferedOutput = void +serializationFormat Envvar + +Envvar.setReader EnvvarReader +Envvar.setWriter EnvvarWriter, PreferredOutput = void template supports*(_: type Envvar, T: type): bool = # The Envvar format should support every type diff --git a/vendor/nim-confutils/confutils/winreg/winreg_serialization.nim b/vendor/nim-confutils/confutils/winreg/winreg_serialization.nim index 6906f52d1..762692d32 100644 --- a/vendor/nim-confutils/confutils/winreg/winreg_serialization.nim +++ b/vendor/nim-confutils/confutils/winreg/winreg_serialization.nim @@ -5,10 +5,10 @@ import export serialization, reader, writer, types -serializationFormat Winreg, - Reader = WinregReader, - Writer = WinregWriter, - PreferedOutput = void +serializationFormat Winreg + +Winreg.setReader WinregReader +Winreg.setWriter WinregWriter, PreferredOutput = void template supports*(_: type Winreg, T: type): bool = # The Winreg format should support every type diff --git a/vendor/nim-eth/eth/p2p/discoveryv5/protocol.nim b/vendor/nim-eth/eth/p2p/discoveryv5/protocol.nim index 1071a6b5a..c38967137 100644 --- a/vendor/nim-eth/eth/p2p/discoveryv5/protocol.nim +++ b/vendor/nim-eth/eth/p2p/discoveryv5/protocol.nim @@ -428,7 +428,7 @@ proc receive*(d: Protocol, a: Address, packet: openArray[byte]) {.gcsafe, # CatchableErrors, in fact, we really don't, but hey, they might, considering we # can't enforce it. proc processClient(transp: DatagramTransport, raddr: TransportAddress): - Future[void] {.async, gcsafe, raises: [Exception, Defect].} = + Future[void] {.async, gcsafe.} = let proto = getUserData[Protocol](transp) # TODO: should we use `peekMessage()` to avoid allocation? diff --git a/vendor/nim-eth/eth/p2p/private/p2p_types.nim b/vendor/nim-eth/eth/p2p/private/p2p_types.nim index aa23227c3..0be9f98d0 100644 --- a/vendor/nim-eth/eth/p2p/private/p2p_types.nim +++ b/vendor/nim-eth/eth/p2p/private/p2p_types.nim @@ -139,13 +139,14 @@ type MessageHandlerDecorator* = proc(msgId: int, n: NimNode): NimNode ThunkProc* = proc(x: Peer, msgId: int, data: Rlp): Future[void] {.gcsafe.} MessageContentPrinter* = proc(msg: pointer): string {.gcsafe.} - RequestResolver* = proc(msg: pointer, future: FutureBase) {.gcsafe.} + RequestResolver* = proc(msg: pointer, future: FutureBase) + {.gcsafe, raises:[Defect].} NextMsgResolver* = proc(msgData: Rlp, future: FutureBase) {.gcsafe.} PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe.} NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe.} HandshakeStep* = proc(peer: Peer): Future[void] {.gcsafe.} - DisconnectionHandler* = proc(peer: Peer, - reason: DisconnectionReason): Future[void] {.gcsafe.} + DisconnectionHandler* = proc(peer: Peer, reason: DisconnectionReason): + Future[void] {.gcsafe.} ConnectionState* = enum None, diff --git a/vendor/nim-eth/eth/p2p/rlpx.nim b/vendor/nim-eth/eth/p2p/rlpx.nim index c9a736dad..a4692abae 100644 --- a/vendor/nim-eth/eth/p2p/rlpx.nim +++ b/vendor/nim-eth/eth/p2p/rlpx.nim @@ -60,7 +60,8 @@ chronicles.formatIt(Peer): $(it.remote) include p2p_backends_helpers -proc requestResolver[MsgType](msg: pointer, future: FutureBase) {.gcsafe.} = +proc requestResolver[MsgType](msg: pointer, future: FutureBase) + {.gcsafe, raises:[Defect].} = var f = Future[Option[MsgType]](future) if not f.finished: if msg != nil: @@ -72,10 +73,13 @@ proc requestResolver[MsgType](msg: pointer, future: FutureBase) {.gcsafe.} = # here. The only reasonable explanation is that the request should # have timed out. if msg != nil: - if f.read.isSome: - doAssert false, "trying to resolve a request twice" - else: - doAssert false, "trying to resolve a timed out request with a value" + try: + if f.read.isSome: + doAssert false, "trying to resolve a request twice" + else: + doAssert false, "trying to resolve a timed out request with a value" + except CatchableError as e: + debug "Exception in requestResolver()", exc = e.name, err = e.msg else: try: if not f.read.isSome: @@ -88,9 +92,8 @@ proc requestResolver[MsgType](msg: pointer, future: FutureBase) {.gcsafe.} = trace "TransportOsError during request", err = e.msg except TransportError: trace "Transport got closed during request" - except Exception as e: + except CatchableError as e: debug "Exception in requestResolver()", exc = e.name, err = e.msg - raise e proc linkSendFailureToReqFuture[S, R](sendFut: Future[S], resFut: Future[R]) = sendFut.addCallback() do (arg: pointer): @@ -360,7 +363,8 @@ proc registerRequest(peer: Peer, doAssert(not peer.dispatcher.isNil) let requestResolver = peer.dispatcher.messages[responseMsgId].requestResolver - proc timeoutExpired(udata: pointer) = requestResolver(nil, responseFuture) + proc timeoutExpired(udata: pointer) {.gcsafe, raises:[Defect].} = + requestResolver(nil, responseFuture) addTimer(timeoutAt, timeoutExpired, nil) diff --git a/vendor/nim-eth/eth/p2p/rlpx_protocols/whisper/whisper_types.nim b/vendor/nim-eth/eth/p2p/rlpx_protocols/whisper/whisper_types.nim index a7136ab81..abc6533a0 100644 --- a/vendor/nim-eth/eth/p2p/rlpx_protocols/whisper/whisper_types.nim +++ b/vendor/nim-eth/eth/p2p/rlpx_protocols/whisper/whisper_types.nim @@ -110,7 +110,7 @@ type ## XXX: really big messages can cause excessive mem usage when using msg \ ## count - FilterMsgHandler* = proc(msg: ReceivedMessage) {.gcsafe, closure.} + FilterMsgHandler* = proc(msg: ReceivedMessage) {.gcsafe, raises: [Defect].} Filter* = object src*: Option[PublicKey] diff --git a/vendor/nim-http-utils/httputils.nim b/vendor/nim-http-utils/httputils.nim index 27ac55350..94943c967 100644 --- a/vendor/nim-http-utils/httputils.nim +++ b/vendor/nim-http-utils/httputils.nim @@ -1146,3 +1146,54 @@ proc checkHeaderValue*(value: string): bool = if (ch == CR) or (ch == LF): result = false break + +proc toInt*(code: HttpCode): int = + ## Returns ``code`` as integer value. + case code + of Http100: 100 + of Http101: 101 + of Http200: 200 + of Http201: 201 + of Http202: 202 + of Http203: 203 + of Http204: 204 + of Http205: 205 + of Http206: 206 + of Http300: 300 + of Http301: 301 + of Http302: 302 + of Http303: 303 + of Http304: 304 + of Http305: 305 + of Http307: 307 + of Http400: 400 + of Http401: 401 + of Http403: 403 + of Http404: 404 + of Http405: 405 + of Http406: 406 + of Http407: 407 + of Http408: 408 + of Http409: 409 + of Http410: 410 + of Http411: 411 + of Http412: 412 + of Http413: 413 + of Http414: 414 + of Http415: 415 + of Http416: 416 + of Http417: 417 + of Http418: 418 + of Http421: 421 + of Http422: 422 + of Http426: 426 + of Http428: 428 + of Http429: 429 + of Http431: 431 + of Http451: 451 + of Http500: 500 + of Http501: 501 + of Http502: 502 + of Http503: 503 + of Http504: 504 + of Http505: 505 diff --git a/vendor/nim-json-serialization/json_serialization.nim b/vendor/nim-json-serialization/json_serialization.nim index fbe48c8d4..d60bc8f5b 100644 --- a/vendor/nim-json-serialization/json_serialization.nim +++ b/vendor/nim-json-serialization/json_serialization.nim @@ -1,16 +1,6 @@ import - serialization, json_serialization/[reader, writer] + serialization, json_serialization/[format, reader, writer] export - serialization, reader, writer - -serializationFormat Json, - Reader = JsonReader, - Writer = JsonWriter, - PreferedOutput = string, - mimeType = "application/json" - -template supports*(_: type Json, T: type): bool = - # The JSON format should support every type - true + serialization, format, reader, writer diff --git a/vendor/nim-json-serialization/json_serialization/format.nim b/vendor/nim-json-serialization/json_serialization/format.nim new file mode 100644 index 000000000..05c889b6e --- /dev/null +++ b/vendor/nim-json-serialization/json_serialization/format.nim @@ -0,0 +1,10 @@ +import + serialization/formats + +serializationFormat Json, + mimeType = "application/json" + +template supports*(_: type Json, T: type): bool = + # The JSON format should support every type + true + diff --git a/vendor/nim-json-serialization/json_serialization/reader.nim b/vendor/nim-json-serialization/json_serialization/reader.nim index 8ed597fd8..bc97337d9 100644 --- a/vendor/nim-json-serialization/json_serialization/reader.nim +++ b/vendor/nim-json-serialization/json_serialization/reader.nim @@ -1,17 +1,17 @@ {.experimental: "notnil".} import - tables, strutils, typetraits, macros, strformat, - faststreams/inputs, serialization/[object_serialization, errors], - types, lexer + std/[tables, strutils, typetraits, macros, strformat], + faststreams/inputs, serialization/[formats, object_serialization, errors], + format, types, lexer from json import JsonNode, JsonNodeKind export - types, errors + format, types, errors type - JsonReader* = object + JsonReader*[Flavor = DefaultFlavor] = object lexer*: JsonLexer allowUnknownFields: bool @@ -50,6 +50,8 @@ type isNegative: bool absIntVal: uint64 +Json.setReader JsonReader + func valueStr(err: ref IntOverflowError): string = if err.isNegative: result.add '-' @@ -176,7 +178,10 @@ proc parseJsonNode(r: var JsonReader): JsonNode = r.lexer.next() if r.lexer.tok != tkCurlyRi: while r.lexer.tok == tkString: - r.readJsonNodeField(result.fields.mgetOrPut(r.lexer.strVal, nil)) + try: + r.readJsonNodeField(result.fields.mgetOrPut(r.lexer.strVal, nil)) + except KeyError: + raiseAssert "mgetOrPut should never raise a KeyError" if r.lexer.tok == tkComma: r.lexer.next() else: @@ -368,6 +373,7 @@ template isCharArray(v: auto): bool = false proc readValue*[T](r: var JsonReader, value: var T) {.raises: [SerializationError, IOError, Defect].} = mixin readValue + type ReaderType = type r let tok {.used.} = r.lexer.tok @@ -495,7 +501,7 @@ proc readValue*[T](r: var JsonReader, value: var T) r.skipToken tkCurlyLe when T.totalSerializedFields > 0: - let fields = T.fieldReadersTable(JsonReader) + let fields = T.fieldReadersTable(ReaderType) var expectedFieldPos = 0 while r.lexer.tok == tkString: when T is tuple: diff --git a/vendor/nim-json-serialization/json_serialization/writer.nim b/vendor/nim-json-serialization/json_serialization/writer.nim index f64d4dcf7..dcf490220 100644 --- a/vendor/nim-json-serialization/json_serialization/writer.nim +++ b/vendor/nim-json-serialization/json_serialization/writer.nim @@ -1,7 +1,10 @@ import - typetraits, + std/typetraits, faststreams/[outputs, textio], serialization, json, - types + format, types + +export + format, JsonString, DefaultFlavor type JsonWriterState = enum @@ -9,23 +12,23 @@ type RecordStarted AfterField - JsonWriter* = object + JsonWriter*[Flavor = DefaultFlavor] = object stream*: OutputStream hasTypeAnnotations: bool hasPrettyOutput*: bool # read-only nestingLevel*: int # read-only state: JsonWriterState -export - JsonString +Json.setWriter JsonWriter, + PreferredOutput = string -proc init*(T: type JsonWriter, stream: OutputStream, - pretty = false, typeAnnotations = false): T = - result.stream = stream - result.hasPrettyOutput = pretty - result.hasTypeAnnotations = typeAnnotations - result.nestingLevel = if pretty: 0 else: -1 - result.state = RecordExpected +proc init*(W: type JsonWriter, stream: OutputStream, + pretty = false, typeAnnotations = false): W = + W(stream: stream, + hasPrettyOutput: pretty, + hasTypeAnnotations: typeAnnotations, + nestingLevel: if pretty: 0 else: -1, + state: RecordExpected) proc beginRecord*(w: var JsonWriter, T: type) proc beginRecord*(w: var JsonWriter) @@ -228,7 +231,7 @@ proc toJson*(v: auto, pretty = false, typeAnnotations = false): string = mixin writeValue var s = memoryOutput() - var w = JsonWriter.init(s, pretty, typeAnnotations) + var w = JsonWriter[DefaultFlavor].init(s, pretty, typeAnnotations) w.writeValue v return s.getOutput(string) diff --git a/vendor/nim-json-serialization/tests/test_json_flavor.nim b/vendor/nim-json-serialization/tests/test_json_flavor.nim new file mode 100644 index 000000000..8921831d2 --- /dev/null +++ b/vendor/nim-json-serialization/tests/test_json_flavor.nim @@ -0,0 +1,41 @@ +import + strutils, + serialization, + ../json_serialization + +Json.createFlavor StringyJson + +proc writeValue*(w: var JsonWriter[StringyJson], val: SomeInteger) = + writeValue(w, $val) + +proc readValue*(r: var JsonReader[StringyJson], v: var SomeSignedInt) = + try: + v = type(v) parseBiggestInt readValue(r, string) + except ValueError as err: + r.raiseUnexpectedValue("A signed integer encoded as string") + +proc readValue*(r: var JsonReader[StringyJson], v: var SomeUnsignedInt) = + try: + v = type(v) parseBiggestUInt readValue(r, string) + except ValueError as err: + r.raiseUnexpectedValue("An unsigned integer encoded as string") + +type + Container = object + name: string + x: int + y: uint64 + list: seq[int64] + +let c = Container(name: "c", x: -10, y: 20, list: @[1'i64, 2, 25]) +let encoded = StringyJson.encode(c) +echo "Encoded: ", encoded + +let decoded = try: + StringyJson.decode(encoded, Container) +except SerializationError as err: + echo err.formatMsg("") + quit 1 + +echo "Decoded: ", decoded + diff --git a/vendor/nim-libbacktrace/vendor/libbacktrace-upstream/libtool b/vendor/nim-libbacktrace/vendor/libbacktrace-upstream/libtool index 6dc06684d..cf78fa613 100755 --- a/vendor/nim-libbacktrace/vendor/libbacktrace-upstream/libtool +++ b/vendor/nim-libbacktrace/vendor/libbacktrace-upstream/libtool @@ -2,7 +2,7 @@ # libtool - Provide generalized library-building support services. # Generated automatically by config.status (libbacktrace) version-unused -# Libtool was configured on host fv-az231-108: +# Libtool was configured on host fv-az193-526: # NOTE: Changes made to this file will be lost: look at ltmain.sh. # # Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004, 2005, diff --git a/vendor/nim-libp2p/.github/workflows/ci.yml b/vendor/nim-libp2p/.github/workflows/ci.yml index 10a442304..18f3e231d 100644 --- a/vendor/nim-libp2p/.github/workflows/ci.yml +++ b/vendor/nim-libp2p/.github/workflows/ci.yml @@ -207,6 +207,7 @@ jobs: nimble test bumpNBC-stable: + if: github.ref == 'refs/heads/master' needs: build runs-on: ubuntu-latest steps: @@ -221,6 +222,7 @@ jobs: uses: actions/checkout@v2 with: repository: status-im/nimbus-eth2 + ref: unstable path: nbc submodules: true fetch-depth: 0 @@ -246,6 +248,7 @@ jobs: title: nim-libp2p auto bump bumpNBC-unstable: + if: github.ref == 'refs/heads/unstable' needs: build runs-on: ubuntu-latest steps: diff --git a/vendor/nim-libp2p/libp2p/stream/connection.nim b/vendor/nim-libp2p/libp2p/stream/connection.nim index ba888c9db..005149c72 100644 --- a/vendor/nim-libp2p/libp2p/stream/connection.nim +++ b/vendor/nim-libp2p/libp2p/stream/connection.nim @@ -41,7 +41,7 @@ proc isUpgraded*(s: Connection): bool = if not isNil(s.upgraded): return s.upgraded.finished -proc upgrade*(s: Connection, failed: ref Exception = nil) = +proc upgrade*(s: Connection, failed: ref CatchableError = nil) = if not isNil(s.upgraded): if not isNil(failed): s.upgraded.fail(failed) diff --git a/vendor/nim-libp2p/libp2p/switch.nim b/vendor/nim-libp2p/libp2p/switch.nim index a947e0708..0274b1dd7 100644 --- a/vendor/nim-libp2p/libp2p/switch.nim +++ b/vendor/nim-libp2p/libp2p/switch.nim @@ -85,8 +85,6 @@ proc removePeerEventHandler*(s: Switch, kind: PeerEventKind) = s.connManager.removePeerEventHandler(handler, kind) -proc disconnect*(s: Switch, peerId: PeerID) {.async, gcsafe.} - proc isConnected*(s: Switch, peerId: PeerID): bool = ## returns true if the peer has one or more ## associated connections (sockets) diff --git a/vendor/nim-libp2p/libp2p/transports/tcptransport.nim b/vendor/nim-libp2p/libp2p/transports/tcptransport.nim index 50513b19c..97c977972 100644 --- a/vendor/nim-libp2p/libp2p/transports/tcptransport.nim +++ b/vendor/nim-libp2p/libp2p/transports/tcptransport.nim @@ -35,7 +35,7 @@ type opened*: uint64 closed*: uint64 -proc setupTcpTransportTracker(): TcpTransportTracker {.gcsafe.} +proc setupTcpTransportTracker(): TcpTransportTracker {.gcsafe, raises: [Defect].} proc getTcpTransportTracker(): TcpTransportTracker {.gcsafe.} = result = cast[TcpTransportTracker](getTracker(TcpTransportTrackerName)) diff --git a/vendor/nim-serialization/README.md b/vendor/nim-serialization/README.md index 634911b7d..265a1ebbb 100644 --- a/vendor/nim-serialization/README.md +++ b/vendor/nim-serialization/README.md @@ -33,11 +33,11 @@ serializationFormat Json, # This is the name of the for Reader = JsonReader, # The associated Reader type. Writer = JsonWriter, # The associated Writer type. - PreferedOutput = string, # APIs such as `Json.encode` will return this type. + PreferredOutput = string, # APIs such as `Json.encode` will return this type. # The type must support the following operations: # proc initWithCapacity(_: type T, n: int) # proc add(v: var T, bytes: openarray[byte]) - # By default, the PreferedOutput is `seq[byte]`. + # By default, the PreferredOutput is `seq[byte]`. mimeType = "application/json", # Mime type associated with the format (Optional). fileExt = "json" # File extension associated with the format (Optional). @@ -48,7 +48,7 @@ serializationFormat Json, # This is the name of the for Most of the time, you'll be using the following high-level APIs when encoding and decoding values: -#### `Format.encode(value: auto, params: varargs): Format.PreferedOutput` +#### `Format.encode(value: auto, params: varargs): Format.PreferredOutput` Encodes a value in the specified format returning the preferred output type for the format (usually `string` or `seq[byte]`). All extra params will be diff --git a/vendor/nim-serialization/serialization.nim b/vendor/nim-serialization/serialization.nim index f48672601..aa066ee90 100644 --- a/vendor/nim-serialization/serialization.nim +++ b/vendor/nim-serialization/serialization.nim @@ -1,31 +1,13 @@ import typetraits, stew/shims/macros, faststreams, - serialization/[object_serialization, errors] + serialization/[object_serialization, errors, formats] export - faststreams, object_serialization, errors - -template serializationFormatImpl(Name: untyped, - Reader, Writer, PreferedOutput: distinct type, - mimeTypeName: static string = "") {.dirty.} = - # This indirection is required in order to be able to generate the - # `mimeType` accessor template. Without the indirection, the template - # mechanism of Nim will try to expand the `mimeType` param in the position - # of the `mimeType` template name which will result in error. - type Name* = object - template ReaderType*(T: type Name): type = Reader - template WriterType*(T: type Name): type = Writer - template PreferedOutputType*(T: type Name): type = PreferedOutput - template mimeType*(T: type Name): string = mimeTypeName - -template serializationFormat*(Name: untyped, - Reader, Writer, PreferedOutput: distinct type, - mimeType: static string = "") = - serializationFormatImpl(Name, Reader, Writer, PreferedOutput, mimeType) + faststreams, object_serialization, errors, formats template encode*(Format: type, value: auto, params: varargs[untyped]): auto = - mixin init, WriterType, writeValue, PreferedOutputType + mixin init, Writer, writeValue, PreferredOutputType {.noSideEffect.}: # We assume that there is no side-effects here, because we are # using a `memoryOutput`. The computed side-effects are coming @@ -33,9 +15,10 @@ template encode*(Format: type, value: auto, params: varargs[untyped]): auto = # faststreams may be writing to a file or a network device. try: var s = memoryOutput() - var writer = unpackArgs(init, [WriterType(Format), s, params]) + type WriterType = Writer(Format) + var writer = unpackArgs(init, [WriterType, s, params]) writeValue writer, value - s.getOutput PreferedOutputType(Format) + s.getOutput PreferredOutputType(Format) except IOError: raise (ref Defect)() # a memoryOutput cannot have an IOError @@ -52,7 +35,7 @@ template decode*(Format: distinct type, params: varargs[untyped]): auto = # TODO, this is dusplicated only due to a Nim bug: # If `input` was `string|openarray[byte]`, it won't match `seq[byte]` - mixin init, ReaderType + mixin init, Reader {.noSideEffect.}: # We assume that there are no side-effects here, because we are # using a `memoryInput`. The computed side-effects are coming @@ -60,7 +43,8 @@ template decode*(Format: distinct type, # faststreams may be reading from a file or a network device. try: var stream = unsafeMemoryInput(input) - var reader = unpackArgs(init, [ReaderType(Format), stream, params]) + type ReaderType = Reader(Format) + var reader = unpackArgs(init, [ReaderType, stream, params]) reader.readValue(RecordType) except IOError: raise (ref Defect)() # memory inputs cannot raise an IOError @@ -71,7 +55,7 @@ template decode*(Format: distinct type, params: varargs[untyped]): auto = # TODO, this is dusplicated only due to a Nim bug: # If `input` was `string|openarray[byte]`, it won't match `seq[byte]` - mixin init, ReaderType + mixin init, Reader {.noSideEffect.}: # We assume that there are no side-effects here, because we are # using a `memoryInput`. The computed side-effects are coming @@ -79,7 +63,8 @@ template decode*(Format: distinct type, # faststreams may be reading from a file or a network device. try: var stream = unsafeMemoryInput(input) - var reader = unpackArgs(init, [ReaderType(Format), stream, params]) + type ReaderType = Reader(Format) + var reader = unpackArgs(init, [ReaderType, stream, params]) reader.readValue(RecordType) except IOError: raise (ref Defect)() # memory inputs cannot raise an IOError @@ -88,11 +73,12 @@ template loadFile*(Format: distinct type, filename: string, RecordType: distinct type, params: varargs[untyped]): auto = - mixin init, ReaderType, readValue + mixin init, Reader, readValue var stream = memFileInput(filename) try: - var reader = unpackArgs(init, [ReaderType(Format), stream, params]) + type ReaderType = Reader(Format) + var reader = unpackArgs(init, [ReaderType, stream, params]) reader.readValue(RecordType) finally: close stream @@ -104,11 +90,12 @@ template loadFile*[RecordType](Format: type, record = loadFile(Format, filename, RecordType, params) template saveFile*(Format: type, filename: string, value: auto, params: varargs[untyped]) = - mixin init, WriterType, writeValue + mixin init, Writer, writeValue var stream = fileOutput(filename) try: - var writer = unpackArgs(init, [WriterType(Format), stream, params]) + type WriterType = Writer(Format) + var writer = unpackArgs(init, [WriterType, stream, params]) writer.writeValue(value) finally: close stream @@ -137,16 +124,16 @@ template borrowSerialization*(Alias: distinct type, template serializesAsBase*(SerializedType: distinct type, Format: distinct type) = - mixin ReaderType, WriterType + mixin Reader, Writer - type Reader = ReaderType(Format) - type Writer = WriterType(Format) + type ReaderType = Reader(Format) + type WriterType = Writer(Format) - template writeValue*(writer: var Writer, value: SerializedType) = + template writeValue*(writer: var WriterType, value: SerializedType) = mixin writeValue writeValue(writer, distinctBase value) - template readValue*(reader: var Reader, value: var SerializedType) = + template readValue*(reader: var ReaderType, value: var SerializedType) = mixin readValue value = SerializedType reader.readValue(distinctBase SerializedType) @@ -160,15 +147,17 @@ template readValue*(stream: InputStream, Format: type, ValueType: type, params: varargs[untyped]): untyped = - mixin ReaderType, init, readValue - var reader = unpackArgs(init, [ReaderType(Format), stream, params]) + mixin Reader, init, readValue + type ReaderType = Reader(Format) + var reader = unpackArgs(init, [ReaderType, stream, params]) readValue reader, ValueType template writeValue*(stream: OutputStream, Format: type, value: auto, params: varargs[untyped]) = - mixin WriterType, init, writeValue - var writer = unpackArgs(init, [WriterType(Format), stream]) + mixin Writer, init, writeValue + type WriterType = Writer(Format) + var writer = unpackArgs(init, [WriterType, stream, params]) writeValue writer, value diff --git a/vendor/nim-serialization/serialization/formats.nim b/vendor/nim-serialization/serialization/formats.nim new file mode 100644 index 000000000..6700b784d --- /dev/null +++ b/vendor/nim-serialization/serialization/formats.nim @@ -0,0 +1,40 @@ +import + std/typetraits + +template serializationFormatImpl(Name: untyped, + mimeTypeName: static string = "") {.dirty.} = + # This indirection is required in order to be able to generate the + # `mimeType` accessor template. Without the indirection, the template + # mechanism of Nim will try to expand the `mimeType` param in the position + # of the `mimeType` template name which will result in error. + type Name* = object + template mimeType*(T: type Name): string = mimeTypeName + +template serializationFormat*(Name: untyped, mimeType: static string = "") = + serializationFormatImpl(Name, mimeType) + +template setReader*(Format, FormatReader: distinct type) = + when arity(FormatReader) > 1: + template ReaderType*(T: type Format, F: distinct type = DefaultFlavor): type = FormatReader[F] + template Reader*(T: type Format, F: distinct type = DefaultFlavor): type = FormatReader[F] + else: + template ReaderType*(T: type Format): type = FormatReader + template Reader*(T: type Format): type = FormatReader + +template setWriter*(Format, FormatWriter, PreferredOutput: distinct type) = + when arity(FormatWriter) > 1: + template WriterType*(T: type Format, F: distinct type = DefaultFlavor): type = FormatWriter[F] + template Writer*(T: type Format, F: distinct type = DefaultFlavor): type = FormatWriter[F] + else: + template WriterType*(T: type Format): type = FormatWriter + template Writer*(T: type Format): type = FormatWriter + + template PreferredOutputType*(T: type Format): type = PreferredOutput + +template createFlavor*(ModifiedFormat, FlavorName: untyped) = + type FlavorName* = object + template Reader*(T: type FlavorName): type = Reader(ModifiedFormat, FlavorName) + template Writer*(T: type FlavorName): type = Writer(ModifiedFormat, FlavorName) + template PreferredOutputType*(T: type FlavorName): type = PreferredOutputType(ModifiedFormat) + template mimeType*(T: type FlavorName): string = mimeType(ModifiedFormat) + diff --git a/vendor/nim-serialization/serialization/object_serialization.nim b/vendor/nim-serialization/serialization/object_serialization.nim index 84255c456..0d6509779 100644 --- a/vendor/nim-serialization/serialization/object_serialization.nim +++ b/vendor/nim-serialization/serialization/object_serialization.nim @@ -3,6 +3,7 @@ import errors type + DefaultFlavor* = object FieldTag*[RecordType; fieldName: static string; FieldType] = distinct void let @@ -190,12 +191,12 @@ template writeFieldIMPL*[Writer](writer: var Writer, mixin writeValue writer.writeValue(fieldVal) -proc makeFieldReadersTable(RecordType, Reader: distinct type): - seq[FieldReader[RecordType, Reader]] = +proc makeFieldReadersTable(RecordType, ReaderType: distinct type): + seq[FieldReader[RecordType, ReaderType]] = mixin enumAllSerializedFields, readFieldIMPL, handleReadException enumAllSerializedFields(RecordType): - proc readField(obj: var RecordType, reader: var Reader) + proc readField(obj: var RecordType, reader: var ReaderType) {.gcsafe, nimcall, raises: [SerializationError, Defect].} = when RecordType is tuple: const i = fieldName.parseInt @@ -221,17 +222,17 @@ proc makeFieldReadersTable(RecordType, Reader: distinct type): result.add((fieldName, readField)) -proc fieldReadersTable*(RecordType, Reader: distinct type): - ptr seq[FieldReader[RecordType, Reader]] = +proc fieldReadersTable*(RecordType, ReaderType: distinct type): + ptr seq[FieldReader[RecordType, ReaderType]] = mixin readValue # careful: https://github.com/nim-lang/Nim/issues/17085 # TODO why is this even here? one could just return the function pointer # to the field reader directly instead of going through this seq etc - var tbl {.threadvar.}: ref seq[FieldReader[RecordType, Reader]] + var tbl {.threadvar.}: ref seq[FieldReader[RecordType, ReaderType]] if tbl == nil: tbl = new typeof(tbl) - tbl[] = makeFieldReadersTable(RecordType, Reader) + tbl[] = makeFieldReadersTable(RecordType, ReaderType) return addr(tbl[]) proc findFieldReader*(fieldsTable: FieldReadersTable, @@ -341,16 +342,16 @@ proc genCustomSerializationForField(Format, field, if readBody != nil: result.add quote do: - type Reader = ReaderType(`Format`) + type ReaderType = Reader(`Format`) proc readFieldIMPL*(F: type FieldTag[`RecordType`, `fieldName`, auto], - `readerSym`: var Reader): `FieldType` + `readerSym`: var ReaderType): `FieldType` {.raises: [IOError, SerializationError, Defect].} = `readBody` if writeBody != nil: result.add quote do: - type Writer = WriterType(`Format`) - proc writeFieldIMPL*(`writerSym`: var Writer, + type WriterType = Writer(`Format`) + proc writeFieldIMPL*(`writerSym`: var WriterType, F: type FieldTag[`RecordType`, `fieldName`, auto], `valueSym`: auto, `holderSym`: `RecordType`) @@ -363,15 +364,15 @@ proc genCustomSerializationForType(Format, typ: NimNode, if readBody != nil: result.add quote do: - type Reader = ReaderType(`Format`) - proc readValue*(`readerSym`: var Reader, T: type `typ`): `typ` + type ReaderType = Reader(`Format`) + proc readValue*(`readerSym`: var ReaderType, T: type `typ`): `typ` {.raises: [IOError, SerializationError, Defect].} = `readBody` if writeBody != nil: result.add quote do: - type Writer = WriterType(`Format`) - proc writeValue*(`writerSym`: var Writer, `valueSym`: `typ`) + type WriterType = Writer(`Format`) + proc writeValue*(`writerSym`: var WriterType, `valueSym`: `typ`) {.raises: [IOError, SerializationError, Defect].} = `writeBody` diff --git a/vendor/nim-serialization/serialization/testing/generic_suite.nim b/vendor/nim-serialization/serialization/testing/generic_suite.nim index b78969b7a..dae05568f 100644 --- a/vendor/nim-serialization/serialization/testing/generic_suite.nim +++ b/vendor/nim-serialization/serialization/testing/generic_suite.nim @@ -317,14 +317,14 @@ proc executeRoundtripTests*(Format: type) = roundtrip namedT proc executeReaderWriterTests*(Format: type) = - mixin init, ReaderType, WriterType + mixin init, Reader, Writer type - Reader = ReaderType Format + ReaderType = Reader Format suite(typetraits.name(Format) & " read/write tests"): test "Low-level field reader test": - let barFields = fieldReadersTable(Bar, Reader) + let barFields = fieldReadersTable(Bar, ReaderType) var idx = 0 var fieldReader = findFieldReader(barFields[], "b", idx) @@ -336,7 +336,7 @@ proc executeReaderWriterTests*(Format: type) = var bytes = Format.encode("test") var stream = unsafeMemoryInput(bytes) - var reader = Reader.init(stream) + var reader = ReaderType.init(stream) var bar: Bar fieldReader(bar, reader) @@ -345,7 +345,7 @@ proc executeReaderWriterTests*(Format: type) = test "Ignored fields should not be included in the field readers table": var pos = 0 - let bazFields = fieldReadersTable(Baz, Reader) + let bazFields = fieldReadersTable(Baz, ReaderType) check: len(bazFields[]) == 2 findFieldReader(bazFields[], "f", pos) != nil diff --git a/vendor/nim-stew/stew/byteutils.nim b/vendor/nim-stew/stew/byteutils.nim index ce0e54a09..977362ae9 100644 --- a/vendor/nim-stew/stew/byteutils.nim +++ b/vendor/nim-stew/stew/byteutils.nim @@ -63,6 +63,11 @@ func hexToByteArray*[N: static[int]](hexStr: string): array[N, byte] ## Read an hex string and store it in a byte array. No "endianness" reordering is done. hexToByteArray(hexStr, result) +func hexToByteArray*(hexStr: string, N: static int): array[N, byte] + {.raises: [ValueError, Defect], noInit, inline.}= + ## Read an hex string and store it in a byte array. No "endianness" reordering is done. + hexToByteArray(hexStr, result) + func fromHex*[N](A: type array[N, byte], hexStr: string): A {.raises: [ValueError, Defect], noInit, inline.}= ## Read an hex string and store it in a byte array. No "endianness" reordering is done. diff --git a/vendor/nim-testutils/.appveyor.yml b/vendor/nim-testutils/.appveyor.yml new file mode 100644 index 000000000..ae28fe693 --- /dev/null +++ b/vendor/nim-testutils/.appveyor.yml @@ -0,0 +1,39 @@ +version: '{build}' + +image: Visual Studio 2015 + +cache: + - NimBinaries + +matrix: + # We always want 32 and 64-bit compilation + fast_finish: false + +platform: + - x86 + - x64 + +# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" +clone_depth: 10 + +install: + # use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin + - IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH% + - IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries + - SET PATH=%CD%\Nim\bin;%USERPROFILE%\.nimble\bin;%PATH% + +build_script: + - cd C:\projects\%APPVEYOR_PROJECT_SLUG% + - nimble install -y + +test_script: + - nimble test + +deploy: off + diff --git a/vendor/nim-testutils/.editorconfig b/vendor/nim-testutils/.editorconfig new file mode 100644 index 000000000..6cc19ffab --- /dev/null +++ b/vendor/nim-testutils/.editorconfig @@ -0,0 +1,5 @@ +[*] +indent_style = space +insert_final_newline = true +indent_size = 2 +trim_trailing_whitespace = true diff --git a/vendor/nim-testutils/.gitignore b/vendor/nim-testutils/.gitignore new file mode 100644 index 000000000..cc935b3d1 --- /dev/null +++ b/vendor/nim-testutils/.gitignore @@ -0,0 +1,8 @@ +# ignore all executable files +* +!*.* +!*/ +*.exe + +nimcache/ + diff --git a/vendor/nim-testutils/.travis.yml b/vendor/nim-testutils/.travis.yml new file mode 100644 index 000000000..03fab1195 --- /dev/null +++ b/vendor/nim-testutils/.travis.yml @@ -0,0 +1,26 @@ +language: c + +# https://docs.travis-ci.com/user/caching/ +cache: + directories: + - NimBinaries + +git: + # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" + depth: 10 + +os: + - linux + - osx + +install: + # build nim from our own branch - this to avoid the day-to-day churn and + # regressions of the fast-paced Nim development while maintaining the + # flexibility to apply patches + - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + - env MAKE="make -j2" bash build_nim.sh Nim csources dist/nimble NimBinaries + - export PATH=$PWD/Nim/bin:$PATH + +script: + - nimble install -y + - nimble test diff --git a/vendor/nim-testutils/README.md b/vendor/nim-testutils/README.md new file mode 100644 index 000000000..516972d5f --- /dev/null +++ b/vendor/nim-testutils/README.md @@ -0,0 +1,12 @@ +# Testutils + +Testutils now is a home to: + +* [Testrunner](testutils/readme.md) +[![Build Status](https://travis-ci.org/status-im/nim-testutils.svg?branch=master)](https://travis-ci.org/status-im/nim-testutils) +[![Build status](https://ci.appveyor.com/api/projects/status/ayqsnuvcpwo2nh6m/branch/master?svg=true)](https://ci.appveyor.com/project/nimbus/nim-testutils/branch/master) +* [Fuzzing](testutils/fuzzing/readme.md) + * [Fuzzing on Windows](testutils/fuzzing/fuzzing_on_windows.md) + +## License +Apache2 or MIT diff --git a/vendor/nim-testutils/nim.cfg b/vendor/nim-testutils/nim.cfg new file mode 100644 index 000000000..3191a4bfb --- /dev/null +++ b/vendor/nim-testutils/nim.cfg @@ -0,0 +1 @@ +-d:nimOldCaseObjects diff --git a/vendor/nim-testutils/ntu.nim b/vendor/nim-testutils/ntu.nim new file mode 100644 index 000000000..954ea7cdb --- /dev/null +++ b/vendor/nim-testutils/ntu.nim @@ -0,0 +1,470 @@ +import + std/[hashes, random, tables, sequtils, strtabs, strutils, + os, osproc, terminal, times, pegs, algorithm], + testutils/[spec, config, helpers, fuzzing_engines] + +#[ + +The runner will look recursively for all *.test files at given path. A +test file should have at minimum a program name. This is the name of the +nim source minus the .nim extension) + +]# + + +# Code is here and there influenced by nim testament tester and unittest +# module. + +const + # defaultOptions = "--verbosity:1 --warnings:off --hint[Processing]:off " & + # "--hint[Conf]:off --hint[XDeclaredButNotUsed]:off " & + # "--hint[Link]:off --hint[Pattern]:off" + defaultOptions = "--verbosity:1 --warnings:on " + backendOrder = @["c", "cpp", "js"] + +type + TestStatus* = enum + OK + FAILED + SKIPPED + INVALID + + #[ + If needed, pass more info to the logresult via a TestResult object + TestResult = object + status: TestStatus + compileTime: float + fileSize: uint + ]# + + ThreadPayload = object + core: int + spec: TestSpec + + TestThread = Thread[ThreadPayload] + TestError* = enum + SourceFileNotFound + ExeFileNotFound + OutputFileNotFound + CompileError + RuntimeError + OutputsDiffer + FileSizeTooLarge + CompileErrorDiffers + + BackendTests = TableRef[string, seq[TestSpec]] + +proc logFailure(test: TestSpec; error: TestError; + data: varargs[string] = [""]) = + case error + of SourceFileNotFound: + styledEcho(fgYellow, styleBright, "source file not found: ", + resetStyle, test.source) + of ExeFileNotFound: + styledEcho(fgYellow, styleBright, "executable file not found: ", + resetStyle, test.binary) + of OutputFileNotFound: + styledEcho(fgYellow, styleBright, "file not found: ", + resetStyle, data[0]) + of CompileError: + styledEcho(fgYellow, styleBright, "compile error:\p", + resetStyle, data[0]) + of RuntimeError: + styledEcho(fgYellow, styleBright, "runtime error:\p", + resetStyle, data[0]) + of OutputsDiffer: + styledEcho(fgYellow, styleBright, "outputs are different:\p", + resetStyle,"Expected output to $#:\p$#" % [data[0], data[1]], + "Resulted output to $#:\p$#" % [data[0], data[2]]) + of FileSizeTooLarge: + styledEcho(fgYellow, styleBright, "file size is too large: ", + resetStyle, data[0] & " > " & $test.maxSize) + of CompileErrorDiffers: + styledEcho(fgYellow, styleBright, "compile error is different:\p", + resetStyle, data[0]) + + styledEcho(fgCyan, styleBright, "compiler: ", resetStyle, + "$# $# $# $#" % [defaultOptions, + test.flags, + test.config.compilationFlags, + test.source]) + +template withinDir(dir: string; body: untyped): untyped = + ## run the body with a specified directory, returning to current dir + let + cwd = getCurrentDir() + setCurrentDir(dir) + try: + body + finally: + setCurrentDir(cwd) + +proc logResult(testName: string, status: TestStatus, time: float) = + var color = block: + case status + of OK: fgGreen + of FAILED: fgRed + of SKIPPED: fgYellow + of INVALID: fgRed + styledEcho(styleBright, color, "[", $status, "] ", + resetStyle, testName, + fgYellow, " ", time.formatFloat(ffDecimal, 3), " s") + +proc logResult(testName: string, status: TestStatus) = + var color = block: + case status + of OK: fgGreen + of FAILED: fgRed + of SKIPPED: fgYellow + of INVALID: fgRed + styledEcho(styleBright, color, "[", $status, "] ", + resetStyle, testName) + +template time(duration, body): untyped = + let t0 = epochTime() + block: + body + duration = epochTime() - t0 + +proc composeOutputs(test: TestSpec, stdout: string): TestOutputs = + ## collect the outputs for the given test + result = newTestOutputs() + for name, expected in test.outputs.pairs: + if name == "stdout": + result[name] = stdout + else: + if not existsFile(name): + continue + result[name] = readFile(name) + removeFile(name) + +proc cmpOutputs(test: TestSpec, outputs: TestOutputs): TestStatus = + ## compare the test program's outputs to those expected by the test + result = OK + for name, expected in test.outputs.pairs: + if name notin outputs: + logFailure(test, OutputFileNotFound, name) + result = FAILED + continue + + let + testOutput = outputs[name] + + # Would be nice to do a real diff here instead of simple compare + if test.timestampPeg.len > 0: + if not cmpIgnorePegs(testOutput, expected, + peg(test.timestampPeg), pegXid): + logFailure(test, OutputsDiffer, name, expected, testOutput) + result = FAILED + else: + if not cmpIgnoreDefaultTimestamps(testOutput, expected): + logFailure(test, OutputsDiffer, name, expected, testOutput) + result = FAILED + +proc compile(test: TestSpec; backend: string): TestStatus = + ## compile the test program for the requested backends + block: + if not existsFile(test.source): + logFailure(test, SourceFileNotFound) + result = FAILED + break + + let + binary = test.binary(backend) + var + cmd = findExe("nim") + cmd &= " " & backend + cmd &= " --nimcache:" & test.config.cache(backend) + cmd &= " --out:" & binary + cmd &= " " & defaultOptions + cmd &= " " & test.flags + cmd &= " " & test.config.compilationFlags + cmd &= " " & test.source.quoteShell + var + c = parseCmdLine(cmd) + p = startProcess(command=c[0], args=c[1.. ^1], + options={poStdErrToStdOut, poUsePath}) + + try: + let + compileInfo = parseCompileStream(p, p.outputStream) + + if compileInfo.exitCode != 0: + if test.compileError.len == 0: + logFailure(test, CompileError, compileInfo.fullMsg) + result = FAILED + break + else: + if test.compileError == compileInfo.msg and + (test.errorFile.len == 0 or test.errorFile == compileInfo.errorFile) and + (test.errorLine == 0 or test.errorLine == compileInfo.errorLine) and + (test.errorColumn == 0 or test.errorColumn == compileInfo.errorColumn): + result = OK + else: + logFailure(test, CompileErrorDiffers, compileInfo.fullMsg) + result = FAILED + break + + # Lets also check file size here as it kinda belongs to the + # compilation result + if test.maxSize != 0: + var size = getFileSize(binary) + if size > test.maxSize: + logFailure(test, FileSizeTooLarge, $size) + result = FAILED + break + + result = OK + finally: + close(p) + +proc threadedExecute(payload: ThreadPayload) {.thread.} + +proc spawnTest(child: var Thread[ThreadPayload]; test: TestSpec; + core: int): bool = + ## invoke a single test on the given thread/core; true if we + ## pinned the test to the given core + assert core >= 0 + child.createThread(threadedExecute, + ThreadPayload(core: core, spec: test)) + # set cpu affinity if requested (and cores remain) + if CpuAffinity in test.config.flags: + if core < countProcessors(): + child.pinToCpu core + result = true + +proc execute(test: TestSpec): TestStatus = + ## invoke a single test and return a status + var + # FIXME: pass a backend + cmd = test.binary + # output the test stage if necessary + if test.stage.len > 0: + echo 20.spaces & test.stage + + if not fileExists(cmd): + result = FAILED + logFailure(test, ExeFileNotFound) + else: + withinDir parentDir(test.path): + cmd = cmd.quoteShell & " " & test.args + let + (output, exitCode) = execCmdEx(cmd) + if exitCode != 0: + # parseExecuteOutput() # Need to parse the run time failures? + logFailure(test, RuntimeError, output) + result = FAILED + else: + let + outputs = test.composeOutputs(output) + result = test.cmpOutputs(outputs) + # perform an update of the testfile if requested and required + if UpdateOutputs in test.config.flags and result == FAILED: + test.rewriteTestFile(outputs) + # we'll call this a `skip` because it's not strictly a failure + # and we want any dependent testing to proceed as usual. + result = SKIPPED + +proc executeAll(test: TestSpec): TestStatus = + ## run a test and any dependent children, yielding a single status + when compileOption("threads"): + try: + var + thread: TestThread + # we spawn and join the test here so that it can receive + # cpu affinity via the standard thread.pinToCpu method + discard thread.spawnTest(test, 0) + thread.joinThreads + except: + # any thread(?) exception is a failure + result = FAILED + else: + # unthreaded serial test execution + result = SKIPPED + while test != nil and result in {OK, SKIPPED}: + result = test.execute + test = test.child + +proc threadedExecute(payload: ThreadPayload) {.thread.} = + ## a thread in which we'll perform a test execution given the payload + var + result = FAILED + if payload.spec.child == nil: + {.gcsafe.}: + result = payload.spec.execute + else: + try: + var + child: TestThread + discard child.spawnTest(payload.spec.child, payload.core + 1) + {.gcsafe.}: + result = payload.spec.execute + child.joinThreads + except: + result = FAILED + if result == FAILED: + raise newException(CatchableError, payload.spec.stage & " failed") + +proc optimizeOrder(tests: seq[TestSpec]; + order: set[SortBy]): seq[TestSpec] = + ## order the tests by how recently each was modified + template whenWritten(path: string): Time = + path.getFileInfo(followSymlink = true).lastWriteTime + + result = tests + for s in SortBy.low .. SortBy.high: + if s in order: + case s + of Test: + result = result.sortedByIt it.path.whenWritten + of Source: + result = result.sortedByIt it.source.whenWritten + of Reverse: + result.reverse + of Random: + result.shuffle + +proc scanTestPath(path: string): seq[string] = + ## add any tests found at the given path + if fileExists(path): + result.add path + else: + for file in walkDirRec path: + if file.endsWith ".test": + result.add file + +proc test(test: TestSpec; backend: string): TestStatus = + let + config = test.config + var + duration: float + + try: + time duration: + # perform all tests in the test file + result = test.executeAll + finally: + logResult(test.name, result, duration) + +proc buildBackendTests(config: TestConfig; + tests: seq[TestSpec]): BackendTests = + ## build the table mapping backend to test inputs + result = newTable[string, seq[TestSpec]](4) + for spec in tests.items: + for backend in config.backends.items: + assert backend != "" + if backend in result: + if spec notin result[backend]: + result[backend].add spec + else: + result[backend] = @[spec] + +proc removeCaches(config: TestConfig; backend: string) = + ## cleanup nimcache directories between backend runs + removeDir config.cache(backend) + +# we want to run tests on "native", first. +proc performTesting(config: TestConfig; + backend: string; tests: seq[TestSpec]): TestStatus = + var + successful, skipped, invalid, failed = 0 + dedupe: CountTable[Hash] + + assert backend != "" + + # perform each test in an optimized order + for spec in tests.optimizeOrder(config.orderBy).items: + + block escapeBlock: + if spec.program.len == 0: + # a program name is bare minimum of a test file + result = INVALID + invalid.inc + logResult(spec.program & " for " & spec.name, result) + break escapeBlock + + if spec.skip or hostOS notin spec.os or config.shouldSkip(spec.name): + result = SKIPPED + skipped.inc + logResult(spec.program & " for " & spec.name, result) + break escapeBlock + + let + build = spec.binaryHash(backend) + if build notin dedupe: + dedupe.inc build + # compile the test program for all backends + var + duration: float + try: + time duration: + result = compile(spec, backend) + if result != OK or spec.compileError.len != 0: + failed.inc + break escapeBlock + finally: + logResult("compiled " & spec.program & " for " & spec.name, + result, duration) + + if result == OK: + successful.inc + + let nonSuccesful = skipped + invalid + failed + styledEcho(styleBright, "Finished run for $#: $#/$# OK, $# SKIPPED, $# FAILED, $# INVALID" % + [backend, $successful, $(tests.len), + $skipped, $failed, $invalid]) + + for spec in tests.items: + try: + # this may fail in 64-bit AppVeyor images with "The process cannot + # access the file because it is being used by another process. + # [OSError]" + let + fn = spec.binary(backend) + if fileExists(fn): + removeFile(fn) + except CatchableError as e: + echo e.msg + + if 0 == tests.len - successful - nonSuccesful: + config.removeCaches(backend) + +proc main(): int = + let config = processArguments() + + case config.cmd + of Command.test: + let testFiles = scanTestPath(config.path) + if testFiles.len == 0: + styledEcho(styleBright, "No test files found") + result = 1 + else: + var + tests = testFiles.mapIt config.parseTestFile(it) + backends = config.buildBackendTests(tests) + + # c > cpp > js + for backend in backendOrder: + assert backend != "" + # if we actually need to do anything on the given backend + if backend notin backends: + continue + let + tests = backends[backend] + try: + if OK != config.performTesting(backend, tests): + break + finally: + backends.del(backend) + + for backend, tests in backends.pairs: + assert backend != "" + if OK != config.performTesting(backend, tests): + break + of Command.fuzz: + runFuzzer(config.target, config.fuzzer, config.corpusDir) + of noCommand: + discard + +when isMainModule: + quit main() diff --git a/vendor/nim-testutils/ntu.nim.cfg b/vendor/nim-testutils/ntu.nim.cfg new file mode 100644 index 000000000..05e5c5cf9 --- /dev/null +++ b/vendor/nim-testutils/ntu.nim.cfg @@ -0,0 +1,2 @@ +--threads:on +--path="$config" diff --git a/vendor/nim-testutils/scripts/install_honggfuzz.sh b/vendor/nim-testutils/scripts/install_honggfuzz.sh new file mode 100755 index 000000000..f41a6d41c --- /dev/null +++ b/vendor/nim-testutils/scripts/install_honggfuzz.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eu + +sudo apt-get update +sudo apt-get install binutils-dev +sudo apt-get install libunwind8-dev + +git clone https://github.com/google/honggfuzz.git /tmp/honggfuzz + +pushd /tmp/honggfuzz + make + sudo make install DESTDIR=/opt/honggfuzz +popd + +rm -rf /tmp/honggfuzz + diff --git a/vendor/nim-testutils/tests/hello.nim b/vendor/nim-testutils/tests/hello.nim new file mode 100644 index 000000000..b5e533743 --- /dev/null +++ b/vendor/nim-testutils/tests/hello.nim @@ -0,0 +1,6 @@ +import std/os + +if paramCount() == 1: + echo "hello ", paramStr(1) +else: + echo "hello world" diff --git a/vendor/nim-testutils/tests/hello/hello.test b/vendor/nim-testutils/tests/hello/hello.test new file mode 100644 index 000000000..90522deaf --- /dev/null +++ b/vendor/nim-testutils/tests/hello/hello.test @@ -0,0 +1,3 @@ +program = "../hello" +[Output] +stdout = "hello world\n" diff --git a/vendor/nim-testutils/tests/hello/hello_multiple.test b/vendor/nim-testutils/tests/hello/hello_multiple.test new file mode 100644 index 000000000..21ac86b6e --- /dev/null +++ b/vendor/nim-testutils/tests/hello/hello_multiple.test @@ -0,0 +1,15 @@ +program = "../hello" + +# my aim is true +[Output] +stdout = "hello world\n" + +# option 2 +[Output_larry_is_a_good_boy] +args = "larry" +stdout = "hello larry\n" + +# option 47 +[Output_stevie_is_a_good_boy] +args = "stephen" +stdout = "hello stephen\n" diff --git a/vendor/nim-testutils/tests/hello/hello_size.test b/vendor/nim-testutils/tests/hello/hello_size.test new file mode 100644 index 000000000..ea6f4bf11 --- /dev/null +++ b/vendor/nim-testutils/tests/hello/hello_size.test @@ -0,0 +1,5 @@ +program = "../hello" +max_size = 60000 +release +--opt:size +os = "linux,macosx" diff --git a/vendor/nim-testutils/tests/threads.nim b/vendor/nim-testutils/tests/threads.nim new file mode 100644 index 000000000..b1542e6cf --- /dev/null +++ b/vendor/nim-testutils/tests/threads.nim @@ -0,0 +1,4 @@ +when compileOption("threads"): + quit(0) +else: + quit(1) diff --git a/vendor/nim-testutils/tests/threads.test b/vendor/nim-testutils/tests/threads.test new file mode 100644 index 000000000..00c5df647 --- /dev/null +++ b/vendor/nim-testutils/tests/threads.test @@ -0,0 +1 @@ +program = "threads" diff --git a/vendor/nim-testutils/tests/tunits.nim b/vendor/nim-testutils/tests/tunits.nim new file mode 100644 index 000000000..073d0dae6 --- /dev/null +++ b/vendor/nim-testutils/tests/tunits.nim @@ -0,0 +1,11 @@ +import unittest2 + +suite "goats": + test "pigs": + echo "oink" + check true + + test "horses": + expect ValueError: + echo "ney" + raise newException(ValueError, "you made an error") diff --git a/vendor/nim-testutils/tests/tunits.test b/vendor/nim-testutils/tests/tunits.test new file mode 100644 index 000000000..83fd0d894 --- /dev/null +++ b/vendor/nim-testutils/tests/tunits.test @@ -0,0 +1 @@ +program = "tunits" diff --git a/vendor/nim-testutils/testutils.nim b/vendor/nim-testutils/testutils.nim new file mode 100644 index 000000000..36dedafdd --- /dev/null +++ b/vendor/nim-testutils/testutils.nim @@ -0,0 +1,6 @@ +import + testutils/unittests + +export + unittests + diff --git a/vendor/nim-testutils/testutils.nimble b/vendor/nim-testutils/testutils.nimble new file mode 100644 index 000000000..5a8df0d74 --- /dev/null +++ b/vendor/nim-testutils/testutils.nimble @@ -0,0 +1,39 @@ +mode = ScriptMode.Verbose + +packageName = "testutils" +version = "0.4.0" +author = "Status Research & Development GmbH" +description = "A unittest framework" +license = "Apache License 2.0" +skipDirs = @["tests"] +bin = @["ntu"] +#srcDir = "testutils" + +requires "nim >= 1.2.0", + "https://github.com/status-im/nim-unittest2.git#head" + +proc execCmd(cmd: string) = + echo "execCmd: " & cmd + exec cmd + +proc execTest(test: string) = + let + test = "ntu test " & test + when true: + execCmd "nim c -f -r " & test + execCmd "nim c -d:release -r " & test + execCmd "nim c -d:danger -r " & test + execCmd "nim cpp -r " & test + execCmd "nim cpp -d:danger -r " & test + #when NimMajor >= 1 and NimMinor >= 1 and not defined(macosx): + when false: + # we disable gc:arc test here because Nim cgen + # generate something not acceptable for clang + # and failed on windows 64 bit too + execCmd "nim c --gc:arc --exceptions:goto -r " & test + execCmd "nim cpp --gc:arc --exceptions:goto -r " & test + else: + execCmd "nim c -f -r " & test + +task test, "run tests for travis": + execTest("tests") diff --git a/vendor/nim-testutils/testutils/config.nim b/vendor/nim-testutils/testutils/config.nim new file mode 100644 index 000000000..e23dde1a9 --- /dev/null +++ b/vendor/nim-testutils/testutils/config.nim @@ -0,0 +1,204 @@ +import + std/[sequtils, hashes, os, parseopt, strutils, algorithm], + fuzzing_engines + +const + Usage = """ + + Usage: + ntu COMMAND [options] + + Available commands: + + $ ntu test [options] + + Run the test(s) specified at path. Will search recursively for test files + provided path is a directory. + + Options: + --backends:"c cpp js objc" Run tests for specified targets + --include:"test1 test2" Run only listed tests (space/comma separated) + --exclude:"test1 test2" Skip listed tests (space/comma separated) + --update Rewrite failed tests with new output + --sort:"source,test" Sort the tests by program and/or test mtime + --reverse Reverse the order of tests + --random Shuffle the order of tests + --help Display this help and exit + + $ ntu fuzz [options] + + Start a fuzzing test with a Nim module based on testutils/fuzzing. + + Options: + --fuzzer:libFuzzer The fuzzing engine to use. + Possible values: libFuzzer, honggfuzz, afl + --corpus: A directory with initial input cases + + """.unindent.strip + +type + FlagKind* = enum + UpdateOutputs = "--update" + UseThreads = "--threads:on" + DebugBuild = "--define:debug" + ReleaseBuild = "--define:release" + DangerBuild = "--define:danger" + CpuAffinity = "--affinity" + + SortBy* {.pure.} = enum + Random = "random" + Source = "source" + Test = "test" + Reverse = "reverse" + + Command* = enum + noCommand + test + fuzz + + TestConfig* = object + case cmd*: Command + of test: + path*: string + includedTests*: seq[string] + excludedTests*: seq[string] + + flags*: set[FlagKind] + # options + backendNames*: seq[string] + orderBy*: set[SortBy] + of fuzz: + fuzzer*: FuzzingEngine + corpusDir*: string + target*: string + of noCommand: + discard + +const + defaultFlags = {UseThreads} + compilerFlags* = {DebugBuild, ReleaseBuild, DangerBuild, UseThreads} + # --define:testutilsBackends="cpp js" + testutilsBackends* {.strdefine.} = "c" + defaultSort = {Source, Reverse} + +proc `backends=`*(config: var TestConfig; inputs: seq[string]) = + config.backendNames = inputs.sorted + +proc `backends=`*(config: var TestConfig; input: string) = + config.backends = input.split(" ") + +proc backends*(config: TestConfig): seq[string] = + result = config.backendNames + +proc hash*(config: TestConfig): Hash = + var h: Hash = 0 + h = h !& config.backends.hash + h = h !& hash(ReleaseBuild in config.flags) + h = h !& hash(DangerBuild in config.flags) + h = h !& hash(UseThreads notin config.flags) + result = !$h + +proc compilationFlags*(config: TestConfig): string = + for flag in compilerFlags * config.flags: + result &= " " & $flag + +proc cache*(config: TestConfig; backend: string): string = + ## return the path to the nimcache for the given backend and + ## compile-time flags + result = getTempDir() + result = result / "testutils-nimcache-$#-$#" % [ backend, + $getCurrentProcessId() ] + +proc processArguments*(): TestConfig = + ## consume the arguments supplied to ntu and yield a computed + ## configuration object + var + opt = initOptParser() + + func toSet[SortBy](list: seq[SortBy]): set[SortBy] = + for element in list.items: + result.incl element + + for kind, key, value in opt.getOpt: + if result.cmd == noCommand: + doAssert kind == cmdArgument + result.cmd = parseEnum[Command](key) + if result.cmd == test: + result.flags = defaultFlags + result.backends = testutilsBackends + result.orderBy = defaultSort + continue + + case result.cmd + of test: + case kind + of cmdArgument: + if result.path == "": + result.path = absolutePath(key) + of cmdLongOption, cmdShortOption: + case key.toLowerAscii + of "help", "h": + quit(Usage, QuitSuccess) + of "reverse", "random": + let + flag = parseEnum[SortBy](value) + if flag in result.orderBy: + result.orderBy.excl flag + else: + result.orderBy.incl flag + of "sort": + result.orderBy = toSet value.split(",").mapIt parseEnum[SortBy](it) + of "backend", "backends", "targets", "t": + result.backends = value + of "release", "danger": + result.flags.incl ReleaseBuild + result.flags.incl DangerBuild + of "nothreads": + result.flags.excl UseThreads + of "update": + result.flags.incl UpdateOutputs + of "include": + result.includedTests.add value.split(Whitespace + {','}) + of "exclude": + result.excludedTests.add value.split(Whitespace + {','}) + else: + quit(Usage) + of cmdEnd: + quit(Usage) + + of fuzz: + case kind + of cmdArgument: + result.target = key + of cmdLongOption, cmdShortOption: + case key.toLowerAscii: + of "f", "fuzzer": + result.fuzzer = parseEnum[FuzzingEngine](value) + of "c", "corpus": + result.corpusDir = absolutePath(value) + else: + quit(Usage) + else: + echo "got kind ", kind + quit(Usage) + + of noCommand: + discard + + case result.cmd + of test: + if result.path == "": + quit(Usage) + of fuzz: + if result.target == "": + quit(Usage) + else: + quit(Usage) + +func shouldSkip*(config: TestConfig, name: string): bool = + ## true if the named test should be skipped + if name in config.excludedTests: + result = true + elif config.includedTests.len > 0: + if name notin config.includedTests: + result = true diff --git a/vendor/nim-testutils/testutils/fuzzing.nim b/vendor/nim-testutils/testutils/fuzzing.nim new file mode 100644 index 000000000..d8b4eea49 --- /dev/null +++ b/vendor/nim-testutils/testutils/fuzzing.nim @@ -0,0 +1,122 @@ +import os, streams, strutils, chronicles, macros, stew/ranges/ptr_arith + +when not defined(windows): + import posix + +# if user forget to import chronicles +# they still can compile without mysterious +# error such as "undeclared identifier: 'activeChroniclesStream'" +export chronicles + +proc suicide() = + # For code we want to fuzz, SIGSEGV is needed on unwanted exceptions. + # However, this is only needed when fuzzing with afl. + when not defined(windows): + discard kill(getpid(), SIGSEGV) + else: + discard + +template fuzz(body) = + when defined(llvmFuzzer): + body + else: + try: + body + except Exception as e: + error "Fuzzer input created exception", exception=e.name, trace=e.repr, + msg=e.msg + suicide() + +when not defined(llvmFuzzer): + proc readStdin(): seq[byte] = + let s = if paramCount() > 0: newFileStream(paramStr(1)) + else: newFileStream(stdin) + if s.isNil: + error "Error opening input stream" + suicide() + # We use binary files as with hex we can get lots of "not hex" failures + var input = s.readAll() + s.close() + # Remove newline if it is there + input.removeSuffix + result = cast[seq[byte]](input) + +proc NimMain() {.importc: "NimMain".} + +# The default init, gets redefined when init template is used. +template initImpl(): untyped = + when defined(llvmFuzzer): + proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} = + NimMain() + + return 0 + else: + discard + +template init*(body: untyped) {.dirty.} = + ## Init block to do any initialisation for the fuzzing test. + ## + ## For AFL this is currently only cosmetic and will be run each time, before + ## the test block. + ## + ## For LLVM fuzzers this will only be run once. So only put data which is + ## stateless or make sure everything gets properply reset for each new run + ## in the test block. + when defined(llvmFuzzer): + template initImpl() {.dirty.} = + bind NimMain + + proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} = + NimMain() + + body + + return 0 + else: + template initImpl(): untyped {.dirty.} = + bind fuzz + fuzz: body + +template test*(body: untyped): untyped = + ## Test block to do the actual test that will be fuzzed in a loop. + ## + ## Within this test block there is access to the payload OpenArray which + ## contains the payload provided by the fuzzer. + mixin initImpl + initImpl() + when defined(llvmFuzzer): + proc fuzzerCall(data: ptr byte, len: csize): + cint {.exportc: "LLVMFuzzerTestOneInput".} = + template payload(): auto = + makeOpenArray(data, len) + + body + else: + when not defined(windows): + var payload {.inject.} = readStdin() + + fuzz: body + else: + proc fuzzerCall() {.exportc: "AFLmain", dynlib, cdecl.} = + var payload {.inject.} = readStdin() + fuzz: body + + fuzzerCall() + +when defined(clangfast) and not defined(llvmFuzzer): + ## Can be used for deferred instrumentation. + ## Should be placed on a suitable location in the code where the delayed + ## cloning can take place (e.g. NOT after creation of threads) + proc aflInit*() {.importc: "__AFL_INIT", noDecl.} + ## Can be used for persistent mode. + ## Should be used as value for controlling a loop around a test case. + ## Test case should be able to handle repeated inputs. No repeated fork() will + ## be done. + # TODO: Lets use this in the test block when afl-clang-fast is used? + proc aflLoopImpl(count: cuint): cint {.importc: "__AFL_LOOP", noDecl.} + template aflLoop*(body: untyped): untyped = + while aflLoopImpl(1000) != 0: + `body` +else: + proc aflInit*() = discard + template aflLoop*(body: untyped): untyped = `body` diff --git a/vendor/nim-testutils/testutils/fuzzing/fuzzing_on_windows.md b/vendor/nim-testutils/testutils/fuzzing/fuzzing_on_windows.md new file mode 100644 index 000000000..3a4f413be --- /dev/null +++ b/vendor/nim-testutils/testutils/fuzzing/fuzzing_on_windows.md @@ -0,0 +1,148 @@ +# Fuzzing on Windows + +This is a supplemental guide to fuzzing on windows platform. + +## Windows Subsystem for Linux(WSL) Ubuntu 20.04 + +Grab Ubuntu from Windows Store and install libFuzzer and afl. +But don't forget to update or upgrade the database if you start with a 'blank' Ubuntu. + +```sh +sudo apt update +sudo apt upgrade +``` + +### Install clang and libFuzzer + +Pick your clang version: 9, 10, or 11. In this example I'll use clang-10. + +```sh +sudo apt install build-essential +sudo apt-get install clang-10 lldb-10 lld-10 +sudo apt-get install libfuzzer-10-dev +``` + +Now copy the symlink in `/usr/bin` or whatever location of clang-10. + +```sh +$> which clang-10 +/usr/bin/clang-10 # this is the result of 'which clang-10' +$> sudo cp -P /usr/bin/clang-10 /usr/bin/clang +``` + +### Install afl + +```sh +git clone https://github.com/google/AFL +cd AFL +make +sudo make install +``` + +Now go back to [Fuzzing instructions for Linux](readme.md) + +## Real Windows instructions + +There are a lot of things you need to install on Windows. + +### Compiling with libFuzzer + +* Download and install Clang 11 for Windows [here](https://llvm.org/builds/) +* Download and install Visual Studio 2019 [here](https://visualstudio.microsoft.com/downloads/) + +You don't need to install all of the Visual Studio components, you only need to +choose “Desktop development with C++”. That will be enough and only download less than 2GB instead of 4GB+. +Perhaps you wonder why need to install two compiler? The answer is: libFuzzer does not work with MingW-GCC. + +If you already prepare your test case, the instruction to build the binary is exactly the same with Linux version. + +```Nim +nim c -d:libFuzzer -d:release -d:chronicles_log_level=fatal --noMain --cc=clang --passC="-fsanitize=fuzzer" --passL="-fsanitize=fuzzer" testcase +``` + +Now go back to [Starting the Fuzzer using libFuzzer](readme.md#Starting-the-Fuzzer) + + +### Compiling with winafl + +We will use the same Visual Studio compiler like libFuzzer. + +* Download and install Visual Studio 2019 [here](https://visualstudio.microsoft.com/downloads/) + +Now open one of this terminal from VS 2019: + +* Developer PowerShell for VS 2019 +* x64 Native Tools Command Prompt for VS 2019 +* x86 Native Tools Command Prompt for VS 2019 + +### Download and build winafl + +No need to install cmake, VS 2019 already included cmake in it's installation package. + +```sh +git clone https://github.com/googleprojectzero/winafl +cd winafl +git submodule update --init --recursive +``` + +#### 32/64 bit build using VS 2017 + +```sh +mkdir build32 +cd build32 +cmake -G"Visual Studio 15 2017" .. -DINTELPT=1 +cmake --build . --config Release + +mkdir build64 +cd build64 +cmake -G"Visual Studio 15 2017 Win64" .. -DINTELPT=1 +cmake --build . --config Release +``` + +#### 32/64 bit build using VS 2019 + +``` +mkdir build32 +cd build32 +cmake -G"Visual Studio 16 2019" .. -DINTELPT=1 -Ax86 +cmake --build . --config Release + +mkdir build64 +cd build64 +cmake -G"Visual Studio 16 2019" .. -DINTELPT=1 -Ax64 +cmake --build . --config Release +``` + +Either you use VS 2017 or VS 2019, you'll get the binary in: + +`winafl/build64/bin/Release` or `winafl/build32/bin/Release` + +If you only need to use it occasionally, you can use this command to add the winafl binary path to +you env `PATH` instead of polluting it system wide. + +* PowerShell: ```$env:path = ($pwd).path + "\bin\Release;" + $env:path``` +* CMD Command Prompt: ```set PATH=%CD%\bin\Release;%PATH%``` + +#### Compiling testcase + +Compiling the testcase is simpler than Linux version, you don't need to use afl-gcc or afl-clang, +you can use clang, vcc, or mingw-gcc as you like. + +```Nim +nim c -d:afl -d:noSignalHandler -d:release -d:chronicles_log_level=fatal testcase +``` + +#### Starting the Fuzzer + +Now run the command from Command Prompt terminal, the `@@` will not work with PowerShell. +Winafl needs the input data to be read from a file, not from stdin, that's why the presence of `@@`. + +```sh +afl-fuzz.exe -i inDir -o outDir -P -t 20000 -- -coverage_module testcase.exe -fuzz_iterations 20 -target_module testcase.exe -target_method AFLmain -nargs 2 -- testcase.exe @@ +``` + +* `inDir` is a directory containing a small but valid input file that makes sense to the program. +* `outDir` will be the location of generated testcase corpus. +* replace both `testcase.exe` with your executable binary. +* `-P` is Intel PT selector +* `-t` timeout in msec diff --git a/vendor/nim-testutils/testutils/fuzzing/readme.md b/vendor/nim-testutils/testutils/fuzzing/readme.md new file mode 100644 index 000000000..9d9b6aa7a --- /dev/null +++ b/vendor/nim-testutils/testutils/fuzzing/readme.md @@ -0,0 +1,207 @@ +# Fuzzing +## tldr: +* [Install afl](#Install-afl). +* Create a testcase. +* Run: `nim fuzz.nims afl testfolder/testcase.nim` + +Or + +* [Install libFuzzer](#Install-libFuzzer) (comes with LLVM). +* Create a testcase. +* Run: `nim fuzz.nims libFuzzer testfolder/testcase.nim` + +## Fuzzing Helpers +There are two convenience templates which will help you set up a quick fuzzing +test. + +These are the mandatory `test` block and the optional `init` block. + +Example usage: +```nim +test: + var rlp = rlpFromBytes(payload) + discard rlp.inspect() +``` + +Any unhandled `Exception` will result in a failure of the testcase. If certain +`Exception`s are to be allowed to occur within the test, they should be caught. + +E.g.: +```nim +test: + try: + var rlp = rlpFromBytes(payload) + discard rlp.inspect() + except RlpError as e: + debug "Inspect failed", err = e.msg +``` + +## Supported Fuzzers +The two templates can prepare the code for both +[afl](http://lcamtuf.coredump.cx/afl/), +[afl++](https://github.com/AFLplusplus/AFLplusplus) and +[libFuzzer](http://llvm.org/docs/LibFuzzer.html). + +You will need to install first the fuzzer you want to use. + +### Install afl + +```sh +# Ubuntu / Debian +sudo apt-get install afl++ + +# Fedora +dnf install american-fuzzy-lop +# for usage with clang & clang-fast you will have to install +# american-fuzzy-lop-clang or american-fuzzy-lop-clang-fast + +# Arch Linux +pacman -S afl + +# NixOS +nix-env -i afl + +``` + +### Install libFuzzer + +LibFuzzer is part of llvm and will be installed together with llvm-libs in +recent versions. Installing clang should install llvm-libs. +```sh +# Ubuntu / Debian +sudo apt-get install clang + +# Fedora +dnf install clang + +# Arch Linux +pacman -S clang + +# NixOS +nix-env -iA nixos.clang_7 nixos.llvm_7 +``` + +## Compiling & Starting the Fuzzer +### Scripted helper +There is a nimscript helper to compile & start the fuzzer: +```sh +# for afl +nim fuzz.nims afl testcase.nim + +# for libFuzzer +nim fuzz.nims libFuzzer testcase.nim +``` +### Manually with afl +#### Compiling +With gcc: +```sh +nim c -d:afl -d:release -d:chronicles_log_level=fatal -d:noSignalHandler --cc=gcc --gcc.exe=afl-gcc --gcc.linkerexe=afl-gcc testcase.nim +``` +The `afl` define is specifically required for the `init` and `test` +templates. + +You typically want to fuzz in `-d:release` and probably also want to lower down +the logging. But this is not strictly necessary. + +There is also a nimscript task in `config.nims` for this: +``` +nim c build_afl testcase.nim +``` + +With clang: +```sh +# afl-clang +nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang --clang.linkerexe=afl-clang ftestcase.nim +# afl-clang-fast +nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang-fast --clang.linkerexe=afl-clang-fast testcase.nim +``` + +#### Starting the Fuzzer + +To start the fuzzer: +```sh +afl-fuzz -i input -o results -- ./testcase +``` + +To rerun it without losing previous results/corpus: +```sh +afl-fuzz -i - -o results -- ./testcase +``` + +To run several parallel fuzzing sessions: +```sh +# Start master fuzzer +afl-fuzz -i input -o results -M fuzzer01 -- ./testcase +# Start slaves (usually 1 per core available) +afl-fuzz -i input -o results -S fuzzer02 -- ./testcase +afl-fuzz -i input -o results -S fuzzer03 -- ./testcase +# add more if needed +``` + +When compiled with `-d:afl` the resulting application can also be run +manually by providing it input data, e.g.: +```sh +./testcase < testfile +``` + +During debugging you might not want the testcase to generate a segmentation +fault on exceptions. You can do this by rebuilding the test without the `-d:afl` +flag. Changing to `-d:debug` will also help but might also change the +behaviour. + +### Manually with libFuzzer +#### Compiling +```sh +nim c -d:libFuzzer -d:release -d:chronicles_log_level=fatal --noMain --cc=clang --passC="-fsanitize=fuzzer" --passL="-fsanitize=fuzzer" testcase.nim +``` + +The `libFuzzer` define is specifically required for the `init` and `test` +templates. + +You typically want to fuzz in `-d:release` and probably also want to lower down +the logging. But this is not strictly necessary. + +There is also a nimscript task in `config.nims` for compiling: +``` +nim c build_libFuzzer testcase.nim +``` + +#### Starting the Fuzzer +Starting the fuzzer is as simple as running the compiled program: +```sh +./testcase corpus_dir -runs=1000000 +``` + +To see the available options: +```sh +./testcase test=1 +``` + +Parallel fuzzing on 8 cores: +```sh +./fuzz-libfuzzer -jobs=8 -workers=8 +``` + +You can also use the application to verify a specific test case: +```sh +./testcase input_file +``` + +## Additional notes +The `init` template, when used with **afl**, is only cosmetic. It will be +run before each test block, compared to libFuzzer, where it will be run only +once. + +In case of using afl with `alf-clang-fast` you can make use of `aflInit()` proc +and `aflLoop()` template. + +`aflInit()` will allow using what is called deferred instrumentation. Basically, +the forking of the process will only happen after this call, where normally it +is done right before `main()`. + +`aflLoop:` will allow for (experimental) persistant mode. It will run the test +in loop (1000 iterations) with different payloads. This is more comparable with +libFuzzer. + +These calls are enabled with `-d:clangfast`, and have to be manually added. +They are currently not part of the `test` or `init` templates. diff --git a/vendor/nim-testutils/testutils/fuzzing_engines.nim b/vendor/nim-testutils/testutils/fuzzing_engines.nim new file mode 100644 index 000000000..f3772ef8f --- /dev/null +++ b/vendor/nim-testutils/testutils/fuzzing_engines.nim @@ -0,0 +1,153 @@ +import strformat +import os except dirExists + +const + aflGcc = "--cc=gcc " & + "--gcc.exe=afl-gcc " & + "--gcc.linkerexe=afl-gcc" + + aflClang = "--cc=clang " & + "--clang.exe=afl-clang " & + "--clang.linkerexe=afl-clang" + + aflClangFast = "--cc=clang " & + "--clang.exe=afl-clang-fast " & + "--clang.linkerexe=afl-clang-fast " & + "-d:clangfast" + + libFuzzerClang = "--cc=clang " & + "--passC='-fsanitize=fuzzer,address' " & + "--passL='-fsanitize=fuzzer,address'" + + honggfuzzClang = "--cc=clang " & + "--clang.exe=hfuzz-clang " & + "--clang.linkerexe=hfuzz-clang " + + # Can also test in debug mode obviously, but might be slower + # Can turn on more logging, in case of libFuzzer it will get very verbose though + defaultFlags = "-d:release -d:chronicles_log_level=fatal " & + "--hints:off --warnings:off --verbosity:0" + +type + FuzzingEngine* = enum + libFuzzer + honggfuzz + afl + + AflCompiler* = enum + gcc = aflGcc, + clang = aflClang, + clangFast = aflClangFast + +const + defaultFuzzingEngine* = libFuzzer + +when not defined(nimscript): + import os, osproc + + template exec(cmd: string) = + discard execCmd(cmd) + + template mkDir(dir: string) = + createDir dir + + template withDir*(dir: string; body: untyped): untyped = + ## Changes the current directory temporarily. + ## + ## If you need a permanent change, use the `cd() <#cd,string>`_ proc. + ## Usage example: + ## + ## .. code-block:: nim + ## withDir "foo": + ## # inside foo + ## #back to last dir + var curDir = getCurrentDir() + try: + setCurrentDir(dir) + body + finally: + setCurrentDir(curDir) + +template q(x: string): string = + quoteShell x + +proc aflCompile*(target: string, c: AflCompiler) = + let aflOptions = &"-d:afl -d:noSignalHandler {$c}" + let compileCmd = &"nim c {defaultFlags} {aflOptions} {q target}" + exec compileCmd + +proc aflExec*(target: string, + inputDir: string, + resultsDir: string, + cleanStart = false) = + let exe = target.addFileExt(ExeExt) + if not dirExists(inputDir): + # create a input dir with one 0 file for afl + mkDir(inputDir) + # TODO: improve + withDir inputDir: exec "echo '0' > test" + + var fuzzCmd: string + # if there is an output dir already, continue fuzzing from previous run + if (not dirExists(resultsDir)) or cleanStart: + fuzzCmd = &"afl-fuzz -i {q inputDir} -o {q resultsDir} -M fuzzer01 -- {q exe}" + else: + fuzzCmd = &"afl-fuzz -i - -o {q resultsDir} -M fuzzer01 -- {q exe}" + exec fuzzCmd + +proc libFuzzerCompile*(target: string) = + let libFuzzerOptions = &"-d:llvmFuzzer --noMain {libFuzzerClang}" + let compileCmd = &"nim c {defaultFlags} {libFuzzerOptions} {q target}" + exec compileCmd + +proc libFuzzerExec*(target: string, corpusDir: string) = + if not dirExists(corpusDir): + # libFuzzer is OK when starting with empty corpus dir + mkDir(corpusDir) + + exec &"{q target} {q corpusDir}" + +proc honggfuzzCompile*(target: string) = + let honggfuzzOptions = &"-d:llvmFuzzer --noMain {honggfuzzClang}" + let compileCmd = &"nim c {defaultFlags} {honggfuzzOptions} {q target}" + exec compileCmd + +proc honggfuzzExec*(target: string, corpusDir: string, outputDir: string) = + #if not dirExists(corpusDir): + # # libFuzzer is OK when starting with empty corpus dir + # mkDir(corpusDir) + + # TODO: + # Other useful parameters: + # --threads|-n VALUE + # Number of concurrent fuzzing threads (default: number of CPUs / 2) + # --workspace|-W VALUE + # Workspace directory to save crashes & runtime files (default: '.') + # --crashdir VALUE + # Directory where crashes are saved to (default: workspace directory) + # --covdir_new VALUE + # New coverage (beyond the dry-run fuzzing phase) is written to this separate directory + # --dict|-w VALUE + # Dictionary file. Format:http://llvm.org/docs/LibFuzzer.html#dictionaries + exec &"honggfuzz --persistent --input {q corpusDir} --output {q outputDir} -- {q target}" + +proc runFuzzer*(targetPath: string, fuzzer: FuzzingEngine, corpusDir: string) = + let + (path, target, ext) = splitFile(targetPath) + compiledExe = addFileExt(path / target, ExeExt) + corpusDir = if corpusDir.len > 0: corpusDir + else: path / "corpus" + + case fuzzer + of afl: + aflCompile(targetPath, clang) + aflExec(compiledExe, corpusDir, path / "results") + + of libFuzzer: + libFuzzerCompile(targetPath) + libFuzzerExec(compiledExe, corpusDir) + + of honggfuzz: + honggfuzzCompile(targetPath) + honggfuzzExec(compiledExe, corpusDir, path / "results") + diff --git a/vendor/nim-testutils/testutils/helpers.nim b/vendor/nim-testutils/testutils/helpers.nim new file mode 100644 index 000000000..6ef9eebd6 --- /dev/null +++ b/vendor/nim-testutils/testutils/helpers.nim @@ -0,0 +1,103 @@ +import std/os +import std/osproc +import std/strutils +import std/streams +import std/pegs + +type + CompileInfo* = object + templFile*: string + errorFile*: string + errorLine*, errorColumn*: int + templLine*, templColumn*: int + msg*: string + fullMsg*: string + compileTime*: float + exitCode*: int + +let + # Error pegs, taken from testament tester + pegLineTemplate = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*" + pegLineError = + peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" + pegOtherError = peg"'Error:' \s* {.*}" + pegError = pegLineError / pegOtherError + pegSuccess = peg"'Hint: operation successful' {[^;]*} '; ' {\d+} '.' {\d+} .*" + + # Timestamp pegs + # peg for unix timestamp, basically any float with 6 digits after the decimal + # Not ideal - could also improve by checking for the location in the line + pegUnixTimestamp = peg"{\d+} '.' {\d\d\d\d\d\d} \s" + # peg for timestamp with format yyyy-MM-dd HH:mm:sszzz + pegRfcTimestamp = peg"{\d\d\d\d} '-' {\d\d} '-' {\d\d} ' ' {\d\d} ':' {\d\d} ':' {\d\d} {'+' / '-'} {\d\d} ':' {\d\d} \s" + # Thread/process id is unpredictable.. + pegXid* = peg"""'tid' (('=') / ('":') / (': ') / (': ') / ('=') / ('>')) \d+""" + +proc cmpIgnorePegs*(a, b: string, pegs: varargs[Peg]): bool = + ## true when input strings are equal without regard to supplied pegs + var + aa = a + bb = b + for peg in pegs: + aa = aa.replace(peg, "dummy") + bb = bb.replace(peg, "dummy") + result = aa == bb + +proc cmpIgnoreTimestamp*(a, b: string, timestamp = ""): bool = + ## true when input strings are equal without regard to supplied timestamp form + if timestamp.len == 0: + result = cmpIgnorePegs(a, b, pegXid) + elif timestamp == "RfcTime": + result = cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid) + elif timestamp == "UnixTime": + result = cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid) + +proc cmpIgnoreDefaultTimestamps*(a, b: string): bool = + ## true when input strings are equal without regard to timestamp + if cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid): + result = true + elif cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid): + result = true + +proc parseCompileStream*(p: Process, output: Stream): CompileInfo = + ## parsing compiler output (based on testament tester) + result.exitCode = -1 + var + line = newStringOfCap(120).TaintedString + suc, err, tmpl = "" + + while true: + if output.readLine(line): + if line =~ pegError: + # `err` should contain the last error/warning message + err = line + elif line =~ pegLineTemplate and err == "": + # `tmpl` contains the last template expansion before the error + tmpl = line + elif line =~ pegSuccess: + suc = line + + if err != "": + result.fullMsg.add(line.string & "\p") + else: + result.exitCode = peekExitCode(p) + if result.exitCode != -1: + break + + if tmpl =~ pegLineTemplate: + result.templFile = extractFilename(matches[0]) + result.templLine = parseInt(matches[1]) + result.templColumn = parseInt(matches[2]) + if err =~ pegLineError: + result.errorFile = extractFilename(matches[0]) + result.errorLine = parseInt(matches[1]) + result.errorColumn = parseInt(matches[2]) + result.msg = matches[3] + elif err =~ pegOtherError: + result.msg = matches[0] + elif suc =~ pegSuccess: + result.msg = suc + result.compileTime = parseFloat(matches[1] & "." & matches[2]) + +proc parseExecuteOutput*() = discard diff --git a/vendor/nim-testutils/testutils/markdown_reports.nim b/vendor/nim-testutils/testutils/markdown_reports.nim new file mode 100644 index 000000000..34aa8b479 --- /dev/null +++ b/vendor/nim-testutils/testutils/markdown_reports.nim @@ -0,0 +1,44 @@ +import algorithm, sequtils, strutils, strformat, tables + +type + Status* {.pure.} = enum OK, Fail, Skip + +proc generateReport*(title: string; data: OrderedTable[string, OrderedTable[string, Status]]; + width = 63) = + ## Generate a markdown report from test data and write it to a file with the given title. + ## The table keys are sections, and the nested tables map tests to statuses. + let symbol: array[Status, string] = ["+", "-", " "] + var raw = "" + var okCountTotal = 0 + var failCountTotal = 0 + var skipCountTotal = 0 + raw.add(title & "\n") + raw.add("===\n") + for section, statuses in data: + raw.add("## " & section & "\n") + raw.add("```diff\n") + var sortedStatuses = statuses + sortedStatuses.sort do (a: (string, Status), b: (string, Status)) -> int: + cmp(a[0], b[0]) + var okCount = 0 + var failCount = 0 + var skipCount = 0 + for name, final in sortedStatuses: + let padded = alignLeft(name, width) + raw.add(&"{symbol[final]} {padded[0 ..< width]} {$final}\n") + case final + of Status.OK: okCount += 1 + of Status.Fail: failCount += 1 + of Status.Skip: skipCount += 1 + raw.add("```\n") + let sum = okCount + failCount + skipCount + okCountTotal += okCount + failCountTotal += failCount + skipCountTotal += skipCount + raw.add("OK: $1/$4 Fail: $2/$4 Skip: $3/$4\n" % [$okCount, $failCount, $skipCount, $sum]) + + let sumTotal = okCountTotal + failCountTotal + skipCountTotal + raw.add("\n---TOTAL---\n") + raw.add("OK: $1/$4 Fail: $2/$4 Skip: $3/$4\n" % [$okCountTotal, $failCountTotal, + $skipCountTotal, $sumTotal]) + writeFile(title & ".md", raw) diff --git a/vendor/nim-testutils/testutils/moduletests.nim b/vendor/nim-testutils/testutils/moduletests.nim new file mode 100644 index 000000000..6400a65e0 --- /dev/null +++ b/vendor/nim-testutils/testutils/moduletests.nim @@ -0,0 +1,22 @@ +{.used.} + +template tests*(body: untyped) = + template payload = + when not declared(unittest2): + import unittest2 + + body + + when defined(testutils_test_build): + payload() + else: + when not compiles(payload()): + payload() + +template programMain*(body: untyped) = + proc main = + body + + when not defined(testutils_test_build): + main() + diff --git a/vendor/nim-testutils/testutils/nimbletasks.nim b/vendor/nim-testutils/testutils/nimbletasks.nim new file mode 100644 index 000000000..fe84bbcea --- /dev/null +++ b/vendor/nim-testutils/testutils/nimbletasks.nim @@ -0,0 +1,5 @@ +template addTestutilsTasks* = + task moduleTests, "Run all module tests": + let (files, errCode) = gorgeEx("git grep -l 'tests:'") + echo files + diff --git a/vendor/nim-testutils/testutils/readme.md b/vendor/nim-testutils/testutils/readme.md new file mode 100644 index 000000000..482189138 --- /dev/null +++ b/vendor/nim-testutils/testutils/readme.md @@ -0,0 +1,189 @@ +# Testrunner [![Build Status](https://travis-ci.org/status-im/nim-testutils.svg?branch=master)](https://travis-ci.org/status-im/nim-testutils) +[![Build status](https://ci.appveyor.com/api/projects/status/ayqsnuvcpwo2nh6m/branch/master?svg=true)](https://ci.appveyor.com/project/nimbus/nim-testutils/branch/master) + +## Usage + +Command syntax: + +```sh + Usage: + ntu COMMAND [options] + + Available commands: + + $ ntu test [options] + + Run the test(s) specified at path. Will search recursively for test files + provided path is a directory. + + Options: + --backends:"c cpp js objc" Run tests for specified targets + --include:"test1 test2" Run only listed tests (space/comma separated) + --exclude:"test1 test2" Skip listed tests (space/comma separated) + --update Rewrite failed tests with new output + --sort:"source,test" Sort the tests by program and/or test mtime + --reverse Reverse the order of tests + --random Shuffle the order of tests + --help Display this help and exit + + $ ntu fuzz [options] + + Start a fuzzing test with a Nim module based on testutils/fuzzing. + + Options: + --fuzzer:libFuzzer The fuzzing engine to use. + Possible values: libFuzzer, honggfuzz, afl + --corpus: A directory with initial input cases +``` + +The runner will look recursively for all `*.test` files at given path. + +## Test file options + +The test files follow the configuration file syntax (similar as `.ini`), see +also [nim parsecfg module](https://nim-lang.org/docs/parsecfg.html). + +### Required + +- **program**: A test file should have at minimum a program name. This is the name +of the nim source minus the `.nim` extension. + +### Optional + +- **max_size**: To check the maximum size of the binary, in bytes. +- **timestamp_peg**: If you don't want to use the default timestamps, you can define +your own timestamp peg here. +- **compile_error**: When expecting a compilation failure, the error message that +should be expected. +- **error_file**: When expecting a compilation failure, the source file where the +error should occur. +- **os**: Space and/or comma separated list of operating systems for which the +test should be run. Defaults to `"linux, macosx, windows"`. Tests meant for a +different OS than the host will be marked as `SKIPPED`. +- **--skip**: This will simply skip the test (will not be marked as failure). + +### Forwarded Options +Any other options or key-value pairs will be forwarded to the nim compiler. + +A **key-value** pair will become a conditional symbol + value (`-d:SYMBOL(:VAL)`) +for the nim compiler, e.g. for `-d:chronicles_timestamps="UnixTime"` the test +file requires: +```ini +chronicles_timestamps="UnixTime" +``` +If only a key is given, an empty value will be forwarded. + +An **option** will be forwarded as is to the nim compiler, e.g. this can be +added in a test file: +```ini +--opt:size +``` + +### Verifying Expected Output + +For outputs to be compared, the output string should be set to the output name +(`stdout` or _filename_) from within an _Output_ section: + +```ini +[Output] +stdout="""expected stdout output""" +file.log="""expected file output""" +``` + +Triple quotes can be used for multiple lines. + +### Supplying Command-line Arguments + +Optionally specify command-line arguments as an escaped string in the following +syntax inside any _Output_ section: + +```ini +[Output] +args = "--title \"useful title\"" +``` + +### Multiple Invocations + +Multiple _Output_ sections denote multiple test program invocations. Any +failure of the test program to match its expected outputs will short-circuit +and fail the test. + +```ini +[Output] +stdout = "" +args = "--no-output" + +[Output_newlines] +stdout = "\n\n" +args = "--newlines" +``` + +### Updating Expected Outputs + +Pass the `--update` argument to `ntu` to rewrite any failing test with +the new outputs of the test. + +### Concurrent Test Execution + +When built with threads, `ntu` will run multiple test invocations +defined in each test file simultaneously. You can specify `nothreads` +in the _preamble_ to disable this behavior. + +```ini +nothreads = true + +[Output_1st_serial] +args = "--first" + +[Output_2nd_serial] +args = "--second" +``` + +The failure of any test will, when possible, short-circuit all other tests +defined in the same file. + +### CPU Affinity + +Specify `affinity` to clamp the first _N_ concurrent test threads to the first +_N_ CPU cores. + +```ini +affinity = true + +[Output_1st_core] +args = "--first" + +[Output_2nd_core] +args = "--second" +``` + +### Testing Alternate Backends + +By default, `ntu` builds tests using Nim's C backend. +Specify the `--backends` command-line option to build and run run tests with +the backends of your choice. + +```sh +$ ntu test --backends="c cpp" tests +``` + +### Setting the Order of Tests + +By default, `ntu` will order test compilation and execution according to the +modification time of the test program source. You can choose to sort by test +program mtime, too. + +```sh +$ ntu test --sort:test suite/ +``` + +You can `--reverse` or `--random`ize the order of tests, too. + +### More Examples + +See `chonicles`, where `testutils` was born: +- https://github.com/status-im/nim-chronicles/tree/master/tests + + +## License +Apache2 or MIT diff --git a/vendor/nim-testutils/testutils/spec.nim b/vendor/nim-testutils/testutils/spec.nim new file mode 100644 index 000000000..837007c34 --- /dev/null +++ b/vendor/nim-testutils/testutils/spec.nim @@ -0,0 +1,203 @@ +import std/hashes +import std/os +import std/parsecfg +import std/strutils +import std/streams +import std/strtabs + +import testutils/config + +type + TestOutputs* = StringTableRef + TestSpec* = ref object + section*: string + args*: string + config*: TestConfig + path*: string + pathComponents*: tuple[dir, name, ext: string] + skip*: bool + program*: string + flags*: string + outputs*: TestOutputs + timestampPeg*: string + errorMsg*: string + maxSize*: int64 + compileError*: string + errorFile*: string + errorLine*: int + errorColumn*: int + os*: seq[string] + child*: TestSpec + +const + DefaultOses = @["linux", "macosx", "windows"] + +proc hash*(spec: TestSpec): Hash = + var h: Hash = 0 + h = h !& spec.config.hash + h = h !& spec.flags.hash + h = h !& spec.os.hash + result = !$h + +proc binaryHash*(spec: TestSpec; backend: string): Hash = + ## hash the backend, any compilation flags, and defines, etc. + var h: Hash = 0 + h = h !& backend.hash + h = h !& spec.os.hash + h = h !& hash(spec.config.flags * compilerFlags) + h = h !& hash(spec.flags) + h = h !& spec.program.hash + h = h !& spec.pathComponents.name.hash + result = !$h + +template name*(spec: TestSpec): string = + spec.pathComponents.name + +proc newTestOutputs*(): StringTableRef = + result = newStringTable(mode = modeStyleInsensitive) + +proc clone*(spec: TestSpec): TestSpec = + ## create the parent of this test and set the child reference appropriately + result = new(TestSpec) + result[] = spec[] + result.outputs = newTestOutputs() + result.args = "" + result.child = spec + +func stage*(spec: TestSpec): string = + ## the name of the output section for the test + ## Output_test_section_name + let + # @["", "test_section_name"] + names = spec.section.split("Output") + result = names[^1].replace("_", " ").strip + +proc source*(spec: TestSpec): string = + result = absolutePath(spec.pathComponents.dir / spec.program.addFileExt(".nim")) + +proc binary*(spec: TestSpec; backend: string): string = + ## some day this will make more sense + result = (spec.pathComponents.dir / spec.pathComponents.name).addFileExt(ExeExt) + if dirExists(result): + result = result.addFileExt("out") + +proc binary*(spec: TestSpec): string {.deprecated.} = + ## the output binary (execution input) of the test + result = spec.binary("c") + +iterator binaries*(spec: TestSpec): string = + ## enumerate binary targets for each backend specified by the test + for backend in spec.config.backends.items: + yield spec.binary(backend) + +proc defaults(spec: var TestSpec) = + ## assert some default values for a given spec + spec.os = DefaultOses + spec.outputs = newTestOutputs() + +proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) = + ## parse a specification supplied prior to any sections + case event.key + of "program": + spec.program = event.value + of "timestamp_peg": + spec.timestampPeg = event.value + of "max_size": + try: + spec.maxSize = parseInt(event.value) + except ValueError: + echo "Parsing warning: value of " & event.key & + " is not a number (value = " & event.value & ")." + of "compile_error": + spec.compileError = event.value + of "error_file": + spec.errorFile = event.value + of "os": + spec.os = event.value.normalize.split({','} + Whitespace) + of "affinity": + spec.config.flags.incl CpuAffinity + of "threads": + spec.config.flags.incl UseThreads + of "nothreads": + spec.config.flags.excl UseThreads + of "release", "danger", "debug": + spec.config.flags.incl parseEnum[FlagKind]("--define:" & event.key) + else: + let + flag = "--define:$#:$#" % [event.key, event.value] + spec.flags.add flag.quoteShell & " " + +proc rewriteTestFile*(spec: TestSpec; outputs: TestOutputs) = + ## rewrite a test file with updated outputs after having run the tests + var + test = loadConfig(spec.path) + # take the opportunity to update an args statement if necessary + if spec.args != "": + test.setSectionKey(spec.section, "args", spec.args) + else: + test.delSectionKey(spec.section, "args") + # delete the old test outputs for completeness + for name, expected in spec.outputs.pairs: + test.delSectionKey(spec.section, name) + # add the new test outputs + for name, expected in outputs.pairs: + test.setSectionKey(spec.section, name, expected) + test.writeConfig(spec.path) + +proc parseTestFile*(config: TestConfig; filePath: string): TestSpec = + ## parse a test input file into a spec + result = new(TestSpec) + result.defaults + result.path = absolutePath(filePath) + result.pathComponents = splitFile result.path + result.config = config + block: + var + f = newFileStream(result.path, fmRead) + if f == nil: + # XXX crash? + echo "Parsing error: cannot open " & result.path + break + + var + outputSection = false + p: CfgParser + p.open(f, result.path) + try: + while true: + var e = next(p) + case e.kind + of cfgEof: + break + of cfgError: + # XXX crash? + echo "Parsing warning:" & e.msg + of cfgSectionStart: + # starts with Output + if e.section[0..len"Output"-1].cmpIgnoreCase("Output") == 0: + if outputSection: + # create our parent; the eternal chain + result = result.clone + outputSection = true + result.section = e.section + of cfgKeyValuePair: + if outputSection: + if e.key.cmpIgnoreStyle("args") == 0: + result.args = e.value + else: + result.outputs[e.key] = e.value + else: + result.consumeConfigEvent(e) + of cfgOption: + case e.key + of "skip": + result.skip = true + else: + # this for for, eg. --opt:size + result.flags &= ("--$#:$#" % [e.key, e.value]).quoteShell & " " + finally: + close p + + # we catch this in ntu and crash there if needed + if result.program == "": + echo "Parsing error: no program value" diff --git a/vendor/nim-testutils/testutils/unittests.nim b/vendor/nim-testutils/testutils/unittests.nim new file mode 100644 index 000000000..2d94d83fd --- /dev/null +++ b/vendor/nim-testutils/testutils/unittests.nim @@ -0,0 +1,17 @@ +import + unittest2 + +export + unittest2 + +template procSuite*(name: string, body: untyped) = + proc suitePayload = + suite name, body + + suitePayload() + +template asyncTest*(name, body: untyped) = + test name: + proc scenario {.async.} = body + waitFor scenario() + diff --git a/vendor/nim-unittest2/.github/workflows/ci.yml b/vendor/nim-unittest2/.github/workflows/ci.yml new file mode 100644 index 000000000..77c8c297b --- /dev/null +++ b/vendor/nim-unittest2/.github/workflows/ci.yml @@ -0,0 +1,148 @@ +name: nim-unittest2 CI +on: [push, pull_request] + +jobs: + build: + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 20 + matrix: + target: + - os: linux + cpu: amd64 + - os: linux + cpu: i386 + - os: macos + cpu: amd64 + - os: windows + cpu: amd64 + branch: [version-1-2, version-1-4, devel] + include: + - target: + os: linux + builder: ubuntu-20.04 + - target: + os: macos + builder: macos-10.15 + - target: + os: windows + builder: windows-2019 + name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})' + runs-on: ${{ matrix.builder }} + steps: + - name: Git checkout + uses: actions/checkout@v2 + + - name: Derive environment variables + shell: bash + run: | + if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then + ARCH=64 + PLATFORM=x64 + else + ARCH=32 + PLATFORM=x86 + fi + echo "ARCH=$ARCH" >> $GITHUB_ENV + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + + if [[ '${{ matrix.target.os }}' == 'windows' ]]; then + MAKE_CMD="mingw32-make" + else + MAKE_CMD="make" + fi + echo "MAKE_CMD=$MAKE_CMD" >> $GITHUB_ENV + + ncpu= + case '${{ runner.os }}' in + 'Linux') + ncpu=$(nproc) + ;; + 'macOS') + ncpu=$(sysctl -n hw.ncpu) + ;; + 'Windows') + ncpu=$NUMBER_OF_PROCESSORS + ;; + esac + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 + echo "ncpu=$ncpu" >> $GITHUB_ENV + + - name: Install build dependencies (Linux i386) + if: runner.os == 'Linux' && matrix.target.cpu == 'i386' + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update -qq + sudo DEBIAN_FRONTEND='noninteractive' apt-get install \ + --no-install-recommends -yq gcc-multilib g++-multilib + mkdir -p external/bin + cat << EOF > external/bin/gcc + #!/bin/bash + exec $(which gcc) -m32 -mno-adx "\$@" + EOF + cat << EOF > external/bin/g++ + #!/bin/bash + exec $(which g++) -m32 -mno-adx "\$@" + EOF + chmod 755 external/bin/gcc external/bin/g++ + echo "${{ github.workspace }}/external/bin" >> $GITHUB_PATH + + - name: Restore MinGW-W64 (Windows) from cache + if: runner.os == 'Windows' + id: windows-mingw-cache + uses: actions/cache@v2 + with: + path: external/mingw-${{ matrix.target.cpu }} + key: 'mingw-${{ matrix.target.cpu }}' + + - name: Restore Nim DLLs dependencies (Windows) from cache + if: runner.os == 'Windows' + id: windows-dlls-cache + uses: actions/cache@v2 + with: + path: external/dlls-${{ matrix.target.cpu }} + key: 'dlls-${{ matrix.target.cpu }}' + + - name: Install MinGW64 dependency (Windows) + if: > + steps.windows-mingw-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + shell: bash + run: | + mkdir -p external + curl -L "https://nim-lang.org/download/mingw$ARCH-6.3.0.7z" -o "external/mingw-${{ matrix.target.cpu }}.7z" + 7z x -y "external/mingw-${{ matrix.target.cpu }}.7z" -oexternal/ + mv external/mingw$ARCH external/mingw-${{ matrix.target.cpu }} + + - name: Install DLLs dependencies (Windows) + if: > + steps.windows-dlls-cache.outputs.cache-hit != 'true' && + runner.os == 'Windows' + shell: bash + run: | + mkdir -p external + curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip + 7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }} + + - name: Path to cached dependencies (Windows) + if: > + runner.os == 'Windows' + shell: bash + run: | + echo "${{ github.workspace }}/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH + + - name: Build the Nim compiler + shell: bash + run: | + git clone -b ${{ matrix.branch }} --depth 1 git://github.com/nim-lang/nim nim/ + curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} CC=gcc QUICK_AND_DIRTY_COMPILER=1 bash build_nim.sh nim csources dist/nimble NimBinaries + echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + + - name: Run tests + shell: bash + run: | + nim --hints:off --verbosity:0 test + diff --git a/vendor/nim-unittest2/.gitignore b/vendor/nim-unittest2/.gitignore new file mode 100644 index 000000000..5f3c68e1a --- /dev/null +++ b/vendor/nim-unittest2/.gitignore @@ -0,0 +1,2 @@ +nimcache/ + diff --git a/vendor/nim-unittest2/LICENSE.txt b/vendor/nim-unittest2/LICENSE.txt new file mode 100644 index 000000000..20091f0f9 --- /dev/null +++ b/vendor/nim-unittest2/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2011-2018 Zahary Karadjov. All rights reserved. +Copyright (c) 2018-2019 Ștefan Talpalaru + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +[ MIT license: http://www.opensource.org/licenses/mit-license.php ] diff --git a/vendor/nim-unittest2/README.md b/vendor/nim-unittest2/README.md new file mode 100644 index 000000000..7a8c14773 --- /dev/null +++ b/vendor/nim-unittest2/README.md @@ -0,0 +1,68 @@ +## Introduction + +**`unittest2`** is a library for writing unit tests for your [Nim](https://nim-lang.org/) programs in the spirit of [xUnit](https://en.wikipedia.org/wiki/XUnit). + +Features of `unittest2` include: + +* [Parallel test execution](https://status-im.github.io/nim-unittest2/unittest2.html#running-tests-in-parallel) +* Test separation with each test running in its own procedure +* Strict exception handling with support for [exception tracking](https://nim-lang.org/docs/manual.html#effect-system-exception-tracking) + +`unittest2` started as a [pull request](https://github.com/nim-lang/Nim/pull/9724) to evolve the [unittest](https://nim-lang.org/docs/unittest.html) module in Nim and has since grown into a separate library. + +## Installing + +```text +nimble install unittest2 +``` + +or add a dependency in your `.nimble` file: + +```text +requires "unittest2" +``` + +## Usage + +See [unittest2.html](https://status-im.github.io/nim-unittest2/unittest2.html) documentation generated by `nim doc unittest2.nim`. + +Create a file that contains your unit tests: + +```nim +import unittest2 + +suite "Suites can be used to group tests": + test "A test": + check: 1 + 1 == 2 +``` + +Compile and run the unit tests: +```bash +nim c -r test.nim +``` + +See the [tests](./tests) for more examples! + +## Porting code from `unittest` + +* Replace `import unittest` with `import unittest2` +* `unittest2` places each test in a separate `proc` which changes the way templates inside tests are interpreted - some code changes may be necessary + +## Testing `unittest2` + +```text +# this calls a task in "config.nims" +nim test +``` + +## License + +MIT + +## Credits + +- original author: Zahary Karadjov + +- fork author: Ștefan Talpalaru \ + +- homepage: https://github.com/status-im/nim-unittest2 diff --git a/vendor/nim-unittest2/config.nims b/vendor/nim-unittest2/config.nims new file mode 100644 index 000000000..4238552cc --- /dev/null +++ b/vendor/nim-unittest2/config.nims @@ -0,0 +1,8 @@ +task test, "Run tests": + for f in listFiles("tests"): + if f.len > 4 and f[^4..^1] == ".nim": + let cmd = "nim c -r -f --threads:on --hints:off --verbosity:0 --skipParentCfg:on --skipUserCfg:on " & f + echo cmd + exec cmd + rmFile(f[0..^5].toExe()) + diff --git a/vendor/nim-unittest2/nim.cfg b/vendor/nim-unittest2/nim.cfg new file mode 100644 index 000000000..23b72e21a --- /dev/null +++ b/vendor/nim-unittest2/nim.cfg @@ -0,0 +1,8 @@ +@if release: + nimcache = "nimcache/release/$projectName" +@else: + nimcache = "nimcache/debug/$projectName" +@end + +--threads:on + diff --git a/vendor/nim-unittest2/tests/nim.cfg b/vendor/nim-unittest2/tests/nim.cfg new file mode 100644 index 000000000..33312679d --- /dev/null +++ b/vendor/nim-unittest2/tests/nim.cfg @@ -0,0 +1,2 @@ +--path:"$projectDir/.." + diff --git a/vendor/nim-unittest2/tests/tunittest.nim b/vendor/nim-unittest2/tests/tunittest.nim new file mode 100644 index 000000000..370f753aa --- /dev/null +++ b/vendor/nim-unittest2/tests/tunittest.nim @@ -0,0 +1,183 @@ +discard """ + output: '''[Suite] suite with only teardown + +[Suite] suite with only setup + +[Suite] suite with none + +[Suite] suite with both + +[Suite] bug #4494 + +[Suite] bug #5571 + +[Suite] bug #5784 + +[Suite] test suite + +[Suite] test name filtering + +''' +""" + +import unittest2, sequtils + +proc doThings(spuds: var int): int = + spuds = 24 + return 99 +test "#964": + var spuds = 0 + check doThings(spuds) == 99 + check spuds == 24 + + +from strutils import toUpperAscii +test "#1384": + check(@["hello", "world"].map(toUpperAscii) == @["HELLO", "WORLD"]) + + +import options +test "unittest typedescs": + check(none(int) == none(int)) + check(none(int) != some(1)) + + +test "unittest multiple requires": + require(true) + require(true) + + +import random +from strutils import parseInt +proc defectiveRobot() = + randomize() + case rand(1..4) + of 1: raise newException(OSError, "CANNOT COMPUTE!") + of 2: discard parseInt("Hello World!") + of 3: raise newException(IOError, "I can't do that Dave.") + else: assert 2 + 2 == 5 +test "unittest expect": + expect IOError, OSError, ValueError, AssertionError: + defectiveRobot() + +var + a = 1 + b = -1 + c = 1 + +#unittests are sequential right now +suite "suite with only teardown": + teardown: + b = 2 + + test "unittest with only teardown 1": + check a == c + + test "unittest with only teardown 2": + check b > a + +suite "suite with only setup": + setup: + var testVar {.used.} = "from setup" + + test "unittest with only setup 1": + check testVar == "from setup" + check b > a + b = -1 + + test "unittest with only setup 2": + check b < a + +suite "suite with none": + test "unittest with none": + check b < a + +suite "suite with both": + setup: + a = -2 + + teardown: + c = 2 + + test "unittest with both 1": + check b > a + + test "unittest with both 2": + check c == 2 + +suite "bug #4494": + test "Uniqueness check": + var tags = @[1, 2, 3, 4, 5] + check: + allIt(0..3, tags[it] != tags[it + 1]) + +suite "bug #5571": + test "can define gcsafe procs within tests": + proc doTest {.gcsafe.} = + let line = "a" + check: line == "a" + doTest() + +suite "bug #5784": + test "`or` should short circuit": + type Obj = ref object + field: int + var obj: Obj + check obj.isNil or obj.field == 0 + +type + SomeType = object + value: int + children: seq[SomeType] + +# bug #5252 + +proc `==`(a, b: SomeType): bool = + return a.value == b.value + +suite "test suite": + test "test": + let a = SomeType(value: 10) + let b = SomeType(value: 10) + + check(a == b) + +when defined(testing): + suite "test name filtering": + test "test name": + check matchFilter("suite1", "foo", "") + check matchFilter("suite1", "foo", "foo") + check matchFilter("suite1", "foo", "::") + check matchFilter("suite1", "foo", "*") + check matchFilter("suite1", "foo", "::foo") + check matchFilter("suite1", "::foo", "::foo") + + test "test name - glob": + check matchFilter("suite1", "foo", "f*") + check matchFilter("suite1", "foo", "*oo") + check matchFilter("suite1", "12345", "12*345") + check matchFilter("suite1", "q*wefoo", "q*wefoo") + check false == matchFilter("suite1", "foo", "::x") + check false == matchFilter("suite1", "foo", "::x*") + check false == matchFilter("suite1", "foo", "::*x") + # overlap + check false == matchFilter("suite1", "12345", "123*345") + check matchFilter("suite1", "ab*c::d*e::f", "ab*c::d*e::f") + + test "suite name": + check matchFilter("suite1", "foo", "suite1::") + check false == matchFilter("suite1", "foo", "suite2::") + check matchFilter("suite1", "qwe::foo", "qwe::foo") + check matchFilter("suite1", "qwe::foo", "suite1::qwe::foo") + + test "suite name - glob": + check matchFilter("suite1", "foo", "::*") + check matchFilter("suite1", "foo", "*::*") + check matchFilter("suite1", "foo", "*::foo") + check false == matchFilter("suite1", "foo", "*ite2::") + check matchFilter("suite1", "q**we::foo", "q**we::foo") + check matchFilter("suite1", "a::b*c::d*e", "a::b*c::d*e") + +# Also supposed to work outside tests: +check 1 == 1 + diff --git a/vendor/nim-unittest2/tests/tunittestparallel.nim b/vendor/nim-unittest2/tests/tunittestparallel.nim new file mode 100644 index 000000000..e7ed1fc6f --- /dev/null +++ b/vendor/nim-unittest2/tests/tunittestparallel.nim @@ -0,0 +1,134 @@ +discard """ + output: '''[Suite] suite #1 + +[Suite] suite #2 +''' +""" + +# Unfortunately, it's not possible to decouple the thread execution order from +# the number of available cores, due to how threadpool dynamically (and lazily) +# adjusts the number of worker threads, so we can't have a PRINT_ALL output in +# the verification section above. + +import unittest2, os + +test "independent test #1": + sleep(1000) + check 1 == 1 + # check 1 == 2 + # require 1 == 2 + # var g {.global.}: seq[int] + # g.add(1) + +test "independent test #2": + sleep(800) + check 1 == 1 + +test "independent test #3": + ## nested tests + # we might as well keep this futile attempt at finding a problem with + # uninitialized `flowVars` in child threads + test "independent test #4": + test "independent test #5": + test "independent test #8": + test "independent test #9": + test "independent test #10": + test "independent test #11": + test "independent test #12": + test "independent test #13": + test "independent test #14": + test "independent test #15": + sleep(200) + check 1 == 1 + test "independent test #16": + sleep(200) + check 1 == 1 + test "independent test #17": + sleep(200) + check 1 == 1 + test "independent test #18": + sleep(200) + check 1 == 1 + test "independent test #19": + sleep(200) + check 1 == 1 + test "independent test #20": + sleep(200) + check 1 == 1 + test "independent test #21": + sleep(200) + check 1 == 1 + test "independent test #22": + sleep(200) + check 1 == 1 + test "independent test #23": + sleep(200) + check 1 == 1 + test "independent test #24": + sleep(200) + check 1 == 1 + test "independent test #25": + test "independent test #26": + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(200) + check 1 == 1 + sleep(400) + check 1 == 1 + sleep(600) + check 1 == 1 + +suite "suite #1": + test "suite #1, test #1": + sleep(400) + check 1 == 1 + + test "suite #1, test #2": + sleep(300) + check 1 == 1 + +# change the output formatter just for the next suite +resetOutputFormatters() +addOutputFormatter(newConsoleOutputFormatter(colorOutput=false)) + +suite "suite #2": + test "suite #2, test #1": + sleep(200) + check 1 == 1 + + test "suite #2, test #2": + sleep(100) + check 1 == 1 + + suiteTeardown: + echo "suite teardown" + + echo "this will be shown first" + +# go back to the default one +resetOutputFormatters() + +test "independent test #6": + sleep(200) + check 1 == 1 + +test "independent test #7": + sleep(100) + check 1 == 1 + diff --git a/vendor/nim-unittest2/tests/tunittestparallel.nim.cfg b/vendor/nim-unittest2/tests/tunittestparallel.nim.cfg new file mode 100644 index 000000000..650d3b5ec --- /dev/null +++ b/vendor/nim-unittest2/tests/tunittestparallel.nim.cfg @@ -0,0 +1,2 @@ +-d:nimtestParallel + diff --git a/vendor/nim-unittest2/unittest2.html b/vendor/nim-unittest2/unittest2.html new file mode 100644 index 000000000..3b19fdb62 --- /dev/null +++ b/vendor/nim-unittest2/unittest2.html @@ -0,0 +1,1830 @@ + + + + + + + + + + + + + + + + + +unittest2 + + + + + + + + +
+
+

unittest2

+
+
+ +
+ Search: +
+
+ Group by: + +
+ + +
+
+
+

+
Authors: Zahary Karadjov, Ștefan Talpalaru

This module implements boilerplate to make unit testing easy.

+

The test status and name is printed after any output or traceback.

+

Tests can be nested, however failure of a nested test will not mark the parent test as failed. Setup and teardown are inherited. Setup can be overridden locally.

+

Compiled test files return the number of failed test as exit code, while

+
nim c -r testfile.nim

exits with 0 or 1.

+ +

Running individual tests

Specify the test names as command line arguments.

+
+nim c -r test "my test name" "another test"

Multiple arguments can be used.

+ +

Running a single test suite

Specify the suite name delimited by "::".

+
+nim c -r test "my suite name::"
+

Selecting tests by pattern

A single "*" can be used for globbing.

+

Delimit the end of a suite name with "::".

+

Tests matching any of the arguments are executed.

+
+nim c -r test fast_suite::mytest1 fast_suite::mytest2
+nim c -r test "fast_suite::mytest*"
+nim c -r test "auth*::" "crypto::hashing*"
+# Run suites starting with 'bug #' and standalone tests starting with '#'
+nim c -r test 'bug #*::' '::#*'
+

Running tests in parallel

To enable the threadpool-based test parallelisation, "--threads:on" needs to be passed to the compiler, along with "-d:nimtestParallel" or the NIMTEST_PARALLEL environment variable:

+
+nim c -r --threads:on -d:nimtestParallel testfile.nim
+# or
+NIMTEST_PARALLEL=1 nim c -r --threads:on testfile.nim

There are some implicit barriers where we wait for all the spawned jobs to complete: before and after each test suite and at the main thread's exit.

+

The suite-related barriers are there to avoid mixing test output, but they also affect which groups of tests can be run in parallel, so keep them in mind when deciding how many tests to place in different suites (or between suites).

+

You may sometimes need to disable test parallelisation for a specific test, even though it was enabled in some configuration file in a parent dir. Do this with "-d:nimtestParallelDisabled" which overrides everything else.

+ +

Example

suite "description for this stuff":
+  echo "suite setup: run once before the tests"
+  
+  setup:
+    echo "run before each test"
+  
+  teardown:
+    echo "run after each test"
+  
+  test "essential truths":
+    # give up and stop if this fails
+    require(true)
+  
+  test "slightly less obvious stuff":
+    # print a nasty message and move on, skipping
+    # the remainder of this block
+    check(1 != 1)
+    check("asd"[2] == 'd')
+  
+  test "out of bounds error is thrown on bad access":
+    let v = @[1, 2, 3]  # you can do initialization here
+    expect(IndexError):
+      discard v[4]
+  
+  suiteTeardown:
+    echo "suite teardown: run once after the tests"

+
+

Types

+
+ +
TestStatus = enum
+  OK, FAILED, SKIPPED
+
+The status of a test when it is done. + +
+ +
OutputLevel = enum
+  PRINT_ALL,                  ## Print as much as possible.
+  PRINT_FAILURES,             ## Print only the failed tests.
+  PRINT_NONE                  ## Print nothing.
+
+The output verbosity of the tests. + +
+ +
TestResult = object
+  suiteName*: string ## Name of the test suite that contains this test case.
+                   ## Can be ``nil`` if the test case is not in a suite.
+  testName*: string            ## Name of the test case
+  status*: TestStatus
+
+
+ + +
+ +
OutputFormatter = ref object of RootObj
+
+ + +
+ +
ConsoleOutputFormatter = ref object of OutputFormatter
+  colorOutput: bool            ## Have test results printed in color.
+                  ## Default is true for the non-js target,
+                  ## for which ``stdout`` is a tty.
+                  ## Setting the environment variable
+                  ## ``NIMTEST_COLOR`` to ``always`` or
+                  ## ``never`` changes the default for the
+                  ## non-js target to true or false respectively.
+                  ## The deprecated environment variable
+                  ## ``NIMTEST_NO_COLOR``, when set,
+                  ## changes the defualt to true, if
+                  ## ``NIMTEST_COLOR`` is undefined.
+  outputLevel: OutputLevel     ## Set the verbosity of test results.
+                         ## Default is ``PRINT_ALL``, unless
+                         ## the ``NIMTEST_OUTPUT_LVL`` environment
+                         ## variable is set for the non-js target.
+  isInSuite: bool
+  isInTest: bool
+
+
+ + +
+ +
JUnitOutputFormatter = ref object of OutputFormatter
+  stream: Stream
+  testErrors: seq[string]
+  testStartTime: float
+  testStackTrace: string
+
+
+ + +
+ +
+
+

Vars

+
+ +
abortOnError: bool
+
+Set to true in order to quit immediately on fail. Default is false, unless the NIMTEST_ABORT_ON_ERROR environment variable is set for the non-js target. + +
+ +
+
+

Consts

+
+ +
paralleliseTests = false
+
+Whether parallel test running was enabled (set at compile time). This constant might be useful in custom output formatters. + +
+ +
+
+

Procs

+
+ +
proc clearOutputFormatters() {...}{.raises: [], tags: [].}
+
+ + +
+ +
proc addOutputFormatter(formatter: OutputFormatter) {...}{.raises: [], tags: [].}
+
+ + +
+ +
proc newConsoleOutputFormatter(outputLevel: OutputLevel = PRINT_ALL;
+                              colorOutput = true): ConsoleOutputFormatter {...}{.
+    raises: [], tags: [].}
+
+ + +
+ +
proc defaultConsoleFormatter(): ConsoleOutputFormatter {...}{.raises: [],
+    tags: [ReadEnvEffect].}
+
+ + +
+ +
proc newJUnitOutputFormatter(stream: Stream): JUnitOutputFormatter {...}{.
+    raises: [Exception], tags: [WriteIOEffect].}
+
+Creates a formatter that writes report to the specified stream in JUnit format. The stream is NOT closed automatically when the test are finished, because the formatter has no way to know when all tests are finished. You should invoke formatter.close() to finalize the report. + +
+ +
proc close(formatter: JUnitOutputFormatter) {...}{.raises: [Exception],
+    tags: [WriteIOEffect].}
+
+Completes the report and closes the underlying stream. + +
+ +
proc checkpoint(msg: string) {...}{.raises: [], tags: [].}
+
+Set a checkpoint identified by msg. Upon test failure all checkpoints encountered so far are printed out. Example:
checkpoint("Checkpoint A")
+check((42, "the Answer to life and everything") == (1, "a"))
+checkpoint("Checkpoint B")

outputs "Checkpoint A" once it fails.

+ + +
+ +
proc disableParamFiltering() {...}{.raises: [], tags: [].}
+
+disables filtering tests with the command line params + +
+ +
+
+

Methods

+
+ +
method suiteStarted(formatter: OutputFormatter; suiteName: string) {...}{.base, gcsafe,
+    raises: [], tags: [].}
+
+ + +
+ +
method testStarted(formatter: OutputFormatter; testName: string) {...}{.base, gcsafe,
+    raises: [], tags: [].}
+
+ + +
+ +
method failureOccurred(formatter: OutputFormatter; checkpoints: seq[string];
+                      stackTrace: string) {...}{.base, gcsafe, raises: [], tags: [].}
+
+stackTrace is provided only if the failure occurred due to an exception. checkpoints is never nil. + +
+ +
method testEnded(formatter: OutputFormatter; testResult: TestResult) {...}{.base, gcsafe,
+    raises: [], tags: [].}
+
+ + +
+ +
method suiteEnded(formatter: OutputFormatter) {...}{.base, gcsafe, raises: [], tags: [].}
+
+ + +
+ +
method suiteStarted(formatter: ConsoleOutputFormatter; suiteName: string) {...}{.
+    raises: [IOError], tags: [WriteIOEffect].}
+
+ + +
+ +
method testStarted(formatter: ConsoleOutputFormatter; testName: string) {...}{.raises: [],
+    tags: [].}
+
+ + +
+ +
method failureOccurred(formatter: ConsoleOutputFormatter; checkpoints: seq[string];
+                      stackTrace: string) {...}{.raises: [], tags: [].}
+
+ + +
+ +
method testEnded(formatter: ConsoleOutputFormatter; testResult: TestResult) {...}{.
+    raises: [IOError], tags: [WriteIOEffect].}
+
+ + +
+ +
method suiteEnded(formatter: ConsoleOutputFormatter) {...}{.raises: [], tags: [].}
+
+ + +
+ +
method suiteStarted(formatter: JUnitOutputFormatter; suiteName: string) {...}{.
+    raises: [Exception, ValueError], tags: [WriteIOEffect].}
+
+ + +
+ +
method testStarted(formatter: JUnitOutputFormatter; testName: string) {...}{.raises: [],
+    tags: [TimeEffect].}
+
+ + +
+ +
method failureOccurred(formatter: JUnitOutputFormatter; checkpoints: seq[string];
+                      stackTrace: string) {...}{.raises: [], tags: [].}
+
+stackTrace is provided only if the failure occurred due to an exception. checkpoints is never nil. + +
+ +
method testEnded(formatter: JUnitOutputFormatter; testResult: TestResult) {...}{.
+    raises: [Exception, ValueError], tags: [TimeEffect, WriteIOEffect].}
+
+ + +
+ +
method suiteEnded(formatter: JUnitOutputFormatter) {...}{.raises: [Exception],
+    tags: [WriteIOEffect].}
+
+ + +
+ +
+
+

Macros

+
+ +
macro check(conditions: untyped): untyped
+
+Verify if a statement or a list of statements is true. A helpful error message and set checkpoints are printed out on failure (if outputLevel is not PRINT_NONE). Example:
import strutils
+
+check("AKB48".toLowerAscii() == "akb48")
+
+let teams = {'A', 'K', 'B', '4', '8'}
+
+check:
+  "AKB48".toLowerAscii() == "akb48"
+  'C' in teams
+ +
+ +
macro expect(exceptions: varargs[typed]; body: untyped): untyped
+
+Test if body raises an exception found in the passed exceptions. The test passes if the raised exception is part of the acceptable exceptions. Otherwise, it fails. Example:
import math, random
+proc defectiveRobot() =
+  randomize()
+  case random(1..4)
+  of 1: raise newException(OSError, "CANNOT COMPUTE!")
+  of 2: discard parseInt("Hello World!")
+  of 3: raise newException(IOError, "I can't do that Dave.")
+  else: assert 2 + 2 == 5
+
+expect IOError, OSError, ValueError, AssertionError:
+  defectiveRobot()
+ +
+ +
+
+

Templates

+
+ +
template suite(name, body) {...}{.dirty.}
+
+

Declare a test suite identified by name with optional setup and/or teardown section.

+

A test suite is a series of one or more related tests sharing a common fixture (setup, teardown). The fixture is executed for EACH test.

+
suite "test suite for addition":
+  setup:
+    let result = 4
+  
+  test "2 + 2 = 4":
+    check(2+2 == result)
+  
+  test "(2 + -2) != 4":
+    check(2 + -2 != result)
+  
+  # No teardown needed

The suite will run the individual test cases in the order in which they were listed. With default global settings the above code prints:

+
[Suite] test suite for addition
+  [OK] 2 + 2 = 4
+  [OK] (2 + -2) != 4
+ +
+ +
template test(name, body)
+
+Define a single test case identified by name.
test "roses are red":
+  let roses = "red"
+  check(roses == "red")

The above code outputs:

+
[OK] roses are red
+ +
+ +
template fail()
+
+Print out the checkpoints encountered so far and quit if abortOnError is true. Otherwise, erase the checkpoints and indicate the test has failed (change exit code and test status). This template is useful for debugging, but is otherwise mostly used internally. Example:
checkpoint("Checkpoint A")
+complicatedProcInThread()
+fail()

outputs "Checkpoint A" before quitting.

+ + +
+ +
template skip()
+
+Mark the test as skipped. Should be used directly in case when it is not possible to perform test for reasons depending on outer environment, or certain application logic conditions or configurations. The test code is still executed.
if not isGLConextCreated():
+  skip()
+ +
+ +
template require(conditions: untyped)
+
+Same as check except any failed test causes the program to quit immediately. Any teardown statements are not executed and the failed test output is not generated. + +
+ +
+ +
+
+ +
+
+ +
+ Made with Nim. Generated: 2019-05-29 14:32:15 UTC +
+
+
+
+ + + diff --git a/vendor/nim-unittest2/unittest2.nim b/vendor/nim-unittest2/unittest2.nim new file mode 100644 index 000000000..1718f809f --- /dev/null +++ b/vendor/nim-unittest2/unittest2.nim @@ -0,0 +1,909 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# (c) Copyright 2019-2021 Ștefan Talpalaru +# (c) Copyright 2021-Onwards Status Research and Development +# + +{.push raises: [Defect].} + +## :Authors: Zahary Karadjov, Stefan Talpalaru +## +## This module implements boilerplate to make unit testing easy. +## +## The test status and name is printed after any output or traceback. +## +## Tests can be nested, however failure of a nested test will not mark the +## parent test as failed. Setup and teardown are inherited. Setup can be +## overridden locally. +## +## Compiled test files return the number of failed test as exit code, while +## +## .. code:: +## nim c -r testfile.nim +## +## exits with 0 or 1. +## +## Running individual tests +## ======================== +## +## Specify the test names as command line arguments. +## +## .. code:: +## +## nim c -r test "my test name" "another test" +## +## Multiple arguments can be used. +## +## Running a single test suite +## =========================== +## +## Specify the suite name delimited by ``"::"``. +## +## .. code:: +## +## nim c -r test "my suite name::" +## +## Selecting tests by pattern +## ========================== +## +## A single ``"*"`` can be used for globbing. +## +## Delimit the end of a suite name with ``"::"``. +## +## Tests matching **any** of the arguments are executed. +## +## .. code:: +## +## nim c -r test fast_suite::mytest1 fast_suite::mytest2 +## nim c -r test "fast_suite::mytest*" +## nim c -r test "auth*::" "crypto::hashing*" +## # Run suites starting with 'bug #' and standalone tests starting with '#' +## nim c -r test 'bug #*::' '::#*' +## +## Running tests in parallel +## ========================= +## +## To enable the threadpool-based test parallelisation, "--threads:on" needs to +## be passed to the compiler, along with "-d:nimtestParallel" or the +## NIMTEST_PARALLEL environment variable: +## +## .. code:: +## +## nim c -r --threads:on -d:nimtestParallel testfile.nim +## # or +## NIMTEST_PARALLEL=1 nim c -r --threads:on testfile.nim +## +## There are some implicit barriers where we wait for all the spawned jobs to +## complete: before and after each test suite and at the main thread's exit. +## +## The suite-related barriers are there to avoid mixing test output, but they +## also affect which groups of tests can be run in parallel, so keep them in +## mind when deciding how many tests to place in different suites (or between +## suites). +## +## You may sometimes need to disable test parallelisation for a specific test, +## even though it was enabled in some configuration file in a parent dir. Do +## this with "-d:nimtestParallelDisabled" which overrides everything else. +## +## Example +## ------- +## +## .. code:: nim +## +## suite "description for this stuff": +## echo "suite setup: run once before the tests" +## +## setup: +## echo "run before each test" +## +## teardown: +## echo "run after each test" +## +## test "essential truths": +## # give up and stop if this fails +## require(true) +## +## test "slightly less obvious stuff": +## # print a nasty message and move on, skipping +## # the remainder of this block +## check(1 != 1) +## check("asd"[2] == 'd') +## +## test "out of bounds error is thrown on bad access": +## let v = @[1, 2, 3] # you can do initialization here +## expect(IndexError): +## discard v[4] +## +## suiteTeardown: +## echo "suite teardown: run once after the tests" + +import std/[locks, macros, sets, strutils, streams, times] + +when declared(stdout): + import std/os + +const useTerminal = not defined(js) + +when useTerminal: + import std/terminal + +when declared(stdout): + const paralleliseTests* = (existsEnv("NIMTEST_PARALLEL") or defined(nimtestParallel) and not defined(nimtestParallelDisabled)) + ## Whether parallel test running was enabled (set at compile time). + ## This constant might be useful in custom output formatters. +else: + const paralleliseTests* = false + +when paralleliseTests: + import threadpool + + # repeatedly calling sync() without waiting for results - on procs that don't + # return any - doesn't work properly (probably due to gSomeReady getting its + # counter increased back to the pre-call value) so we're stuck with these + # dummy flowvars + # (`flowVars` will be initialized in each child thread, when using nested tests, by the compiler) + # TODO: try getting rid of them when nim-0.20.0 is released + var flowVars {.threadvar.}: seq[FlowVarBase] + proc repeatableSync*() = + sync() + for flowVar in flowVars: + when (NimMajor, NimMinor, NimPatch) >= (1, 4, 0): + blockUntil(flowVar[]) + else: + blockUntil(flowVar) + flowVars = @[] + + # make sure all the spawned tests are done before exiting + # (this will be the last sync, so no need for repeatability) + let mainThreadID = getThreadId() + proc quitProc() {.noconv.} = + # "require" can exit from a worker thread and syncing in there would block + if getThreadId() == mainThreadID: + sync() + addQuitProc(quitProc) + + var outputLock: Lock # used by testEnded() to avoid mixed test outputs + initLock(outputLock) + +type + TestStatus* = enum ## The status of a test when it is done. + OK, + FAILED, + SKIPPED + + OutputLevel* = enum ## The output verbosity of the tests. + PRINT_ALL, ## Print as much as possible. + PRINT_FAILURES, ## Print only the failed tests. + PRINT_NONE ## Print nothing. + + TestResult* = object + suiteName*: string + ## Name of the test suite that contains this test case. + ## Can be ``nil`` if the test case is not in a suite. + testName*: string + ## Name of the test case + status*: TestStatus + + OutputFormatter* = ref object of RootObj + + ConsoleOutputFormatter* = ref object of OutputFormatter + colorOutput: bool + ## Have test results printed in color. + ## Default is `auto` depending on `isatty(stdout)`, or override it with + ## `-d:nimUnittestColor:auto|on|off`. + ## + ## Deprecated: Setting the environment variable `NIMTEST_COLOR` to `always` + ## or `never` changes the default for the non-js target to true or false respectively. + ## Deprecated: the environment variable `NIMTEST_NO_COLOR`, when set, changes the + ## default to true, if `NIMTEST_COLOR` is undefined. + outputLevel: OutputLevel + ## Set the verbosity of test results. + ## Default is `PRINT_ALL`, or override with: + ## `-d:nimUnittestOutputLevel:PRINT_ALL|PRINT_FAILURES|PRINT_NONE`. + ## + ## Deprecated: the `NIMTEST_OUTPUT_LVL` environment variable is set for the non-js target. + isInSuite: bool + isInTest: bool + + JUnitOutputFormatter* = ref object of OutputFormatter + stream: Stream + testErrors: seq[string] + testStartTime: float + testStackTrace: string + +var + abortOnError* {.threadvar.}: bool ## Set to true in order to quit + ## immediately on fail. Default is false, + ## or override with `-d:nimUnittestAbortOnError:on|off`. + ## + ## Deprecated: can also override depending on whether + ## `NIMTEST_ABORT_ON_ERROR` environment variable is set. + + checkpoints {.threadvar.}: seq[string] + formattersLock: Lock + formatters {.guard: formattersLock.}: seq[OutputFormatter] + testFiltersLock: Lock + testsFilters {.guard: testFiltersLock.}: HashSet[string] + disabledParamFiltering: bool + +const + outputLevelDefault = PRINT_ALL + nimUnittestOutputLevel {.strdefine.} = $outputLevelDefault + nimUnittestColor {.strdefine.} = "auto" ## auto|on|off + nimUnittestAbortOnError {.booldefine.} = false + +initLock(formattersLock) +initLock(testFiltersLock) + +template deprecateEnvVarHere() = + # xxx issue a runtime warning to deprecate this envvar. + discard + +abortOnError = nimUnittestAbortOnError +when declared(stdout): + if existsEnv("NIMTEST_ABORT_ON_ERROR"): + deprecateEnvVarHere() + abortOnError = true + +method suiteStarted*(formatter: OutputFormatter, suiteName: string) {.base, gcsafe.} = + discard +method testStarted*(formatter: OutputFormatter, testName: string) {.base, gcsafe.} = + discard +method failureOccurred*(formatter: OutputFormatter, checkpoints: seq[string], + stackTrace: string) {.base, gcsafe.} = + ## ``stackTrace`` is provided only if the failure occurred due to an exception. + ## ``checkpoints`` is never ``nil``. + discard +method testEnded*(formatter: OutputFormatter, testResult: TestResult) {.base, gcsafe.} = + discard +method suiteEnded*(formatter: OutputFormatter) {.base, gcsafe.} = + discard + +proc addOutputFormatter*(formatter: OutputFormatter) = + withLock formattersLock: + {.gcsafe.}: + formatters.add(formatter) + +proc resetOutputFormatters*() = + withLock formattersLock: + {.gcsafe.}: + formatters = @[] + +proc newConsoleOutputFormatter*(outputLevel: OutputLevel = outputLevelDefault, + colorOutput = true): ConsoleOutputFormatter = + ConsoleOutputFormatter( + outputLevel: outputLevel, + colorOutput: colorOutput + ) + +proc colorOutput(): bool = + let color = nimUnittestColor + case color + of "auto": + when declared(stdout): result = isatty(stdout) + else: result = false + of "on": result = true + of "off": result = false + else: doAssert false, $color + + when declared(stdout): + if existsEnv("NIMTEST_COLOR"): + let colorEnv = getEnv("NIMTEST_COLOR") + if colorEnv == "never": + result = false + elif colorEnv == "always": + result = true + elif existsEnv("NIMTEST_NO_COLOR"): + result = false + +proc defaultConsoleFormatter*(): ConsoleOutputFormatter = + var colorOutput = colorOutput() + var outputLevel = static: nimUnittestOutputLevel.parseEnum[:OutputLevel] + when declared(stdout): + const a = "NIMTEST_OUTPUT_LVL" + if existsEnv(a): + try: + outputLevel = getEnv(a).parseEnum[:OutputLevel] + except ValueError as exc: + echo "Cannot parse NIMTEST_OUTPUT_LVL: ", exc.msg + quit 1 + + result = newConsoleOutputFormatter(outputLevel, colorOutput) + +method suiteStarted*(formatter: ConsoleOutputFormatter, suiteName: string) = + template rawPrint() = echo("\n[Suite] ", suiteName) + when useTerminal: + if formatter.colorOutput: + try: + styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, suiteName + except Exception: rawPrint() # Work around exceptions in `terminal.nim` + else: rawPrint() + else: rawPrint() + formatter.isInSuite = true + +method testStarted*(formatter: ConsoleOutputFormatter, testName: string) = + formatter.isInTest = true + +method failureOccurred*(formatter: ConsoleOutputFormatter, + checkpoints: seq[string], stackTrace: string) = + if stackTrace.len > 0: + echo stackTrace + let prefix = if formatter.isInSuite: " " else: "" + for msg in items(checkpoints): + echo prefix, msg + +method testEnded*(formatter: ConsoleOutputFormatter, testResult: TestResult) = + formatter.isInTest = false + + if formatter.outputLevel != OutputLevel.PRINT_NONE and + (formatter.outputLevel == OutputLevel.PRINT_ALL or testResult.status == TestStatus.FAILED): + let prefix = if testResult.suiteName.len > 0: " " else: "" + template rawPrint() = echo(prefix, "[", $testResult.status, "] ", + testResult.testName) + when useTerminal: + if formatter.colorOutput: + var color = case testResult.status + of TestStatus.OK: fgGreen + of TestStatus.FAILED: fgRed + of TestStatus.SKIPPED: fgYellow + try: + styledEcho styleBright, color, prefix, "[", $testResult.status, "] ", + resetStyle, testResult.testName + except Exception: rawPrint() # Work around exceptions in `terminal.nim` + else: + rawPrint() + else: + rawPrint() + +method suiteEnded*(formatter: ConsoleOutputFormatter) = + formatter.isInSuite = false + +proc xmlEscape(s: string): string = + result = newStringOfCap(s.len) + for c in items(s): + case c: + of '<': result.add("<") + of '>': result.add(">") + of '&': result.add("&") + of '"': result.add(""") + of '\'': result.add("'") + else: + if ord(c) < 32: + result.add("&#" & $ord(c) & ';') + else: + result.add(c) + +proc newJUnitOutputFormatter*(stream: Stream): JUnitOutputFormatter = + ## Creates a formatter that writes report to the specified stream in + ## JUnit format. + ## The ``stream`` is NOT closed automatically when the test are finished, + ## because the formatter has no way to know when all tests are finished. + ## You should invoke formatter.close() to finalize the report. + result = JUnitOutputFormatter( + stream: stream, + testErrors: @[], + testStackTrace: "", + testStartTime: 0.0 + ) + try: + stream.writeLine("") + stream.writeLine("") + except CatchableError as exc: + echo "Cannot write JUnit: ", exc.msg + quit 1 + +proc close*(formatter: JUnitOutputFormatter) = + ## Completes the report and closes the underlying stream. + try: + formatter.stream.writeLine("") + formatter.stream.close() + except Exception as exc: # Work around Exception raised in stream + echo "Cannot write JUnit: ", exc.msg + quit 1 + +method suiteStarted*(formatter: JUnitOutputFormatter, suiteName: string) = + try: + formatter.stream.writeLine("\t" % xmlEscape(suiteName)) + except CatchableError as exc: + echo "Cannot write JUnit: ", exc.msg + quit 1 + +method testStarted*(formatter: JUnitOutputFormatter, testName: string) = + formatter.testErrors.setLen(0) + formatter.testStackTrace.setLen(0) + formatter.testStartTime = epochTime() + +method failureOccurred*(formatter: JUnitOutputFormatter, + checkpoints: seq[string], stackTrace: string) = + ## ``stackTrace`` is provided only if the failure occurred due to an exception. + ## ``checkpoints`` is never ``nil``. + formatter.testErrors.add(checkpoints) + if stackTrace.len > 0: + formatter.testStackTrace = stackTrace + +method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) = + let time = epochTime() - formatter.testStartTime + let timeStr = time.formatFloat(ffDecimal, precision = 8) + try: + formatter.stream.writeLine("\t\t" % [ + xmlEscape(testResult.testName), timeStr]) + case testResult.status + of TestStatus.OK: + discard + of TestStatus.SKIPPED: + formatter.stream.writeLine("") + of TestStatus.FAILED: + let failureMsg = if formatter.testStackTrace.len > 0 and + formatter.testErrors.len > 0: + xmlEscape(formatter.testErrors[^1]) + elif formatter.testErrors.len > 0: + xmlEscape(formatter.testErrors[0]) + else: "The test failed without outputting an error" + + var errs = "" + if formatter.testErrors.len > 1: + var startIdx = if formatter.testStackTrace.len > 0: 0 else: 1 + var endIdx = if formatter.testStackTrace.len > 0: + formatter.testErrors.len - 2 + else: formatter.testErrors.len - 1 + + for errIdx in startIdx..endIdx: + if errs.len > 0: + errs.add("\n") + errs.add(xmlEscape(formatter.testErrors[errIdx])) + + if formatter.testStackTrace.len > 0: + formatter.stream.writeLine("\t\t\t$#" % [ + failureMsg, xmlEscape(formatter.testStackTrace)]) + if errs.len > 0: + formatter.stream.writeLine("\t\t\t$#" % errs) + else: + formatter.stream.writeLine("\t\t\t$#" % + [failureMsg, errs]) + + formatter.stream.writeLine("\t\t") + except CatchableError as exc: + echo "Cannot write JUnit: ", exc.msg + quit 1 + +method suiteEnded*(formatter: JUnitOutputFormatter) = + try: + formatter.stream.writeLine("\t") + except CatchableError as exc: + echo "Cannot write JUnit: ", exc.msg + quit 1 + +proc glob(matcher, filter: string): bool = + ## Globbing using a single `*`. Empty `filter` matches everything. + if filter.len == 0: + return true + + if not filter.contains('*'): + return matcher == filter + + let beforeAndAfter = filter.split('*', maxsplit=1) + if beforeAndAfter.len == 1: + # "foo*" + return matcher.startsWith(beforeAndAfter[0]) + + if matcher.len < filter.len - 1: + return false # "12345" should not match "123*345" + + return matcher.startsWith(beforeAndAfter[0]) and matcher.endsWith( + beforeAndAfter[1]) + +proc matchFilter(suiteName, testName, filter: string): bool = + if filter == "": + return true + if testName == filter: + # corner case for tests containing "::" in their name + return true + let suiteAndTestFilters = filter.split("::", maxsplit=1) + + if suiteAndTestFilters.len == 1: + # no suite specified + let testFilter = suiteAndTestFilters[0] + return glob(testName, testFilter) + + return glob(suiteName, suiteAndTestFilters[0]) and + glob(testName, suiteAndTestFilters[1]) + +when defined(testing): export matchFilter + +proc shouldRun(currentSuiteName, testName: string): bool = + ## Check if a test should be run by matching suiteName and testName against + ## test filters. + withLock testFiltersLock: + {.gcsafe.}: + if testsFilters.len == 0: + return true + + for f in testsFilters: + if matchFilter(currentSuiteName, testName, f): + return true + + return false + +proc ensureInitialized() = + withLock formattersLock: + {.gcsafe.}: + if formatters.len == 0: + formatters = @[OutputFormatter(defaultConsoleFormatter())] + + withLock testFiltersLock: + {.gcsafe.}: + if not disabledParamFiltering: + when declared(paramCount): + # Read tests to run from the command line. + for i in 1 .. paramCount(): + testsFilters.incl(paramStr(i)) + +proc suiteStarted(name: string) = + when paralleliseTests: + repeatableSync() # wait for any independent tests from the threadpool before starting the suite + withLock formattersLock: + {.gcsafe.}: + for formatter in formatters: + formatter.suiteStarted(name) + +proc suiteEnded() = + when paralleliseTests: + repeatableSync() # wait for a suite's tests from the threadpool before moving on to the next suite + withLock formattersLock: + {.gcsafe.}: + for formatter in formatters: + formatter.suiteEnded() + +proc testStarted(name: string) = + withLock formattersLock: + {.gcsafe.}: + for formatter in formatters: + if not formatter.isNil: + # Useless check that somehow prevents a method dispatch failure on macOS + formatter.testStarted(name) + +proc testEnded(testResult: TestResult) = + withLock formattersLock: + {.gcsafe.}: + for formatter in formatters: + when paralleliseTests: + withLock outputLock: + formatter.testEnded(testResult) + else: + formatter.testEnded(testResult) + +template suite*(name, body) {.dirty.} = + ## Declare a test suite identified by `name` with optional ``setup`` + ## and/or ``teardown`` section. + ## + ## A test suite is a series of one or more related tests sharing a + ## common fixture (``setup``, ``teardown``). The fixture is executed + ## for EACH test. + ## + ## .. code-block:: nim + ## suite "test suite for addition": + ## setup: + ## let result = 4 + ## + ## test "2 + 2 = 4": + ## check(2+2 == result) + ## + ## test "(2 + -2) != 4": + ## check(2 + -2 != result) + ## + ## # No teardown needed + ## + ## The suite will run the individual test cases in the order in which + ## they were listed. With default global settings the above code prints: + ## + ## .. code-block:: + ## + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 + bind formatters, ensureInitialized, suiteStarted, suiteEnded + + block: + template setup(setupBody: untyped) {.dirty, used.} = + var testSetupIMPLFlag {.used.} = true + template testSetupIMPL: untyped {.dirty.} = setupBody + + template teardown(teardownBody: untyped) {.dirty, used.} = + var testTeardownIMPLFlag {.used.} = true + template testTeardownIMPL: untyped {.dirty.} = teardownBody + + template suiteTeardown(suiteTeardownBody: untyped) {.dirty, used.} = + var testSuiteTeardownIMPLFlag {.used.} = true + template testSuiteTeardownIMPL: untyped {.dirty.} = suiteTeardownBody + + let testSuiteName {.used.} = name + + ensureInitialized() + try: + suiteStarted(name) + body + when declared(testSuiteTeardownIMPLFlag): + when paralleliseTests: + repeatableSync() + testSuiteTeardownIMPL() + finally: + suiteEnded() + +template exceptionTypeName(e: typed): string = $e.name + +template test*(name, body) = + ## Define a single test case identified by `name`. + ## + ## .. code-block:: nim + ## + ## test "roses are red": + ## let roses = "red" + ## check(roses == "red") + ## + ## The above code outputs: + ## + ## .. code-block:: + ## + ## [OK] roses are red + bind shouldRun, checkpoints, ensureInitialized, testEnded, exceptionTypeName + withLock formattersLock: + {.gcsafe.}: + bind formatters + + # `gensym` can't be in here because it's not a first-class pragma + when paralleliseTests: + # We use "fastcall" to get proper error messages about variable access that + # would make runTest() a closure - which we can't have in a spawned proc. + # "nimcall" doesn't work here, because of https://github.com/nim-lang/Nim/issues/8473 + {.pragma: testrunner, gcsafe, fastcall.} + else: + {.pragma: testrunner.} + + proc runTest(testSuiteName: string, testName: string): int {.gensym, testrunner.} = + ensureInitialized() + + checkpoints = @[] + var testStatusIMPL {.inject.} = TestStatus.OK + let testName {.inject.} = testName + + testStarted(testName) + try: + when declared(testSetupIMPLFlag): testSetupIMPL() + when declared(testTeardownIMPLFlag): + defer: testTeardownIMPL() + + body + + except Exception as e: # This will also catch Defect which may or may not work + let eTypeDesc = "[" & exceptionTypeName(e) & "]" + checkpoint("Unhandled exception: " & e.msg & " " & eTypeDesc) + if e == nil: # foreign + fail() + else: + var stackTrace {.inject.} = e.getStackTrace() + fail() + + finally: + if testStatusIMPL == TestStatus.FAILED: + programResult = 1 + let testResult = TestResult( + suiteName: testSuiteName, + testName: name, + status: testStatusIMPL + ) + testEnded(testResult) + checkpoints = @[] + + let optionalTestSuiteName = when declared(testSuiteName): testSuiteName else: "" + if shouldRun(optionalTestSuiteName, name): + when paralleliseTests: + flowVars.add(spawn runTest(optionalTestSuiteName, name)) + else: + discard runTest(optionalTestSuiteName, name) + +proc checkpoint*(msg: string) = + ## Set a checkpoint identified by `msg`. Upon test failure all + ## checkpoints encountered so far are printed out. Example: + ## + ## .. code-block:: nim + ## + ## checkpoint("Checkpoint A") + ## check((42, "the Answer to life and everything") == (1, "a")) + ## checkpoint("Checkpoint B") + ## + ## outputs "Checkpoint A" once it fails. + checkpoints.add(msg) + # TODO: add support for something like SCOPED_TRACE from Google Test + +template fail* = + ## Print out the checkpoints encountered so far and quit if ``abortOnError`` + ## is true. Otherwise, erase the checkpoints and indicate the test has + ## failed (change exit code and test status). This template is useful + ## for debugging, but is otherwise mostly used internally. Example: + ## + ## .. code-block:: nim + ## + ## checkpoint("Checkpoint A") + ## complicatedProcInThread() + ## fail() + ## + ## outputs "Checkpoint A" before quitting. + bind ensureInitialized + when declared(testStatusIMPL): + testStatusIMPL = TestStatus.FAILED + else: + programResult = 1 + + ensureInitialized() + + withLock formattersLock: + {.gcsafe.}: + for formatter in formatters: + when declared(stackTrace): + formatter.failureOccurred(checkpoints, stackTrace) + else: + formatter.failureOccurred(checkpoints, "") + + if abortOnError: quit(1) + + checkpoints = @[] + +template skip* = + ## Mark the test as skipped. Should be used directly + ## in case when it is not possible to perform test + ## for reasons depending on outer environment, + ## or certain application logic conditions or configurations. + ## The test code is still executed. + ## + ## .. code-block:: nim + ## + ## if not isGLContextCreated(): + ## skip() + bind checkpoints + + testStatusIMPL = TestStatus.SKIPPED + checkpoints = @[] + +{.pop.} # raises: [Defect] + +macro check*(conditions: untyped): untyped = + ## Verify if a statement or a list of statements is true. + ## A helpful error message and set checkpoints are printed out on + ## failure (if ``outputLevel`` is not ``PRINT_NONE``). + runnableExamples: + import std/strutils + + check("AKB48".toLowerAscii() == "akb48") + + let teams = {'A', 'K', 'B', '4', '8'} + + check: + "AKB48".toLowerAscii() == "akb48" + 'C' notin teams + + let checked = callsite()[1] + + template asgn(a: untyped, value: typed) = + var a = value # XXX: we need "var: var" here in order to + # preserve the semantics of var params + + template print(name: untyped, value: typed) = + when compiles(string($value)): + checkpoint(name & " was " & $value) + + proc inspectArgs(exp: NimNode): tuple[assigns, check, printOuts: NimNode] = + result.check = copyNimTree(exp) + result.assigns = newNimNode(nnkStmtList) + result.printOuts = newNimNode(nnkStmtList) + + var counter = 0 + + if exp[0].kind in {nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym} and + $exp[0] in ["not", "in", "notin", "==", "<=", + ">=", "<", ">", "!=", "is", "isnot"]: + + for i in 1 ..< exp.len: + if exp[i].kind notin nnkLiterals: + inc counter + let argStr = exp[i].toStrLit + let paramAst = exp[i] + if exp[i].kind == nnkIdent: + result.printOuts.add getAst(print(argStr, paramAst)) + if exp[i].kind in nnkCallKinds + {nnkDotExpr, nnkBracketExpr, nnkPar} and + (exp[i].typeKind notin {ntyTypeDesc} or $exp[0] notin ["is", "isnot"]): + let callVar = newIdentNode(":c" & $counter) + result.assigns.add getAst(asgn(callVar, paramAst)) + result.check[i] = callVar + result.printOuts.add getAst(print(argStr, callVar)) + if exp[i].kind == nnkExprEqExpr: + # ExprEqExpr + # Ident "v" + # IntLit 2 + result.check[i] = exp[i][1] + if exp[i].typeKind notin {ntyTypeDesc}: + let arg = newIdentNode(":p" & $counter) + result.assigns.add getAst(asgn(arg, paramAst)) + result.printOuts.add getAst(print(argStr, arg)) + if exp[i].kind != nnkExprEqExpr: + result.check[i] = arg + else: + result.check[i][1] = arg + + case checked.kind + of nnkCallKinds: + + let (assigns, check, printOuts) = inspectArgs(checked) + let lineinfo = newStrLitNode(checked.lineInfo) + let callLit = checked.toStrLit + result = quote do: + block: + `assigns` + if not `check`: + checkpoint(`lineinfo` & ": Check failed: " & `callLit`) + `printOuts` + fail() + + of nnkStmtList: + result = newNimNode(nnkStmtList) + for node in checked: + if node.kind != nnkCommentStmt: + result.add(newCall(newIdentNode("check"), node)) + + else: + let lineinfo = newStrLitNode(checked.lineInfo) + let callLit = checked.toStrLit + + result = quote do: + if not `checked`: + checkpoint(`lineinfo` & ": Check failed: " & `callLit`) + fail() + +template require*(conditions: untyped) = + ## Same as `check` except any failed test causes the program to quit + ## immediately. Any teardown statements are not executed and the failed + ## test output is not generated. + let savedAbortOnError = abortOnError + block: + abortOnError = true + check conditions + abortOnError = savedAbortOnError + +macro expect*(exceptions: varargs[typed], body: untyped): untyped = + ## Test if `body` raises an exception found in the passed `exceptions`. + ## The test passes if the raised exception is part of the acceptable + ## exceptions. Otherwise, it fails. + runnableExamples: + import std/[math, random, strutils] + proc defectiveRobot() = + randomize() + case rand(1..4) + of 1: raise newException(OSError, "CANNOT COMPUTE!") + of 2: discard parseInt("Hello World!") + of 3: raise newException(IOError, "I can't do that Dave.") + else: assert 2 + 2 == 5 + + expect IOError, OSError, ValueError, AssertionDefect: + defectiveRobot() + + template expectBody(errorTypes, lineInfoLit, body): NimNode {.dirty.} = + try: + body + checkpoint(lineInfoLit & ": Expect Failed, no exception was thrown.") + fail() + except errorTypes: + discard + except: + checkpoint(lineInfoLit & ": Expect Failed, unexpected exception was thrown.") + fail() + + var errorTypes = newNimNode(nnkBracket) + for exp in exceptions: + errorTypes.add(exp) + + result = getAst(expectBody(errorTypes, errorTypes.lineInfo, body)) + +proc disableParamFiltering* = + ## disables filtering tests with the command line params + disabledParamFiltering = true diff --git a/vendor/nim-unittest2/unittest2.nimble b/vendor/nim-unittest2/unittest2.nimble new file mode 100644 index 000000000..d0fa48a83 --- /dev/null +++ b/vendor/nim-unittest2/unittest2.nimble @@ -0,0 +1,8 @@ +mode = ScriptMode.Verbose + +version = "0.0.2" +author = "Ștefan Talpalaru" +description = "unittest fork with support for parallel test execution" +license = "MIT" +requires "nim >= 0.19.4" + diff --git a/waku/v1/node/wakunode1.nim b/waku/v1/node/wakunode1.nim index a98163f3b..3eeaa6b30 100644 --- a/waku/v1/node/wakunode1.nim +++ b/waku/v1/node/wakunode1.nim @@ -83,7 +83,9 @@ proc run(config: WakuNodeConf, rng: ref BrHmacDrbgContext) = if config.logAccounting: - proc logPeerAccounting(udata: pointer) {.closure, gcsafe.} = + # https://github.com/nim-lang/Nim/issues/17369 + var logPeerAccounting: proc(udata: pointer) {.gcsafe, raises: [Defect].} + logPeerAccounting = proc(udata: pointer) = {.gcsafe.}: for peer in node.peerPool.peers: let @@ -105,7 +107,9 @@ proc run(config: WakuNodeConf, rng: ref BrHmacDrbgContext) = metrics.startHttpServer($address, Port(port)) if config.logMetrics: - proc logMetrics(udata: pointer) {.closure, gcsafe.} = + # https://github.com/nim-lang/Nim/issues/17369 + var logMetrics: proc(udata: pointer) {.gcsafe, raises: [Defect].} + logMetrics = proc(udata: pointer) = {.gcsafe.}: let connectedPeers = connected_peers diff --git a/waku/v2/node/wakunode2.nim b/waku/v2/node/wakunode2.nim index 009021eb7..228bf17b2 100644 --- a/waku/v2/node/wakunode2.nim +++ b/waku/v2/node/wakunode2.nim @@ -517,7 +517,9 @@ when isMainModule: metrics.startHttpServer($serverIp, serverPort) proc startMetricsLog() = - proc logMetrics(udata: pointer) {.closure, gcsafe.} = + # https://github.com/nim-lang/Nim/issues/17369 + var logMetrics: proc(udata: pointer) {.gcsafe, raises: [Defect].} + logMetrics = proc(udata: pointer) = {.gcsafe.}: # TODO: libp2p_pubsub_peers is not public, so we need to make this either # public in libp2p or do our own peer counting after all. @@ -525,7 +527,10 @@ when isMainModule: totalMessages = 0.float64 for key in waku_node_messages.metrics.keys(): - totalMessages = totalMessages + waku_node_messages.value(key) + try: + totalMessages = totalMessages + waku_node_messages.value(key) + except KeyError: + discard info "Node metrics", totalMessages discard setTimer(Moment.fromNow(2.seconds), logMetrics)