Bump chronos and fix exception tracking issues (#436)

* Bump nim-chronos and fix exception tracking issues

* Bump other Nim submodules to latest

* Fix repeatMessage properly through proc type fix in nim-eth

Also add and use unittest2 through testutils to avoid extra
annotations.

.gitmodules

@@ -110,3 +110,13 @@
 	path = vendor/rln
 	url = https://github.com/kilic/rln
 	branch = full-node
+[submodule "vendor/nim-testutils"]
+	path = vendor/nim-testutils
+	url = https://github.com/status-im/nim-testutils.git
+	ignore = untracked
+	branch = master
+[submodule "vendor/nim-unittest2"]
+	path = vendor/nim-unittest2
+	url = https://github.com/status-im/nim-unittest2.git
+	ignore = untracked
+	branch = master

examples/v1/example.nim

@@ -10,99 +10,103 @@ import
 
 const clientId = "Waku example v1"
 
-let
-  # Load the cli configuration from `config_example.nim`.
-  config = WakuNodeConf.load()
-  # Seed the rng.
-  rng = keys.newRng()
+proc run(config: WakuNodeConf, rng: ref BrHmacDrbgContext) =
   # Set up the address according to NAT information.
-  (ipExt, tcpPortExt, udpPortExt) = setupNat(config.nat, clientId,
+  let (ipExt, tcpPortExt, udpPortExt) = setupNat(config.nat, clientId,
     Port(config.tcpPort + config.portsShift),
     Port(config.udpPort + config.portsShift))
   # TODO: EthereumNode should have a better split of binding address and
   # external address. Also, can't have different ports as it stands now.
-  address = if ipExt.isNone():
-              Address(ip: parseIpAddress("0.0.0.0"),
-                tcpPort: Port(config.tcpPort + config.portsShift),
-                udpPort: Port(config.udpPort + config.portsShift))
-            else:
-              Address(ip: ipExt.get(),
-                tcpPort: Port(config.tcpPort + config.portsShift),
-                udpPort: Port(config.udpPort + config.portsShift))
+  let address = if ipExt.isNone():
+                 Address(ip: parseIpAddress("0.0.0.0"),
+                    tcpPort: Port(config.tcpPort + config.portsShift),
+                    udpPort: Port(config.udpPort + config.portsShift))
+                else:
+                  Address(ip: ipExt.get(),
+                    tcpPort: Port(config.tcpPort + config.portsShift),
+                    udpPort: Port(config.udpPort + config.portsShift))
 
-# Create Ethereum Node
-var node = newEthereumNode(config.nodekey, # Node identifier
-  address, # Address reachable for incoming requests
-  NetworkId(1), # Network Id, only applicable for ETH protocol
-  nil, # Database, not required for Waku
-  clientId, # Client id string
-  addAllCapabilities = false, # Disable default all RLPx capabilities
-  rng = rng)
+  # Create Ethereum Node
+  var node = newEthereumNode(config.nodekey, # Node identifier
+    address, # Address reachable for incoming requests
+    NetworkId(1), # Network Id, only applicable for ETH protocol
+    nil, # Database, not required for Waku
+    clientId, # Client id string
+    addAllCapabilities = false, # Disable default all RLPx capabilities
+    rng = rng)
 
-node.addCapability Waku # Enable only the Waku protocol.
+  node.addCapability Waku # Enable only the Waku protocol.
 
-# Set up the Waku configuration.
-let wakuConfig = WakuConfig(powRequirement: 0.002,
-  bloom: some(fullBloom()), # Full bloom filter
-  isLightNode: false, # Full node
-  maxMsgSize: waku_protocol.defaultMaxMsgSize,
-  topics: none(seq[waku_protocol.Topic]) # empty topic interest
-  )
-node.configureWaku(wakuConfig)
+  # Set up the Waku configuration.
+  let wakuConfig = WakuConfig(powRequirement: 0.002,
+    bloom: some(fullBloom()), # Full bloom filter
+    isLightNode: false, # Full node
+    maxMsgSize: waku_protocol.defaultMaxMsgSize,
+    topics: none(seq[waku_protocol.Topic]) # empty topic interest
+    )
+  node.configureWaku(wakuConfig)
 
-# Optionally direct connect to a set of nodes.
-if config.staticnodes.len > 0:
-  connectToNodes(node, config.staticnodes)
+  # Optionally direct connect to a set of nodes.
+  if config.staticnodes.len > 0:
+    connectToNodes(node, config.staticnodes)
 
-# Connect to the network, which will make the node start listening and/or
-# connect to bootnodes, and/or start discovery.
-# This will block until first connection is made, which in this case can only
-# happen if we directly connect to nodes (step above) or if an incoming
-# connection occurs, which is why we use a callback to exit on errors instead of
-# using `await`.
-# TODO: This looks a bit awkward and the API should perhaps be altered here.
-let connectedFut = node.connectToNetwork(@[],
-  true, # Enable listening
-  false # Disable discovery (only discovery v4 is currently supported)
-  )
-connectedFut.callback = proc(data: pointer) {.gcsafe.} =
-  {.gcsafe.}:
-    if connectedFut.failed:
-      fatal "connectToNetwork failed", msg = connectedFut.readError.msg
-      quit(1)
+  # Connect to the network, which will make the node start listening and/or
+  # connect to bootnodes, and/or start discovery.
+  # This will block until first connection is made, which in this case can only
+  # happen if we directly connect to nodes (step above) or if an incoming
+  # connection occurs, which is why we use a callback to exit on errors instead of
+  # using `await`.
+  # TODO: This looks a bit awkward and the API should perhaps be altered here.
+  let connectedFut = node.connectToNetwork(@[],
+    true, # Enable listening
+    false # Disable discovery (only discovery v4 is currently supported)
+    )
+  connectedFut.callback = proc(data: pointer) {.gcsafe.} =
+    {.gcsafe.}:
+      if connectedFut.failed:
+        fatal "connectToNetwork failed", msg = connectedFut.readError.msg
+        quit(1)
 
-# Using a hardcoded symmetric key for encryption of the payload for the sake of
-# simplicity.
-var symKey: SymKey
-symKey[31] = 1
-# Asymmetric keypair to sign the payload.
-let signKeyPair = KeyPair.random(rng[])
+  # Using a hardcoded symmetric key for encryption of the payload for the sake of
+  # simplicity.
+  var symKey: SymKey
+  symKey[31] = 1
+  # Asymmetric keypair to sign the payload.
+  let signKeyPair = KeyPair.random(rng[])
 
-# Code to be executed on receival of a message on filter.
-proc handler(msg: ReceivedMessage) =
-  if msg.decoded.src.isSome():
-    echo "Received message from ", $msg.decoded.src.get(), ": ",
-      string.fromBytes(msg.decoded.payload)
+  # Code to be executed on receival of a message on filter.
+  proc handler(msg: ReceivedMessage) =
+    if msg.decoded.src.isSome():
+      echo "Received message from ", $msg.decoded.src.get(), ": ",
+        string.fromBytes(msg.decoded.payload)
 
-# Create and subscribe filter with above handler.
-let
-  topic = [byte 0, 0, 0, 0]
-  filter = initFilter(symKey = some(symKey), topics = @[topic])
-discard node.subscribeFilter(filter, handler)
+  # Create and subscribe filter with above handler.
+  let
+    topic = [byte 0, 0, 0, 0]
+    filter = initFilter(symKey = some(symKey), topics = @[topic])
+  discard node.subscribeFilter(filter, handler)
 
-# Repeat the posting of a message every 5 seconds.
-proc repeatMessage(udata: pointer) {.gcsafe.} =
-  {.gcsafe.}:
-    # Post a waku message on the network, encrypted with provided symmetric key,
-    # signed with asymmetric key, on topic and with ttl of 30 seconds.
-    let posted = node.postMessage(
-      symKey = some(symKey), src = some(signKeyPair.seckey),
-      ttl = 30, topic = topic, payload = @[byte 0x48, 0x65, 0x6C, 0x6C, 0x6F])
+  # Repeat the posting of a message every 5 seconds.
+  # https://github.com/nim-lang/Nim/issues/17369
+  var repeatMessage: proc(udata: pointer) {.gcsafe, raises: [Defect].}
+  repeatMessage = proc(udata: pointer) =
+    {.gcsafe.}:
+      # Post a waku message on the network, encrypted with provided symmetric key,
+      # signed with asymmetric key, on topic and with ttl of 30 seconds.
+      let posted = node.postMessage(
+        symKey = some(symKey), src = some(signKeyPair.seckey),
+        ttl = 30, topic = topic, payload = @[byte 0x48, 0x65, 0x6C, 0x6C, 0x6F])
 
-    if posted: echo "Posted message as ", $signKeyPair.pubkey
-    else: echo "Posting message failed."
+      if posted: echo "Posted message as ", $signKeyPair.pubkey
+      else: echo "Posting message failed."
 
+    discard setTimer(Moment.fromNow(5.seconds), repeatMessage)
   discard setTimer(Moment.fromNow(5.seconds), repeatMessage)
-discard setTimer(Moment.fromNow(5.seconds), repeatMessage)
 
-runForever()
+  runForever()
+
+when isMainModule:
+  let
+    rng = keys.newRng()
+    conf = WakuNodeConf.load()
+  run(conf, rng)

tests/test_helpers.nim

@@ -1,5 +1,5 @@
 import
-  unittest, chronos, bearssl,
+  chronos, bearssl,
   eth/[keys, p2p]
 
 import libp2p/crypto/crypto
@@ -21,18 +21,6 @@ proc setupTestNode*(
   for capability in capabilities:
     result.addCapability capability
 
-template asyncTest*(name, body: untyped) =
-  test name:
-    proc scenario {.async.} = body
-    waitFor scenario()
-
-template procSuite*(name, body: untyped) =
-  proc suitePayload =
-    suite name:
-      body
-
-  suitePayload()
-
 # Copied from here: https://github.com/status-im/nim-libp2p/blob/d522537b19a532bc4af94fcd146f779c1f23bad0/tests/helpers.nim#L28
 type RngWrap = object
   rng: ref BrHmacDrbgContext

tests/v1/test_waku_bridge.nim

@@ -9,8 +9,8 @@
 {.used.}
 
 import
-  std/[sequtils, unittest, tables],
-  chronos, eth/p2p, eth/p2p/peer_pool,
+  std/[sequtils, tables],
+  chronos, testutils/unittests, eth/p2p, eth/p2p/peer_pool,
   eth/p2p/rlpx_protocols/whisper_protocol as whisper,
   ../../waku/v1/protocol/waku_protocol as waku,
   ../../waku/v1/protocol/waku_bridge,

tests/v1/test_waku_connect.nim

@@ -9,8 +9,8 @@
 {.used.}
 
 import
-  std/[sequtils, tables, unittest],
-  chronos, eth/[keys, p2p], eth/p2p/peer_pool,
+  std/[sequtils, tables],
+  chronos, testutils/unittests, eth/[keys, p2p], eth/p2p/peer_pool,
   ../../waku/v1/protocol/waku_protocol,
   ../test_helpers
 

tests/v1/test_waku_mail.nim

@@ -1,8 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, tables, sequtils, times],
-  chronos, eth/[p2p, async_utils], eth/p2p/peer_pool,
+  std/[tables, sequtils, times],
+  chronos, testutils/unittests, eth/[p2p, async_utils], eth/p2p/peer_pool,
   ../../waku/v1/protocol/[waku_protocol, waku_mail],
   ../test_helpers
 

tests/v2/test_jsonrpc_waku.nim

@@ -1,8 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, options, sets, tables, os, strutils, sequtils],
-  stew/shims/net as stewNet,
+  std/[options, sets, tables, os, strutils, sequtils],
+  testutils/unittests, stew/shims/net as stewNet,
   json_rpc/[rpcserver, rpcclient],
   eth/[keys, rlp], eth/common/eth_types,
   libp2p/[standard_setup, switch, multiaddress],

tests/v2/test_peer_manager.nim

@@ -1,8 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, options, sets, tables, sequtils],
-  stew/shims/net as stewNet,
+  std/[options, sets, tables, sequtils],
+  testutils/unittests, stew/shims/net as stewNet,
   json_rpc/[rpcserver, rpcclient],
   eth/[keys, rlp], eth/common/eth_types,
   libp2p/[standard_setup, switch, multiaddress],

tests/v2/test_waku_bridge.nim

@@ -1,7 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, strutils],
+  std/strutils,
+  testutils/unittests,
   chronicles, chronos, stew/shims/net as stewNet, stew/byteutils,
   libp2p/crypto/crypto,
   libp2p/crypto/secp,

tests/v2/test_waku_filter.nim

@@ -1,8 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, options, tables, sets],
-  chronos, chronicles,
+  std/[options, tables, sets],
+  testutils/unittests, chronos, chronicles,
   libp2p/switch,
   libp2p/protobuf/minprotobuf,
   libp2p/stream/[bufferstream, connection],

tests/v2/test_waku_pagination.nim

@@ -1,7 +1,7 @@
 {.used.}
 import
-  std/[unittest,algorithm,options],
-  nimcrypto/sha2,
+  std/[algorithm, options],
+  testutils/unittests, nimcrypto/sha2,
   ../../waku/v2/protocol/waku_store/waku_store,
   ../test_helpers
 

tests/v2/test_waku_payload.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import
-  std/unittest,
+  testutils/unittests,
   ../../waku/v2/protocol/waku_message,
   ../../waku/v2/node/waku_payload,
   ../test_helpers

tests/v2/test_waku_rln_relay.nim

@@ -1,17 +1,14 @@
 {.used.}
 
 import
-  chronos, chronicles, options, stint, unittest,
-  web3,
+  std/options,
+  testutils/unittests, chronos, chronicles, stint, web3,
   stew/byteutils, stew/shims/net as stewNet,
   libp2p/crypto/crypto,
   ../../waku/v2/protocol/waku_rln_relay/[rln, waku_rln_relay_utils],
   ../../waku/v2/node/wakunode2,
   ../test_helpers,
-  test_utils
-
-
-  
+  ./test_utils
 
 # the address of Ethereum client (ganache-cli for now)
 # TODO this address in hardcoded in the code, we may need to take it as input from the user

tests/v2/test_waku_store.nim

@@ -1,8 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, options, tables, sets],
-  chronos, chronicles,
+  std/[options, tables, sets],
+  testutils/unittests, chronos, chronicles,
   libp2p/switch,
   libp2p/protobuf/minprotobuf,
   libp2p/stream/[bufferstream, connection],

tests/v2/test_waku_swap.nim

@@ -1,7 +1,8 @@
 {.used.}
 
 import
-  std/[unittest, options, tables, sets],
+  std/[options, tables, sets],
+  testutils/unittests,
   chronos, chronicles, stew/shims/net as stewNet, stew/byteutils,
   libp2p/switch,
   libp2p/protobuf/minprotobuf,

tests/v2/test_wakunode.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import
-  std/unittest,
+  testutils/unittests,
   chronicles, chronos, stew/shims/net as stewNet, stew/byteutils,
   libp2p/crypto/crypto,
   libp2p/crypto/secp,

vendor/nim-chronicles

@@ -1 +1 @@
-Subproject commit b42899070a7daa5cf6f0843faf3d6d41659e9591
+Subproject commit 8b1419b4a37a3a8995a9a0a992b4705427056d98

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit 0b78606e4142affbdc0e0e94bc1b8c39a8705737
+Subproject commit c206d2bc191712e4e5f89ecd87df7ef014bbb484

vendor/nim-confutils

@@ -1 +1 @@
-Subproject commit cfa95661913b0ff8b1609e3954894f8ab31bbf3e
+Subproject commit f091a70a5bf95ec772c8b4d9978e81b8ae89af0c

vendor/nim-eth

@@ -1 +1 @@
-Subproject commit be5e088b21e06a85cac4826454412db8459ed4f1
+Subproject commit 8c27f291f535f261274be9f69de1216981ac93e4

vendor/nim-http-utils

@@ -1 +1 @@
-Subproject commit 422026688405940a1304d5cdaf68353d93263035
+Subproject commit 613ad40f00ab3d0ee839f9db9c4d25e5e0248dee

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit 7999d2522565d88499b9d7f99c4175a8eb3f2b41
+Subproject commit fe8a82ca76150b60a950d5aa4e5baa382441ada4

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit 3bf6acef2348162448b1b988a2ba06f0f2e10103
+Subproject commit df497660bcf7aa23005f22aa7daced15b5668e3a

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit 261de741b73601821cb6e749fc9b4092f1cc5377
+Subproject commit f9a1121b8733eb75e624ab59f8d79e707f15f76f

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 6bcb21184aeb96ce6c62e187a64d678b74609f1e
+Subproject commit ee78822e057ac5f39804ecb6ac1096734be13ef8

vendor/nim-testutils

@@ -0,0 +1 @@
+Subproject commit 6d76fc5e51ce6580289e872558a3b858775e310f

vendor/nim-unittest2

@@ -0,0 +1 @@
+Subproject commit e788deab3d59ff8a4fe103aeb5d82d3d82fcac7d

waku/v1/node/wakunode1.nim

@@ -83,7 +83,9 @@ proc run(config: WakuNodeConf, rng: ref BrHmacDrbgContext) =
 
 
   if config.logAccounting:
-    proc logPeerAccounting(udata: pointer) {.closure, gcsafe.} =
+    # https://github.com/nim-lang/Nim/issues/17369
+    var logPeerAccounting: proc(udata: pointer) {.gcsafe, raises: [Defect].}
+    logPeerAccounting = proc(udata: pointer) =
       {.gcsafe.}:
         for peer in node.peerPool.peers:
           let
@@ -105,7 +107,9 @@ proc run(config: WakuNodeConf, rng: ref BrHmacDrbgContext) =
       metrics.startHttpServer($address, Port(port))
 
   if config.logMetrics:
-    proc logMetrics(udata: pointer) {.closure, gcsafe.} =
+    # https://github.com/nim-lang/Nim/issues/17369
+    var logMetrics: proc(udata: pointer) {.gcsafe, raises: [Defect].}
+    logMetrics = proc(udata: pointer) =
       {.gcsafe.}:
         let
           connectedPeers = connected_peers

waku/v2/node/wakunode2.nim

@@ -517,7 +517,9 @@ when isMainModule:
       metrics.startHttpServer($serverIp, serverPort)
 
   proc startMetricsLog() =
-    proc logMetrics(udata: pointer) {.closure, gcsafe.} =
+    # https://github.com/nim-lang/Nim/issues/17369
+    var logMetrics: proc(udata: pointer) {.gcsafe, raises: [Defect].}
+    logMetrics = proc(udata: pointer) =
       {.gcsafe.}:
         # TODO: libp2p_pubsub_peers is not public, so we need to make this either
         # public in libp2p or do our own peer counting after all.
@@ -525,7 +527,10 @@ when isMainModule:
           totalMessages = 0.float64
 
         for key in waku_node_messages.metrics.keys():
-          totalMessages = totalMessages + waku_node_messages.value(key)
+          try:
+            totalMessages = totalMessages + waku_node_messages.value(key)
+          except KeyError:
+            discard
 
       info "Node metrics", totalMessages
       discard setTimer(Moment.fromNow(2.seconds), logMetrics)