REST server metrics and connections API endpoints. (#5193)

* Add new REST endpoints to monitor REST server connections and new chronos metrics.

* Bump head versions of chronos and presto.

* Bump chronos with regression fix.

* Remove outdated tests which was supposed to test pipeline mode.

* Disable pipeline mode in resttest.

* Update copyright year.

* Upgrade test_signing_node to start use AsyncProcess instead of std library's osproc.
Bump chronos to check graceful shutdown.

* Update AllTests.

* Bump chronos.

AllTests-mainnet.md

@@ -382,8 +382,6 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 ## Nimbus remote signer/signing test (web3signer)
 ```diff
 + Connection timeout test                                                                    OK
-+ Connections pool stress test                                                               OK
-+ Idle connection test                                                                       OK
 + Public keys enumeration (/api/v1/eth2/publicKeys) test                                     OK
 + Public keys reload (/reload) test                                                          OK
 + Signing BeaconBlock (getBlockSignature(bellatrix))                                         OK
@@ -401,7 +399,7 @@ OK: 4/4 Fail: 0/4 Skip: 0/4
 + Signing voluntary exit (getValidatorExitSignature())                                       OK
 + Waiting for signing node (/upcheck) test                                                   OK
 ```
-OK: 19/19 Fail: 0/19 Skip: 0/19
+OK: 17/17 Fail: 0/17 Skip: 0/17
 ## Old database versions [Preset: mainnet]
 ```diff
 + pre-1.1.0                                                                                  OK
@@ -700,4 +698,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 393/398 Fail: 0/398 Skip: 5/398
+OK: 391/396 Fail: 0/396 Skip: 5/396

beacon_chain/rpc/rest_nimbus_api.nim

@@ -7,8 +7,9 @@
 
 import
   std/[sequtils],
-  stew/results,
+  stew/[results, base10],
   chronicles,
+  chronos/apps/http/httpdebug,
   libp2p/[multiaddress, multicodec, peerstore],
   libp2p/protocols/pubsub/pubsubpeer,
   ./rest_utils,
@@ -61,10 +62,22 @@ type
     http_client_connections*: uint64
     http_client_requests*: uint64
     http_client_responses*: uint64
-    http_server_connections*: uint64
+    http_server_secure_connections*: uint64
+    http_server_unsecure_connections*: uint64
+    http_server_requests*: uint64
+    http_server_responses*: uint64
     http_body_readers*: uint64
     http_body_writers*: uint64
 
+  RestConnectionInfo* = object
+    handle*: string
+    connection_type*: string
+    connection_state*: string
+    remote_address*: string
+    local_address*: string
+    since_accept*: string
+    since_create*: string
+
   RestPubSubPeer* = object
     peerId*: PeerId
     score*: float64
@@ -279,30 +292,77 @@ proc installNimbusApiHandlers*(router: var RestRouter, node: BeaconNode) =
   router.api(MethodGet, "/nimbus/v1/debug/chronos/metrics") do (
     ) -> RestApiResponse:
 
-    template getCount(ttype: untyped, name: string): uint64 =
+    template getCount(name: string): uint64 =
-      let res = ttype(getTracker(name))
+      let res = getTrackerCounter(name)
-      if res.isNil(): 0'u64 else: uint64(res.opened - res.closed)
+      uint64(res.opened - res.closed)
 
     let res = RestChronosMetricsInfo(
-      tcp_transports: getCount(StreamTransportTracker, "stream.transport"),
+      tcp_transports: getCount(StreamTransportTrackerName),
-      udp_transports: getCount(DgramTransportTracker, "datagram.transport"),
+      udp_transports: getCount(DgramTransportTrackerName),
-      tcp_servers: getCount(StreamServerTracker, "stream.server"),
+      tcp_servers: getCount(StreamServerTrackerName),
-      stream_readers: getCount(AsyncStreamTracker,
+      stream_readers: getCount(AsyncStreamReaderTrackerName),
-                               AsyncStreamReaderTrackerName),
+      stream_writers: getCount(AsyncStreamWriterTrackerName),
-      stream_writers: getCount(AsyncStreamTracker,
+      http_client_connections: getCount(HttpClientConnectionTrackerName),
-                               AsyncStreamWriterTrackerName),
+      http_client_requests: getCount(HttpClientRequestTrackerName),
-      http_client_connections: getCount(HttpClientTracker,
+      http_client_responses: getCount(HttpClientResponseTrackerName),
-                                        HttpClientConnectionTrackerName),
+      http_server_secure_connections:
-      http_client_requests: getCount(HttpClientTracker,
+        getCount(HttpServerSecureConnectionTrackerName),
-                                     HttpClientRequestTrackerName),
+      http_server_unsecure_connections:
-      http_client_responses: getCount(HttpClientTracker,
+        getCount(HttpServerUnsecureConnectionTrackerName),
-                                      HttpClientResponseTrackerName),
+      http_server_requests: getCount(HttpServerRequestTrackerName),
-      http_server_connections: lenu64(node.restServer.server.connections),
+      http_server_responses: getCount(HttpServerResponseTrackerName),
-      http_body_readers: getCount(HttpBodyTracker, HttpBodyReaderTrackerName),
+      http_body_readers: getCount(HttpBodyReaderTrackerName),
-      http_body_writers: getCount(HttpBodyTracker, HttpBodyWriterTrackerName)
+      http_body_writers: getCount(HttpBodyWriterTrackerName)
     )
     return RestApiResponse.jsonResponse(res)
 
+  router.api(MethodGet, "/nimbus/v1/debug/chronos/restserver/connections") do (
+    ) -> RestApiResponse:
+    var res: seq[RestConnectionInfo]
+    for connection in node.restServer.server.getConnections():
+      let
+        connectionState =
+          case connection.connectionState
+          of httpdebug.ConnectionState.Accepted: "accepted"
+          of httpdebug.ConnectionState.Alive: "alive"
+          of httpdebug.ConnectionState.Closing: "closing"
+          of httpdebug.ConnectionState.Closed: "closed"
+        connectionType =
+          case connection.connectionType
+          of ConnectionType.Secure: "secure"
+          of ConnectionType.NonSecure: "non-secure"
+        localAddress =
+          if connection.localAddress.isNone():
+            "not available"
+          else:
+            $connection.localAddress.get()
+        remoteAddress =
+          if connection.remoteAddress.isNone():
+            "not available"
+          else:
+            $connection.remoteAddress.get()
+        handle = Base10.toString(uint64(connection.handle))
+        moment = Moment.now()
+        sinceAccept = $(moment - connection.acceptMoment)
+        sinceCreate =
+          if connection.createMoment.isSome():
+            $(moment - connection.createMoment.get())
+          else:
+            "not available"
+
+      res.add(
+        RestConnectionInfo(
+          handle: handle,
+          connection_state: connectionState,
+          connection_type: connectionType,
+          local_address: localAddress,
+          remote_address: remoteAddress,
+          since_accept: sinceAccept,
+          since_create: sinceCreate
+        )
+      )
+    return RestApiResponse.jsonResponse(res)
+
   router.api(MethodPost, "/nimbus/v1/validator/activity/{epoch}") do (
     epoch: Epoch, contentBody: Option[ContentBody]) -> RestApiResponse:
     let indexList =

ncli/resttest.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2021-2022 Status Research & Development GmbH
+# Copyright (c) 2021-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -993,9 +993,8 @@ proc workerLoop(address: TransportAddress, uri: Uri, worker: int,
       let testRes = await runTest(conn, uri, test.rule, worker, test.index)
       let caseRes = TestCaseResult(index: test.index, data: testRes)
       await outputQueue.addLast(caseRes)
-      if ResetConnection in testRes.flags:
+      await conn.closeWait()
-        await conn.closeWait()
+      conn = nil
-        conn = nil
       index = 0
     except CancelledError:
       if not(isNil(conn)):

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit 0035f4fa6692e85756aa192b4df84c21d3cacacb
+Subproject commit e04c042e8acfe0025c780de8a025aa4c4e042130

vendor/nim-presto

@@ -1 +1 @@
-Subproject commit 35652ed19ccbbf042e95941bc2f8bab39e3f6030
+Subproject commit 42552611cfe04ca7838243d0a0e88fb9a5e9701a