chore(networkmonitor): use nim-presto + add timeout (#1389)

tools/networkmonitor/networkmonitor.nim

@@ -4,17 +4,18 @@ else:
   {.push raises: [].}
 
 import
-  std/[tables,strutils,times,sequtils,httpclient],
+  std/[tables,strutils,times,sequtils],
   chronicles,
   chronicles/topics_registry,
   chronos,
+  chronos/timer as ctime,
   confutils,
   eth/keys,
   eth/p2p/discoveryv5/enr,
   libp2p/crypto/crypto,
   metrics,
   metrics/chronos_httpserver,
-  presto/[route, server],
+  presto/[route, server, client],
   stew/shims/net
 
 import
@@ -42,7 +43,7 @@ proc setDiscoveredPeersCapabilities(
 proc setConnectedPeersMetrics(discoveredNodes: seq[Node],
                               node: WakuNode,
                               timeout: chronos.Duration,
-                              client: HttpClient,
+                              restClient: RestClientRef,
                               allPeers: CustomPeersTableRef) {.async.} =
 
   let currentTime = $getTime()
@@ -81,13 +82,19 @@ proc setConnectedPeersMetrics(discoveredNodes: seq[Node],
     allPeers[peerId].ip = ip
 
     # get more info the peers from its ip address
-    let location = await ipToLocation(ip, client)
+    var location: NodeLocation
-    if not location.isOk():
+    try:
+      # IP-API endpoints are now limited to 45 HTTP requests per minute
+      # TODO: As network grows, find a better way to now block the whole app
+      await sleepAsync(1400)
+      let response = await restClient.ipToLocation(ip)
+      location = response.data
+    except:
       warn "could not get location", ip=ip
       continue
 
-    allPeers[peerId].country = location.get().country
+    allPeers[peerId].country = location.country
-    allPeers[peerId].city = location.get().city
+    allPeers[peerId].city = location.city
           
     let peer = toRemotePeerInfo(discNode.record)
     if not peer.isOk():
@@ -145,11 +152,11 @@ proc setConnectedPeersMetrics(discoveredNodes: seq[Node],
 # crawls the network discovering peers and trying to connect to them
 # metrics are processed and exposed
 proc crawlNetwork(node: WakuNode,
+                  restClient: RestClientRef,
                   conf: NetworkMonitorConf,
                   allPeersRef: CustomPeersTableRef) {.async.} = 
 
   let crawlInterval = conf.refreshInterval * 1000
-  let client = newHttpClient()
   while true:
     # discover new random nodes
     let discoveredNodes = await node.wakuDiscv5.protocol.queryRandom()
@@ -163,7 +170,7 @@ proc crawlNetwork(node: WakuNode,
     # tries to connect to all newly discovered nodes
     # and populates metrics related to peers we could connect
     # note random discovered nodes can be already known
-    await setConnectedPeersMetrics(discoveredNodes, node, conf.timeout, client, allPeersRef)
+    await setConnectedPeersMetrics(discoveredNodes, node, conf.timeout, restClient, allPeersRef)
 
     let totalNodes = flatNodes.len
     let seenNodes = flatNodes.countIt(it.seen)
@@ -281,6 +288,15 @@ when isMainModule:
   let res = startRestApiServer(conf, allPeersInfo, msgPerContentTopic)
   if res.isErr():
     error "could not start rest api server", err=res.error
+    quit(1)
+
+  # create a rest client
+  let clientRest = RestClientRef.new(url="http://ip-api.com",
+                                     connectTimeout=ctime.seconds(2))
+  if clientRest.isErr():
+    error "could not start rest api client", err=res.error
+    quit(1)
+  let restClient = clientRest.get()
 
   # start waku node
   let nodeRes = initAndStartNode(conf)
@@ -297,6 +313,6 @@ when isMainModule:
 
   # spawn the routine that crawls the network
   # TODO: split into 3 routines (discovery, connections, ip2location)
-  asyncSpawn crawlNetwork(node, conf, allPeersInfo)
+  asyncSpawn crawlNetwork(node, restClient, conf, allPeersInfo)
   
   runForever()

tools/networkmonitor/networkmonitor_utils.nim

@@ -4,14 +4,16 @@ else:
   {.push raises: [].}
   
 import
-  std/[json,httpclient],
+  std/json,
+  stew/results,
+  stew/shims/net,
   chronicles,
   chronicles/topics_registry,
   chronos,
-  stew/results
+  presto/[client,common]
 
 type
-  NodeLocation = object
+  NodeLocation* = object
     country*: string
     city*: string
     lat*: string
@@ -24,30 +26,28 @@ proc flatten*[T](a: seq[seq[T]]): seq[T] =
     aFlat &= subseq
   return aFlat
 
-# using an external api retrieves some data associated with the ip
+proc decodeBytes*(t: typedesc[NodeLocation], value: openArray[byte],
-# TODO: use a cache
+                  contentType: Opt[ContentTypeData]): RestResult[NodeLocation] =
-# TODO: use nim-presto's HTTP asynchronous client
+  var res: string
-proc ipToLocation*(ip: string,
+  if len(value) > 0:
-                   client: Httpclient):
+    res = newString(len(value))
-                   Future[Result[NodeLocation, string]] {.async.} =
+    copyMem(addr res[0], unsafeAddr value[0], len(value))
-  # naive mechanism to avoid hitting the rate limit
-  # IP-API endpoints are now limited to 45 HTTP requests per minute
-  await sleepAsync(1400)
     try:
-    let content = client.getContent("http://ip-api.com/json/" & ip)
+      let jsonContent = parseJson(res)
-    let jsonContent = parseJson(content)
-
       if $jsonContent["status"].getStr() != "success":
         error "query failed", result=jsonContent
-      return err("query failed: " & $jsonContent)
+        return err("query failed")
-
       return ok(NodeLocation(
         country: jsonContent["country"].getStr(),
         city:    jsonContent["city"].getStr(),
-      lat:     jsonContent["lat"].getStr(),
+        lat:     $jsonContent["lat"].getFloat(),
-      long:    jsonContent["lon"].getStr(),
+        long:    $jsonContent["lon"].getFloat(),
         isp:     jsonContent["isp"].getStr()
       ))
     except:
-    error "failed to get the location for IP", ip=ip, error=getCurrentExceptionMsg()
+      return err("failed to get the location: " & getCurrentExceptionMsg())
-    return err("failed to get the location for IP '" & ip & "':" & getCurrentExceptionMsg())
+
+proc encodeString*(value: string): RestResult[string] =
+  ok(value)
+
+proc ipToLocation*(ip: string): RestResponse[NodeLocation] {.rest, endpoint: "json/{ip}", meth: MethodGet.}